<commit_before>// A machine-local concurrent LSM tree based on B-tree implementation

use super::btree::level::*;
use super::btree::*;
use crate::client::AsyncClient;
use crate::ram::cell::Cell;
use crate::ram::schema::{Field, Schema};
use crate::ram::types::*;
use crossbeam_epoch::*;
use lightning::map::HashSet as LFHashSet;
use std::collections::HashSet as StdHashSet;
use std::mem;
use std::sync::atomic::Ordering::{Acquire, Release};
use std::sync::Arc;

pub const LSM_TREE_SCHEMA_NAME: &'static str = "NEB_LSM_TREE";
pub const LSM_TREE_LEVELS_NAME: &'static str = "levels";
pub const LSM_TREE_MIGRATION_NAME: &'static str = "migration";
pub const LAST_LEVEL_MULT_FACTOR: usize = 2;

type LevelTrees = [Box<dyn LevelTree>; NUM_LEVELS];
type LevelCusors = [Box<dyn Cursor>; NUM_LEVELS + 2]; // 2 for mem and trans mem
pub type DeletionSet = LFHashSet<EntryKey>;

lazy_static! {
    pub static ref LSM_TREE_SCHEMA_ID: u32 = key_hash(LSM_TREE_SCHEMA_NAME) as u32;
    pub static ref LSM_TREE_LEVELS_HASH: u64 = key_hash(LSM_TREE_LEVELS_NAME);
    pub static ref LSM_TREE_MIGRATION_HASH: u64 = key_hash(LSM_TREE_MIGRATION_NAME);
    pub static ref LSM_TREE_SCHEMA: Schema = lsm_treee_schema();
}

pub struct LSMTree {
    pub mem_tree: Atomic<Box<dyn LevelTree>>,
    pub trans_mem_tree: Atomic<Box<dyn LevelTree>>,
    pub disk_trees: LevelTrees,
    pub deletion: Arc<DeletionSet>,
}

impl LSMTree {
    pub async fn create(neb_client: &Arc<AsyncClient>, id: &Id) -> Self {
        let deletion_ref = Arc::new(LFHashSet::with_capacity(16));
        let tree_m = LevelMTree::new(&deletion_ref);
        let tree_0 = Level0Tree::new(&deletion_ref);
        let tree_1 = Level1Tree::new(&deletion_ref);
        tree_0.persist_root(neb_client).await;
        tree_1.persist_root(neb_client).await;
        let level_ids = vec![tree_m.head_id(), tree_1.head_id()];
        let lsm_tree_cell = lsm_tree_cell(&level_ids, id, None);
        neb_client.write_cell(lsm_tree_cell).await.unwrap().unwrap();
        Self {
            mem_tree: Atomic::new(box tree_m),
            trans_mem_tree: Atomic::null(),
            disk_trees: [box tree_0, box tree_1],
            deletion: deletion_ref,
        }
    }

    pub async fn recover(neb_client: &Arc<AsyncClient>, lsm_tree_id: &Id) -> Self {
        info!("Recovering LSM tree {:?}", lsm_tree_id);
        let deletion_ref = Arc::new(LFHashSet::with_capacity(16));
        let cell = neb_client.read_cell(*lsm_tree_id).await.unwrap().unwrap();
        let trees = &cell.data[*LSM_TREE_LEVELS_HASH];
        let trees_0_val = &trees[0usize];
        let trees_1_val = &trees[1usize];
        info!("Record shows trees {:?}", trees);
        let tree_0 =
            Level0Tree::from_head_id(trees_0_val.Id().unwrap(), neb_client, &deletion_ref).await;
        let tree_1 =
            Level1Tree::from_head_id(trees_1_val.Id().unwrap(), neb_client, &deletion_ref).await;
        Self {
            mem_tree: Atomic::new(box LevelMTree::new(&deletion_ref)),
            trans_mem_tree: Atomic::null(),
            disk_trees: [box tree_0, box tree_1],
            deletion: deletion_ref,
        }
    }

    pub fn insert(&self, entry: &EntryKey) -> bool {
        let guard = crossbeam_epoch::pin();
        let mem_tree_ptr = self.mem_tree.load(Acquire, &guard);
        let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
        mem_tree.insert_into(entry)
    }

    pub fn delete(&self, entry: &EntryKey) -> bool {
        if let Some(k) = self.seek(entry, Ordering::Forward).current() {
            if k == entry {
                self.deletion.insert(entry);
                return true;
            }
        }
        return false;
    }

    pub fn seek(&self, entry: &EntryKey, ordering: Ordering) -> LSMTreeCursor {
        LSMTreeCursor::new(entry, self, ordering)
    }

    pub async fn merge_levels(&self) -> bool {
        let mut merged = false;
        let mut deleted = StdHashSet::new();
        {
            let guard = crossbeam_epoch::pin();
            let mem_tree_ptr = self.mem_tree.load(Acquire, &guard);
            let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
            if mem_tree.oversized() {
                info!("Memory tree oversized, moving everything to disk tree");
                // Memory tree oversized
                // Will construct a new tree and swap the old one to trans_mem_tree for query and move all
                // keys in old tree to disk tree level 0
                let new_mem_tree: Box<dyn LevelTree> = box LevelMTree::new(&self.deletion);
                let new_mem_tree_ptr = Owned::new(new_mem_tree).into_shared(&guard);
                self.trans_mem_tree.store(mem_tree_ptr, Release);
                self.mem_tree.store(new_mem_tree_ptr, Release);
                info!("Starting moving memory tree...");
                mem_tree.merge_all_to(0, &*self.disk_trees[0], &mut deleted, false);
                info!("Memory tree merge completed, reset trans tree and destory old tree");
                self.trans_mem_tree.store(Shared::null(), Release);
                unsafe {
                    guard.defer_destroy(mem_tree_ptr);
                }
                merged = true;
                info!("Memory tree merging completed");
            } else {
                trace!("Memory tree not oversized");
            }
        }
        storage::wait_until_updated().await;
        for i in 0..self.disk_trees.len() - 1 {
            let level = i + 1;
            if self.disk_trees[i].oversized() {
                info!("Level {} tree oversized, merging", level);
                self.disk_trees[i].merge_to(level, &*self.disk_trees[i + 1], &mut deleted, true);
                storage::wait_until_updated().await;
                info!("Level {} merge completed", level);
                merged = true;
            } else {
                trace!("Level {} tree not oversized", level);
            }
        }
        if !deleted.is_empty() {
            debug!(
                "LSM merges collected {} deleted keys, remove them from deletion set",
                deleted.len()
            );
            for dk in deleted {
                self.deletion.remove(&dk);
            }
        }
        merged
    }

    pub fn oversized(&self) -> bool {
        self.last_level_tree().count() > self.ideal_capacity()
    }

    pub fn mid_key(&self) -> Option<EntryKey> {
        self.last_level_tree().mid_key()
    }

    pub async fn mark_migration(&self, id: &Id, migration: Option<Id>, client: &Arc<AsyncClient>) {
        let lsm_tree_cell = lsm_tree_cell(
            &self.disk_trees.iter().map(|tree| tree.head_id()).collect(),
            id,
            migration,
        );
        client.update_cell(lsm_tree_cell).await.unwrap().unwrap();
    }

    pub fn merge_keys(&self, keys: Vec<EntryKey>) {
        self.last_level_tree().merge_with_keys(keys);
    }

    pub fn ideal_capacity(&self) -> usize {
        self.last_level_tree().ideal_capacity() * LAST_LEVEL_MULT_FACTOR
    }

    pub fn count(&self) -> usize {
        // Only count keys in disk trees
        self.disk_trees.iter().map(|t| t.count()).sum()
    }

    fn last_level_tree(&self) -> &Box<dyn LevelTree> {
        self.disk_trees.last().unwrap()
    }
}

pub struct LSMTreeCursor {
    cursors: LevelCusors,
    current: Option<(usize, EntryKey)>,
    ordering: Ordering,
    deletion: Arc<DeletionSet>,
}

impl LSMTreeCursor {
    fn new(key: &EntryKey, lsm_tree: &LSMTree, ordering: Ordering) -> Self {
        let guard = crossbeam_epoch::pin();
        let disk_trees = &lsm_tree.disk_trees;
        let mem_tree_ptr = lsm_tree.mem_tree.load(Acquire, &guard);
        let trans_mem_tree_ptr = lsm_tree.trans_mem_tree.load(Acquire, &guard);
        let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
        let mut cursors = [
            mem_tree.seek_for(key, ordering),
            disk_trees[0].seek_for(key, ordering),
            disk_trees[1].seek_for(key, ordering),
            box DummyCursor, // for trans mem
        ];
        if !trans_mem_tree_ptr.is_null() && trans_mem_tree_ptr != mem_tree_ptr {
            let trans_mem_tree = unsafe { trans_mem_tree_ptr.as_ref().unwrap() };
            *cursors.last_mut().unwrap() = trans_mem_tree.seek_for(key, ordering);
        }
        let current = Self::leading_tree_key(&cursors, ordering);
        let deletion = lsm_tree.deletion.clone();
        Self {
            cursors,
            current,
            ordering,
            deletion,
        }
    }

    fn leading_tree_key(cursors: &LevelCusors, ordering: Ordering) -> Option<(usize, EntryKey)> {
        match ordering {
            Ordering::Forward => cursors
                .iter()
                .enumerate()
                .filter_map(|(i, c)| c.current().map(|c| (i, c)))
                .min_by(|(_, x), (_, y)| x.cmp(y))
                .map(|(i, k)| (i, k.clone())),
            Ordering::Backward => cursors
                .iter()
                .enumerate()
                .filter_map(|(i, c)| c.current().map(|c| (i, c)))
                .max_by(|(_, x), (_, y)| x.cmp(y))
                .map(|(i, k)| (i, k.clone())),
        }
    }
}

impl Cursor for LSMTreeCursor {
    fn current(&self) -> Option<&EntryKey> {
        self.current.as_ref().map(|(_, k)| k)
    }
    fn next(&mut self) -> Option<EntryKey> {
        loop {
            if let Some((tree_id, _)) = self.current {
                self.cursors[tree_id].next();
                let mut other_entry = Self::leading_tree_key(&self.cursors, self.ordering);
                mem::swap(&mut other_entry, &mut self.current);
                let res = other_entry.map(|(_, key)| key);
                if let Some(k) = &res {
                    if self.deletion.contains(k) {
                        // Skip keys in deletion set
                        continue;
                    }
                }
                return res;
            } else {
                return None;
            }
        }
    }
}

fn lsm_treee_schema() -> Schema {
    Schema {
        id: *LSM_TREE_SCHEMA_ID,
        name: String::from(LSM_TREE_SCHEMA_NAME),
        key_field: None,
        str_key_field: None,
        is_dynamic: false,
        fields: Field::new(
            "*",
            0,
            false,
            false,
            Some(vec![
                Field::new(
                    LSM_TREE_LEVELS_NAME,
                    type_id_of(Type::Id),
                    false,
                    true,
                    None,
                    vec![],
                ),
                Field::new(
                    LSM_TREE_MIGRATION_NAME,
                    type_id_of(Type::Id),
                    true,
                    false,
                    None,
                    vec![],
                ),
            ]),
            vec![],
        ),
    }
}

fn lsm_tree_cell(level_ids: &Vec<Id>, id: &Id, migration: Option<Id>) -> Cell {
    let mut cell_map = Map::new();
    cell_map.insert_key_id(
        *LSM_TREE_LEVELS_HASH,
        Value::Array(level_ids.iter().map(|id| id.value()).collect()),
    );
    cell_map.insert_key_id(
        *LSM_TREE_MIGRATION_HASH,
        migration.map(|id| Value::Id(id)).unwrap_or(Value::Null),
    );
    Cell::new_with_id(*LSM_TREE_SCHEMA_ID, id, Value::Map(cell_map))
}

impl_btree_level!(LEVEL_M);
type LevelMTreeKeySlice = [EntryKey; LEVEL_M];
type LevelMTreePtrSlice = [NodeCellRef; LEVEL_M + 1];
type LevelMTree = BPlusTree<LevelMTreeKeySlice, LevelMTreePtrSlice>;

impl_btree_level!(LEVEL_0);
type Level0TreeKeySlice = [EntryKey; LEVEL_0];
type Level0TreePtrSlice = [NodeCellRef; LEVEL_0 + 1];
type Level0Tree = BPlusTree<Level0TreeKeySlice, Level0TreePtrSlice>;

impl_btree_level!(LEVEL_1);
type Level1TreeKeySlice = [EntryKey; LEVEL_1];
type Level1TreePtrSlice = [NodeCellRef; LEVEL_1 + 1];
type Level1Tree = BPlusTree<Level1TreeKeySlice, Level1TreePtrSlice>;

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

struct DummyCursor;

impl Cursor for DummyCursor {
    fn next(&mut self) -> Option<EntryKey> {
        None
    }
    fn current(&self) -> Option<&EntryKey> {
        None
    }
}
<commit_msg>log recovering on trees<commit_after>// A machine-local concurrent LSM tree based on B-tree implementation

use super::btree::level::*;
use super::btree::*;
use crate::client::AsyncClient;
use crate::ram::cell::Cell;
use crate::ram::schema::{Field, Schema};
use crate::ram::types::*;
use crossbeam_epoch::*;
use lightning::map::HashSet as LFHashSet;
use std::collections::HashSet as StdHashSet;
use std::mem;
use std::sync::atomic::Ordering::{Acquire, Release};
use std::sync::Arc;

pub const LSM_TREE_SCHEMA_NAME: &'static str = "NEB_LSM_TREE";
pub const LSM_TREE_LEVELS_NAME: &'static str = "levels";
pub const LSM_TREE_MIGRATION_NAME: &'static str = "migration";
pub const LAST_LEVEL_MULT_FACTOR: usize = 2;

type LevelTrees = [Box<dyn LevelTree>; NUM_LEVELS];
type LevelCusors = [Box<dyn Cursor>; NUM_LEVELS + 2]; // 2 for mem and trans mem
pub type DeletionSet = LFHashSet<EntryKey>;

lazy_static! {
    pub static ref LSM_TREE_SCHEMA_ID: u32 = key_hash(LSM_TREE_SCHEMA_NAME) as u32;
    pub static ref LSM_TREE_LEVELS_HASH: u64 = key_hash(LSM_TREE_LEVELS_NAME);
    pub static ref LSM_TREE_MIGRATION_HASH: u64 = key_hash(LSM_TREE_MIGRATION_NAME);
    pub static ref LSM_TREE_SCHEMA: Schema = lsm_treee_schema();
}

pub struct LSMTree {
    pub mem_tree: Atomic<Box<dyn LevelTree>>,
    pub trans_mem_tree: Atomic<Box<dyn LevelTree>>,
    pub disk_trees: LevelTrees,
    pub deletion: Arc<DeletionSet>,
}

impl LSMTree {
    pub async fn create(neb_client: &Arc<AsyncClient>, id: &Id) -> Self {
        let deletion_ref = Arc::new(LFHashSet::with_capacity(16));
        let tree_m = LevelMTree::new(&deletion_ref);
        let tree_0 = Level0Tree::new(&deletion_ref);
        let tree_1 = Level1Tree::new(&deletion_ref);
        tree_0.persist_root(neb_client).await;
        tree_1.persist_root(neb_client).await;
        let level_ids = vec![tree_m.head_id(), tree_1.head_id()];
        let lsm_tree_cell = lsm_tree_cell(&level_ids, id, None);
        neb_client.write_cell(lsm_tree_cell).await.unwrap().unwrap();
        Self {
            mem_tree: Atomic::new(box tree_m),
            trans_mem_tree: Atomic::null(),
            disk_trees: [box tree_0, box tree_1],
            deletion: deletion_ref,
        }
    }

    pub async fn recover(neb_client: &Arc<AsyncClient>, lsm_tree_id: &Id) -> Self {
        info!("Recovering LSM tree {:?}", lsm_tree_id);
        let deletion_ref = Arc::new(LFHashSet::with_capacity(16));
        let cell = neb_client.read_cell(*lsm_tree_id).await.unwrap().unwrap();
        let trees = &cell.data[*LSM_TREE_LEVELS_HASH].prim_array().unwrap().Id().unwrap();
        let tree_0_id = &trees[0usize];
        let tree_1_id = &trees[1usize];
        info!("Record shows trees {:?}", trees);
        debug!("Recovering level 0 tree {:?}", tree_0_id);
        let tree_0 =
            Level0Tree::from_head_id(tree_0_id, neb_client, &deletion_ref).await;
        debug!("Recovering level 1 tree {:?}", tree_1_id);
        let tree_1 =
            Level1Tree::from_head_id(tree_1_id, neb_client, &deletion_ref).await;
        Self {
            mem_tree: Atomic::new(box LevelMTree::new(&deletion_ref)),
            trans_mem_tree: Atomic::null(),
            disk_trees: [box tree_0, box tree_1],
            deletion: deletion_ref,
        }
    }

    pub fn insert(&self, entry: &EntryKey) -> bool {
        let guard = crossbeam_epoch::pin();
        let mem_tree_ptr = self.mem_tree.load(Acquire, &guard);
        let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
        mem_tree.insert_into(entry)
    }

    pub fn delete(&self, entry: &EntryKey) -> bool {
        if let Some(k) = self.seek(entry, Ordering::Forward).current() {
            if k == entry {
                self.deletion.insert(entry);
                return true;
            }
        }
        return false;
    }

    pub fn seek(&self, entry: &EntryKey, ordering: Ordering) -> LSMTreeCursor {
        LSMTreeCursor::new(entry, self, ordering)
    }

    pub async fn merge_levels(&self) -> bool {
        let mut merged = false;
        let mut deleted = StdHashSet::new();
        {
            let guard = crossbeam_epoch::pin();
            let mem_tree_ptr = self.mem_tree.load(Acquire, &guard);
            let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
            if mem_tree.oversized() {
                info!("Memory tree oversized, moving everything to disk tree");
                // Memory tree oversized
                // Will construct a new tree and swap the old one to trans_mem_tree for query and move all
                // keys in old tree to disk tree level 0
                let new_mem_tree: Box<dyn LevelTree> = box LevelMTree::new(&self.deletion);
                let new_mem_tree_ptr = Owned::new(new_mem_tree).into_shared(&guard);
                self.trans_mem_tree.store(mem_tree_ptr, Release);
                self.mem_tree.store(new_mem_tree_ptr, Release);
                info!("Starting moving memory tree...");
                mem_tree.merge_all_to(0, &*self.disk_trees[0], &mut deleted, false);
                info!("Memory tree merge completed, reset trans tree and destory old tree");
                self.trans_mem_tree.store(Shared::null(), Release);
                unsafe {
                    guard.defer_destroy(mem_tree_ptr);
                }
                merged = true;
                info!("Memory tree merging completed");
            } else {
                trace!("Memory tree not oversized");
            }
        }
        storage::wait_until_updated().await;
        for i in 0..self.disk_trees.len() - 1 {
            let level = i + 1;
            if self.disk_trees[i].oversized() {
                info!("Level {} tree oversized, merging", level);
                self.disk_trees[i].merge_to(level, &*self.disk_trees[i + 1], &mut deleted, true);
                storage::wait_until_updated().await;
                info!("Level {} merge completed", level);
                merged = true;
            } else {
                trace!("Level {} tree not oversized", level);
            }
        }
        if !deleted.is_empty() {
            debug!(
                "LSM merges collected {} deleted keys, remove them from deletion set",
                deleted.len()
            );
            for dk in deleted {
                self.deletion.remove(&dk);
            }
        }
        merged
    }

    pub fn oversized(&self) -> bool {
        self.last_level_tree().count() > self.ideal_capacity()
    }

    pub fn mid_key(&self) -> Option<EntryKey> {
        self.last_level_tree().mid_key()
    }

    pub async fn mark_migration(&self, id: &Id, migration: Option<Id>, client: &Arc<AsyncClient>) {
        let lsm_tree_cell = lsm_tree_cell(
            &self.disk_trees.iter().map(|tree| tree.head_id()).collect(),
            id,
            migration,
        );
        client.update_cell(lsm_tree_cell).await.unwrap().unwrap();
    }

    pub fn merge_keys(&self, keys: Vec<EntryKey>) {
        self.last_level_tree().merge_with_keys(keys);
    }

    pub fn ideal_capacity(&self) -> usize {
        self.last_level_tree().ideal_capacity() * LAST_LEVEL_MULT_FACTOR
    }

    pub fn count(&self) -> usize {
        // Only count keys in disk trees
        self.disk_trees.iter().map(|t| t.count()).sum()
    }

    fn last_level_tree(&self) -> &Box<dyn LevelTree> {
        self.disk_trees.last().unwrap()
    }
}

pub struct LSMTreeCursor {
    cursors: LevelCusors,
    current: Option<(usize, EntryKey)>,
    ordering: Ordering,
    deletion: Arc<DeletionSet>,
}

impl LSMTreeCursor {
    fn new(key: &EntryKey, lsm_tree: &LSMTree, ordering: Ordering) -> Self {
        let guard = crossbeam_epoch::pin();
        let disk_trees = &lsm_tree.disk_trees;
        let mem_tree_ptr = lsm_tree.mem_tree.load(Acquire, &guard);
        let trans_mem_tree_ptr = lsm_tree.trans_mem_tree.load(Acquire, &guard);
        let mem_tree = unsafe { mem_tree_ptr.as_ref().unwrap() };
        let mut cursors = [
            mem_tree.seek_for(key, ordering),
            disk_trees[0].seek_for(key, ordering),
            disk_trees[1].seek_for(key, ordering),
            box DummyCursor, // for trans mem
        ];
        if !trans_mem_tree_ptr.is_null() && trans_mem_tree_ptr != mem_tree_ptr {
            let trans_mem_tree = unsafe { trans_mem_tree_ptr.as_ref().unwrap() };
            *cursors.last_mut().unwrap() = trans_mem_tree.seek_for(key, ordering);
        }
        let current = Self::leading_tree_key(&cursors, ordering);
        let deletion = lsm_tree.deletion.clone();
        Self {
            cursors,
            current,
            ordering,
            deletion,
        }
    }

    fn leading_tree_key(cursors: &LevelCusors, ordering: Ordering) -> Option<(usize, EntryKey)> {
        match ordering {
            Ordering::Forward => cursors
                .iter()
                .enumerate()
                .filter_map(|(i, c)| c.current().map(|c| (i, c)))
                .min_by(|(_, x), (_, y)| x.cmp(y))
                .map(|(i, k)| (i, k.clone())),
            Ordering::Backward => cursors
                .iter()
                .enumerate()
                .filter_map(|(i, c)| c.current().map(|c| (i, c)))
                .max_by(|(_, x), (_, y)| x.cmp(y))
                .map(|(i, k)| (i, k.clone())),
        }
    }
}

impl Cursor for LSMTreeCursor {
    fn current(&self) -> Option<&EntryKey> {
        self.current.as_ref().map(|(_, k)| k)
    }
    fn next(&mut self) -> Option<EntryKey> {
        loop {
            if let Some((tree_id, _)) = self.current {
                self.cursors[tree_id].next();
                let mut other_entry = Self::leading_tree_key(&self.cursors, self.ordering);
                mem::swap(&mut other_entry, &mut self.current);
                let res = other_entry.map(|(_, key)| key);
                if let Some(k) = &res {
                    if self.deletion.contains(k) {
                        // Skip keys in deletion set
                        continue;
                    }
                }
                return res;
            } else {
                return None;
            }
        }
    }
}

fn lsm_treee_schema() -> Schema {
    Schema {
        id: *LSM_TREE_SCHEMA_ID,
        name: String::from(LSM_TREE_SCHEMA_NAME),
        key_field: None,
        str_key_field: None,
        is_dynamic: false,
        fields: Field::new(
            "*",
            0,
            false,
            false,
            Some(vec![
                Field::new(
                    LSM_TREE_LEVELS_NAME,
                    type_id_of(Type::Id),
                    false,
                    true,
                    None,
                    vec![],
                ),
                Field::new(
                    LSM_TREE_MIGRATION_NAME,
                    type_id_of(Type::Id),
                    true,
                    false,
                    None,
                    vec![],
                ),
            ]),
            vec![],
        ),
    }
}

fn lsm_tree_cell(level_ids: &Vec<Id>, id: &Id, migration: Option<Id>) -> Cell {
    let mut cell_map = Map::new();
    cell_map.insert_key_id(
        *LSM_TREE_LEVELS_HASH,
        Value::Array(level_ids.iter().map(|id| id.value()).collect()),
    );
    cell_map.insert_key_id(
        *LSM_TREE_MIGRATION_HASH,
        migration.map(|id| Value::Id(id)).unwrap_or(Value::Null),
    );
    Cell::new_with_id(*LSM_TREE_SCHEMA_ID, id, Value::Map(cell_map))
}

impl_btree_level!(LEVEL_M);
type LevelMTreeKeySlice = [EntryKey; LEVEL_M];
type LevelMTreePtrSlice = [NodeCellRef; LEVEL_M + 1];
type LevelMTree = BPlusTree<LevelMTreeKeySlice, LevelMTreePtrSlice>;

impl_btree_level!(LEVEL_0);
type Level0TreeKeySlice = [EntryKey; LEVEL_0];
type Level0TreePtrSlice = [NodeCellRef; LEVEL_0 + 1];
type Level0Tree = BPlusTree<Level0TreeKeySlice, Level0TreePtrSlice>;

impl_btree_level!(LEVEL_1);
type Level1TreeKeySlice = [EntryKey; LEVEL_1];
type Level1TreePtrSlice = [NodeCellRef; LEVEL_1 + 1];
type Level1Tree = BPlusTree<Level1TreeKeySlice, Level1TreePtrSlice>;

unsafe impl Send for LSMTree {}
unsafe impl Sync for LSMTree {}

struct DummyCursor;

impl Cursor for DummyCursor {
    fn next(&mut self) -> Option<EntryKey> {
        None
    }
    fn current(&self) -> Option<&EntryKey> {
        None
    }
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

//! Types required to send and receive bytes over an I/O channel.
//!
//! The core type is the `TTransport` trait, through which a `TProtocol` can
//! send and receive primitives over the wire. While `TProtocol` instances deal
//! with primitive types, `TTransport` instances understand only bytes.

use std::cell::RefCell;
use std::io;
use std::rc::Rc;

mod buffered;
mod framed;
mod passthru;
mod socket;

#[cfg(test)]
pub mod mem;

pub use self::buffered::{TBufferedTransport, TBufferedTransportFactory};
pub use self::framed::{TFramedTransport, TFramedTransportFactory};
pub use self::passthru::TPassThruTransport;
pub use self::socket::TTcpTransport;

/// Identifies an I/O channel that can be used to send and receive bytes.
pub trait TTransport: io::Read + io::Write {}
impl<I: io::Read + io::Write> TTransport for I {}

/// Helper type used by servers to create `TTransport` instances for accepted
/// client connections.
pub trait TTransportFactory {
    /// Create a `TTransport` that wraps an `inner` transport, thus creating
    /// a transport stack.
    fn create(&self, inner: Rc<RefCell<Box<TTransport>>>) -> Box<TTransport>;
}
<commit_msg>THRIFT-4113: export thrift::mem::TBufferTransport Client: rs<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

//! Types required to send and receive bytes over an I/O channel.
//!
//! The core type is the `TTransport` trait, through which a `TProtocol` can
//! send and receive primitives over the wire. While `TProtocol` instances deal
//! with primitive types, `TTransport` instances understand only bytes.

use std::cell::RefCell;
use std::io;
use std::rc::Rc;

mod buffered;
mod framed;
mod passthru;
mod socket;

pub mod mem;

pub use self::mem::TBufferTransport;
pub use self::buffered::{TBufferedTransport, TBufferedTransportFactory};
pub use self::framed::{TFramedTransport, TFramedTransportFactory};
pub use self::passthru::TPassThruTransport;
pub use self::socket::TTcpTransport;

/// Identifies an I/O channel that can be used to send and receive bytes.
pub trait TTransport: io::Read + io::Write {}
impl<I: io::Read + io::Write> TTransport for I {}

/// Helper type used by servers to create `TTransport` instances for accepted
/// client connections.
pub trait TTransportFactory {
    /// Create a `TTransport` that wraps an `inner` transport, thus creating
    /// a transport stack.
    fn create(&self, inner: Rc<RefCell<Box<TTransport>>>) -> Box<TTransport>;
}
<|endoftext|>


<commit_before>extern crate byteorder;

use std::io::Cursor;
use self::byteorder::{ReadBytesExt, WriteBytesExt, BigEndian, LittleEndian};

#[derive(Debug, Clone, PartialEq)]
pub struct Value {
    data: Vec<u8>
}

impl From<i64> for Value {
    fn from(val: i64) -> Value {
        let mut buffer = Vec::new();
        buffer.write_i64::<BigEndian>(val).unwrap();
        Value { data: buffer }
    }
}

impl From<String> for Value {
    fn from(val: String) -> Value {
        Value{data:val.into_bytes()}
    }
}

impl<'a> From<&'a str> for Value {
    fn from(val: &'a str) -> Value {
        let tmp = val.as_bytes();
        let mut v : Vec<u8> = Vec::new();
        v.extend_from_slice(&tmp);
        Value{data:v}
    }
}<commit_msg>equality checking working for int<commit_after>extern crate byteorder;

use std::io::Cursor;
use self::byteorder::{ReadBytesExt, WriteBytesExt, BigEndian, LittleEndian};

#[derive(Debug, Clone, PartialEq)]
pub struct Value {
    data: Vec<u8>
}

impl From<i64> for Value {
    fn from(val: i64) -> Value {
        let mut buffer = Vec::new();
        buffer.write_i64::<BigEndian>(val).unwrap();
        Value { data: buffer }
    }
}

impl From<String> for Value {
    fn from(val: String) -> Value {
        Value{data:val.into_bytes()}
    }
}

impl<'a> From<&'a str> for Value {
    fn from(val: &'a str) -> Value {
        let tmp = val.as_bytes();
        let mut v : Vec<u8> = Vec::new();
        v.extend_from_slice(&tmp);
        Value{data:v}
    }
}

#[cfg(test)]
mod tests {
    use super::Value;
    #[test]
    fn ints_should_compare_equal() {
        let mut x = Value::from(1);
        let mut y = Value::from(1);
        assert_eq!(x, y);

        y = Value::from(2);
        assert!(x != y);
    }
}<|endoftext|>


<commit_before>// The following are copied from cairo/src/{cairo-fixed-private.h, cairo-fixed-type-private.h}

const CAIRO_FIXED_FRAC_BITS: u64 = 8;
const CAIRO_MAGIC_NUMBER_FIXED: f64 = (1u64 << (52 - CAIRO_FIXED_FRAC_BITS)) as f64 * 1.5;

fn cairo_magic_double(d: f64) -> f64 {
    d + CAIRO_MAGIC_NUMBER_FIXED
}

fn cairo_fixed_from_double(d: f64) -> i32 {
    let bits = cairo_magic_double(d).to_bits();
    let lower = bits & 0xffffffff;
    lower as i32
}

/// Implements a method to check whether two `f64` numbers would have
/// the same fixed-point representation in Cairo.
///
/// This generally means that the absolute difference between them,
/// when taken as floating-point numbers, is less than the smallest
/// representable fraction that Cairo can represent in fixed-point.
///
/// Implementation detail: Cairo fixed-point numbers use 24 bits for
/// the integral part, and 8 bits for the fractional part.  That is,
/// the smallest fraction they can represent is 1/256.
pub trait FixedEqCairo {
    fn fixed_eq_cairo(&self, other: &Self) -> bool;
}

impl FixedEqCairo for f64 {
    fn fixed_eq_cairo(&self, other: &f64) -> bool {
        // FIXME: Here we have the same problem as Cairo itself: we
        // don't check for overflow in the conversion of double to
        // fixed-point.
        cairo_fixed_from_double(*self) == cairo_fixed_from_double(*other)
    }
}

/// Checks whether two floating-point numbers are approximately equal,
/// considering Cairo's limitations on numeric representation.
///
/// Cairo uses fixed-point numbers internally.  We implement this
/// trait for `f64`, so that two numbers can be considered "close
/// enough to equal" if their absolute difference is smaller than the
/// smallest fixed-point fraction that Cairo can represent.
///
/// Note that this trait is reliable even if the given numbers are
/// outside of the range that Cairo's fixed-point numbers can
/// represent.
pub trait ApproxEqCairo {
    fn approx_eq_cairo(&self, other: &Self) -> bool;
}

impl ApproxEqCairo for f64 {
    fn approx_eq_cairo(&self, other: &f64) -> bool {
        let abs_diff = (self - other).abs();
        let cairo_smallest_fraction = 1.0 / f64::from(1 << CAIRO_FIXED_FRAC_BITS);
        abs_diff < cairo_smallest_fraction
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn numbers_equal_in_cairo_fixed_point() {
        assert!(1.0_f64.fixed_eq_cairo(&1.0_f64));

        assert!(1.0_f64.fixed_eq_cairo(&1.001953125_f64)); // 1 + 1/512 - cairo rounds to 1

        assert!(!1.0_f64.fixed_eq_cairo(&1.00390625_f64)); // 1 + 1/256 - cairo can represent it
    }

    #[test]
    fn numbers_approx_equal() {
        assert!(0.0_f64.approx_eq_cairo(&0.001953125_f64)); // 1/512
        assert!(1.0_f64.approx_eq_cairo(&1.001953125_f64)); // 1 + 1/512
    }
}
<commit_msg>Use ApproxEq again in ApproxEqCairo.  This lets us compare big numbers, too.<commit_after>use float_cmp::ApproxEq;

// The following are copied from cairo/src/{cairo-fixed-private.h, cairo-fixed-type-private.h}

const CAIRO_FIXED_FRAC_BITS: u64 = 8;
const CAIRO_MAGIC_NUMBER_FIXED: f64 = (1u64 << (52 - CAIRO_FIXED_FRAC_BITS)) as f64 * 1.5;

fn cairo_magic_double(d: f64) -> f64 {
    d + CAIRO_MAGIC_NUMBER_FIXED
}

fn cairo_fixed_from_double(d: f64) -> i32 {
    let bits = cairo_magic_double(d).to_bits();
    let lower = bits & 0xffffffff;
    lower as i32
}

/// Implements a method to check whether two `f64` numbers would have
/// the same fixed-point representation in Cairo.
///
/// This generally means that the absolute difference between them,
/// when taken as floating-point numbers, is less than the smallest
/// representable fraction that Cairo can represent in fixed-point.
///
/// Implementation detail: Cairo fixed-point numbers use 24 bits for
/// the integral part, and 8 bits for the fractional part.  That is,
/// the smallest fraction they can represent is 1/256.
pub trait FixedEqCairo {
    fn fixed_eq_cairo(&self, other: &Self) -> bool;
}

impl FixedEqCairo for f64 {
    fn fixed_eq_cairo(&self, other: &f64) -> bool {
        // FIXME: Here we have the same problem as Cairo itself: we
        // don't check for overflow in the conversion of double to
        // fixed-point.
        cairo_fixed_from_double(*self) == cairo_fixed_from_double(*other)
    }
}

/// Checks whether two floating-point numbers are approximately equal,
/// considering Cairo's limitations on numeric representation.
///
/// Cairo uses fixed-point numbers internally.  We implement this
/// trait for `f64`, so that two numbers can be considered "close
/// enough to equal" if their absolute difference is smaller than the
/// smallest fixed-point fraction that Cairo can represent.
///
/// Note that this trait is reliable even if the given numbers are
/// outside of the range that Cairo's fixed-point numbers can
/// represent.  In that case, we check for the absolute difference,
/// and finally allow a difference of 1 unit-in-the-last-place (ULP)
/// for very large f64 values.
pub trait ApproxEqCairo: ApproxEq {
    fn approx_eq_cairo(&self, other: &Self) -> bool;
}

impl ApproxEqCairo for f64 {
    fn approx_eq_cairo(&self, other: &f64) -> bool {
        let cairo_smallest_fraction = 1.0 / f64::from(1 << CAIRO_FIXED_FRAC_BITS);
        self.approx_eq(other, cairo_smallest_fraction, 1)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn numbers_equal_in_cairo_fixed_point() {
        assert!(1.0_f64.fixed_eq_cairo(&1.0_f64));

        assert!(1.0_f64.fixed_eq_cairo(&1.001953125_f64)); // 1 + 1/512 - cairo rounds to 1

        assert!(!1.0_f64.fixed_eq_cairo(&1.00390625_f64)); // 1 + 1/256 - cairo can represent it
    }

    #[test]
    fn numbers_approx_equal() {
        // 0 == 1/256 - cairo can represent it, so not equal
        assert!(!0.0_f64.approx_eq_cairo(&0.00390635_f64));

        // 1 == 1 + 1/256 - cairo can represent it, so not equal
        assert!(!1.0_f64.approx_eq_cairo(&1.00390635_f64));

        // 0 == 1/256 - cairo can represent it, so not equal
        assert!(!0.0_f64.approx_eq_cairo(&-0.00390635_f64));

        // 1 == 1 - 1/256 - cairo can represent it, so not equal
        assert!(!1.0_f64.approx_eq_cairo(&0.99609365_f64));

        // 0 == 1/512 - cairo approximates to 0, so equal
        assert!(0.0_f64.approx_eq_cairo(&0.001953125_f64));

        // 1 == 1 + 1/512 - cairo approximates to 1, so equal
        assert!(1.0_f64.approx_eq_cairo(&1.001953125_f64));

        // 0 == -1/512 - cairo approximates to 0, so equal
        assert!(0.0_f64.approx_eq_cairo(&-0.001953125_f64));

        // 1 == 1 - 1/512 - cairo approximates to 1, so equal
        assert!(1.0_f64.approx_eq_cairo(&0.998046875_f64));

        // This is 2^53 compared to (2^53 + 2).  When represented as
        // f64, they are 1 unit-in-the-last-place (ULP) away from each
        // other, since the mantissa has 53 bits (52 bits plus 1
        // "hidden" bit).  The first number is an exact double, and
        // the second one is the next biggest double.  We consider a
        // difference of 1 ULP to mean that numbers are "equal", to
        // account for slight imprecision in floating-point
        // calculations.  Most of the time, for small values, we will
        // be using the cairo_smallest_fraction from the
        // implementation of approx_eq_cairo() above.  For large
        // values, we want the ULPs.
        //
        // In the second assertion, we compare 2^53 with (2^53 + 4).  Those are
        // 2 ULPs away, and we don't consider them equal.
        assert!(9_007_199_254_740_992.0.approx_eq_cairo(&9_007_199_254_740_994.0));
        assert!(!9_007_199_254_740_992.0.approx_eq_cairo(&9_007_199_254_740_996.0));
    }
}
<|endoftext|>


<commit_before>/**
 * The tutorial guessing game at https://doc.rust-lang.org/book/guessing-game.html
 */
extern crate rand;

use std::io;
use std::cmp::Ordering;
use rand::Rng;

fn main() {
    println!("Guess the number!");

    let secret_number = rand::thread_rng().gen_range(1, 101);
    println!("The secret number is: {}", secret_number);

    println!("Please input your guess.");

    let mut guess = String::new();

    io::stdin().read_line(&mut guess).expect("Failed to read line");

    let guess: u32 = guess.trim().parse()
                        .expect("Please type a number!");

    println!("You guessed: {}", guess);

    match guess.cmp(&secret_number) {
        Ordering::Less => println!("Too small!"),
        Ordering::Greater => println!("Too big!"),
        Ordering::Equal => println!("You win!"),
    }
}
<commit_msg>Completed the small tutorial "guessing game" from the rust tutorial<commit_after>/**
 * The tutorial guessing game at https://doc.rust-lang.org/book/guessing-game.html
 */
extern crate rand;

use std::io;
use std::cmp::Ordering;
use rand::Rng;

fn main() {
    println!("Guess the number!");

    let secret_number = rand::thread_rng().gen_range(1, 101);
    //Too easy if we display the secret number !
    //println!("The secret number is: {}", secret_number);

    loop {
        println!("Please input your guess.");

        let mut guess = String::new();

        io::stdin().read_line(&mut guess).expect("Failed to read line");

        let guess: u32 = match guess.trim().parse() {
                                                        Ok(num) => num,
                                                        Err(_) => continue,
                                                    };

        println!("You guessed: {}", guess);

        match guess.cmp(&secret_number) {
            Ordering::Less => println!("Too small!"),
            Ordering::Greater => println!("Too big!"),
            Ordering::Equal => {
                println!("You win!");
                break;
            }
        }
    }
}
<|endoftext|>


<commit_before><commit_msg>Fix show_scrollbars<commit_after><|endoftext|>


<commit_before>#[allow(non_upper_case_globals)]
#[allow(non_camel_case_types)]
mod extern_def {
    /* automatically generated by rust-bindgen */

    pub type ptrdiff_t = ::libc::c_long;
    pub type size_t = ::libc::c_ulong;
    pub type wchar_t = ::libc::c_int;
    pub type byte = ::libc::c_uchar;
    pub type Enum_Arch = ::libc::c_uint;
    pub const x86: ::libc::c_uint = 0;
    pub const x86_64: ::libc::c_uint = 1;
    pub const mips: ::libc::c_uint = 2;
    pub const arm: ::libc::c_uint = 3;
    pub const arm64: ::libc::c_uint = 4;
    pub const thumb: ::libc::c_uint = 5;
    pub const ppc32: ::libc::c_uint = 6;
    #[link(name = "assemble")]
    extern "C" {
        pub fn assemble(arch: Enum_Arch, instructions: *const ::libc::c_char,
                        out: *mut *mut byte, out_len: *mut size_t) -> ::libc::c_int;
        pub fn free_vec(vec: *mut byte);
    }
}

#[allow(non_camel_case_types)]
pub enum Arch {
    X86,
    X86_64,
    Mips,
    Arm,
    Arm64,
    Thumb,
    PPC32
}

impl Arch {
    pub fn to_c(self) -> u32 {
        match self {
            Arch::X86 => extern_def::x86,
            Arch::X86_64 => extern_def::x86_64,
            Arch::Mips => extern_def::mips,
            Arch::Arm => extern_def::arm,
            Arch::Arm64 => extern_def::arm64,
            Arch::Thumb => extern_def::thumb,
            Arch::PPC32 => extern_def::ppc32
        }
    }
}

pub fn assemble(arch: Arch, input: &str) -> Option<Vec<u8>> {
    use std::ffi::CString;
    use std::slice;
    use std::mem;
    use libc;

    let e_arch = arch.to_c();
    let ins = CString::new(input).unwrap();

    unsafe {
        let mut out_len = 0;
        let out_arr: *mut *mut u8 =
            libc::malloc(mem::size_of::<usize>() as libc::size_t)
            as *mut *mut u8;

        let res = extern_def::assemble(e_arch, ins.as_ptr(), out_arr, &mut out_len);
        if res == 0 {
            let vec = Some(
                slice::from_raw_parts(*out_arr, out_len as usize).to_vec());

            extern_def::free_vec(*out_arr);
            libc::free(out_arr as *mut libc::c_void);
            vec
        } else {
            None
        }
    }
}
<commit_msg>improving code (thanks clippy)<commit_after>#[allow(non_upper_case_globals)]
#[allow(non_camel_case_types)]
mod extern_def {
    /* automatically generated by rust-bindgen */

    pub type ptrdiff_t = ::libc::c_long;
    pub type size_t = ::libc::c_ulong;
    pub type wchar_t = ::libc::c_int;
    pub type byte = ::libc::c_uchar;
    pub type Enum_Arch = ::libc::c_uint;
    pub const x86: ::libc::c_uint = 0;
    pub const x86_64: ::libc::c_uint = 1;
    pub const mips: ::libc::c_uint = 2;
    pub const arm: ::libc::c_uint = 3;
    pub const arm64: ::libc::c_uint = 4;
    pub const thumb: ::libc::c_uint = 5;
    pub const ppc32: ::libc::c_uint = 6;
    #[link(name = "assemble")]
    extern "C" {
        pub fn assemble(arch: Enum_Arch, instructions: *const ::libc::c_char,
                        out: *mut *mut byte, out_len: *mut size_t) -> ::libc::c_int;
        pub fn free_vec(vec: *mut byte);
    }
}

#[allow(non_camel_case_types)]
pub enum Arch {
    X86,
    X86_64,
    Mips,
    Arm,
    Arm64,
    Thumb,
    PPC32
}

impl Arch {
    pub fn to_c(&self) -> u32 {
        match *self {
            Arch::X86 => extern_def::x86,
            Arch::X86_64 => extern_def::x86_64,
            Arch::Mips => extern_def::mips,
            Arch::Arm => extern_def::arm,
            Arch::Arm64 => extern_def::arm64,
            Arch::Thumb => extern_def::thumb,
            Arch::PPC32 => extern_def::ppc32
        }
    }
}

pub fn assemble(arch: Arch, input: &str) -> Option<Vec<u8>> {
    use std::ffi::CString;
    use std::slice;
    use std::mem;
    use libc;

    let e_arch = arch.to_c();
    let ins = CString::new(input).unwrap();

    unsafe {
        let mut out_len = 0;
        let out_arr: *mut *mut u8 =
            libc::malloc(mem::size_of::<usize>() as libc::size_t)
            as *mut *mut u8;

        let res = extern_def::assemble(e_arch, ins.as_ptr(), out_arr, &mut out_len);
        if res == 0 {
            let vec = Some(
                slice::from_raw_parts(*out_arr, out_len as usize).to_vec());

            extern_def::free_vec(*out_arr);
            libc::free(out_arr as *mut libc::c_void);
            vec
        } else {
            None
        }
    }
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2015 Johan Johansson
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

//! Emit the AST in some format
//!
//! Which formats should be supported:
//! 	* Rust source code ☑
//! 	* Rust AST ☐
//! 	* LLVM something ☐

use front::*;

pub trait ToRustSrc {
	fn to_rust(&self) -> String;
}

impl ToRustSrc for Path {
	fn to_rust(&self) -> String {
		format!("{}{}{}",
			if self.is_absolute() { "::" } else { "" },
			self.parts()[0],
			self.parts()[1..].iter().fold(String::new(), |acc, s| format!("{}::{}", acc, s)))
	}
}

impl ToRustSrc for Type {
	fn to_rust(&self) -> String {
		match *self {
			Type::Inferred => "_".to_string(),
			Type::Basic(ref ty) => ty.clone(),
			Type::Construct(ref con, ref args) => format!("{}<{}>",
				con,
				args.iter().fold(String::new(), |acc, ty| format!("{}{},", acc, ty.to_rust()))),
			Type::Tuple(ref tys) => format!("({})",
				tys.iter().fold(String::new(), |acc, ty| format!("{}{},", acc, ty.to_rust()))),
			Type::Poly(ref ty) => ty.clone(),
		}
	}
}

impl ToRustSrc for TypedBinding {
	fn to_rust(&self) -> String {
		format!("{}: {}", self.ident, self.type_sig.to_rust())
	}
}

impl ToRustSrc for Use {
	fn to_rust(&self) -> String {
		self.paths.iter()
			.fold(String::new(), |acc, ident| format!("{}use {};", acc, ident.to_rust()))
	}
}

/// Return the different elements of `it`. Each item in returned vec is unique, no doubles.
fn different_elements<'a, I: Iterator<Item=&'a Type>>(it: I) -> Vec<&'a Type> {
	let mut v: Vec<&Type> = Vec::new();

	for ty in it {
		if !v.contains(&ty) {
			v.push(ty)
		}
	}

	v
}

impl ToRustSrc for ConstDef {
	fn to_rust(&self) -> String {
		if let Expr::Lambda(ref lambda) = *self.body.value {
			format!("fn {}<{}>({}) -> {} {{ {} }}",
				self.binding.ident,
				different_elements(lambda.arg_bindings.iter()
						.map(|tb| &tb.type_sig)
						.filter(|ty| ty.is_poly()))
					.into_iter()
					.fold(String::new(), |acc, ty| format!("{}{},", acc, ty.to_rust())),
				lambda.arg_bindings.first()
					.map(|first| lambda.arg_bindings.tail()
						.iter()
						.fold(first.to_rust(), |acc, bnd|
							format!("{}, {}", acc, bnd.to_rust())))
					.unwrap_or("".into()),
				lambda.body.type_.to_rust(),
				lambda.body.to_rust())
		} else {
			format!("const {}: {} = {};",
				self.binding.ident,
				self.binding.type_sig.to_rust(),
				self.body.to_rust())
		}
	}
}

impl ToRustSrc for SExpr {
	fn to_rust(&self) -> String {
		let func = self.func.to_rust();
		match func.as_ref() {
			"+" | "-" | "*" | "/" | ">" | "<" => format!("({}{}{})",
				self.args[0].to_rust(),
				func,
				self.args[1].to_rust()),
			"=" => format!("({} == {})", self.args[0].to_rust(), self.args[1].to_rust()),
			_ => format!("{}({})",
				self.func.to_rust(),
				self.args.first()
					.map(|first| self.args[1..].iter()
						.fold(first.to_rust(), |acc, bnd|
							format!("{}, {}", acc, bnd.to_rust())))
					.unwrap_or("".into()))
		}
	}
}

impl ToRustSrc for Block {
	fn to_rust(&self) -> String {
		self.exprs.first()
			.map(|first| self.exprs[1..].iter()
				.fold(format!("{{ {}", first.to_rust()), |acc, expr|
					format!("{}; {}", acc, expr.to_rust())) + "}")
			.unwrap_or("{ }".into())
	}
}

impl ToRustSrc for Cond {
	fn to_rust(&self) -> String {
		format!("if {} {{ {} }}{}{}",
			self.clauses[0].0.to_rust(),
			self.clauses[0].1.to_rust(),
			self.clauses.tail().iter().fold(String::new(), |acc, &(ref cond, ref conseq)|
				format!("{} else if {} {{ {} }}", acc, cond.to_rust(), conseq.to_rust())),
			self.else_clause.as_ref().map(|conseq| format!(" else {{ {} }}", conseq.to_rust()))
				.unwrap_or("".into()),
		)
	}
}

impl ToRustSrc for Lambda {
	fn to_rust(&self) -> String {
		format!("|{}| -> {} {{ {} }}",
			self.arg_bindings.first()
				.map(|first| self.arg_bindings[1..].iter()
					.fold(first.to_rust(), |acc, bnd|
						format!("{}, {}", acc, bnd.to_rust())))
				.unwrap_or("".into()),
			self.body.type_.to_rust(),
			self.body.to_rust()
		)
	}
}

impl ToRustSrc for VarDef {
	fn to_rust(&self) -> String {
		format!("let{} {}: {} = {}",
			if self.mutable { " mut" } else { "" },
			self.binding.ident,
			self.binding.type_sig.to_rust(),
			if let Expr::Lambda(ref lambda) = *self.body.value {
				format!("|{}| -> {} {{ {} }}",
					lambda.arg_bindings.first()
						.map(|first| lambda.arg_bindings.tail()
							.iter()
							.fold(first.to_rust(), |acc, bnd|
								format!("{}, {}", acc, bnd.to_rust())))
						.unwrap_or("".into()),
					lambda.body.type_.to_rust(),
					lambda.body.to_rust())
			} else {
				self.body.to_rust()
			})
	}
}

impl ToRustSrc for Assign {
	fn to_rust(&self) -> String {
		format!("{} = {}", self.lvalue.ident, self.rvalue.to_rust())
	}
}

impl ToRustSrc for ExprMeta {
	fn to_rust(&self) -> String {
		match *self.value {
			Expr::Nil => "()".into(),
			Expr::NumLit(ref s) => s.clone(),
			Expr::StrLit(ref s) => s.clone(),
			Expr::Bool(b) => if b { "true" } else { "false" }.into(),
			Expr::Binding(ref ident) => ident.to_rust(),
			Expr::SExpr(ref sexpr) => sexpr.to_rust(),
			Expr::Block(ref block) => block.to_rust(),
			Expr::Cond(ref cond) => cond.to_rust(),
			Expr::Lambda(ref λ) => λ.to_rust(),
			Expr::VarDef(ref def) => def.to_rust(),
			Expr::Assign(ref a) => a.to_rust(),
		}
	}
}

impl ToRustSrc for AST {
	fn to_rust(&self) -> String {
		format!("{}{}",
			self.uses.iter()
				.fold(String::new(), |acc, u| format!("{}{};\n", acc, u.to_rust())),
			self.const_defs.iter()
				.fold(String::new(), |acc, def| format!("{}{}\n", acc, def.to_rust())))
	}
}

pub fn generate_rust_src(ast: &AST) -> String {
	ast.to_rust()
}
<commit_msg>Include consts in block when emitting rust src<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2015 Johan Johansson
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

//! Emit the AST in some format
//!
//! Which formats should be supported:
//! 	* Rust source code ☑
//! 	* Rust AST ☐
//! 	* LLVM something ☐

use front::*;

pub trait ToRustSrc {
	fn to_rust(&self) -> String;
}

impl ToRustSrc for Path {
	fn to_rust(&self) -> String {
		format!("{}{}{}",
			if self.is_absolute() { "::" } else { "" },
			self.parts()[0],
			self.parts()[1..].iter().fold(String::new(), |acc, s| format!("{}::{}", acc, s)))
	}
}

impl ToRustSrc for Type {
	fn to_rust(&self) -> String {
		match *self {
			Type::Inferred => "_".to_string(),
			Type::Basic(ref ty) => ty.clone(),
			Type::Construct(ref con, ref args) => format!("{}<{}>",
				con,
				args.iter().fold(String::new(), |acc, ty| format!("{}{},", acc, ty.to_rust()))),
			Type::Tuple(ref tys) => format!("({})",
				tys.iter().fold(String::new(), |acc, ty| format!("{}{},", acc, ty.to_rust()))),
			Type::Poly(ref ty) => ty.clone(),
		}
	}
}

impl ToRustSrc for TypedBinding {
	fn to_rust(&self) -> String {
		format!("{}: {}", self.ident, self.type_sig.to_rust())
	}
}

impl ToRustSrc for Use {
	fn to_rust(&self) -> String {
		self.paths.iter()
			.fold(String::new(), |acc, ident| format!("{}use {};", acc, ident.to_rust()))
	}
}

/// Return the different elements of `it`. Each item in returned vec is unique, no doubles.
fn different_elements<'a, I: Iterator<Item=&'a Type>>(it: I) -> Vec<&'a Type> {
	let mut v: Vec<&Type> = Vec::new();

	for ty in it {
		if !v.contains(&ty) {
			v.push(ty)
		}
	}

	v
}

fn delim_between_items<T: ToRustSrc>(items: &[T], delim: &str) -> String {
	items.first()
		.map(|first| items.tail()
			.iter()
			.fold(first.to_rust(), |acc, item|
				format!("{}{}{}", acc, delim, item.to_rust())))
		.unwrap_or("".into())
}

impl ToRustSrc for ConstDef {
	fn to_rust(&self) -> String {
		if let Expr::Lambda(ref lambda) = *self.body.value {
			format!("fn {}<{}>({}) -> {} {{ {} }}",
				self.binding.ident,
				different_elements(lambda.arg_bindings.iter()
						.map(|tb| &tb.type_sig)
						.filter(|ty| ty.is_poly()))
					.into_iter()
					.fold(String::new(), |acc, ty| format!("{}{},", acc, ty.to_rust())),
				delim_between_items(&lambda.arg_bindings, ", "),
				lambda.body.type_.to_rust(),
				lambda.body.to_rust())
		} else {
			format!("const {}: {} = {};",
				self.binding.ident,
				self.binding.type_sig.to_rust(),
				self.body.to_rust())
		}
	}
}

impl ToRustSrc for SExpr {
	fn to_rust(&self) -> String {
		let func = self.func.to_rust();
		match func.as_ref() {
			"+" | "-" | "*" | "/" | ">" | "<" => format!("({}{}{})",
				self.args[0].to_rust(),
				func,
				self.args[1].to_rust()),
			"=" => format!("({} == {})", self.args[0].to_rust(), self.args[1].to_rust()),
			_ => format!("{}({})", self.func.to_rust(), delim_between_items(&self.args, ", ")),
		}
	}
}

impl ToRustSrc for Block {
	fn to_rust(&self) -> String {
		format!("{{ {} {} {} }}",
			delim_between_items(&self.uses, " "),
			delim_between_items(&self.const_defs, " "),
			delim_between_items(&self.exprs, "; "))
	}
}

impl ToRustSrc for Cond {
	fn to_rust(&self) -> String {
		format!("if {} {{ {} }}{}{}",
			self.clauses[0].0.to_rust(),
			self.clauses[0].1.to_rust(),
			self.clauses.tail().iter().fold(String::new(), |acc, &(ref cond, ref conseq)|
				format!("{} else if {} {{ {} }}", acc, cond.to_rust(), conseq.to_rust())),
			self.else_clause.as_ref().map(|conseq| format!(" else {{ {} }}", conseq.to_rust()))
				.unwrap_or("".into()),
		)
	}
}

impl ToRustSrc for Lambda {
	fn to_rust(&self) -> String {
		format!("|{}| -> {} {{ {} }}",
			delim_between_items(&self.arg_bindings, ", "),
			self.body.type_.to_rust(),
			self.body.to_rust())
	}
}

impl ToRustSrc for VarDef {
	fn to_rust(&self) -> String {
		format!("let{} {}: {} = {}",
			if self.mutable { " mut" } else { "" },
			self.binding.ident,
			self.binding.type_sig.to_rust(),
			if let Expr::Lambda(ref lambda) = *self.body.value {
				format!("|{}| -> {} {{ {} }}",
					lambda.arg_bindings.first()
						.map(|first| lambda.arg_bindings.tail()
							.iter()
							.fold(first.to_rust(), |acc, bnd|
								format!("{}, {}", acc, bnd.to_rust())))
						.unwrap_or("".into()),
					lambda.body.type_.to_rust(),
					lambda.body.to_rust())
			} else {
				self.body.to_rust()
			})
	}
}

impl ToRustSrc for Assign {
	fn to_rust(&self) -> String {
		format!("{} = {}", self.lvalue.ident, self.rvalue.to_rust())
	}
}

impl ToRustSrc for ExprMeta {
	fn to_rust(&self) -> String {
		match *self.value {
			Expr::Nil => "()".into(),
			Expr::NumLit(ref s) => s.clone(),
			Expr::StrLit(ref s) => s.clone(),
			Expr::Bool(b) => if b { "true" } else { "false" }.into(),
			Expr::Binding(ref ident) => ident.to_rust(),
			Expr::SExpr(ref sexpr) => sexpr.to_rust(),
			Expr::Block(ref block) => block.to_rust(),
			Expr::Cond(ref cond) => cond.to_rust(),
			Expr::Lambda(ref λ) => λ.to_rust(),
			Expr::VarDef(ref def) => def.to_rust(),
			Expr::Assign(ref a) => a.to_rust(),
		}
	}
}

impl ToRustSrc for AST {
	fn to_rust(&self) -> String {
		format!("{}{}",
			self.uses.iter()
				.fold(String::new(), |acc, u| format!("{}{};\n", acc, u.to_rust())),
			self.const_defs.iter()
				.fold(String::new(), |acc, def| format!("{}{}\n", acc, def.to_rust())))
	}
}

pub fn generate_rust_src(ast: &AST) -> String {
	ast.to_rust()
}
<|endoftext|>


<commit_before>use std::path::PathBuf;
use std::io::Write;
use temp;
use toml;
use rustup_utils;
use manifest::Component;
use dist::TargetTriple;

declare_errors! {
    types {
        Error, ErrorKind, ChainErr, Result, err;
    }

    links {
        rustup_utils::Error, rustup_utils::ErrorKind, Utils;
    }

    foreign_links {
        temp::Error, Temp;
    }

    errors {
        InvalidFileExtension {
            description("invalid file extension")
        }
        InvalidInstaller {
            description("invalid installer")
        }
        InvalidTargetTriple(t: String) {
            description("invalid target triple")
            display("invalid target triple: '{}'", t)
        }
        InvalidToolchainName(t: String) {
            description("invalid toolchain name")
            display("invalid toolchain name: '{}'", t)
        }
        InvalidCustomToolchainName(t: String) {
            description("invalid custom toolchain name")
            display("invalid custom toolchain name: '{}'", t)
        }
        NotInstalledHere {
            description("not installed here")
        }
        UnsupportedHost(h: String) {
            description("binary package not provided for fost")
            display("a binary package was not provided for: '{}'", h)
        }
        ChecksumFailed {
            url: String,
            expected: String,
            calculated: String,
        } {
            description("checksum failed")
            display("checksum failed, expected: '{}', calculated: '{}'",
                    expected,
                    calculated)
        }
        ComponentConflict {
            name: String,
            path: PathBuf,
        } {
            description("conflicting component")
            display("failed to install component: '{}', detected conflict: '{:?}'",
                    name,
                    path)
        }
        ComponentMissingFile {
            name: String,
            path: PathBuf,
        } {
            description("missing file in component")
            display("failure removing component '{}', directory does not exist: '{:?}'",
                    name,
                    path)
        }
        ComponentMissingDir {
            name: String,
            path: PathBuf,
        } {
            description("missing directory in component")
            display("failure removing component '{}', directory does not exist: '{:?}'",
                    name,
                    path)
        }
        CorruptComponent(name: String) {
            description("corrupt component manifest")
            display("component manifest for '{}' is corrupt", name)
        }
        ExtractingPackage {
            description("failed to extract package")
        }
        ExtensionNotFound(c: Component) {
            description("could not find extension")
            display("could not find extension: '{}-{}'", c.pkg, c.target)
        }
        InvalidChangeSet {
            description("invalid change-set")
        }
        NoGPG {
            description("could not find 'gpg' on PATH")
        }
        BadInstallerVersion(v: String) {
            description("unsupported installer version")
            display("unsupported installer version: {}", v)
        }
        BadInstalledMetadataVersion(v: String) {
            description("unsupported metadata version in existing installation")
            display("unsupported metadata version in existing installation: {}", v)
        }
        ComponentDirPermissionsFailed {
            description("I/O error walking directory during install")
        }
        ComponentFilePermissionsFailed {
            description("error setting file permissions during install")
        }
        ComponentDownloadFailed(c: Component) {
            description("component download failed")
            display("component download failed for {}-{}", c.pkg, c.target)
        }
        ObsoleteDistManifest {
            description("the server unexpectedly provided an obsolete version of the distribution manifest")
        }
        Parsing(e: Vec<toml::ParserError>) {
            description("error parsing manifest")
        }
        MissingKey(k: String) {
            description("missing key")
            display("missing key: '{}'", k)
        }
        ExpectedType(t: &'static str, n: String) {
            description("expected type")
            display("expected type: '{}' for '{}'", t, n)
        }
        PackageNotFound(p: String) {
            description("package not found")
            display("package not found: '{}'", p)
        }
        TargetNotFound(t: TargetTriple) {
            description("target not found")
            display("target not found: '{}'", t)
        }
        MissingRoot {
            description("mamnifest has no root package")
        }
        UnsupportedVersion(v: String) {
            description("unsupported manifest version")
            display("manifest version '{}' is not supported", v)
        }
        MissingPackageForComponent(c: Component) {
            description("missing package for component")
            display("server sent a broken manifest: missing package for component {}", c.name())
        }
        RequestedComponentsUnavailable(c: Vec<Component>) {
            description("some requested components are unavailable to download")
            display("{}", component_unavailable_msg(&c))
        }
        NoManifestFound(ch: String, e: Box<Error>) {
            description("no release found")
            display("{}", no_manifest_found_msg(&ch, &e))
        }
        CreatingFile(p: PathBuf) {
            description("error creating file")
            display("error creating file '{}'", p.display())
        }
    }
}

fn component_unavailable_msg(cs: &[Component]) -> String {
    assert!(!cs.is_empty());

    let mut buf = vec![];

    if cs.len() == 1 {
        let _ = write!(buf, "component '{}' for '{}' is unavailable for download",
                       cs[0].pkg, cs[0].target);
    } else {
        use itertools::Itertools;
        let same_target = cs.iter().all(|c| c.target == cs[0].target);
        if same_target {
            let mut cs_strs = cs.iter().map(|c| format!("'{}'", c.pkg));
            let cs_str = cs_strs.join(", ");
            let _ = write!(buf, "some components unavailable for download: {}",
                           cs_str);
        } else {
            let mut cs_strs = cs.iter().map(|c| format!("'{}' for '{}'", c.pkg, c.target));
            let cs_str = cs_strs.join(", ");
            let _ = write!(buf, "some components unavailable for download: {}",
                           cs_str);
        }
    }

    String::from_utf8(buf).expect("")
}

// FIXME This should be two different errors
fn no_manifest_found_msg(ch: &str, e: &Error) -> String {

    let mut buf = vec![];

    match *e {
        Error(ErrorKind::Utils(rustup_utils::ErrorKind::Download404 { .. }), _ ) => {
            let _ = write!(buf, "no release found for '{}'", ch);
        }
        _ => {
            // FIXME: Need handle other common cases nicely,
            // like dns lookup, network unavailable.
            let _ = write!(buf, "failed to download manifest for '{}': {}", ch, e);
        }
    }

    String::from_utf8(buf).expect("")
}
<commit_msg>Remove unused error variants<commit_after>use std::path::PathBuf;
use std::io::Write;
use temp;
use toml;
use rustup_utils;
use manifest::Component;
use dist::TargetTriple;

declare_errors! {
    types {
        Error, ErrorKind, ChainErr, Result, err;
    }

    links {
        rustup_utils::Error, rustup_utils::ErrorKind, Utils;
    }

    foreign_links {
        temp::Error, Temp;
    }

    errors {
        InvalidToolchainName(t: String) {
            description("invalid toolchain name")
            display("invalid toolchain name: '{}'", t)
        }
        InvalidCustomToolchainName(t: String) {
            description("invalid custom toolchain name")
            display("invalid custom toolchain name: '{}'", t)
        }
        UnsupportedHost(h: String) {
            description("binary package not provided for fost")
            display("a binary package was not provided for: '{}'", h)
        }
        ChecksumFailed {
            url: String,
            expected: String,
            calculated: String,
        } {
            description("checksum failed")
            display("checksum failed, expected: '{}', calculated: '{}'",
                    expected,
                    calculated)
        }
        ComponentConflict {
            name: String,
            path: PathBuf,
        } {
            description("conflicting component")
            display("failed to install component: '{}', detected conflict: '{:?}'",
                    name,
                    path)
        }
        ComponentMissingFile {
            name: String,
            path: PathBuf,
        } {
            description("missing file in component")
            display("failure removing component '{}', directory does not exist: '{:?}'",
                    name,
                    path)
        }
        ComponentMissingDir {
            name: String,
            path: PathBuf,
        } {
            description("missing directory in component")
            display("failure removing component '{}', directory does not exist: '{:?}'",
                    name,
                    path)
        }
        CorruptComponent(name: String) {
            description("corrupt component manifest")
            display("component manifest for '{}' is corrupt", name)
        }
        ExtractingPackage {
            description("failed to extract package")
        }
        NoGPG {
            description("could not find 'gpg' on PATH")
        }
        BadInstallerVersion(v: String) {
            description("unsupported installer version")
            display("unsupported installer version: {}", v)
        }
        BadInstalledMetadataVersion(v: String) {
            description("unsupported metadata version in existing installation")
            display("unsupported metadata version in existing installation: {}", v)
        }
        ComponentDirPermissionsFailed {
            description("I/O error walking directory during install")
        }
        ComponentFilePermissionsFailed {
            description("error setting file permissions during install")
        }
        ComponentDownloadFailed(c: Component) {
            description("component download failed")
            display("component download failed for {}-{}", c.pkg, c.target)
        }
        ObsoleteDistManifest {
            description("the server unexpectedly provided an obsolete version of the distribution manifest")
        }
        Parsing(e: Vec<toml::ParserError>) {
            description("error parsing manifest")
        }
        MissingKey(k: String) {
            description("missing key")
            display("missing key: '{}'", k)
        }
        ExpectedType(t: &'static str, n: String) {
            description("expected type")
            display("expected type: '{}' for '{}'", t, n)
        }
        PackageNotFound(p: String) {
            description("package not found")
            display("package not found: '{}'", p)
        }
        TargetNotFound(t: TargetTriple) {
            description("target not found")
            display("target not found: '{}'", t)
        }
        UnsupportedVersion(v: String) {
            description("unsupported manifest version")
            display("manifest version '{}' is not supported", v)
        }
        MissingPackageForComponent(c: Component) {
            description("missing package for component")
            display("server sent a broken manifest: missing package for component {}", c.name())
        }
        RequestedComponentsUnavailable(c: Vec<Component>) {
            description("some requested components are unavailable to download")
            display("{}", component_unavailable_msg(&c))
        }
        NoManifestFound(ch: String, e: Box<Error>) {
            description("no release found")
            display("{}", no_manifest_found_msg(&ch, &e))
        }
        CreatingFile(p: PathBuf) {
            description("error creating file")
            display("error creating file '{}'", p.display())
        }
    }
}

fn component_unavailable_msg(cs: &[Component]) -> String {
    assert!(!cs.is_empty());

    let mut buf = vec![];

    if cs.len() == 1 {
        let _ = write!(buf, "component '{}' for '{}' is unavailable for download",
                       cs[0].pkg, cs[0].target);
    } else {
        use itertools::Itertools;
        let same_target = cs.iter().all(|c| c.target == cs[0].target);
        if same_target {
            let mut cs_strs = cs.iter().map(|c| format!("'{}'", c.pkg));
            let cs_str = cs_strs.join(", ");
            let _ = write!(buf, "some components unavailable for download: {}",
                           cs_str);
        } else {
            let mut cs_strs = cs.iter().map(|c| format!("'{}' for '{}'", c.pkg, c.target));
            let cs_str = cs_strs.join(", ");
            let _ = write!(buf, "some components unavailable for download: {}",
                           cs_str);
        }
    }

    String::from_utf8(buf).expect("")
}

// FIXME This should be two different errors
fn no_manifest_found_msg(ch: &str, e: &Error) -> String {

    let mut buf = vec![];

    match *e {
        Error(ErrorKind::Utils(rustup_utils::ErrorKind::Download404 { .. }), _ ) => {
            let _ = write!(buf, "no release found for '{}'", ch);
        }
        _ => {
            // FIXME: Need handle other common cases nicely,
            // like dns lookup, network unavailable.
            let _ = write!(buf, "failed to download manifest for '{}': {}", ch, e);
        }
    }

    String::from_utf8(buf).expect("")
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use backup_service::start_backup_service;
use consensus::{consensus_provider::start_consensus, gen_consensus_reconfig_subscription};
use debug_interface::node_debug_service::NodeDebugService;
use executor::{db_bootstrapper::maybe_bootstrap, Executor};
use executor_types::ChunkExecutor;
use futures::{channel::mpsc::channel, executor::block_on};
use libra_config::{
    config::{NetworkConfig, NodeConfig, RoleType},
    network_id::NodeNetworkId,
    utils::get_genesis_txn,
};
use libra_json_rpc::bootstrap_from_config as bootstrap_rpc;
use libra_logger::{prelude::*, Logger};
use libra_mempool::gen_mempool_reconfig_subscription;
use libra_metrics::metric_server;
use libra_types::{
    account_config::libra_root_address, account_state::AccountState, chain_id::ChainId,
    move_resource::MoveStorage,
};
use libra_vm::LibraVM;
use libradb::LibraDB;
use network_builder::builder::NetworkBuilder;
use state_synchronizer::StateSynchronizer;
use std::{boxed::Box, convert::TryFrom, net::ToSocketAddrs, sync::Arc, thread, time::Instant};
use storage_interface::DbReaderWriter;
use storage_service::start_storage_service_with_db;
use tokio::runtime::{Builder, Runtime};

const AC_SMP_CHANNEL_BUFFER_SIZE: usize = 1_024;
const INTRA_NODE_CHANNEL_BUFFER_SIZE: usize = 1;

pub struct LibraHandle {
    _rpc: Runtime,
    _mempool: Runtime,
    _state_synchronizer: StateSynchronizer,
    network_runtimes: Vec<Runtime>,
    _consensus_runtime: Option<Runtime>,
    _debug: NodeDebugService,
    _backup: Runtime,
}

impl LibraHandle {
    pub fn shutdown(&mut self) {
        // Shutdown network runtimes to avoid panic error log after LibraHandle is dropped:
        // thread ‘network-’ panicked at ‘SelectNextSome polled after terminated’,...
        // stack backtrace:
        //    ......
        //    8: network_simple_onchain_discovery::ConfigurationChangeListener::start::{{closure}}
        //      at network/simple-onchain-discovery/src/lib.rs:175
        //    ......
        // Other runtimes don't have same problem.
        while !self.network_runtimes.is_empty() {
            self.network_runtimes.remove(0).shutdown_background();
        }
    }
}

// Fetch chain ID from on-chain resource
fn fetch_chain_id(db: &DbReaderWriter) -> ChainId {
    let blob = db
        .reader
        .get_account_state_with_proof_by_version(
            libra_root_address(),
            (&*db.reader)
                .fetch_synced_version()
                .expect("[libra-node] failed fetching synced version."),
        )
        .expect("[libra-node] failed to get Libra root address account state")
        .0
        .expect("[libra-node] missing Libra root address account state");
    AccountState::try_from(&blob)
        .expect("[libra-node] failed to convert blob to account state")
        .get_chain_id_resource()
        .expect("[libra-node] failed to get chain ID resource")
        .expect("[libra-node] missing chain ID resource")
        .chain_id()
}

fn setup_chunk_executor(db: DbReaderWriter) -> Box<dyn ChunkExecutor> {
    Box::new(Executor::<LibraVM>::new(db))
}

fn setup_debug_interface(config: &NodeConfig, logger: Option<Arc<Logger>>) -> NodeDebugService {
    let addr = format!(
        "{}:{}",
        config.debug_interface.address, config.debug_interface.admission_control_node_debug_port,
    )
    .to_socket_addrs()
    .unwrap()
    .next()
    .unwrap();

    libra_trace::set_libra_trace(&config.debug_interface.libra_trace.sampling)
        .expect("Failed to set libra trace sampling rate.");

    NodeDebugService::new(addr, logger)
}

pub fn setup_environment(node_config: &NodeConfig, logger: Option<Arc<Logger>>) -> LibraHandle {
    let debug_if = setup_debug_interface(&node_config, logger);

    let metrics_port = node_config.debug_interface.metrics_server_port;
    let metric_host = node_config.debug_interface.address.clone();
    thread::spawn(move || metric_server::start_server(metric_host, metrics_port, false));
    let public_metrics_port = node_config.debug_interface.public_metrics_server_port;
    let public_metric_host = node_config.debug_interface.address.clone();
    thread::spawn(move || {
        metric_server::start_server(public_metric_host, public_metrics_port, true)
    });

    let mut instant = Instant::now();
    let (libra_db, db_rw) = DbReaderWriter::wrap(
        LibraDB::open(
            &node_config.storage.dir(),
            false, /* readonly */
            node_config.storage.prune_window,
        )
        .expect("DB should open."),
    );
    let _simple_storage_service =
        start_storage_service_with_db(&node_config, Arc::clone(&libra_db));
    let backup_service = start_backup_service(
        node_config.storage.backup_service_address,
        Arc::clone(&libra_db),
    );

    let waypoint = node_config.base.waypoint.waypoint();
    println!("WWWW1111: {}", waypoint);
    // if there's genesis txn and waypoint, commit it if the result matches.
    if let Some(genesis) = get_genesis_txn(&node_config) {
        maybe_bootstrap::<LibraVM>(&db_rw, genesis, waypoint)
            .expect("Db-bootstrapper should not fail.");
    }

    debug!(
        "Storage service started in {} ms",
        instant.elapsed().as_millis()
    );

    instant = Instant::now();
    let chunk_executor = setup_chunk_executor(db_rw.clone());
    debug!(
        "ChunkExecutor setup in {} ms",
        instant.elapsed().as_millis()
    );
    let chain_id = fetch_chain_id(&db_rw);
    let mut network_runtimes = vec![];
    let mut state_sync_network_handles = vec![];
    let mut mempool_network_handles = vec![];
    let mut consensus_network_handles = None;
    let mut reconfig_subscriptions = vec![];

    let (mempool_reconfig_subscription, mempool_reconfig_events) =
        gen_mempool_reconfig_subscription();
    reconfig_subscriptions.push(mempool_reconfig_subscription);
    // consensus has to subscribe to ALL on-chain configs
    let (consensus_reconfig_subscription, consensus_reconfig_events) =
        gen_consensus_reconfig_subscription();
    if node_config.base.role.is_validator() {
        reconfig_subscriptions.push(consensus_reconfig_subscription);
    }

    // Gather all network configs into a single vector.
    // TODO:  consider explicitly encoding the role in the NetworkConfig
    let mut network_configs: Vec<(RoleType, &NetworkConfig)> = node_config
        .full_node_networks
        .iter()
        .map(|network_config| (RoleType::FullNode, network_config))
        .collect();
    if let Some(network_config) = node_config.validator_network.as_ref() {
        network_configs.push((RoleType::Validator, network_config));
    }

    let mut network_builders = Vec::new();

    // Instantiate every network and collect the requisite endpoints for state_sync, mempool, and consensus.
    for (idx, (role, network_config)) in network_configs.into_iter().enumerate() {
        // Perform common instantiation steps
        let mut network_builder = NetworkBuilder::create(chain_id, role, network_config);
        let network_id = network_config.network_id.clone();

        // Create the endpoints to connect the Network to StateSynchronizer.
        let (state_sync_sender, state_sync_events) = network_builder
            .add_protocol_handler(state_synchronizer::network::network_endpoint_config());
        state_sync_network_handles.push((
            NodeNetworkId::new(network_id.clone(), idx),
            state_sync_sender,
            state_sync_events,
        ));

        // Create the endpoints to connect the Network to mempool.
        let (mempool_sender, mempool_events) =
            network_builder.add_protocol_handler(libra_mempool::network::network_endpoint_config(
                // TODO:  Make this configuration option more clear.
                node_config.mempool.max_broadcasts_per_peer,
            ));
        mempool_network_handles.push((
            NodeNetworkId::new(network_id, idx),
            mempool_sender,
            mempool_events,
        ));

        match role {
            // Perform steps relevant specifically to Validator networks.
            RoleType::Validator => {
                // A valid config is allowed to have at most one ValidatorNetwork
                // TODO:  `expect_none` would be perfect here, once it is stable.
                if consensus_network_handles.is_some() {
                    panic!("There can be at most one validator network!");
                }

                consensus_network_handles =
                    Some(network_builder.add_protocol_handler(
                        consensus::network_interface::network_endpoint_config(),
                    ));
            }
            // Currently no FullNode network specific steps.
            RoleType::FullNode => (),
        }

        reconfig_subscriptions.append(network_builder.reconfig_subscriptions());

        network_builders.push(network_builder);
    }

    // Build the configured networks.
    for network_builder in &mut network_builders {
        debug!("Creating runtime for {}", network_builder.network_context());
        let runtime = Builder::new()
            .thread_name("network-")
            .threaded_scheduler()
            .enable_all()
            .build()
            .expect("Failed to start runtime. Won't be able to start networking.");
        network_builder.build(runtime.handle().clone());
        network_runtimes.push(runtime);
        debug!(
            "Network built for network context: {}",
            network_builder.network_context()
        );
    }

    // Start the configured networks.
    // TODO:  Collect all component starts at the end of this function
    for network_builder in &mut network_builders {
        network_builder.start();
    }

    // TODO set up on-chain discovery network based on UpstreamConfig.fallback_network
    // and pass network handles to mempool/state sync

    // for state sync to send requests to mempool
    let (state_sync_to_mempool_sender, state_sync_requests) =
        channel(INTRA_NODE_CHANNEL_BUFFER_SIZE);
    let state_synchronizer = StateSynchronizer::bootstrap(
        state_sync_network_handles,
        state_sync_to_mempool_sender,
        Arc::clone(&db_rw.reader),
        chunk_executor,
        &node_config,
        waypoint,
        reconfig_subscriptions,
    );
    let (mp_client_sender, mp_client_events) = channel(AC_SMP_CHANNEL_BUFFER_SIZE);

    let rpc_runtime = bootstrap_rpc(&node_config, chain_id, libra_db.clone(), mp_client_sender);

    let mut consensus_runtime = None;
    let (consensus_to_mempool_sender, consensus_requests) = channel(INTRA_NODE_CHANNEL_BUFFER_SIZE);

    instant = Instant::now();
    let mempool = libra_mempool::bootstrap(
        node_config,
        Arc::clone(&db_rw.reader),
        mempool_network_handles,
        mp_client_events,
        consensus_requests,
        state_sync_requests,
        mempool_reconfig_events,
    );
    debug!("Mempool started in {} ms", instant.elapsed().as_millis());

    // StateSync should be instantiated and started before Consensus to avoid a cyclic dependency:
    // network provider -> consensus -> state synchronizer -> network provider.  This has resulted
    // in a deadlock as observed in GitHub issue #749.
    if let Some((consensus_network_sender, consensus_network_events)) = consensus_network_handles {
        // Make sure that state synchronizer is caught up at least to its waypoint
        // (in case it's present). There is no sense to start consensus prior to that.
        // TODO: Note that we need the networking layer to be able to discover & connect to the
        // peers with potentially outdated network identity public keys.
        debug!("Wait until state synchronizer is initialized");
        block_on(state_synchronizer.wait_until_initialized())
            .expect("State synchronizer initialization failure");
        debug!("State synchronizer initialization complete.");

        // Initialize and start consensus.
        instant = Instant::now();
        consensus_runtime = Some(start_consensus(
            node_config,
            consensus_network_sender,
            consensus_network_events,
            state_synchronizer.create_client(),
            consensus_to_mempool_sender,
            libra_db,
            consensus_reconfig_events,
        ));
        debug!("Consensus started in {} ms", instant.elapsed().as_millis());
    }

    LibraHandle {
        network_runtimes,
        _rpc: rpc_runtime,
        _mempool: mempool,
        _state_synchronizer: state_synchronizer,
        _consensus_runtime: consensus_runtime,
        _debug: debug_if,
        _backup: backup_service,
    }
}
<commit_msg>trivial: remove a debug println<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use backup_service::start_backup_service;
use consensus::{consensus_provider::start_consensus, gen_consensus_reconfig_subscription};
use debug_interface::node_debug_service::NodeDebugService;
use executor::{db_bootstrapper::maybe_bootstrap, Executor};
use executor_types::ChunkExecutor;
use futures::{channel::mpsc::channel, executor::block_on};
use libra_config::{
    config::{NetworkConfig, NodeConfig, RoleType},
    network_id::NodeNetworkId,
    utils::get_genesis_txn,
};
use libra_json_rpc::bootstrap_from_config as bootstrap_rpc;
use libra_logger::{prelude::*, Logger};
use libra_mempool::gen_mempool_reconfig_subscription;
use libra_metrics::metric_server;
use libra_types::{
    account_config::libra_root_address, account_state::AccountState, chain_id::ChainId,
    move_resource::MoveStorage,
};
use libra_vm::LibraVM;
use libradb::LibraDB;
use network_builder::builder::NetworkBuilder;
use state_synchronizer::StateSynchronizer;
use std::{boxed::Box, convert::TryFrom, net::ToSocketAddrs, sync::Arc, thread, time::Instant};
use storage_interface::DbReaderWriter;
use storage_service::start_storage_service_with_db;
use tokio::runtime::{Builder, Runtime};

const AC_SMP_CHANNEL_BUFFER_SIZE: usize = 1_024;
const INTRA_NODE_CHANNEL_BUFFER_SIZE: usize = 1;

pub struct LibraHandle {
    _rpc: Runtime,
    _mempool: Runtime,
    _state_synchronizer: StateSynchronizer,
    network_runtimes: Vec<Runtime>,
    _consensus_runtime: Option<Runtime>,
    _debug: NodeDebugService,
    _backup: Runtime,
}

impl LibraHandle {
    pub fn shutdown(&mut self) {
        // Shutdown network runtimes to avoid panic error log after LibraHandle is dropped:
        // thread ‘network-’ panicked at ‘SelectNextSome polled after terminated’,...
        // stack backtrace:
        //    ......
        //    8: network_simple_onchain_discovery::ConfigurationChangeListener::start::{{closure}}
        //      at network/simple-onchain-discovery/src/lib.rs:175
        //    ......
        // Other runtimes don't have same problem.
        while !self.network_runtimes.is_empty() {
            self.network_runtimes.remove(0).shutdown_background();
        }
    }
}

// Fetch chain ID from on-chain resource
fn fetch_chain_id(db: &DbReaderWriter) -> ChainId {
    let blob = db
        .reader
        .get_account_state_with_proof_by_version(
            libra_root_address(),
            (&*db.reader)
                .fetch_synced_version()
                .expect("[libra-node] failed fetching synced version."),
        )
        .expect("[libra-node] failed to get Libra root address account state")
        .0
        .expect("[libra-node] missing Libra root address account state");
    AccountState::try_from(&blob)
        .expect("[libra-node] failed to convert blob to account state")
        .get_chain_id_resource()
        .expect("[libra-node] failed to get chain ID resource")
        .expect("[libra-node] missing chain ID resource")
        .chain_id()
}

fn setup_chunk_executor(db: DbReaderWriter) -> Box<dyn ChunkExecutor> {
    Box::new(Executor::<LibraVM>::new(db))
}

fn setup_debug_interface(config: &NodeConfig, logger: Option<Arc<Logger>>) -> NodeDebugService {
    let addr = format!(
        "{}:{}",
        config.debug_interface.address, config.debug_interface.admission_control_node_debug_port,
    )
    .to_socket_addrs()
    .unwrap()
    .next()
    .unwrap();

    libra_trace::set_libra_trace(&config.debug_interface.libra_trace.sampling)
        .expect("Failed to set libra trace sampling rate.");

    NodeDebugService::new(addr, logger)
}

pub fn setup_environment(node_config: &NodeConfig, logger: Option<Arc<Logger>>) -> LibraHandle {
    let debug_if = setup_debug_interface(&node_config, logger);

    let metrics_port = node_config.debug_interface.metrics_server_port;
    let metric_host = node_config.debug_interface.address.clone();
    thread::spawn(move || metric_server::start_server(metric_host, metrics_port, false));
    let public_metrics_port = node_config.debug_interface.public_metrics_server_port;
    let public_metric_host = node_config.debug_interface.address.clone();
    thread::spawn(move || {
        metric_server::start_server(public_metric_host, public_metrics_port, true)
    });

    let mut instant = Instant::now();
    let (libra_db, db_rw) = DbReaderWriter::wrap(
        LibraDB::open(
            &node_config.storage.dir(),
            false, /* readonly */
            node_config.storage.prune_window,
        )
        .expect("DB should open."),
    );
    let _simple_storage_service =
        start_storage_service_with_db(&node_config, Arc::clone(&libra_db));
    let backup_service = start_backup_service(
        node_config.storage.backup_service_address,
        Arc::clone(&libra_db),
    );

    let waypoint = node_config.base.waypoint.waypoint();
    // if there's genesis txn and waypoint, commit it if the result matches.
    if let Some(genesis) = get_genesis_txn(&node_config) {
        maybe_bootstrap::<LibraVM>(&db_rw, genesis, waypoint)
            .expect("Db-bootstrapper should not fail.");
    }

    debug!(
        "Storage service started in {} ms",
        instant.elapsed().as_millis()
    );

    instant = Instant::now();
    let chunk_executor = setup_chunk_executor(db_rw.clone());
    debug!(
        "ChunkExecutor setup in {} ms",
        instant.elapsed().as_millis()
    );
    let chain_id = fetch_chain_id(&db_rw);
    let mut network_runtimes = vec![];
    let mut state_sync_network_handles = vec![];
    let mut mempool_network_handles = vec![];
    let mut consensus_network_handles = None;
    let mut reconfig_subscriptions = vec![];

    let (mempool_reconfig_subscription, mempool_reconfig_events) =
        gen_mempool_reconfig_subscription();
    reconfig_subscriptions.push(mempool_reconfig_subscription);
    // consensus has to subscribe to ALL on-chain configs
    let (consensus_reconfig_subscription, consensus_reconfig_events) =
        gen_consensus_reconfig_subscription();
    if node_config.base.role.is_validator() {
        reconfig_subscriptions.push(consensus_reconfig_subscription);
    }

    // Gather all network configs into a single vector.
    // TODO:  consider explicitly encoding the role in the NetworkConfig
    let mut network_configs: Vec<(RoleType, &NetworkConfig)> = node_config
        .full_node_networks
        .iter()
        .map(|network_config| (RoleType::FullNode, network_config))
        .collect();
    if let Some(network_config) = node_config.validator_network.as_ref() {
        network_configs.push((RoleType::Validator, network_config));
    }

    let mut network_builders = Vec::new();

    // Instantiate every network and collect the requisite endpoints for state_sync, mempool, and consensus.
    for (idx, (role, network_config)) in network_configs.into_iter().enumerate() {
        // Perform common instantiation steps
        let mut network_builder = NetworkBuilder::create(chain_id, role, network_config);
        let network_id = network_config.network_id.clone();

        // Create the endpoints to connect the Network to StateSynchronizer.
        let (state_sync_sender, state_sync_events) = network_builder
            .add_protocol_handler(state_synchronizer::network::network_endpoint_config());
        state_sync_network_handles.push((
            NodeNetworkId::new(network_id.clone(), idx),
            state_sync_sender,
            state_sync_events,
        ));

        // Create the endpoints to connect the Network to mempool.
        let (mempool_sender, mempool_events) =
            network_builder.add_protocol_handler(libra_mempool::network::network_endpoint_config(
                // TODO:  Make this configuration option more clear.
                node_config.mempool.max_broadcasts_per_peer,
            ));
        mempool_network_handles.push((
            NodeNetworkId::new(network_id, idx),
            mempool_sender,
            mempool_events,
        ));

        match role {
            // Perform steps relevant specifically to Validator networks.
            RoleType::Validator => {
                // A valid config is allowed to have at most one ValidatorNetwork
                // TODO:  `expect_none` would be perfect here, once it is stable.
                if consensus_network_handles.is_some() {
                    panic!("There can be at most one validator network!");
                }

                consensus_network_handles =
                    Some(network_builder.add_protocol_handler(
                        consensus::network_interface::network_endpoint_config(),
                    ));
            }
            // Currently no FullNode network specific steps.
            RoleType::FullNode => (),
        }

        reconfig_subscriptions.append(network_builder.reconfig_subscriptions());

        network_builders.push(network_builder);
    }

    // Build the configured networks.
    for network_builder in &mut network_builders {
        debug!("Creating runtime for {}", network_builder.network_context());
        let runtime = Builder::new()
            .thread_name("network-")
            .threaded_scheduler()
            .enable_all()
            .build()
            .expect("Failed to start runtime. Won't be able to start networking.");
        network_builder.build(runtime.handle().clone());
        network_runtimes.push(runtime);
        debug!(
            "Network built for network context: {}",
            network_builder.network_context()
        );
    }

    // Start the configured networks.
    // TODO:  Collect all component starts at the end of this function
    for network_builder in &mut network_builders {
        network_builder.start();
    }

    // TODO set up on-chain discovery network based on UpstreamConfig.fallback_network
    // and pass network handles to mempool/state sync

    // for state sync to send requests to mempool
    let (state_sync_to_mempool_sender, state_sync_requests) =
        channel(INTRA_NODE_CHANNEL_BUFFER_SIZE);
    let state_synchronizer = StateSynchronizer::bootstrap(
        state_sync_network_handles,
        state_sync_to_mempool_sender,
        Arc::clone(&db_rw.reader),
        chunk_executor,
        &node_config,
        waypoint,
        reconfig_subscriptions,
    );
    let (mp_client_sender, mp_client_events) = channel(AC_SMP_CHANNEL_BUFFER_SIZE);

    let rpc_runtime = bootstrap_rpc(&node_config, chain_id, libra_db.clone(), mp_client_sender);

    let mut consensus_runtime = None;
    let (consensus_to_mempool_sender, consensus_requests) = channel(INTRA_NODE_CHANNEL_BUFFER_SIZE);

    instant = Instant::now();
    let mempool = libra_mempool::bootstrap(
        node_config,
        Arc::clone(&db_rw.reader),
        mempool_network_handles,
        mp_client_events,
        consensus_requests,
        state_sync_requests,
        mempool_reconfig_events,
    );
    debug!("Mempool started in {} ms", instant.elapsed().as_millis());

    // StateSync should be instantiated and started before Consensus to avoid a cyclic dependency:
    // network provider -> consensus -> state synchronizer -> network provider.  This has resulted
    // in a deadlock as observed in GitHub issue #749.
    if let Some((consensus_network_sender, consensus_network_events)) = consensus_network_handles {
        // Make sure that state synchronizer is caught up at least to its waypoint
        // (in case it's present). There is no sense to start consensus prior to that.
        // TODO: Note that we need the networking layer to be able to discover & connect to the
        // peers with potentially outdated network identity public keys.
        debug!("Wait until state synchronizer is initialized");
        block_on(state_synchronizer.wait_until_initialized())
            .expect("State synchronizer initialization failure");
        debug!("State synchronizer initialization complete.");

        // Initialize and start consensus.
        instant = Instant::now();
        consensus_runtime = Some(start_consensus(
            node_config,
            consensus_network_sender,
            consensus_network_events,
            state_synchronizer.create_client(),
            consensus_to_mempool_sender,
            libra_db,
            consensus_reconfig_events,
        ));
        debug!("Consensus started in {} ms", instant.elapsed().as_millis());
    }

    LibraHandle {
        network_runtimes,
        _rpc: rpc_runtime,
        _mempool: mempool,
        _state_synchronizer: state_synchronizer,
        _consensus_runtime: consensus_runtime,
        _debug: debug_if,
        _backup: backup_service,
    }
}
<|endoftext|>