<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

macro_rules! calculate_redundancy {
    ($redundancy:ident) => {
        match $redundancy {
            None | Some(0) => $crate::engine::Redundancy::NONE,
            Some(n) => {
                let message = format!("code {} does not correspond to any redundancy", n);
                return Err($crate::stratis::StratisError::Engine(
                    $crate::stratis::ErrorEnum::Error,
                    message,
                ));
            }
        }
    };
}

macro_rules! get_pool {
    ($s:ident; $uuid:ident) => {
        $s.pools
            .get_by_uuid($uuid)
            .map(|(name, p)| (name.clone(), p as &dyn $crate::engine::Pool))
    };
}

macro_rules! get_mut_pool {
    ($s:ident; $uuid:ident) => {
        $s.pools
            .get_mut_by_uuid($uuid)
            .map(|(name, p)| (name.clone(), p as &mut dyn $crate::engine::Pool))
    };
}

macro_rules! rename_pre {
    ($s:expr; $uuid:ident; $new_name:ident; $not_found:expr; $same:expr) => {{
        let old_name = match $s.get_by_uuid($uuid) {
            Some((name, _)) => name,
            None => return $not_found,
        };

        if &*old_name == $new_name {
            return $same;
        }

        if $s.contains_name($new_name) {
            return Err($crate::stratis::StratisError::Engine(
                $crate::stratis::ErrorEnum::AlreadyExists,
                $new_name.into(),
            ));
        }
        old_name
    }};
}

macro_rules! rename_filesystem_pre {
    ($s:ident; $uuid:ident; $new_name:ident) => {{
        rename_pre!(
            $s.filesystems;
            $uuid;
            $new_name;
            Err($crate::stratis::StratisError::Engine(
                $crate::stratis::ErrorEnum::NotFound,
                format!("Filesystem not found with UUID of {}", $uuid),
            ));
            Ok(None)
        )
    }}
}

macro_rules! rename_pre_idem {
    ($s:expr; $uuid:ident; $new_name:ident) => {{
        rename_pre!(
            $s;
            $uuid;
            $new_name;
            Ok($crate::engine::RenameAction::NoSource);
            Ok($crate::engine::RenameAction::Identity)
        )
    }}
}

macro_rules! rename_filesystem_pre_idem {
    ($s:ident; $uuid:ident; $new_name:ident) => {{
        rename_pre_idem!(
            $s.filesystems;
            $uuid;
            $new_name
        )
    }}
}

macro_rules! rename_pool_pre_idem {
    ($s:ident; $uuid:ident; $new_name:ident) => {{
        rename_pre_idem!(
            $s.pools;
            $uuid;
            $new_name
        )
    }}
}

macro_rules! set_blockdev_user_info {
    ($s:ident; $info:ident) => {
        if $s.user_info.as_ref().map(|x| &**x) != $info {
            $s.user_info = $info.map(|x| x.to_owned());
            true
        } else {
            false
        }
    };
}

macro_rules! device_list_check_num {
    ($vec:ident, ($is_one:tt, $is_many:tt)) => {{
        let joined_string = $vec.join(", ");
        if $vec.len() == 1 {
            format!($is_one, joined_string)
        } else if $vec.len() > 1 {
            format!($is_many, joined_string)
        } else {
            String::new()
        }
    }};
}

macro_rules! create_pool_generate_error_string {
    ($pool_name:ident, $input:ident, $exists:ident) => {
        format!(
            "There was a difference in the blockdevs associated with \
             the existing pool named {} and the input requesting creation \
             of a pool by the same name{}{}",
            $pool_name,
            device_list_check_num!(
                $input,
                (
                    " - the input requests blockdev {} \
                     which does not exist in the current pool",
                    " - the input requests blockdevs {} \
                     which do not exist in the current pool"
                )
            ),
            device_list_check_num!(
                $exists,
                (
                    " - the existing pool contains blockdev {} which was \
                     not requested by the input",
                    " - the existing pool contains blockdevs {} which were \
                     not requested by the input"
                )
            ),
        )
    };
}

#[cfg(test)]
macro_rules! strs_to_paths {
    ($slice:expr) => {
        &$slice.iter().map(Path::new).collect::<Vec<_>>()
    };
}

macro_rules! init_cache_generate_error_string {
    ($input:ident, $exists:ident) => {
        format!(
            "The input requests initialization of a cache with different block devices \
             from the block devices in the existing cache{}{} ; to \
             resolve this error, the block devices requested in the input should be the \
             same as the block devices in the existing cache.",
            device_list_check_num!(
                $input,
                (
                    "; the existing cache contains \
                     the block device {} which the input did not include",
                    "; the existing cache contains \
                     the block devices {} which the input did not include"
                )
            ),
            device_list_check_num!(
                $exists,
                (
                    "; the input requested block device {} which does not exist in the already \
                     initialized cache",
                    "; the input requested block devices {} which do not exist in the already \
                     initialized cache"
                )
            ),
        )
    };
}

macro_rules! convert_int {
    ($expr:expr, $from_type:ty, $to_type:ty) => {{
        let expr = $expr;
        <$to_type as std::convert::TryFrom<$from_type>>::try_from(expr).map_err(|_| {
            StratisError::Error(format!(
                "Failed to convert integer {} from {} to {}",
                expr,
                stringify!($from_type),
                stringify!($to_type)
            ))
        })
    }};
}

macro_rules! convert_const {
    ($expr:expr, $from_type:ty, $to_type:ty) => {
        <$to_type as std::convert::TryFrom<$from_type>>::try_from($expr)
            .expect(format!("{} is a constant", stringify!($expr)).as_str())
    };
}

#[cfg(test)]
macro_rules! convert_test {
    ($expr:expr, $from_type:ty, $to_type:ty) => {
        <$to_type as std::convert::TryFrom<$from_type>>::try_from($expr).unwrap()
    };
}
<commit_msg>Fully qualify name in macro<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

macro_rules! calculate_redundancy {
    ($redundancy:ident) => {
        match $redundancy {
            None | Some(0) => $crate::engine::Redundancy::NONE,
            Some(n) => {
                let message = format!("code {} does not correspond to any redundancy", n);
                return Err($crate::stratis::StratisError::Engine(
                    $crate::stratis::ErrorEnum::Error,
                    message,
                ));
            }
        }
    };
}

macro_rules! get_pool {
    ($s:ident; $uuid:ident) => {
        $s.pools
            .get_by_uuid($uuid)
            .map(|(name, p)| (name.clone(), p as &dyn $crate::engine::Pool))
    };
}

macro_rules! get_mut_pool {
    ($s:ident; $uuid:ident) => {
        $s.pools
            .get_mut_by_uuid($uuid)
            .map(|(name, p)| (name.clone(), p as &mut dyn $crate::engine::Pool))
    };
}

macro_rules! rename_pre {
    ($s:expr; $uuid:ident; $new_name:ident; $not_found:expr; $same:expr) => {{
        let old_name = match $s.get_by_uuid($uuid) {
            Some((name, _)) => name,
            None => return $not_found,
        };

        if &*old_name == $new_name {
            return $same;
        }

        if $s.contains_name($new_name) {
            return Err($crate::stratis::StratisError::Engine(
                $crate::stratis::ErrorEnum::AlreadyExists,
                $new_name.into(),
            ));
        }
        old_name
    }};
}

macro_rules! rename_filesystem_pre {
    ($s:ident; $uuid:ident; $new_name:ident) => {{
        rename_pre!(
            $s.filesystems;
            $uuid;
            $new_name;
            Err($crate::stratis::StratisError::Engine(
                $crate::stratis::ErrorEnum::NotFound,
                format!("Filesystem not found with UUID of {}", $uuid),
            ));
            Ok(None)
        )
    }}
}

macro_rules! rename_pre_idem {
    ($s:expr; $uuid:ident; $new_name:ident) => {{
        rename_pre!(
            $s;
            $uuid;
            $new_name;
            Ok($crate::engine::RenameAction::NoSource);
            Ok($crate::engine::RenameAction::Identity)
        )
    }}
}

macro_rules! rename_filesystem_pre_idem {
    ($s:ident; $uuid:ident; $new_name:ident) => {{
        rename_pre_idem!(
            $s.filesystems;
            $uuid;
            $new_name
        )
    }}
}

macro_rules! rename_pool_pre_idem {
    ($s:ident; $uuid:ident; $new_name:ident) => {{
        rename_pre_idem!(
            $s.pools;
            $uuid;
            $new_name
        )
    }}
}

macro_rules! set_blockdev_user_info {
    ($s:ident; $info:ident) => {
        if $s.user_info.as_ref().map(|x| &**x) != $info {
            $s.user_info = $info.map(|x| x.to_owned());
            true
        } else {
            false
        }
    };
}

macro_rules! device_list_check_num {
    ($vec:ident, ($is_one:tt, $is_many:tt)) => {{
        let joined_string = $vec.join(", ");
        if $vec.len() == 1 {
            format!($is_one, joined_string)
        } else if $vec.len() > 1 {
            format!($is_many, joined_string)
        } else {
            String::new()
        }
    }};
}

macro_rules! create_pool_generate_error_string {
    ($pool_name:ident, $input:ident, $exists:ident) => {
        format!(
            "There was a difference in the blockdevs associated with \
             the existing pool named {} and the input requesting creation \
             of a pool by the same name{}{}",
            $pool_name,
            device_list_check_num!(
                $input,
                (
                    " - the input requests blockdev {} \
                     which does not exist in the current pool",
                    " - the input requests blockdevs {} \
                     which do not exist in the current pool"
                )
            ),
            device_list_check_num!(
                $exists,
                (
                    " - the existing pool contains blockdev {} which was \
                     not requested by the input",
                    " - the existing pool contains blockdevs {} which were \
                     not requested by the input"
                )
            ),
        )
    };
}

#[cfg(test)]
macro_rules! strs_to_paths {
    ($slice:expr) => {
        &$slice.iter().map(Path::new).collect::<Vec<_>>()
    };
}

macro_rules! init_cache_generate_error_string {
    ($input:ident, $exists:ident) => {
        format!(
            "The input requests initialization of a cache with different block devices \
             from the block devices in the existing cache{}{} ; to \
             resolve this error, the block devices requested in the input should be the \
             same as the block devices in the existing cache.",
            device_list_check_num!(
                $input,
                (
                    "; the existing cache contains \
                     the block device {} which the input did not include",
                    "; the existing cache contains \
                     the block devices {} which the input did not include"
                )
            ),
            device_list_check_num!(
                $exists,
                (
                    "; the input requested block device {} which does not exist in the already \
                     initialized cache",
                    "; the input requested block devices {} which do not exist in the already \
                     initialized cache"
                )
            ),
        )
    };
}

macro_rules! convert_int {
    ($expr:expr, $from_type:ty, $to_type:ty) => {{
        let expr = $expr;
        <$to_type as std::convert::TryFrom<$from_type>>::try_from(expr).map_err(|_| {
            $crate::stratis::StratisError::Error(format!(
                "Failed to convert integer {} from {} to {}",
                expr,
                stringify!($from_type),
                stringify!($to_type)
            ))
        })
    }};
}

macro_rules! convert_const {
    ($expr:expr, $from_type:ty, $to_type:ty) => {
        <$to_type as std::convert::TryFrom<$from_type>>::try_from($expr)
            .expect(format!("{} is a constant", stringify!($expr)).as_str())
    };
}

#[cfg(test)]
macro_rules! convert_test {
    ($expr:expr, $from_type:ty, $to_type:ty) => {
        <$to_type as std::convert::TryFrom<$from_type>>::try_from($expr).unwrap()
    };
}
<|endoftext|>


<commit_before>use std::sync::Mutex;

use threadpool::ThreadPool;

use ctxt::VM;
use driver::cmd::Args;
use gc::root::{get_rootset, Slot};
use gc::swiper::card::CardTable;
use gc::swiper::crossing::CrossingMap;
use gc::swiper::full::FullCollector;
use gc::swiper::large::LargeSpace;
use gc::swiper::minor::MinorCollector;
use gc::swiper::old::OldGen;
use gc::swiper::verify::{Verifier, VerifierPhase};
use gc::swiper::young::YoungGen;
use gc::tlab;
use gc::Collector;
use gc::{align_gen, arena, GcReason};
use gc::{formatted_size, Address, Region};
use mem;

pub mod card;
mod controller;
mod crossing;
mod full;
mod large;
mod marking;
mod minor;
pub mod old;
mod paged_old;
mod verify;
pub mod young;

// determines size of young generation in heap
// young generation size = heap size / YOUNG_RATIO
const YOUNG_RATIO: usize = 2;

// heap is divided into cards of size CARD_SIZE.
// card entry determines whether this part of the heap was modified
// in minor collections those parts of the heap need to be analyzed
pub const CARD_SIZE: usize = 512;
pub const CARD_SIZE_BITS: usize = 9;

pub const LARGE_OBJECT_SIZE: usize = 16 * 1024;

pub struct Swiper {
    // contiguous memory for young/old generation and large space
    heap: Region,

    // contains heap and also card table and crossing map
    reserved_area: Region,

    young: YoungGen,
    old: OldGen,
    large: LargeSpace,

    card_table: CardTable,
    crossing_map: CrossingMap,

    card_table_offset: usize,

    // minimum & maximum heap size
    min_heap_size: usize,
    max_heap_size: usize,

    threadpool: ThreadPool,

    stats: GcStats,
}

impl Swiper {
    pub fn new(args: &Args) -> Swiper {
        let min_heap_size = args.min_heap_size();
        let max_heap_size = args.max_heap_size();

        // determine initial young-ratio
        let mut young_ratio = args.flag_gc_young_ratio.unwrap_or(YOUNG_RATIO);

        if young_ratio == 0 {
            young_ratio = YOUNG_RATIO;
        }

        // determine size for generations
        let young_size = align_gen(max_heap_size / young_ratio);
        let old_size = align_gen(max_heap_size - young_size);

        let max_heap_size = young_size + old_size;

        // determine size for card table
        let card_size = mem::page_align((4 * max_heap_size) >> CARD_SIZE_BITS);

        // determine size for crossing map
        let crossing_size = mem::page_align(max_heap_size >> CARD_SIZE_BITS);

        // determine full memory size
        let reserve_size = max_heap_size * 4 + card_size + crossing_size;

        // reserve full memory
        let ptr = arena::reserve(reserve_size);

        // heap is young/old generation & large space
        let heap_start = ptr;
        let heap_end = ptr.offset(4 * max_heap_size);

        // reserved area also contains card table & crossing map
        let reserved_area = heap_start.region_start(reserve_size);

        // determine offset to card table (card table starts right after heap)
        // offset = card_table_start - (heap_start >> CARD_SIZE_BITS)
        let card_table_offset = heap_end.to_usize() - (heap_start.to_usize() >> CARD_SIZE_BITS);

        // determine boundaries for card table
        let card_start = heap_end;
        let card_end = card_start.offset(card_size);

        arena::commit(card_start, card_size, false);

        // determine boundaries for crossing map
        let crossing_start = card_end;
        let crossing_end = crossing_start.offset(crossing_size);

        arena::commit(crossing_start, crossing_size, false);

        // determine boundaries of young generation
        let young_start = heap_start;
        let young_end = young_start.offset(max_heap_size);
        let young = Region::new(young_start, young_end);

        // determine boundaries of old generation
        let old_start = young_end;
        let old_end = old_start.offset(max_heap_size);

        // determine large object space
        let large_start = old_end;
        let large_end = large_start.offset(2 * max_heap_size);

        let card_table = CardTable::new(
            card_start,
            card_end,
            Region::new(old_start, large_end),
            old_end,
            max_heap_size,
        );
        let crossing_map = CrossingMap::new(crossing_start, crossing_end, max_heap_size);
        let young = YoungGen::new(young, young_size, args.flag_gc_verify);
        let old = OldGen::new(
            old_start,
            old_end,
            old_size,
            crossing_map.clone(),
            card_table.clone(),
        );
        let large = LargeSpace::new(large_start, large_end);

        if args.flag_gc_verbose {
            println!(
                "GC: heap info: {}, old {}, young {}, card {}, crossing {}",
                formatted_size(max_heap_size),
                formatted_size(old_size),
                formatted_size(young_size),
                formatted_size(card_size),
                formatted_size(crossing_size)
            );
        }

        let nworkers = if args.flag_gc_worker > 1 {
            args.flag_gc_worker
        } else {
            1
        };

        Swiper {
            heap: Region::new(heap_start, heap_end),
            reserved_area: reserved_area,

            young: young,
            old: old,
            large: large,

            card_table: card_table,
            crossing_map: crossing_map,

            card_table_offset: card_table_offset,

            min_heap_size: min_heap_size,
            max_heap_size: max_heap_size,

            threadpool: ThreadPool::with_name("gc-worker".to_string(), nworkers),
            stats: GcStats::new(),
        }
    }

    fn minor_collect_inner(&self, vm: &VM, reason: GcReason) -> bool {
        // make heap iterable
        tlab::make_iterable(vm);
        let rootset = get_rootset(vm);

        self.verify(vm, VerifierPhase::PreMinor, "pre-minor", &rootset);
        let promotion_failed = self.minor_collect(vm, reason, &rootset);
        self.verify(vm, VerifierPhase::PostMinor, "post-minor", &rootset);

        promotion_failed
    }

    fn minor_collect(&self, vm: &VM, reason: GcReason, rootset: &[Slot]) -> bool {
        let mut collector = MinorCollector::new(
            vm,
            &self.young,
            &self.old,
            &self.large,
            &self.card_table,
            &self.crossing_map,
            rootset,
            reason,
            self.min_heap_size,
            self.max_heap_size,
            &self.stats,
            &self.threadpool,
        );
        collector.collect()
    }

    fn full_collect(&self, vm: &VM, reason: GcReason) {
        // make heap iterable
        tlab::make_iterable(vm);

        let rootset = get_rootset(vm);

        self.verify(vm, VerifierPhase::PreFull, "pre-full", &rootset);

        let mut collector = FullCollector::new(
            vm,
            self.heap.clone(),
            &self.young,
            &self.old,
            &self.large,
            &self.card_table,
            &self.crossing_map,
            &vm.gc.perm_space,
            &rootset,
            reason,
            &self.threadpool,
            self.min_heap_size,
            self.max_heap_size,
            &self.stats,
        );
        collector.collect();

        self.verify(vm, VerifierPhase::PostFull, "post-full", &rootset);
    }

    fn verify(&self, vm: &VM, phase: VerifierPhase, _name: &str, rootset: &[Slot]) {
        if vm.args.flag_gc_verify {
            if vm.args.flag_gc_dev_verbose {
                println!("GC: Verify {}", _name);
            }

            let perm_space = &vm.gc.perm_space;

            let mut verifier = Verifier::new(
                &self.young,
                &self.old,
                &self.card_table,
                &self.crossing_map,
                rootset,
                &self.large,
                &*perm_space,
                self.reserved_area.clone(),
                phase,
            );
            verifier.verify();

            if vm.args.flag_gc_dev_verbose {
                println!("GC: Verify {} finished", _name);
            }
        }
    }
}

impl Collector for Swiper {
    fn alloc_tlab_area(&self, vm: &VM, size: usize) -> Option<Region> {
        let ptr = self.young.bump_alloc(size);

        if !ptr.is_null() {
            return Some(ptr.region_start(size));
        }

        let promotion_failed = self.minor_collect_inner(vm, GcReason::AllocationFailure);

        if promotion_failed {
            self.full_collect(vm, GcReason::PromotionFailure);
            let ptr = self.young.bump_alloc(size);

            return if ptr.is_null() {
                None
            } else {
                Some(ptr.region_start(size))
            };
        }

        let ptr = self.young.bump_alloc(size);

        if !ptr.is_null() {
            return Some(ptr.region_start(size));
        }

        self.full_collect(vm, GcReason::AllocationFailure);
        let ptr = self.young.bump_alloc(size);

        return if ptr.is_null() {
            None
        } else {
            Some(ptr.region_start(size))
        };
    }

    fn alloc_normal(&self, vm: &VM, size: usize, array_ref: bool) -> Address {
        let ptr = self.young.bump_alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        let promotion_failed = self.minor_collect_inner(vm, GcReason::AllocationFailure);

        if promotion_failed {
            self.full_collect(vm, GcReason::PromotionFailure);
        }

        let ptr = self.young.bump_alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        self.old.bump_alloc(size, array_ref)
    }

    fn alloc_large(&self, vm: &VM, size: usize, _: bool) -> Address {
        let ptr = self.large.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        self.full_collect(vm, GcReason::AllocationFailure);

        self.large.alloc(size)
    }

    fn collect(&self, vm: &VM, reason: GcReason) {
        self.full_collect(vm, reason);
    }

    fn minor_collect(&self, vm: &VM, reason: GcReason) {
        tlab::make_iterable(vm);
        self.minor_collect_inner(vm, reason);
    }

    fn needs_write_barrier(&self) -> bool {
        return true;
    }

    fn card_table_offset(&self) -> usize {
        self.card_table_offset
    }

    fn dump_summary(&self, runtime: f32) {
        self.stats.update(|stats| {
            let total_gc = stats.minor_runtime() + stats.full_runtime();
            let gc_percentage = ((total_gc / runtime) * 100.0).round();
            let mutator_percentage = 100.0 - gc_percentage;

            println!(
                "GC summary: {:.1}ms minor ({}), {:.1}ms full ({}), {:.1}ms runtime ({}% mutator, {}% GC)",
                stats.minor_runtime(),
                stats.minor_collections(),
                stats.full_runtime(),
                stats.full_collections(),
                runtime,
                mutator_percentage,
                gc_percentage,
            );
        })
    }
}

#[derive(Copy, Clone, PartialEq, Eq)]
pub struct CardIdx(usize);

impl CardIdx {
    pub fn to_usize(self) -> usize {
        self.0
    }

    pub fn offset(self, val: usize) -> CardIdx {
        (self.0 + val).into()
    }
}

impl From<usize> for CardIdx {
    fn from(val: usize) -> CardIdx {
        CardIdx(val)
    }
}

fn on_different_cards(curr: Address, next: Address) -> bool {
    (curr.to_usize() >> CARD_SIZE_BITS) != (next.to_usize() >> CARD_SIZE_BITS)
}

pub struct GcStats {
    stats: Mutex<GcStatsProtected>,
}

impl GcStats {
    fn new() -> GcStats {
        GcStats {
            stats: Mutex::new(GcStatsProtected::new()),
        }
    }

    fn update<F>(&self, fct: F)
    where
        F: FnOnce(&mut GcStatsProtected),
    {
        let mut stats = self.stats.lock().unwrap();
        fct(&mut *stats);
    }
}

struct GcStatsProtected {
    // number of minor GCs
    minor_collections: usize,

    // total runtime of minor GCs
    minor_runtime: f32,

    // number of full GCs
    full_collections: usize,

    // total runtime of full GCs
    full_runtime: f32,
}

impl GcStatsProtected {
    fn new() -> GcStatsProtected {
        GcStatsProtected {
            minor_collections: 0,
            minor_runtime: 0.0,
            full_collections: 0,
            full_runtime: 0.0,
        }
    }

    fn incr_minor_collections(&mut self) {
        self.minor_collections += 1;
    }

    fn minor_collections(&self) -> usize {
        self.minor_collections
    }

    fn incr_minor_runtime(&mut self, time: f32) {
        self.minor_runtime += time;
    }

    fn minor_runtime(&self) -> f32 {
        self.minor_runtime
    }

    fn incr_full_collections(&mut self) {
        self.full_collections += 1;
    }

    fn full_collections(&self) -> usize {
        self.full_collections
    }

    fn incr_full_runtime(&mut self, time: f32) {
        self.full_runtime += time;
    }

    fn full_runtime(&self) -> f32 {
        self.full_runtime
    }
}
<commit_msg>swiper: young_ratio can't be 1<commit_after>use std::sync::Mutex;

use threadpool::ThreadPool;

use ctxt::VM;
use driver::cmd::Args;
use gc::root::{get_rootset, Slot};
use gc::swiper::card::CardTable;
use gc::swiper::crossing::CrossingMap;
use gc::swiper::full::FullCollector;
use gc::swiper::large::LargeSpace;
use gc::swiper::minor::MinorCollector;
use gc::swiper::old::OldGen;
use gc::swiper::verify::{Verifier, VerifierPhase};
use gc::swiper::young::YoungGen;
use gc::tlab;
use gc::Collector;
use gc::{align_gen, arena, GcReason};
use gc::{formatted_size, Address, Region};
use mem;

pub mod card;
mod controller;
mod crossing;
mod full;
mod large;
mod marking;
mod minor;
pub mod old;
mod paged_old;
mod verify;
pub mod young;

// determines size of young generation in heap
// young generation size = heap size / YOUNG_RATIO
const YOUNG_RATIO: usize = 2;

// heap is divided into cards of size CARD_SIZE.
// card entry determines whether this part of the heap was modified
// in minor collections those parts of the heap need to be analyzed
pub const CARD_SIZE: usize = 512;
pub const CARD_SIZE_BITS: usize = 9;

pub const LARGE_OBJECT_SIZE: usize = 16 * 1024;

pub struct Swiper {
    // contiguous memory for young/old generation and large space
    heap: Region,

    // contains heap and also card table and crossing map
    reserved_area: Region,

    young: YoungGen,
    old: OldGen,
    large: LargeSpace,

    card_table: CardTable,
    crossing_map: CrossingMap,

    card_table_offset: usize,

    // minimum & maximum heap size
    min_heap_size: usize,
    max_heap_size: usize,

    threadpool: ThreadPool,

    stats: GcStats,
}

impl Swiper {
    pub fn new(args: &Args) -> Swiper {
        let min_heap_size = args.min_heap_size();
        let max_heap_size = args.max_heap_size();

        // determine initial young-ratio
        let mut young_ratio = args.flag_gc_young_ratio.unwrap_or(YOUNG_RATIO);

        if young_ratio < 2 {
            young_ratio = YOUNG_RATIO;
        }

        // determine size for generations
        let young_size = align_gen(max_heap_size / young_ratio);
        let old_size = align_gen(max_heap_size - young_size);

        let max_heap_size = young_size + old_size;

        // determine size for card table
        let card_size = mem::page_align((4 * max_heap_size) >> CARD_SIZE_BITS);

        // determine size for crossing map
        let crossing_size = mem::page_align(max_heap_size >> CARD_SIZE_BITS);

        // determine full memory size
        let reserve_size = max_heap_size * 4 + card_size + crossing_size;

        // reserve full memory
        let ptr = arena::reserve(reserve_size);

        // heap is young/old generation & large space
        let heap_start = ptr;
        let heap_end = ptr.offset(4 * max_heap_size);

        // reserved area also contains card table & crossing map
        let reserved_area = heap_start.region_start(reserve_size);

        // determine offset to card table (card table starts right after heap)
        // offset = card_table_start - (heap_start >> CARD_SIZE_BITS)
        let card_table_offset = heap_end.to_usize() - (heap_start.to_usize() >> CARD_SIZE_BITS);

        // determine boundaries for card table
        let card_start = heap_end;
        let card_end = card_start.offset(card_size);

        arena::commit(card_start, card_size, false);

        // determine boundaries for crossing map
        let crossing_start = card_end;
        let crossing_end = crossing_start.offset(crossing_size);

        arena::commit(crossing_start, crossing_size, false);

        // determine boundaries of young generation
        let young_start = heap_start;
        let young_end = young_start.offset(max_heap_size);
        let young = Region::new(young_start, young_end);

        // determine boundaries of old generation
        let old_start = young_end;
        let old_end = old_start.offset(max_heap_size);

        // determine large object space
        let large_start = old_end;
        let large_end = large_start.offset(2 * max_heap_size);

        let card_table = CardTable::new(
            card_start,
            card_end,
            Region::new(old_start, large_end),
            old_end,
            max_heap_size,
        );
        let crossing_map = CrossingMap::new(crossing_start, crossing_end, max_heap_size);
        let young = YoungGen::new(young, young_size, args.flag_gc_verify);
        let old = OldGen::new(
            old_start,
            old_end,
            old_size,
            crossing_map.clone(),
            card_table.clone(),
        );
        let large = LargeSpace::new(large_start, large_end);

        if args.flag_gc_verbose {
            println!(
                "GC: heap info: {}, old {}, young {}, card {}, crossing {}",
                formatted_size(max_heap_size),
                formatted_size(old_size),
                formatted_size(young_size),
                formatted_size(card_size),
                formatted_size(crossing_size)
            );
        }

        let nworkers = if args.flag_gc_worker > 1 {
            args.flag_gc_worker
        } else {
            1
        };

        Swiper {
            heap: Region::new(heap_start, heap_end),
            reserved_area: reserved_area,

            young: young,
            old: old,
            large: large,

            card_table: card_table,
            crossing_map: crossing_map,

            card_table_offset: card_table_offset,

            min_heap_size: min_heap_size,
            max_heap_size: max_heap_size,

            threadpool: ThreadPool::with_name("gc-worker".to_string(), nworkers),
            stats: GcStats::new(),
        }
    }

    fn minor_collect_inner(&self, vm: &VM, reason: GcReason) -> bool {
        // make heap iterable
        tlab::make_iterable(vm);
        let rootset = get_rootset(vm);

        self.verify(vm, VerifierPhase::PreMinor, "pre-minor", &rootset);
        let promotion_failed = self.minor_collect(vm, reason, &rootset);
        self.verify(vm, VerifierPhase::PostMinor, "post-minor", &rootset);

        promotion_failed
    }

    fn minor_collect(&self, vm: &VM, reason: GcReason, rootset: &[Slot]) -> bool {
        let mut collector = MinorCollector::new(
            vm,
            &self.young,
            &self.old,
            &self.large,
            &self.card_table,
            &self.crossing_map,
            rootset,
            reason,
            self.min_heap_size,
            self.max_heap_size,
            &self.stats,
            &self.threadpool,
        );
        collector.collect()
    }

    fn full_collect(&self, vm: &VM, reason: GcReason) {
        // make heap iterable
        tlab::make_iterable(vm);

        let rootset = get_rootset(vm);

        self.verify(vm, VerifierPhase::PreFull, "pre-full", &rootset);

        let mut collector = FullCollector::new(
            vm,
            self.heap.clone(),
            &self.young,
            &self.old,
            &self.large,
            &self.card_table,
            &self.crossing_map,
            &vm.gc.perm_space,
            &rootset,
            reason,
            &self.threadpool,
            self.min_heap_size,
            self.max_heap_size,
            &self.stats,
        );
        collector.collect();

        self.verify(vm, VerifierPhase::PostFull, "post-full", &rootset);
    }

    fn verify(&self, vm: &VM, phase: VerifierPhase, _name: &str, rootset: &[Slot]) {
        if vm.args.flag_gc_verify {
            if vm.args.flag_gc_dev_verbose {
                println!("GC: Verify {}", _name);
            }

            let perm_space = &vm.gc.perm_space;

            let mut verifier = Verifier::new(
                &self.young,
                &self.old,
                &self.card_table,
                &self.crossing_map,
                rootset,
                &self.large,
                &*perm_space,
                self.reserved_area.clone(),
                phase,
            );
            verifier.verify();

            if vm.args.flag_gc_dev_verbose {
                println!("GC: Verify {} finished", _name);
            }
        }
    }
}

impl Collector for Swiper {
    fn alloc_tlab_area(&self, vm: &VM, size: usize) -> Option<Region> {
        let ptr = self.young.bump_alloc(size);

        if !ptr.is_null() {
            return Some(ptr.region_start(size));
        }

        let promotion_failed = self.minor_collect_inner(vm, GcReason::AllocationFailure);

        if promotion_failed {
            self.full_collect(vm, GcReason::PromotionFailure);
            let ptr = self.young.bump_alloc(size);

            return if ptr.is_null() {
                None
            } else {
                Some(ptr.region_start(size))
            };
        }

        let ptr = self.young.bump_alloc(size);

        if !ptr.is_null() {
            return Some(ptr.region_start(size));
        }

        self.full_collect(vm, GcReason::AllocationFailure);
        let ptr = self.young.bump_alloc(size);

        return if ptr.is_null() {
            None
        } else {
            Some(ptr.region_start(size))
        };
    }

    fn alloc_normal(&self, vm: &VM, size: usize, array_ref: bool) -> Address {
        let ptr = self.young.bump_alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        let promotion_failed = self.minor_collect_inner(vm, GcReason::AllocationFailure);

        if promotion_failed {
            self.full_collect(vm, GcReason::PromotionFailure);
        }

        let ptr = self.young.bump_alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        self.old.bump_alloc(size, array_ref)
    }

    fn alloc_large(&self, vm: &VM, size: usize, _: bool) -> Address {
        let ptr = self.large.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        self.full_collect(vm, GcReason::AllocationFailure);

        self.large.alloc(size)
    }

    fn collect(&self, vm: &VM, reason: GcReason) {
        self.full_collect(vm, reason);
    }

    fn minor_collect(&self, vm: &VM, reason: GcReason) {
        tlab::make_iterable(vm);
        self.minor_collect_inner(vm, reason);
    }

    fn needs_write_barrier(&self) -> bool {
        return true;
    }

    fn card_table_offset(&self) -> usize {
        self.card_table_offset
    }

    fn dump_summary(&self, runtime: f32) {
        self.stats.update(|stats| {
            let total_gc = stats.minor_runtime() + stats.full_runtime();
            let gc_percentage = ((total_gc / runtime) * 100.0).round();
            let mutator_percentage = 100.0 - gc_percentage;

            println!(
                "GC summary: {:.1}ms minor ({}), {:.1}ms full ({}), {:.1}ms runtime ({}% mutator, {}% GC)",
                stats.minor_runtime(),
                stats.minor_collections(),
                stats.full_runtime(),
                stats.full_collections(),
                runtime,
                mutator_percentage,
                gc_percentage,
            );
        })
    }
}

#[derive(Copy, Clone, PartialEq, Eq)]
pub struct CardIdx(usize);

impl CardIdx {
    pub fn to_usize(self) -> usize {
        self.0
    }

    pub fn offset(self, val: usize) -> CardIdx {
        (self.0 + val).into()
    }
}

impl From<usize> for CardIdx {
    fn from(val: usize) -> CardIdx {
        CardIdx(val)
    }
}

fn on_different_cards(curr: Address, next: Address) -> bool {
    (curr.to_usize() >> CARD_SIZE_BITS) != (next.to_usize() >> CARD_SIZE_BITS)
}

pub struct GcStats {
    stats: Mutex<GcStatsProtected>,
}

impl GcStats {
    fn new() -> GcStats {
        GcStats {
            stats: Mutex::new(GcStatsProtected::new()),
        }
    }

    fn update<F>(&self, fct: F)
    where
        F: FnOnce(&mut GcStatsProtected),
    {
        let mut stats = self.stats.lock().unwrap();
        fct(&mut *stats);
    }
}

struct GcStatsProtected {
    // number of minor GCs
    minor_collections: usize,

    // total runtime of minor GCs
    minor_runtime: f32,

    // number of full GCs
    full_collections: usize,

    // total runtime of full GCs
    full_runtime: f32,
}

impl GcStatsProtected {
    fn new() -> GcStatsProtected {
        GcStatsProtected {
            minor_collections: 0,
            minor_runtime: 0.0,
            full_collections: 0,
            full_runtime: 0.0,
        }
    }

    fn incr_minor_collections(&mut self) {
        self.minor_collections += 1;
    }

    fn minor_collections(&self) -> usize {
        self.minor_collections
    }

    fn incr_minor_runtime(&mut self, time: f32) {
        self.minor_runtime += time;
    }

    fn minor_runtime(&self) -> f32 {
        self.minor_runtime
    }

    fn incr_full_collections(&mut self) {
        self.full_collections += 1;
    }

    fn full_collections(&self) -> usize {
        self.full_collections
    }

    fn incr_full_runtime(&mut self, time: f32) {
        self.full_runtime += time;
    }

    fn full_runtime(&self) -> f32 {
        self.full_runtime
    }
}
<|endoftext|>


<commit_before>use std::io::net::ip::IpAddr;
use std::io::net::ip::SocketAddr;
use collections::HashSet;
use cluster::Cluster;

/// A server/node within a single gossip cluster. Each server has
/// a fast knowledge of it's cluster, which is all stored here.
#[deriving(Show,Eq)]
pub struct Server {
    addr: SocketAddr,
    cluster: Option<Cluster>,
    metadata: Metadata
}

#[deriving(Show,Eq)]
pub struct Metadata {
    // An eager set contains the peers that the current node will
    // communicate with when a new message comes in. The goal is to form
    // a [Spanning Tree](https://en.wikipedia.org/wiki/Spanning_tree)
    eager_set: HashSet<~str>,

    // The lazy set contains the nodes where if they were within the eager set,
    // would add additional, duplicate edges to the cluster graph. This means that
    // given Node A, B, C, and D:
    //
    // A- B
    // |\/|
    // |/\|
    // C- D
    //
    // Each node has the rest of the nodes within their eager set. This means that
    // given a new broadcast, they'll be duplicate, wasteful messages sent across.
    //
    // This isn't the ideal state that the cluster should be in. We're striving
    // to form a spanning tree where the links to A and D, C and B, and B and D
    // are cut.
    //
    // A --- B
    // |
    // |
    // C --- D
    //
    // This is the most optimal graph our cluster's state would represent.
    //
    // We still need to keep the cut edges, which we'll put inside the lazy set.
    // This is used for healing the tree (when a node goes down or we have a network
    // partition somewhere resulting in some nodes missing broadcasts.)
    lazy_set: HashSet<~str>,
    exchanges: Vec<~str>,
    outstanding: Vec<~str>
}

impl Metadata {
    pub fn new() -> Metadata {
        Metadata {
            eager_set: HashSet::new()
        }
    }
}

impl Server {

    /// Create a new server given an address (ipv4 or ipv6) and a port.
    /// This function will **not** do any connection initializations. This
    /// is handled by further methods.
    pub fn new(ip: IpAddr, port: u16) -> Server {
        Server {
            // We're handling the creation of the SocketAddr to allow
            // for a more friendly API.
            addr: SocketAddr {
                ip: ip,
                port: port
            },

            // By default, we aren't joining a cluster yet.
            cluster: None,
            metadata: Metadata::new()
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::io::net::ip::Ipv4Addr;

    #[test]
    fn new_server() {
        let server = Server::new(Ipv4Addr(127, 0, 0, 1), 4989);

        assert_eq!(server.addr.ip, Ipv4Addr(127, 0, 0, 1));
        assert_eq!(server.addr.port, 4989);
        match server.cluster {
            Some(_) => fail!("Expected a new server without joining a cluster."),
            None => {}
        }
    }
}<commit_msg>fixed the failing test<commit_after>use std::io::net::ip::IpAddr;
use std::io::net::ip::SocketAddr;
use collections::HashSet;
use cluster::Cluster;

/// A server/node within a single gossip cluster. Each server has
/// a fast knowledge of it's cluster, which is all stored here.
#[deriving(Show,Eq)]
pub struct Server {
    addr: SocketAddr,
    cluster: Option<Cluster>,
    metadata: Metadata
}

#[deriving(Show,Eq)]
pub struct Metadata {
    // An eager set contains the peers that the current node will
    // communicate with when a new message comes in. The goal is to form
    // a [Spanning Tree](https://en.wikipedia.org/wiki/Spanning_tree)
    eager_set: HashSet<~str>,

    // The lazy set contains the nodes where if they were within the eager set,
    // would add additional, duplicate edges to the cluster graph. This means that
    // given Node A, B, C, and D:
    //
    // A- B
    // |\/|
    // |/\|
    // C- D
    //
    // Each node has the rest of the nodes within their eager set. This means that
    // given a new broadcast, they'll be duplicate, wasteful messages sent across.
    //
    // This isn't the ideal state that the cluster should be in. We're striving
    // to form a spanning tree where the links to A and D, C and B, and B and D
    // are cut.
    //
    // A --- B
    // |
    // |
    // C --- D
    //
    // This is the most optimal graph our cluster's state would represent.
    //
    // We still need to keep the cut edges, which we'll put inside the lazy set.
    // This is used for healing the tree (when a node goes down or we have a network
    // partition somewhere resulting in some nodes missing broadcasts.)
    lazy_set: HashSet<~str>,
    exchanges: Vec<~str>,
    outstanding: Vec<~str>
}

impl Metadata {
    pub fn new() -> Metadata {
        Metadata {
            eager_set: HashSet::new(),
            lazy_set: HashSet::new(),
            exchanges: Vec::new(),
            outstanding: Vec::new()
        }
    }
}

impl Server {

    /// Create a new server given an address (ipv4 or ipv6) and a port.
    /// This function will **not** do any connection initializations. This
    /// is handled by further methods.
    pub fn new(ip: IpAddr, port: u16) -> Server {
        Server {
            // We're handling the creation of the SocketAddr to allow
            // for a more friendly API.
            addr: SocketAddr {
                ip: ip,
                port: port
            },

            // By default, we aren't joining a cluster yet.
            cluster: None,
            metadata: Metadata::new()
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use std::io::net::ip::Ipv4Addr;

    #[test]
    fn new_server() {
        let server = Server::new(Ipv4Addr(127, 0, 0, 1), 4989);

        assert_eq!(server.addr.ip, Ipv4Addr(127, 0, 0, 1));
        assert_eq!(server.addr.port, 4989);
        match server.cluster {
            Some(_) => fail!("Expected a new server without joining a cluster."),
            None => {}
        }
    }
}<|endoftext|>


<commit_before><commit_msg>Create output dir for benchmarks as necessary<commit_after><|endoftext|>


<commit_before>use std::collections::HashSet;

use enums;
use parse;

pub fn write_descriptor_sets(doc: &parse::Spirv) -> String {
    // TODO: not implemented

    // finding all the descriptors
    let mut descriptors = Vec::new();
    struct Descriptor {
        name: String,
        desc_ty: String,
        bind_ty: String,
        bind_start: String,
        bind_end: String,
        set: u32,
        binding: u32,
    }

    // looping to find all the elements that have the `DescriptorSet` decoration
    for instruction in doc.instructions.iter() {
        let (variable_id, descriptor_set) = match instruction {
            &parse::Instruction::Decorate { target_id, decoration: enums::Decoration::DecorationDescriptorSet, ref params } => {
                (target_id, params[0])
            },
            _ => continue
        };

        // find which type is pointed to by this variable
        let pointed_ty = pointer_variable_ty(doc, variable_id);
        // name of the variable
        let name = ::name_from_id(doc, variable_id);

        // find the binding point of this descriptor
        let binding = doc.instructions.iter().filter_map(|i| {
            match i {
                &parse::Instruction::Decorate { target_id, decoration: enums::Decoration::DecorationBinding, ref params } if target_id == variable_id => {
                    Some(params[0])
                },
                _ => None,      // TODO: other types
            }
        }).next().expect(&format!("Uniform `{}` is missing a binding", name));

        // find informations about the kind of binding for this descriptor
        let (desc_ty, bind_ty, bind_start, bind_end) = doc.instructions.iter().filter_map(|i| {
            match i {
                &parse::Instruction::TypeStruct { result_id, .. } if result_id == pointed_ty => {
                    Some((
                        "::vulkano::descriptor_set::DescriptorType::UniformBuffer",
                        "::vulkano::buffer::AbstractBuffer",
                        "::vulkano::descriptor_set::DescriptorBind::UniformBuffer { buffer: ",
                        ", offset: 0, size: 128 /* FIXME */ }"
                    ))
                },
                &parse::Instruction::TypeImage { result_id, sampled_type_id, ref dim, arrayed, ms,
                                                 sampled, ref format, ref access, .. }
                                        if result_id == pointed_ty && sampled == Some(true) =>
                {
                    Some((
                        "::vulkano::descriptor_set::DescriptorType::SampledImage",
                        "::vulkano::image::AbstractImageView",
                        "::vulkano::descriptor_set::DescriptorBind::SampledImage(",
                        ", ::vulkano::image::Layout::ShaderReadOnlyOptimal)"      // FIXME:
                    ))
                },
                &parse::Instruction::TypeImage { result_id, sampled_type_id, ref dim, arrayed, ms,
                                                 sampled, ref format, ref access, .. }
                                        if result_id == pointed_ty && sampled == Some(false) =>
                {
                    Some((
                        "::vulkano::descriptor_set::DescriptorType::InputAttachment",       // FIXME: can be `StorageImage`
                        "::vulkano::image::AbstractImageView",
                        "::vulkano::descriptor_set::DescriptorBind::InputAttachment(",      // FIXME:
                        ", ::vulkano::image::Layout::ShaderReadOnlyOptimal)"     // FIXME:
                    ))
                },
                &parse::Instruction::TypeSampledImage { result_id, image_type_id }
                                                                    if result_id == pointed_ty =>
                {
                    Some((
                        "::vulkano::descriptor_set::DescriptorType::SampledImage",
                        "::vulkano::image::AbstractImageView",
                        "::vulkano::descriptor_set::DescriptorBind::SampledImage(",
                        ", ::vulkano::image::Layout::ShaderReadOnlyOptimal)"      // FIXME:
                    ))
                },
                _ => None,      // TODO: other types
            }
        }).next().expect(&format!("Couldn't find relevant type for uniform `{}` (type {}, maybe unimplemented)", name, pointed_ty));

        descriptors.push(Descriptor {
            name: name,
            desc_ty: desc_ty.to_owned(),
            bind_ty: bind_ty.to_owned(),
            bind_start: bind_start.to_owned(),
            bind_end: bind_end.to_owned(),
            set: descriptor_set,
            binding: binding,
        });
    }

    let sets_list = descriptors.iter().map(|d| d.set).collect::<HashSet<u32>>();

    let mut output = String::new();

    // iterate once per set that is defined somewhere
    for set in sets_list.iter() {
        let write_ty = descriptors.iter().filter(|d| d.set == *set)
                                  .map(|d| format!("::std::sync::Arc<{}>", d.bind_ty))
                                  .collect::<Vec<_>>();

        let writes = descriptors.iter().enumerate().filter(|&(_, d)| d.set == *set)
                                .map(|(entry, d)| {
                                    let entry = if write_ty.len() == 1 {
                                        "".to_owned()
                                    } else {
                                        format!(".{}", entry)
                                    };

                                    format!("::vulkano::descriptor_set::DescriptorWrite {{
                                                 binding: {binding},
                                                 array_element: 0,
                                                 content: {bind_start} data{entry} {bind_end},
                                             }}", binding = d.binding, bind_start = d.bind_start,
                                                  bind_end = d.bind_end,
                                                  entry = entry)
                                })
                                .collect::<Vec<_>>();

        let write_ty = if write_ty.len() == 1 {
            write_ty.into_iter().next().unwrap()
        } else {
            format!("({})", write_ty.join(", "))
        };

        let descr = descriptors.iter().enumerate().filter(|&(_, d)| d.set == *set)
                               .map(|(entry, d)| {
                                   format!("::vulkano::descriptor_set::DescriptorDesc {{
                                                binding: {binding},
                                                ty: {desc_ty},
                                                array_count: 1,
                                                stages: ::vulkano::descriptor_set::ShaderStages::all_graphics(),        // TODO:
                                            }}", binding = d.binding, desc_ty = d.desc_ty)
                               })
                               .collect::<Vec<_>>();

        output.push_str(&format!(r#"
#[derive(Default)]
pub struct Set{set};

unsafe impl ::vulkano::descriptor_set::SetLayout for Set{set} {{
    fn descriptors(&self) -> Vec<::vulkano::descriptor_set::DescriptorDesc> {{
        vec![
            {descr}
        ]
    }}
}}

unsafe impl ::vulkano::descriptor_set::SetLayoutWrite<{write_ty}> for Set{set} {{
    fn decode(&self, data: {write_ty}) -> Vec<::vulkano::descriptor_set::DescriptorWrite> {{
        vec![
            {writes}
        ]
    }}
}}

unsafe impl ::vulkano::descriptor_set::SetLayoutInit<{write_ty}> for Set{set} {{
    fn decode(&self, data: {write_ty}) -> Vec<::vulkano::descriptor_set::DescriptorWrite> {{
        ::vulkano::descriptor_set::SetLayoutWrite::decode(self, data)
    }}
}}

"#, set = set, write_ty = write_ty, writes = writes.join(","), descr = descr.join(",")));
    }

    let max_set = sets_list.iter().cloned().max().map(|v| v + 1).unwrap_or(0);

    let sets_defs = (0 .. max_set).map(|num| {
        if sets_list.contains(&num) {
            format!("::std::sync::Arc<::vulkano::descriptor_set::DescriptorSet<Set{}>>", num)
        } else {
            "()".to_owned()
        }
    }).collect::<Vec<_>>();

    let sets = (0 .. max_set).map(|num| {
        if sets_list.contains(&num) {
            if sets_defs.len() == 1 {
                format!("sets")
            } else {
                format!("sets.{}", num)
            }
        } else {
            "()".to_owned()     // FIXME: wrong
        }
    }).collect::<Vec<_>>();

    let layouts_defs = (0 .. max_set).map(|num| {
        if sets_list.contains(&num) {
            format!("::std::sync::Arc<::vulkano::descriptor_set::DescriptorSetLayout<Set{}>>", num)
        } else {
            "()".to_owned()
        }
    }).collect::<Vec<_>>();

    let layouts = (0 .. max_set).map(|num| {
        if sets_list.contains(&num) {
            if layouts_defs.len() == 1 {
                format!("layouts")
            } else {
                format!("layouts.{}", num)
            }
        } else {
            "()".to_owned()     // FIXME: wrong
        }
    }).collect::<Vec<_>>();

    output.push_str(&format!(r#"
#[derive(Default)]
pub struct Layout;

unsafe impl ::vulkano::descriptor_set::Layout for Layout {{
    type DescriptorSets = ({sets_defs});
    type DescriptorSetLayouts = ({layouts_defs});
    type PushConstants = ();

    fn decode_descriptor_set_layouts(&self, layouts: Self::DescriptorSetLayouts)
        -> Vec<::std::sync::Arc<::vulkano::descriptor_set::AbstractDescriptorSetLayout>>
    {{
        vec![
            {layouts}
        ]
    }}

    fn decode_descriptor_sets(&self, sets: Self::DescriptorSets)
        -> Vec<::std::sync::Arc<::vulkano::descriptor_set::AbstractDescriptorSet>>
    {{
        vec![
            {sets}
        ]
    }}
}}
"#, sets_defs = sets_defs.join(","), layouts_defs = layouts_defs.join(","),
    layouts = layouts.join(","), sets = sets.join(",")));

    output
}

/// Assumes that `variable` is a variable with a `TypePointer` and returns the id of the pointed
/// type.
fn pointer_variable_ty(doc: &parse::Spirv, variable: u32) -> u32 {
    let var_ty = doc.instructions.iter().filter_map(|i| {
        match i {
            &parse::Instruction::Variable { result_type_id, result_id, .. } if result_id == variable => {
                Some(result_type_id)
            },
            _ => None
        }
    }).next().unwrap();

    doc.instructions.iter().filter_map(|i| {
        match i {
            &parse::Instruction::TypePointer { result_id, type_id, .. } if result_id == var_ty => {
                Some(type_id)
            },
            _ => None
        }
    }).next().unwrap()
}
<commit_msg>Rework the set binding code in the shader parser<commit_after>use std::collections::HashSet;

use enums;
use parse;

pub fn write_descriptor_sets(doc: &parse::Spirv) -> String {
    // TODO: not implemented

    // finding all the descriptors
    let mut descriptors = Vec::new();
    struct Descriptor {
        name: String,
        desc_ty: String,
        bind_ty: String,
        bind: String,
        set: u32,
        binding: u32,
    }

    // looping to find all the elements that have the `DescriptorSet` decoration
    for instruction in doc.instructions.iter() {
        let (variable_id, descriptor_set) = match instruction {
            &parse::Instruction::Decorate { target_id, decoration: enums::Decoration::DecorationDescriptorSet, ref params } => {
                (target_id, params[0])
            },
            _ => continue
        };

        // find which type is pointed to by this variable
        let pointed_ty = pointer_variable_ty(doc, variable_id);
        // name of the variable
        let name = ::name_from_id(doc, variable_id);

        // find the binding point of this descriptor
        let binding = doc.instructions.iter().filter_map(|i| {
            match i {
                &parse::Instruction::Decorate { target_id, decoration: enums::Decoration::DecorationBinding, ref params } if target_id == variable_id => {
                    Some(params[0])
                },
                _ => None,      // TODO: other types
            }
        }).next().expect(&format!("Uniform `{}` is missing a binding", name));

        // find informations about the kind of binding for this descriptor
        let (desc_ty, bind_ty, bind) = doc.instructions.iter().filter_map(|i| {
            match i {
                &parse::Instruction::TypeStruct { result_id, .. } if result_id == pointed_ty => {
                    Some((
                        "::vulkano::descriptor_set::DescriptorType::UniformBuffer",
                        "::vulkano::buffer::AbstractBuffer",
                        "::vulkano::descriptor_set::DescriptorBind::UniformBuffer { buffer: data, offset: 0, size: 128 /* FIXME */ }"
                    ))
                },
                &parse::Instruction::TypeImage { result_id, sampled_type_id, ref dim, arrayed, ms,
                                                 sampled, ref format, ref access, .. }
                                        if result_id == pointed_ty && sampled == Some(true) =>
                {
                    Some((
                        "::vulkano::descriptor_set::DescriptorType::CombinedImageSampler",
                        "::vulkano::image::AbstractImageView",
                        "::vulkano::descriptor_set::DescriptorBind::CombinedImageSampler(data.0, data.1, ::vulkano::image::Layout::ShaderReadOnlyOptimal)"      // FIXME:
                    ))
                },
                &parse::Instruction::TypeImage { result_id, sampled_type_id, ref dim, arrayed, ms,
                                                 sampled, ref format, ref access, .. }
                                        if result_id == pointed_ty && sampled == Some(false) =>
                {
                    Some((
                        "::vulkano::descriptor_set::DescriptorType::InputAttachment",       // FIXME: can be `StorageImage`
                        "::vulkano::image::AbstractImageView",
                        "::vulkano::descriptor_set::DescriptorBind::InputAttachment(data, ::vulkano::image::Layout::ShaderReadOnlyOptimal)"     // FIXME:
                    ))
                },
                &parse::Instruction::TypeSampledImage { result_id, image_type_id }
                                                                    if result_id == pointed_ty =>
                {
                    Some((
                        "::vulkano::descriptor_set::DescriptorType::SampledImage",
                        "::vulkano::image::AbstractImageView",
                        "::vulkano::descriptor_set::DescriptorBind::SampledImage(data, ::vulkano::image::Layout::ShaderReadOnlyOptimal)"      // FIXME:
                    ))
                },
                _ => None,      // TODO: other types
            }
        }).next().expect(&format!("Couldn't find relevant type for uniform `{}` (type {}, maybe unimplemented)", name, pointed_ty));

        descriptors.push(Descriptor {
            name: name,
            desc_ty: desc_ty.to_owned(),
            bind_ty: bind_ty.to_owned(),
            bind: bind.to_owned(),
            set: descriptor_set,
            binding: binding,
        });
    }

    let sets_list = descriptors.iter().map(|d| d.set).collect::<HashSet<u32>>();

    let mut output = String::new();

    // iterate once per set that is defined somewhere
    for set in sets_list.iter() {
        let write_ty = descriptors.iter().filter(|d| d.set == *set)
                                  .map(|d| format!("::std::sync::Arc<{}>", d.bind_ty))
                                  .collect::<Vec<_>>();

        let writes = descriptors.iter().enumerate().filter(|&(_, d)| d.set == *set)
                                .map(|(entry, d)| {
                                    let entry = if write_ty.len() == 1 {
                                        "".to_owned()
                                    } else {
                                        format!(".{}", entry)
                                    };

                                    format!("{{ let data = data{entry};
                                            ::vulkano::descriptor_set::DescriptorWrite {{
                                                 binding: {binding},
                                                 array_element: 0,
                                                 content: {bind},
                                             }} }}", binding = d.binding, bind = d.bind,
                                                     entry = entry)
                                })
                                .collect::<Vec<_>>();

        let write_ty = if write_ty.len() == 1 {
            write_ty.into_iter().next().unwrap()
        } else {
            format!("({})", write_ty.join(", "))
        };

        let descr = descriptors.iter().enumerate().filter(|&(_, d)| d.set == *set)
                               .map(|(entry, d)| {
                                   format!("::vulkano::descriptor_set::DescriptorDesc {{
                                                binding: {binding},
                                                ty: {desc_ty},
                                                array_count: 1,
                                                stages: ::vulkano::descriptor_set::ShaderStages::all_graphics(),        // TODO:
                                            }}", binding = d.binding, desc_ty = d.desc_ty)
                               })
                               .collect::<Vec<_>>();

        output.push_str(&format!(r#"
#[derive(Default)]
pub struct Set{set};

unsafe impl ::vulkano::descriptor_set::SetLayout for Set{set} {{
    fn descriptors(&self) -> Vec<::vulkano::descriptor_set::DescriptorDesc> {{
        vec![
            {descr}
        ]
    }}
}}

unsafe impl ::vulkano::descriptor_set::SetLayoutWrite<{write_ty}> for Set{set} {{
    fn decode(&self, data: {write_ty}) -> Vec<::vulkano::descriptor_set::DescriptorWrite> {{
        vec![
            {writes}
        ]
    }}
}}

unsafe impl ::vulkano::descriptor_set::SetLayoutInit<{write_ty}> for Set{set} {{
    fn decode(&self, data: {write_ty}) -> Vec<::vulkano::descriptor_set::DescriptorWrite> {{
        ::vulkano::descriptor_set::SetLayoutWrite::decode(self, data)
    }}
}}

"#, set = set, write_ty = write_ty, writes = writes.join(","), descr = descr.join(",")));
    }

    let max_set = sets_list.iter().cloned().max().map(|v| v + 1).unwrap_or(0);

    let sets_defs = (0 .. max_set).map(|num| {
        if sets_list.contains(&num) {
            format!("::std::sync::Arc<::vulkano::descriptor_set::DescriptorSet<Set{}>>", num)
        } else {
            "()".to_owned()
        }
    }).collect::<Vec<_>>();

    let sets = (0 .. max_set).map(|num| {
        if sets_list.contains(&num) {
            if sets_defs.len() == 1 {
                format!("sets")
            } else {
                format!("sets.{}", num)
            }
        } else {
            "()".to_owned()     // FIXME: wrong
        }
    }).collect::<Vec<_>>();

    let layouts_defs = (0 .. max_set).map(|num| {
        if sets_list.contains(&num) {
            format!("::std::sync::Arc<::vulkano::descriptor_set::DescriptorSetLayout<Set{}>>", num)
        } else {
            "()".to_owned()
        }
    }).collect::<Vec<_>>();

    let layouts = (0 .. max_set).map(|num| {
        if sets_list.contains(&num) {
            if layouts_defs.len() == 1 {
                format!("layouts")
            } else {
                format!("layouts.{}", num)
            }
        } else {
            "()".to_owned()     // FIXME: wrong
        }
    }).collect::<Vec<_>>();

    output.push_str(&format!(r#"
#[derive(Default)]
pub struct Layout;

unsafe impl ::vulkano::descriptor_set::Layout for Layout {{
    type DescriptorSets = ({sets_defs});
    type DescriptorSetLayouts = ({layouts_defs});
    type PushConstants = ();

    fn decode_descriptor_set_layouts(&self, layouts: Self::DescriptorSetLayouts)
        -> Vec<::std::sync::Arc<::vulkano::descriptor_set::AbstractDescriptorSetLayout>>
    {{
        vec![
            {layouts}
        ]
    }}

    fn decode_descriptor_sets(&self, sets: Self::DescriptorSets)
        -> Vec<::std::sync::Arc<::vulkano::descriptor_set::AbstractDescriptorSet>>
    {{
        vec![
            {sets}
        ]
    }}
}}
"#, sets_defs = sets_defs.join(","), layouts_defs = layouts_defs.join(","),
    layouts = layouts.join(","), sets = sets.join(",")));

    output
}

/// Assumes that `variable` is a variable with a `TypePointer` and returns the id of the pointed
/// type.
fn pointer_variable_ty(doc: &parse::Spirv, variable: u32) -> u32 {
    let var_ty = doc.instructions.iter().filter_map(|i| {
        match i {
            &parse::Instruction::Variable { result_type_id, result_id, .. } if result_id == variable => {
                Some(result_type_id)
            },
            _ => None
        }
    }).next().unwrap();

    doc.instructions.iter().filter_map(|i| {
        match i {
            &parse::Instruction::TypePointer { result_id, type_id, .. } if result_id == var_ty => {
                Some(type_id)
            },
            _ => None
        }
    }).next().unwrap()
}
<|endoftext|>


<commit_before>extern crate itertools;
extern crate time;

use itertools::Itertools;
use time::PreciseTime;

fn check_sum_zero(v: Vec<i32>) -> bool {
    let int_list = v;

    for i in (1..int_list.len()+1).rev() {

        let iter = int_list.iter();

        for item in iter.combinations(i) {
            let a = item.into_iter();
            let sum: i32 = a.sum();

            if sum == 0 {
                return true;
            }
        }
    }
    return false;
}

fn main() {
    // Main function only for benchmarking Optional bonus 2
    let start = PreciseTime::now();
    println!("Running optional bonus 2 test...");

    let list_1 = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25];

    println!("Result: {:?}", check_sum_zero(list_1));

    let end = PreciseTime::now();

    println!("Second to finish: {}", start.to(end));

}

#[cfg(test)]
mod tests {
    use super::check_sum_zero;

    #[test]
    fn base_examples_ok() {

        let list_ok = [vec![-5, -3, -1, 2, 4, 6], vec![-1, 1],
            vec![-97364, -71561, -69336, 19675, 71561, 97863],
            vec![-53974, -39140, -36561, -23935, -15680, 0], vec![-2, 2, 3]];

        for list_vector in list_ok.iter() {
            assert_eq!(true, check_sum_zero(list_vector.to_vec()));
        }

    }

    #[test]
    fn base_examples_no_ok() {
        let list_no_ok = [vec![1, 2, 3], vec![]];

        for list_vector in list_no_ok.iter() {
            assert_eq!(false, check_sum_zero(list_vector.to_vec()));
        }
    }

    #[test]
    fn optional_bonus_ok() {
        let list_ok = [vec![0], vec![-3, 1, 2],
            vec![-98634, -86888, -48841, -40483, 2612, 9225, 17848, 71967, 84319, 88875]];

        for list_vector in list_ok.iter() {
            assert_eq!(true, check_sum_zero(list_vector.to_vec()));
        }
    }

    #[test]
    fn optional_bonus_examples_no_ok() {
        let list_no_ok = [vec![-83314, -82838, -80120, -63468, -62478, -59378, -56958, -50061, -34791, -32264, -21928, -14988, 23767, 24417, 26403, 26511, 36399, 78055],
            vec![-92953, -91613, -89733, -50673, -16067, -9172, 8852, 30883, 46690, 46968, 56772, 58703, 59150, 78476, 84413, 90106, 94777, 95148],
            vec![-94624, -86776, -85833, -80822, -71902, -54562, -38638, -26483, -20207, -1290, 12414, 12627, 19509, 30894, 32505, 46825, 50321, 69294],
            vec![-83964, -81834, -78386, -70497, -69357, -61867, -49127, -47916, -38361, -35772, -29803, -15343, 6918, 19662, 44614, 66049, 93789, 95405],
            vec![-68808, -58968, -45958, -36013, -32810, -28726, -13488, 3986, 26342, 29245, 30686, 47966, 58352, 68610, 74533, 77939, 80520, 87195]];

        for list_vector in list_no_ok.iter() {
            assert_eq!(false, check_sum_zero(list_vector.to_vec()));
        }
    }

    #[test]
    fn optional_bonus_examples_ok() {
        let list_ok = [vec![-97162, -95761, -94672, -87254, -57207, -22163, -20207, -1753, 11646, 13652, 14572, 30580, 52502, 64282, 74896, 83730, 89889, 92200],
            vec![-93976, -93807, -64604, -59939, -44394, -36454, -34635, -16483, 267, 3245, 8031, 10622, 44815, 46829, 61689, 65756, 69220, 70121],
            vec![-92474, -61685, -55348, -42019, -35902, -7815, -5579, 4490, 14778, 19399, 34202, 46624, 55800, 57719, 60260, 71511, 75665, 82754],
            vec![-85029, -84549, -82646, -80493, -73373, -57478, -56711, -42456, -38923, -29277, -3685, -3164, 26863, 29890, 37187, 46607, 69300, 84808],
            vec![-87565, -71009, -49312, -47554, -27197, 905, 2839, 8657, 14622, 32217, 35567, 38470, 46885, 59236, 64704, 82944, 86902, 90487]];

        for list_vector in list_ok.iter() {
            assert_eq!(true, check_sum_zero(list_vector.to_vec()));
        }
    }
}<commit_msg>Adding info to optional test 2<commit_after>extern crate itertools;
extern crate time;

use itertools::Itertools;
use time::PreciseTime;

fn check_sum_zero(v: Vec<i32>) -> bool {
    let int_list = v;

    for i in (1..int_list.len()+1).rev() {

        let iter = int_list.iter();

        for item in iter.combinations(i) {
            let a = item.into_iter();
            let sum: i32 = a.sum();

            if sum == 0 {
                return true;
            }
        }
    }
    return false;
}

fn main() {
    // Main function only for benchmarking Optional bonus 2
    let start = PreciseTime::now();
    let list_1 = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26];

    println!("Running optional bonus 2 test... with {:?} items", list_1.len());
    println!("Result: {:?}", check_sum_zero(list_1));

    let end = PreciseTime::now();

    println!("Second to finish: {}", start.to(end));

}

#[cfg(test)]
mod tests {
    use super::check_sum_zero;

    #[test]
    fn base_examples_ok() {

        let list_ok = [vec![-5, -3, -1, 2, 4, 6], vec![-1, 1],
            vec![-97364, -71561, -69336, 19675, 71561, 97863],
            vec![-53974, -39140, -36561, -23935, -15680, 0], vec![-2, 2, 3]];

        for list_vector in list_ok.iter() {
            assert_eq!(true, check_sum_zero(list_vector.to_vec()));
        }

    }

    #[test]
    fn base_examples_no_ok() {
        let list_no_ok = [vec![1, 2, 3], vec![]];

        for list_vector in list_no_ok.iter() {
            assert_eq!(false, check_sum_zero(list_vector.to_vec()));
        }
    }

    #[test]
    fn optional_bonus_ok() {
        let list_ok = [vec![0], vec![-3, 1, 2],
            vec![-98634, -86888, -48841, -40483, 2612, 9225, 17848, 71967, 84319, 88875]];

        for list_vector in list_ok.iter() {
            assert_eq!(true, check_sum_zero(list_vector.to_vec()));
        }
    }

    #[test]
    fn optional_bonus_examples_no_ok() {
        let list_no_ok = [vec![-83314, -82838, -80120, -63468, -62478, -59378, -56958, -50061, -34791, -32264, -21928, -14988, 23767, 24417, 26403, 26511, 36399, 78055],
            vec![-92953, -91613, -89733, -50673, -16067, -9172, 8852, 30883, 46690, 46968, 56772, 58703, 59150, 78476, 84413, 90106, 94777, 95148],
            vec![-94624, -86776, -85833, -80822, -71902, -54562, -38638, -26483, -20207, -1290, 12414, 12627, 19509, 30894, 32505, 46825, 50321, 69294],
            vec![-83964, -81834, -78386, -70497, -69357, -61867, -49127, -47916, -38361, -35772, -29803, -15343, 6918, 19662, 44614, 66049, 93789, 95405],
            vec![-68808, -58968, -45958, -36013, -32810, -28726, -13488, 3986, 26342, 29245, 30686, 47966, 58352, 68610, 74533, 77939, 80520, 87195]];

        for list_vector in list_no_ok.iter() {
            assert_eq!(false, check_sum_zero(list_vector.to_vec()));
        }
    }

    #[test]
    fn optional_bonus_examples_ok() {
        let list_ok = [vec![-97162, -95761, -94672, -87254, -57207, -22163, -20207, -1753, 11646, 13652, 14572, 30580, 52502, 64282, 74896, 83730, 89889, 92200],
            vec![-93976, -93807, -64604, -59939, -44394, -36454, -34635, -16483, 267, 3245, 8031, 10622, 44815, 46829, 61689, 65756, 69220, 70121],
            vec![-92474, -61685, -55348, -42019, -35902, -7815, -5579, 4490, 14778, 19399, 34202, 46624, 55800, 57719, 60260, 71511, 75665, 82754],
            vec![-85029, -84549, -82646, -80493, -73373, -57478, -56711, -42456, -38923, -29277, -3685, -3164, 26863, 29890, 37187, 46607, 69300, 84808],
            vec![-87565, -71009, -49312, -47554, -27197, 905, 2839, 8657, 14622, 32217, 35567, 38470, 46885, 59236, 64704, 82944, 86902, 90487]];

        for list_vector in list_ok.iter() {
            assert_eq!(true, check_sum_zero(list_vector.to_vec()));
        }
    }
}<|endoftext|>


<commit_before>// Copyright 2016 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Synchronization of the local blockchain with the rest of the network. Used
//! either on a brand new node or when a node is late based on others' heads.
//! Always starts by downloading the header chain before asking either for full
//! blocks or a full UTXO set with related information.

/// How many block bodies to download in parallel
const MAX_BODY_DOWNLOADS: usize = 8;

use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Instant, Duration};

use core::core::hash::{Hash, Hashed};
use chain;
use p2p;
use types::Error;
use util::LOGGER;

pub struct Syncer {
	chain: Arc<chain::Chain>,
	p2p: Arc<p2p::Server>,

	sync: Mutex<bool>,
	last_header_req: Mutex<Instant>,
	blocks_to_download: Mutex<Vec<Hash>>,
	blocks_downloading: Mutex<Vec<(Hash, Instant)>>,
}

impl Syncer {
	pub fn new(chain_ref: Arc<chain::Chain>, p2p: Arc<p2p::Server>) -> Syncer {
		Syncer {
			chain: chain_ref,
			p2p: p2p,
			sync: Mutex::new(true),
			last_header_req: Mutex::new(Instant::now() - Duration::from_secs(2)),
			blocks_to_download: Mutex::new(vec![]),
			blocks_downloading: Mutex::new(vec![]),
		}
	}

	pub fn syncing(&self) -> bool {
		*self.sync.lock().unwrap()
	}

	/// Checks the local chain state, comparing it with our peers and triggers
	/// syncing if required.
	pub fn run(&self) -> Result<(), Error> {
		debug!(LOGGER, "Starting syncer.");
		let start = Instant::now();
		loop {
			let pc = self.p2p.peer_count();
			if pc > 3 {
				break;
			}
			if pc > 0 && (Instant::now() - start > Duration::from_secs(10)) {
				break;
			}
			thread::sleep(Duration::from_millis(200));
		}

		// check if we have missing full blocks for which we already have a header
		self.init_download()?;

		// main syncing loop, requests more headers and bodies periodically as long
		// as a peer with higher difficulty exists and we're not fully caught up
		info!(LOGGER, "Starting sync loop.");
		loop {
			let tip = self.chain.get_header_head()?;
			// TODO do something better (like trying to get more) if we lose peers
			let peer = self.p2p.most_work_peer().unwrap();

			let more_headers = peer.info.total_difficulty > tip.total_difficulty;
			let more_bodies = {
				let blocks_to_download = self.blocks_to_download.lock().unwrap();
				let blocks_downloading = self.blocks_downloading.lock().unwrap();
				blocks_to_download.len() > 0 || blocks_downloading.len() > 0
			};

			{
				let last_header_req = self.last_header_req.lock().unwrap().clone();
				if more_headers && (Instant::now() - Duration::from_secs(2) > last_header_req) {
					self.request_headers()?;
				}
			}
			if more_bodies {
				self.request_bodies();
			}
			if !more_headers && !more_bodies {
				// TODO check we haven't been lied to on the total work
				let mut sync = self.sync.lock().unwrap();
				*sync = false;
				break;
			}

			thread::sleep(Duration::from_secs(2));
		}
		info!(LOGGER, "Sync done.");
		Ok(())
	}

	/// Checks the gap between the header chain and the full block chain and
	/// initializes the blocks_to_download structure with the missing full
	/// blocks
	fn init_download(&self) -> Result<(), Error> {
		// compare the header's head to the full one to see what we're missing
		let header_head = self.chain.get_header_head()?;
		let full_head = self.chain.head()?;
		let mut blocks_to_download = self.blocks_to_download.lock().unwrap();

		// go back the chain and insert for download all blocks we only have the
		// head for
		let mut prev_h = header_head.last_block_h;
		while prev_h != full_head.last_block_h {
			let header = self.chain.get_block_header(&prev_h)?;
			if header.height < full_head.height {
				break;
			}
			blocks_to_download.push(header.hash());
			prev_h = header.previous;
		}

		debug!(
			LOGGER,
			"Added {} full block hashes to download.",
			blocks_to_download.len()
		);
		Ok(())
	}

	/// Asks for the blocks we haven't downloaded yet and place them in the
	/// downloading structure.
	fn request_bodies(&self) {
		let mut blocks_to_download = self.blocks_to_download.lock().unwrap();
		let mut blocks_downloading = self.blocks_downloading.lock().unwrap();

		// clean up potentially dead downloads
		let twenty_sec_ago = Instant::now() - Duration::from_secs(20);
		let too_old_pos = (0..blocks_downloading.len())
			.filter(|p| blocks_downloading[*p].1 < twenty_sec_ago)
			.collect::<Vec<_>>();
		let mut offs = 0;
		for too_old in too_old_pos {
			let block_h = blocks_downloading.remove(too_oldi - offs);
			debug!(
				LOGGER,
				"Download request expired for {}, will re-issue.",
				block_h.0
			);
			blocks_to_download.push(block_h.0);
			offs += 1;
		}

		// consume hashes from blocks to download, place them in downloading and
		// request them from the network
		while blocks_to_download.len() > 0 && blocks_downloading.len() < MAX_BODY_DOWNLOADS {
			let h = blocks_to_download.pop().unwrap();
			let peer = self.p2p.random_peer().unwrap();
			let send_result = peer.send_block_request(h);
			if let Err(e) = send_result {
				debug!(LOGGER, "Error requesting block: {:?}", e);
			}
			blocks_downloading.push((h, Instant::now()));
		}
		debug!(
			LOGGER,
			"Requesting full blocks to download, total left: {}.",
			blocks_to_download.len()
		);
	}

	/// We added a block, clean up the downloading structure
	pub fn block_received(&self, bh: Hash) {
		// just clean up the downloading list
		let mut bds = self.blocks_downloading.lock().unwrap();
		bds.iter().position(|&h| h.0 == bh).map(|n| bds.remove(n));
	}

	/// Request some block headers from a peer to advance us
	fn request_headers(&self) -> Result<(), Error> {
		{
			let mut last_header_req = self.last_header_req.lock().unwrap();
			*last_header_req = Instant::now();
		}

		let tip = self.chain.get_header_head()?;
		let peer = self.p2p.most_work_peer();
		let locator = self.get_locator(&tip)?;
		if let Some(p) = peer {
			debug!(
				LOGGER,
				"Asking peer {} for more block headers starting from {} at {}.",
				p.info.addr,
				tip.last_block_h,
				tip.height
			);
			p.send_header_request(locator)?;
		} else {
			warn!(LOGGER, "Could not get most worked peer to request headers.");
		}
		Ok(())
	}

	/// We added a header, add it to the full block download list
	pub fn headers_received(&self, bhs: Vec<Hash>) {
		let mut blocks_to_download = self.blocks_to_download.lock().unwrap();
		let hs_len = bhs.len();
		for h in bhs {
			// enlist for full block download
			blocks_to_download.insert(0, h);
		}
		// ask for more headers if we got as many as required
		if hs_len == (p2p::MAX_BLOCK_HEADERS as usize) {
			self.request_headers().unwrap();
		}
	}

	/// Builds a vector of block hashes that should help the remote peer sending
	/// us the right block headers.
	fn get_locator(&self, tip: &chain::Tip) -> Result<Vec<Hash>, Error> {
		// Prepare the heights we want as the latests height minus increasing powers
		// of 2 up to max.
		let mut heights = vec![tip.height];
		let mut tail = (1..p2p::MAX_LOCATORS)
			.map(|n| 2u64.pow(n))
			.filter_map(|n| if n > tip.height {
				None
			} else {
				Some(tip.height - n)
			})
			.collect::<Vec<_>>();
		heights.append(&mut tail);

		// Iteratively travel the header chain back from our head and retain the
		// headers at the wanted heights.
		let mut header = self.chain.get_block_header(&tip.last_block_h)?;
		let mut locator = vec![];
		while heights.len() > 0 {
			if header.height == heights[0] {
				heights = heights[1..].to_vec();
				locator.push(header.hash());
			}
			if header.height > 0 {
				header = self.chain.get_block_header(&header.previous)?;
			}
		}
		Ok(locator)
	}
}
<commit_msg>Fix typo (yes I use vi)<commit_after>// Copyright 2016 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Synchronization of the local blockchain with the rest of the network. Used
//! either on a brand new node or when a node is late based on others' heads.
//! Always starts by downloading the header chain before asking either for full
//! blocks or a full UTXO set with related information.

/// How many block bodies to download in parallel
const MAX_BODY_DOWNLOADS: usize = 8;

use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Instant, Duration};

use core::core::hash::{Hash, Hashed};
use chain;
use p2p;
use types::Error;
use util::LOGGER;

pub struct Syncer {
	chain: Arc<chain::Chain>,
	p2p: Arc<p2p::Server>,

	sync: Mutex<bool>,
	last_header_req: Mutex<Instant>,
	blocks_to_download: Mutex<Vec<Hash>>,
	blocks_downloading: Mutex<Vec<(Hash, Instant)>>,
}

impl Syncer {
	pub fn new(chain_ref: Arc<chain::Chain>, p2p: Arc<p2p::Server>) -> Syncer {
		Syncer {
			chain: chain_ref,
			p2p: p2p,
			sync: Mutex::new(true),
			last_header_req: Mutex::new(Instant::now() - Duration::from_secs(2)),
			blocks_to_download: Mutex::new(vec![]),
			blocks_downloading: Mutex::new(vec![]),
		}
	}

	pub fn syncing(&self) -> bool {
		*self.sync.lock().unwrap()
	}

	/// Checks the local chain state, comparing it with our peers and triggers
	/// syncing if required.
	pub fn run(&self) -> Result<(), Error> {
		debug!(LOGGER, "Starting syncer.");
		let start = Instant::now();
		loop {
			let pc = self.p2p.peer_count();
			if pc > 3 {
				break;
			}
			if pc > 0 && (Instant::now() - start > Duration::from_secs(10)) {
				break;
			}
			thread::sleep(Duration::from_millis(200));
		}

		// check if we have missing full blocks for which we already have a header
		self.init_download()?;

		// main syncing loop, requests more headers and bodies periodically as long
		// as a peer with higher difficulty exists and we're not fully caught up
		info!(LOGGER, "Starting sync loop.");
		loop {
			let tip = self.chain.get_header_head()?;
			// TODO do something better (like trying to get more) if we lose peers
			let peer = self.p2p.most_work_peer().unwrap();

			let more_headers = peer.info.total_difficulty > tip.total_difficulty;
			let more_bodies = {
				let blocks_to_download = self.blocks_to_download.lock().unwrap();
				let blocks_downloading = self.blocks_downloading.lock().unwrap();
				blocks_to_download.len() > 0 || blocks_downloading.len() > 0
			};

			{
				let last_header_req = self.last_header_req.lock().unwrap().clone();
				if more_headers && (Instant::now() - Duration::from_secs(2) > last_header_req) {
					self.request_headers()?;
				}
			}
			if more_bodies {
				self.request_bodies();
			}
			if !more_headers && !more_bodies {
				// TODO check we haven't been lied to on the total work
				let mut sync = self.sync.lock().unwrap();
				*sync = false;
				break;
			}

			thread::sleep(Duration::from_secs(2));
		}
		info!(LOGGER, "Sync done.");
		Ok(())
	}

	/// Checks the gap between the header chain and the full block chain and
	/// initializes the blocks_to_download structure with the missing full
	/// blocks
	fn init_download(&self) -> Result<(), Error> {
		// compare the header's head to the full one to see what we're missing
		let header_head = self.chain.get_header_head()?;
		let full_head = self.chain.head()?;
		let mut blocks_to_download = self.blocks_to_download.lock().unwrap();

		// go back the chain and insert for download all blocks we only have the
		// head for
		let mut prev_h = header_head.last_block_h;
		while prev_h != full_head.last_block_h {
			let header = self.chain.get_block_header(&prev_h)?;
			if header.height < full_head.height {
				break;
			}
			blocks_to_download.push(header.hash());
			prev_h = header.previous;
		}

		debug!(
			LOGGER,
			"Added {} full block hashes to download.",
			blocks_to_download.len()
		);
		Ok(())
	}

	/// Asks for the blocks we haven't downloaded yet and place them in the
	/// downloading structure.
	fn request_bodies(&self) {
		let mut blocks_to_download = self.blocks_to_download.lock().unwrap();
		let mut blocks_downloading = self.blocks_downloading.lock().unwrap();

		// clean up potentially dead downloads
		let twenty_sec_ago = Instant::now() - Duration::from_secs(20);
		let too_old_pos = (0..blocks_downloading.len())
			.filter(|p| blocks_downloading[*p].1 < twenty_sec_ago)
			.collect::<Vec<_>>();
		let mut offs = 0;
		for too_old in too_old_pos {
			let block_h = blocks_downloading.remove(too_old - offs);
			debug!(
				LOGGER,
				"Download request expired for {}, will re-issue.",
				block_h.0
			);
			blocks_to_download.push(block_h.0);
			offs += 1;
		}

		// consume hashes from blocks to download, place them in downloading and
		// request them from the network
		while blocks_to_download.len() > 0 && blocks_downloading.len() < MAX_BODY_DOWNLOADS {
			let h = blocks_to_download.pop().unwrap();
			let peer = self.p2p.random_peer().unwrap();
			let send_result = peer.send_block_request(h);
			if let Err(e) = send_result {
				debug!(LOGGER, "Error requesting block: {:?}", e);
			}
			blocks_downloading.push((h, Instant::now()));
		}
		debug!(
			LOGGER,
			"Requesting full blocks to download, total left: {}.",
			blocks_to_download.len()
		);
	}

	/// We added a block, clean up the downloading structure
	pub fn block_received(&self, bh: Hash) {
		// just clean up the downloading list
		let mut bds = self.blocks_downloading.lock().unwrap();
		bds.iter().position(|&h| h.0 == bh).map(|n| bds.remove(n));
	}

	/// Request some block headers from a peer to advance us
	fn request_headers(&self) -> Result<(), Error> {
		{
			let mut last_header_req = self.last_header_req.lock().unwrap();
			*last_header_req = Instant::now();
		}

		let tip = self.chain.get_header_head()?;
		let peer = self.p2p.most_work_peer();
		let locator = self.get_locator(&tip)?;
		if let Some(p) = peer {
			debug!(
				LOGGER,
				"Asking peer {} for more block headers starting from {} at {}.",
				p.info.addr,
				tip.last_block_h,
				tip.height
			);
			p.send_header_request(locator)?;
		} else {
			warn!(LOGGER, "Could not get most worked peer to request headers.");
		}
		Ok(())
	}

	/// We added a header, add it to the full block download list
	pub fn headers_received(&self, bhs: Vec<Hash>) {
		let mut blocks_to_download = self.blocks_to_download.lock().unwrap();
		let hs_len = bhs.len();
		for h in bhs {
			// enlist for full block download
			blocks_to_download.insert(0, h);
		}
		// ask for more headers if we got as many as required
		if hs_len == (p2p::MAX_BLOCK_HEADERS as usize) {
			self.request_headers().unwrap();
		}
	}

	/// Builds a vector of block hashes that should help the remote peer sending
	/// us the right block headers.
	fn get_locator(&self, tip: &chain::Tip) -> Result<Vec<Hash>, Error> {
		// Prepare the heights we want as the latests height minus increasing powers
		// of 2 up to max.
		let mut heights = vec![tip.height];
		let mut tail = (1..p2p::MAX_LOCATORS)
			.map(|n| 2u64.pow(n))
			.filter_map(|n| if n > tip.height {
				None
			} else {
				Some(tip.height - n)
			})
			.collect::<Vec<_>>();
		heights.append(&mut tail);

		// Iteratively travel the header chain back from our head and retain the
		// headers at the wanted heights.
		let mut header = self.chain.get_block_header(&tip.last_block_h)?;
		let mut locator = vec![];
		while heights.len() > 0 {
			if header.height == heights[0] {
				heights = heights[1..].to_vec();
				locator.push(header.hash());
			}
			if header.height > 0 {
				header = self.chain.get_block_header(&header.previous)?;
			}
		}
		Ok(locator)
	}
}
<|endoftext|>