<commit_before>use std::fmt;

use ctxt::SemContext;
use driver::cmd::Args;
use gc::arena;
use gc::root::{get_rootset, IndirectObj};
use gc::swiper::card::CardTable;
use gc::swiper::crossing::CrossingMap;
use gc::swiper::full::FullCollector;
use gc::swiper::large::LargeSpace;
use gc::swiper::minor::MinorCollector;
use gc::swiper::old::OldGen;
use gc::swiper::verify::{Verifier, VerifierPhase};
use gc::swiper::young::YoungGen;
use gc::Address;
use gc::Collector;
use gc::{TLAB_OBJECT_SIZE, TLAB_SIZE};
use mem;
use os;
use vtable::VTable;

pub mod card;
mod crossing;
mod full;
mod large;
mod minor;
pub mod old;
mod verify;
pub mod young;

// determines size of young generation in heap
// young generation size = heap size / YOUNG_RATIO
const YOUNG_RATIO: usize = 4;

// heap is divided into cards of size CARD_SIZE.
// card entry determines whether this part of the heap was modified
// in minor collections those parts of the heap need to be analyzed
pub const CARD_SIZE: usize = 512;
pub const CARD_SIZE_BITS: usize = 9;

pub const LARGE_OBJECT_SIZE: usize = 16 * 1024;

pub struct Swiper {
    // contiguous memory for young/old generation and large space
    heap: Region,

    // contains heap and also card table and crossing map
    reserved_area: Region,

    young: YoungGen,
    old: OldGen,
    large: LargeSpace,

    card_table: CardTable,
    crossing_map: CrossingMap,

    card_table_offset: usize,

    // maximum heap size
    heap_size: usize,

    // committed memory size
    committed_size: usize,
}

impl Swiper {
    pub fn new(args: &Args) -> Swiper {
        let heap_size = args.heap_size();

        // set heap size to multiple of page
        let heap_size = mem::page_align(heap_size);

        // determine sizes of young/old-gen
        let alignment = os::page_size() as usize * 2;
        let young_size = mem::align_usize(heap_size / YOUNG_RATIO, alignment);
        let old_size = heap_size - young_size;

        // determine size for card table
        let card_size = mem::page_align((4 * heap_size) >> CARD_SIZE_BITS);

        // determine size for crossing map
        let crossing_size = mem::page_align(heap_size >> CARD_SIZE_BITS);

        // determine full memory size
        let heap_reserve_size = heap_size * 4 + card_size + crossing_size;

        // reserve full memory
        let ptr = arena::reserve(heap_reserve_size);

        // heap is young/old generation & large space
        let heap_start = ptr;
        let heap_end = ptr.offset(4 * heap_size);

        // reserved area also contains card table & crossing map
        let reserved_area = heap_start.region_start(heap_reserve_size);

        // determine offset to card table (card table starts right after heap)
        // offset = card_table_start - (heap_start >> CARD_SIZE_BITS)
        let card_table_offset = heap_end.to_usize() - (heap_start.to_usize() >> CARD_SIZE_BITS);

        // determine boundaries for card table
        let card_start = heap_end;
        let card_end = card_start.offset(card_size);

        arena::commit(card_start, card_size, false);

        // determine boundaries for crossing map
        let crossing_start = card_end;
        let crossing_end = crossing_start.offset(crossing_size);

        arena::commit(crossing_start, crossing_size, false);

        // determine boundaries of young generation
        let young_start = heap_start;
        let young_end = young_start.offset(heap_size);

        arena::commit(young_start, heap_size, false);

        // determine boundaries of old generation
        let old_start = young_end;
        let old_end = old_start.offset(heap_size);

        arena::commit(old_start, heap_size, false);

        // determine large object space
        let large_start = old_end;
        let large_end = large_start.offset(2 * heap_size);

        let card_table = CardTable::new(
            card_start,
            card_end,
            Region::new(old_start, large_end),
            old_end,
            heap_size,
        );
        let crossing_map = CrossingMap::new(crossing_start, crossing_end, heap_size);
        let young = YoungGen::new(young_start, young_end, args.flag_gc_verify);
        let old = OldGen::new(old_start, old_end, crossing_map.clone(), card_table.clone());
        let large = LargeSpace::new(large_start, large_end);

        if args.flag_gc_verbose {
            println!(
                "GC: heap info: {}, old {}, young {}, card {}, crossing {}",
                formatted_size(heap_size),
                formatted_size(old_size),
                formatted_size(young_size),
                formatted_size(card_size),
                formatted_size(crossing_size)
            );
        }

        Swiper {
            heap: Region::new(heap_start, heap_end),
            reserved_area: reserved_area,

            young: young,
            old: old,
            large: large,

            card_table: card_table,
            crossing_map: crossing_map,

            card_table_offset: card_table_offset,
            heap_size: heap_size,
            committed_size: 0,
        }
    }

    fn minor_collect_inner(&self, ctxt: &SemContext) -> bool {
        // make heap iterable
        self.fill_current_tlab(ctxt);

        let rootset = get_rootset(ctxt);

        self.verify(ctxt, VerifierPhase::PreMinor, "pre-minor", &rootset);

        let mut collector = MinorCollector::new(
            ctxt,
            &self.young,
            &self.old,
            &self.large,
            &self.card_table,
            &self.crossing_map,
            &rootset,
        );
        collector.collect();

        let promotion_failed = collector.promotion_failed();
        self.verify(ctxt, VerifierPhase::PostMinor, "post-minor", &rootset);

        promotion_failed
    }

    fn full_collect(&self, ctxt: &SemContext) {
        // make heap iterable
        self.fill_current_tlab(ctxt);

        let rootset = get_rootset(ctxt);

        self.verify(ctxt, VerifierPhase::PreFull, "pre-full", &rootset);

        let mut collector = FullCollector::new(
            ctxt,
            self.heap.clone(),
            &self.young,
            &self.old,
            &self.large,
            &self.card_table,
            &self.crossing_map,
            &ctxt.gc.perm_space,
            &rootset,
        );
        collector.collect();

        self.verify(ctxt, VerifierPhase::PostFull, "post-full", &rootset);
    }

    fn verify(
        &self,
        ctxt: &SemContext,
        phase: VerifierPhase,
        _name: &str,
        rootset: &[IndirectObj],
    ) {
        if ctxt.args.flag_gc_verify {
            if ctxt.args.flag_gc_verbose {
                println!("VERIFY: {}", _name);
            }

            let perm_space = &ctxt.gc.perm_space;

            let mut verifier = Verifier::new(
                &self.young,
                &self.old,
                &self.card_table,
                &self.crossing_map,
                rootset,
                &self.large,
                &*perm_space,
                self.reserved_area.clone(),
                phase,
            );
            verifier.verify();
        }
    }
}

impl Collector for Swiper {
    fn alloc(&self, ctxt: &SemContext, size: usize, array_ref: bool) -> Address {
        if ctxt.args.flag_gc_stress_minor {
            self.minor_collect(ctxt);
        }

        if ctxt.args.flag_gc_stress {
            self.full_collect(ctxt);
        }

        if size < TLAB_OBJECT_SIZE && !ctxt.args.flag_disable_tlab {
            self.alloc_tlab(ctxt, size, array_ref)
        } else if size < LARGE_OBJECT_SIZE {
            self.alloc_normal(ctxt, size, array_ref)
        } else {
            self.alloc_large(ctxt, size, array_ref)
        }
    }

    fn collect(&self, ctxt: &SemContext) {
        self.full_collect(ctxt);
    }

    fn minor_collect(&self, ctxt: &SemContext) {
        self.fill_current_tlab(ctxt);
        self.minor_collect_inner(ctxt);
    }

    fn needs_write_barrier(&self) -> bool {
        return true;
    }

    fn card_table_offset(&self) -> usize {
        self.card_table_offset
    }
}

impl Swiper {
    fn alloc_tlab(&self, ctxt: &SemContext, size: usize, array_ref: bool) -> Address {
        let tlab = ctxt.tld.borrow().tlab_region();

        // try to allocate in current tlab
        if size <= tlab.size() {
            ctxt.tld
                .borrow_mut()
                .tlab_initialize(tlab.start.offset(size), tlab.end);
            return tlab.start;
        }

        // if there is not enough space, make heap iterable by filling tlab with unused objects
        self.fill_tlab(ctxt, tlab.start, tlab.end);

        // allocate new tlab
        let tlab_start = self.alloc_tlab_area(ctxt, size);

        if tlab_start.is_null() {
            // allocate object in old generation instead
            self.old.alloc(size, array_ref)
        } else {
            let object_start = tlab_start;
            let tlab_start = tlab_start.offset(size);
            let tlab_end = tlab_start.offset(TLAB_SIZE);

            // initialize TLAB to new boundaries
            ctxt.tld.borrow_mut().tlab_initialize(tlab_start, tlab_end);

            // object is allocated before TLAB
            object_start
        }
    }

    fn alloc_tlab_area(&self, ctxt: &SemContext, extend: usize) -> Address {
        assert!(extend < TLAB_OBJECT_SIZE);
        let size = TLAB_SIZE + extend;
        let ptr = self.young.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        let promotion_failed = self.minor_collect_inner(ctxt);

        if promotion_failed {
            self.full_collect(ctxt);
            return self.young.alloc(size);
        }

        let ptr = self.young.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        self.full_collect(ctxt);
        self.young.alloc(TLAB_SIZE)
    }

    fn fill_current_tlab(&self, ctxt: &SemContext) {
        let tlab = ctxt.tld.borrow().tlab_region();
        self.fill_tlab(ctxt, tlab.start, tlab.end);
    }

    fn fill_tlab(&self, ctxt: &SemContext, start: Address, end: Address) {
        if start == end {
            // nothing to do

        } else if end.offset_from(start) == mem::ptr_width_usize() {
            // fill with object
            let cls_id = ctxt.vips.obj(ctxt);
            let cls = ctxt.class_defs[cls_id].borrow();
            let vtable: *const VTable = &**cls.vtable.as_ref().unwrap();

            unsafe {
                *start.to_mut_ptr::<usize>() = vtable as usize;
            }
        } else {
            // fill with int array
            let cls_id = ctxt.vips.int_array(ctxt);
            let cls = ctxt.class_defs[cls_id].borrow();
            let vtable: *const VTable = &**cls.vtable.as_ref().unwrap();
            let length: usize = end.offset_from(start.add_ptr(2)) / 4;

            unsafe {
                *start.to_mut_ptr::<usize>() = vtable as usize;
                *start.add_ptr(1).to_mut_ptr::<usize>() = length;
            }
        }

        let n = Address::null();
        ctxt.tld.borrow_mut().tlab_initialize(n, n);
    }

    fn alloc_normal(&self, ctxt: &SemContext, size: usize, array_ref: bool) -> Address {
        let ptr = self.young.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        let promotion_failed = self.minor_collect_inner(ctxt);

        if promotion_failed {
            self.full_collect(ctxt);
        }

        let ptr = self.young.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        self.old.alloc(size, array_ref)
    }

    fn alloc_large(&self, ctxt: &SemContext, size: usize, _: bool) -> Address {
        let ptr = self.large.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        self.full_collect(ctxt);

        self.large.alloc(size)
    }
}

#[derive(Copy, Clone, PartialEq, Eq)]
pub struct CardIdx(usize);

impl CardIdx {
    pub fn to_usize(self) -> usize {
        self.0
    }

    pub fn offset(self, val: usize) -> CardIdx {
        (self.0 + val).into()
    }
}

impl From<usize> for CardIdx {
    fn from(val: usize) -> CardIdx {
        CardIdx(val)
    }
}

#[derive(Clone)]
pub struct Region {
    pub start: Address,
    pub end: Address,
}

impl Region {
    pub fn new(start: Address, end: Address) -> Region {
        Region {
            start: start,
            end: end,
        }
    }

    #[inline(always)]
    pub fn contains(&self, addr: Address) -> bool {
        self.start <= addr && addr < self.end
    }

    #[inline(always)]
    pub fn valid_top(&self, addr: Address) -> bool {
        self.start <= addr && addr <= self.end
    }

    #[inline(always)]
    pub fn size(&self) -> usize {
        self.end.to_usize() - self.start.to_usize()
    }
}

impl fmt::Display for Region {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}-{}", self.start, self.end)
    }
}

struct FormattedSize {
    size: usize,
}

impl fmt::Display for FormattedSize {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let ksize = (self.size as f64) / 1024f64;

        if ksize < 1f64 {
            return write!(f, "{}B", self.size);
        }

        let msize = ksize / 1024f64;

        if msize < 1f64 {
            return write!(f, "{:.1}K", ksize);
        }

        let gsize = msize / 1024f64;

        if gsize < 1f64 {
            write!(f, "{:.1}M", msize)
        } else {
            write!(f, "{:.1}G", gsize)
        }
    }
}

fn formatted_size(size: usize) -> FormattedSize {
    FormattedSize { size }
}

fn on_different_cards(curr: Address, next: Address) -> bool {
    (curr.to_usize() >> CARD_SIZE_BITS) != (next.to_usize() >> CARD_SIZE_BITS)
}
<commit_msg>swiper: was allocating wrong size<commit_after>use std::fmt;

use ctxt::SemContext;
use driver::cmd::Args;
use gc::arena;
use gc::root::{get_rootset, IndirectObj};
use gc::swiper::card::CardTable;
use gc::swiper::crossing::CrossingMap;
use gc::swiper::full::FullCollector;
use gc::swiper::large::LargeSpace;
use gc::swiper::minor::MinorCollector;
use gc::swiper::old::OldGen;
use gc::swiper::verify::{Verifier, VerifierPhase};
use gc::swiper::young::YoungGen;
use gc::Address;
use gc::Collector;
use gc::{TLAB_OBJECT_SIZE, TLAB_SIZE};
use mem;
use os;
use vtable::VTable;

pub mod card;
mod crossing;
mod full;
mod large;
mod minor;
pub mod old;
mod verify;
pub mod young;

// determines size of young generation in heap
// young generation size = heap size / YOUNG_RATIO
const YOUNG_RATIO: usize = 4;

// heap is divided into cards of size CARD_SIZE.
// card entry determines whether this part of the heap was modified
// in minor collections those parts of the heap need to be analyzed
pub const CARD_SIZE: usize = 512;
pub const CARD_SIZE_BITS: usize = 9;

pub const LARGE_OBJECT_SIZE: usize = 16 * 1024;

pub struct Swiper {
    // contiguous memory for young/old generation and large space
    heap: Region,

    // contains heap and also card table and crossing map
    reserved_area: Region,

    young: YoungGen,
    old: OldGen,
    large: LargeSpace,

    card_table: CardTable,
    crossing_map: CrossingMap,

    card_table_offset: usize,

    // maximum heap size
    heap_size: usize,

    // committed memory size
    committed_size: usize,
}

impl Swiper {
    pub fn new(args: &Args) -> Swiper {
        let heap_size = args.heap_size();

        // set heap size to multiple of page
        let heap_size = mem::page_align(heap_size);

        // determine sizes of young/old-gen
        let alignment = os::page_size() as usize * 2;
        let young_size = mem::align_usize(heap_size / YOUNG_RATIO, alignment);
        let old_size = heap_size - young_size;

        // determine size for card table
        let card_size = mem::page_align((4 * heap_size) >> CARD_SIZE_BITS);

        // determine size for crossing map
        let crossing_size = mem::page_align(heap_size >> CARD_SIZE_BITS);

        // determine full memory size
        let heap_reserve_size = heap_size * 4 + card_size + crossing_size;

        // reserve full memory
        let ptr = arena::reserve(heap_reserve_size);

        // heap is young/old generation & large space
        let heap_start = ptr;
        let heap_end = ptr.offset(4 * heap_size);

        // reserved area also contains card table & crossing map
        let reserved_area = heap_start.region_start(heap_reserve_size);

        // determine offset to card table (card table starts right after heap)
        // offset = card_table_start - (heap_start >> CARD_SIZE_BITS)
        let card_table_offset = heap_end.to_usize() - (heap_start.to_usize() >> CARD_SIZE_BITS);

        // determine boundaries for card table
        let card_start = heap_end;
        let card_end = card_start.offset(card_size);

        arena::commit(card_start, card_size, false);

        // determine boundaries for crossing map
        let crossing_start = card_end;
        let crossing_end = crossing_start.offset(crossing_size);

        arena::commit(crossing_start, crossing_size, false);

        // determine boundaries of young generation
        let young_start = heap_start;
        let young_end = young_start.offset(heap_size);

        arena::commit(young_start, heap_size, false);

        // determine boundaries of old generation
        let old_start = young_end;
        let old_end = old_start.offset(heap_size);

        arena::commit(old_start, heap_size, false);

        // determine large object space
        let large_start = old_end;
        let large_end = large_start.offset(2 * heap_size);

        let card_table = CardTable::new(
            card_start,
            card_end,
            Region::new(old_start, large_end),
            old_end,
            heap_size,
        );
        let crossing_map = CrossingMap::new(crossing_start, crossing_end, heap_size);
        let young = YoungGen::new(young_start, young_end, args.flag_gc_verify);
        let old = OldGen::new(old_start, old_end, crossing_map.clone(), card_table.clone());
        let large = LargeSpace::new(large_start, large_end);

        if args.flag_gc_verbose {
            println!(
                "GC: heap info: {}, old {}, young {}, card {}, crossing {}",
                formatted_size(heap_size),
                formatted_size(old_size),
                formatted_size(young_size),
                formatted_size(card_size),
                formatted_size(crossing_size)
            );
        }

        Swiper {
            heap: Region::new(heap_start, heap_end),
            reserved_area: reserved_area,

            young: young,
            old: old,
            large: large,

            card_table: card_table,
            crossing_map: crossing_map,

            card_table_offset: card_table_offset,
            heap_size: heap_size,
            committed_size: 0,
        }
    }

    fn minor_collect_inner(&self, ctxt: &SemContext) -> bool {
        // make heap iterable
        self.fill_current_tlab(ctxt);

        let rootset = get_rootset(ctxt);

        self.verify(ctxt, VerifierPhase::PreMinor, "pre-minor", &rootset);

        let mut collector = MinorCollector::new(
            ctxt,
            &self.young,
            &self.old,
            &self.large,
            &self.card_table,
            &self.crossing_map,
            &rootset,
        );
        collector.collect();

        let promotion_failed = collector.promotion_failed();
        self.verify(ctxt, VerifierPhase::PostMinor, "post-minor", &rootset);

        promotion_failed
    }

    fn full_collect(&self, ctxt: &SemContext) {
        // make heap iterable
        self.fill_current_tlab(ctxt);

        let rootset = get_rootset(ctxt);

        self.verify(ctxt, VerifierPhase::PreFull, "pre-full", &rootset);

        let mut collector = FullCollector::new(
            ctxt,
            self.heap.clone(),
            &self.young,
            &self.old,
            &self.large,
            &self.card_table,
            &self.crossing_map,
            &ctxt.gc.perm_space,
            &rootset,
        );
        collector.collect();

        self.verify(ctxt, VerifierPhase::PostFull, "post-full", &rootset);
    }

    fn verify(
        &self,
        ctxt: &SemContext,
        phase: VerifierPhase,
        _name: &str,
        rootset: &[IndirectObj],
    ) {
        if ctxt.args.flag_gc_verify {
            if ctxt.args.flag_gc_verbose {
                println!("VERIFY: {}", _name);
            }

            let perm_space = &ctxt.gc.perm_space;

            let mut verifier = Verifier::new(
                &self.young,
                &self.old,
                &self.card_table,
                &self.crossing_map,
                rootset,
                &self.large,
                &*perm_space,
                self.reserved_area.clone(),
                phase,
            );
            verifier.verify();
        }
    }
}

impl Collector for Swiper {
    fn alloc(&self, ctxt: &SemContext, size: usize, array_ref: bool) -> Address {
        if ctxt.args.flag_gc_stress_minor {
            self.minor_collect(ctxt);
        }

        if ctxt.args.flag_gc_stress {
            self.full_collect(ctxt);
        }

        if size < TLAB_OBJECT_SIZE && !ctxt.args.flag_disable_tlab {
            self.alloc_tlab(ctxt, size, array_ref)
        } else if size < LARGE_OBJECT_SIZE {
            self.alloc_normal(ctxt, size, array_ref)
        } else {
            self.alloc_large(ctxt, size, array_ref)
        }
    }

    fn collect(&self, ctxt: &SemContext) {
        self.full_collect(ctxt);
    }

    fn minor_collect(&self, ctxt: &SemContext) {
        self.fill_current_tlab(ctxt);
        self.minor_collect_inner(ctxt);
    }

    fn needs_write_barrier(&self) -> bool {
        return true;
    }

    fn card_table_offset(&self) -> usize {
        self.card_table_offset
    }
}

impl Swiper {
    fn alloc_tlab(&self, ctxt: &SemContext, size: usize, array_ref: bool) -> Address {
        let tlab = ctxt.tld.borrow().tlab_region();

        // try to allocate in current tlab
        if size <= tlab.size() {
            ctxt.tld
                .borrow_mut()
                .tlab_initialize(tlab.start.offset(size), tlab.end);
            return tlab.start;
        }

        // if there is not enough space, make heap iterable by filling tlab with unused objects
        self.fill_tlab(ctxt, tlab.start, tlab.end);

        // allocate new tlab
        let tlab_start = self.alloc_tlab_area(ctxt, size);

        if tlab_start.is_null() {
            // allocate object in old generation instead
            self.old.alloc(size, array_ref)
        } else {
            let object_start = tlab_start;
            let tlab_start = tlab_start.offset(size);
            let tlab_end = tlab_start.offset(TLAB_SIZE);

            // initialize TLAB to new boundaries
            ctxt.tld.borrow_mut().tlab_initialize(tlab_start, tlab_end);

            // object is allocated before TLAB
            object_start
        }
    }

    fn alloc_tlab_area(&self, ctxt: &SemContext, extend: usize) -> Address {
        assert!(extend < TLAB_OBJECT_SIZE);
        let size = TLAB_SIZE + extend;
        let ptr = self.young.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        let promotion_failed = self.minor_collect_inner(ctxt);

        if promotion_failed {
            self.full_collect(ctxt);
            return self.young.alloc(size);
        }

        let ptr = self.young.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        self.full_collect(ctxt);
        self.young.alloc(size)
    }

    fn fill_current_tlab(&self, ctxt: &SemContext) {
        let tlab = ctxt.tld.borrow().tlab_region();
        self.fill_tlab(ctxt, tlab.start, tlab.end);
    }

    fn fill_tlab(&self, ctxt: &SemContext, start: Address, end: Address) {
        if start == end {
            // nothing to do

        } else if end.offset_from(start) == mem::ptr_width_usize() {
            // fill with object
            let cls_id = ctxt.vips.obj(ctxt);
            let cls = ctxt.class_defs[cls_id].borrow();
            let vtable: *const VTable = &**cls.vtable.as_ref().unwrap();

            unsafe {
                *start.to_mut_ptr::<usize>() = vtable as usize;
            }
        } else {
            // fill with int array
            let cls_id = ctxt.vips.int_array(ctxt);
            let cls = ctxt.class_defs[cls_id].borrow();
            let vtable: *const VTable = &**cls.vtable.as_ref().unwrap();
            let length: usize = end.offset_from(start.add_ptr(2)) / 4;

            unsafe {
                *start.to_mut_ptr::<usize>() = vtable as usize;
                *start.add_ptr(1).to_mut_ptr::<usize>() = length;
            }
        }

        let n = Address::null();
        ctxt.tld.borrow_mut().tlab_initialize(n, n);
    }

    fn alloc_normal(&self, ctxt: &SemContext, size: usize, array_ref: bool) -> Address {
        let ptr = self.young.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        let promotion_failed = self.minor_collect_inner(ctxt);

        if promotion_failed {
            self.full_collect(ctxt);
        }

        let ptr = self.young.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        self.old.alloc(size, array_ref)
    }

    fn alloc_large(&self, ctxt: &SemContext, size: usize, _: bool) -> Address {
        let ptr = self.large.alloc(size);

        if !ptr.is_null() {
            return ptr;
        }

        self.full_collect(ctxt);

        self.large.alloc(size)
    }
}

#[derive(Copy, Clone, PartialEq, Eq)]
pub struct CardIdx(usize);

impl CardIdx {
    pub fn to_usize(self) -> usize {
        self.0
    }

    pub fn offset(self, val: usize) -> CardIdx {
        (self.0 + val).into()
    }
}

impl From<usize> for CardIdx {
    fn from(val: usize) -> CardIdx {
        CardIdx(val)
    }
}

#[derive(Clone)]
pub struct Region {
    pub start: Address,
    pub end: Address,
}

impl Region {
    pub fn new(start: Address, end: Address) -> Region {
        Region {
            start: start,
            end: end,
        }
    }

    #[inline(always)]
    pub fn contains(&self, addr: Address) -> bool {
        self.start <= addr && addr < self.end
    }

    #[inline(always)]
    pub fn valid_top(&self, addr: Address) -> bool {
        self.start <= addr && addr <= self.end
    }

    #[inline(always)]
    pub fn size(&self) -> usize {
        self.end.to_usize() - self.start.to_usize()
    }
}

impl fmt::Display for Region {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}-{}", self.start, self.end)
    }
}

struct FormattedSize {
    size: usize,
}

impl fmt::Display for FormattedSize {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let ksize = (self.size as f64) / 1024f64;

        if ksize < 1f64 {
            return write!(f, "{}B", self.size);
        }

        let msize = ksize / 1024f64;

        if msize < 1f64 {
            return write!(f, "{:.1}K", ksize);
        }

        let gsize = msize / 1024f64;

        if gsize < 1f64 {
            write!(f, "{:.1}M", msize)
        } else {
            write!(f, "{:.1}G", gsize)
        }
    }
}

fn formatted_size(size: usize) -> FormattedSize {
    FormattedSize { size }
}

fn on_different_cards(curr: Address, next: Address) -> bool {
    (curr.to_usize() >> CARD_SIZE_BITS) != (next.to_usize() >> CARD_SIZE_BITS)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

//! Synchronization primitives for Vulkan objects.
//!
//! In Vulkan, you have to manually ensure two things:
//!
//! - That a buffer or an image are not read and written simultaneously (similarly to the CPU).
//! - That writes to a buffer or an image are propagated to other queues by inserting memory
//!   barriers.
//!
//! But don't worry ; this is automatically enforced by this library (as long as you don't use
//! any unsafe function). See the `memory` module for more info.
//!
use std::sync::Arc;

use device::Queue;

pub use self::event::Event;
pub use self::fence::Fence;
pub use self::fence::FenceWaitError;
pub use self::semaphore::Semaphore;

mod event;
mod fence;
mod semaphore;

/// Base trait for objects that can be used as resources and must be synchronized.
// TODO: remove
pub unsafe trait Resource {
    /// Returns in which queue family or families this resource can be used.
    fn sharing_mode(&self) -> &SharingMode;

    /// Returns true if the `gpu_access` function should be passed a fence.
    #[inline]
    fn requires_fence(&self) -> bool {
        true
    }

    /// Returns true if the `gpu_access` function should be passed a semaphore.
    #[inline]
    fn requires_semaphore(&self) -> bool {
        true
    }
}

/// Declares in which queue(s) a resource can be used.
///
/// When you create a buffer or an image, you have to tell the Vulkan library in which queue
/// families it will be used. The vulkano library requires you to tell in which queue famiily
/// the resource will be used, even for exclusive mode.
#[derive(Debug, Clone, PartialEq, Eq)]
// TODO: remove
pub enum SharingMode {
    /// The resource is used is only one queue family.
    Exclusive(u32),
    /// The resource is used in multiple queue families. Can be slower than `Exclusive`.
    Concurrent(Vec<u32>),       // TODO: Vec is too expensive here
}

impl<'a> From<&'a Arc<Queue>> for SharingMode {
    #[inline]
    fn from(queue: &'a Arc<Queue>) -> SharingMode {
        SharingMode::Exclusive(queue.family().id())
    }
}

impl<'a> From<&'a [&'a Arc<Queue>]> for SharingMode {
    #[inline]
    fn from(queues: &'a [&'a Arc<Queue>]) -> SharingMode {
        SharingMode::Concurrent(queues.iter().map(|queue| {
            queue.family().id()
        }).collect())
    }
}

/// Declares in which queue(s) a resource can be used.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Sharing<I> where I: Iterator<Item = u32> {
    /// The resource is used is only one queue family.
    Exclusive,
    /// The resource is used in multiple queue families. Can be slower than `Exclusive`.
    Concurrent(I),
}
<commit_msg>Add ShaderStages and AccessFlagBits<commit_after>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

//! Synchronization primitives for Vulkan objects.
//!
//! In Vulkan, you have to manually ensure two things:
//!
//! - That a buffer or an image are not read and written simultaneously (similarly to the CPU).
//! - That writes to a buffer or an image are propagated to other queues by inserting memory
//!   barriers.
//!
//! But don't worry ; this is automatically enforced by this library (as long as you don't use
//! any unsafe function). See the `memory` module for more info.
//!
use std::sync::Arc;
use device::Queue;
use vk;

pub use self::event::Event;
pub use self::fence::Fence;
pub use self::fence::FenceWaitError;
pub use self::semaphore::Semaphore;

mod event;
mod fence;
mod semaphore;

/// Base trait for objects that can be used as resources and must be synchronized.
// TODO: remove
pub unsafe trait Resource {
    /// Returns in which queue family or families this resource can be used.
    fn sharing_mode(&self) -> &SharingMode;

    /// Returns true if the `gpu_access` function should be passed a fence.
    #[inline]
    fn requires_fence(&self) -> bool {
        true
    }

    /// Returns true if the `gpu_access` function should be passed a semaphore.
    #[inline]
    fn requires_semaphore(&self) -> bool {
        true
    }
}

/// Declares in which queue(s) a resource can be used.
///
/// When you create a buffer or an image, you have to tell the Vulkan library in which queue
/// families it will be used. The vulkano library requires you to tell in which queue famiily
/// the resource will be used, even for exclusive mode.
#[derive(Debug, Clone, PartialEq, Eq)]
// TODO: remove
pub enum SharingMode {
    /// The resource is used is only one queue family.
    Exclusive(u32),
    /// The resource is used in multiple queue families. Can be slower than `Exclusive`.
    Concurrent(Vec<u32>),       // TODO: Vec is too expensive here
}

impl<'a> From<&'a Arc<Queue>> for SharingMode {
    #[inline]
    fn from(queue: &'a Arc<Queue>) -> SharingMode {
        SharingMode::Exclusive(queue.family().id())
    }
}

impl<'a> From<&'a [&'a Arc<Queue>]> for SharingMode {
    #[inline]
    fn from(queues: &'a [&'a Arc<Queue>]) -> SharingMode {
        SharingMode::Concurrent(queues.iter().map(|queue| {
            queue.family().id()
        }).collect())
    }
}

/// Declares in which queue(s) a resource can be used.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Sharing<I> where I: Iterator<Item = u32> {
    /// The resource is used is only one queue family.
    Exclusive,
    /// The resource is used in multiple queue families. Can be slower than `Exclusive`.
    Concurrent(I),
}

macro_rules! pipeline_stages {
    ($($elem:ident => $val:expr,)+) => (
        #[derive(Debug, Copy, Clone)]
        pub struct PipelineStages {
            $(
                pub $elem: bool,
            )+
        }

        #[doc(hidden)]
        impl Into<vk::PipelineStageFlagBits> for PipelineStages {
            #[inline]
            fn into(self) -> vk::PipelineStageFlagBits {
                let mut result = 0;
                $(
                    if self.$elem { result |= $val }
                )+
                result
            }
        }
    );
}

pipeline_stages!{
    top_of_pipe => vk::PIPELINE_STAGE_TOP_OF_PIPE_BIT,
    draw_indirect => vk::PIPELINE_STAGE_DRAW_INDIRECT_BIT,
    vertex_input => vk::PIPELINE_STAGE_VERTEX_INPUT_BIT,
    vertex_shader => vk::PIPELINE_STAGE_VERTEX_SHADER_BIT,
    tessellation_control_shader => vk::PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT,
    tessellation_evaluation_shader => vk::PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT,
    geometry_shader => vk::PIPELINE_STAGE_GEOMETRY_SHADER_BIT,
    fragment_shader => vk::PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
    early_fragment_tests => vk::PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT,
    late_fragment_tests => vk::PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT,
    color_attachment_output => vk::PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
    compute_shader => vk::PIPELINE_STAGE_COMPUTE_SHADER_BIT,
    transfer => vk::PIPELINE_STAGE_TRANSFER_BIT,
    bottom_of_pipe => vk::PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
    host => vk::PIPELINE_STAGE_HOST_BIT,
    all_graphics => vk::PIPELINE_STAGE_ALL_GRAPHICS_BIT,
    all_commands => vk::PIPELINE_STAGE_ALL_COMMANDS_BIT,
}

macro_rules! access_flags {
    ($($elem:ident => $val:expr,)+) => (
        #[derive(Debug, Copy, Clone)]
        pub struct AccessFlagBits {
            $(
                pub $elem: bool,
            )+
        }

        #[doc(hidden)]
        impl Into<vk::AccessFlagBits> for AccessFlagBits {
            #[inline]
            fn into(self) -> vk::AccessFlagBits {
                let mut result = 0;
                $(
                    if self.$elem { result |= $val }
                )+
                result
            }
        }
    );
}

access_flags!{
    indirect_command_read => vk::ACCESS_INDIRECT_COMMAND_READ_BIT,
    index_read => vk::ACCESS_INDEX_READ_BIT,
    vertex_attribute_read => vk::ACCESS_VERTEX_ATTRIBUTE_READ_BIT,
    uniform_read => vk::ACCESS_UNIFORM_READ_BIT,
    input_attachment_read => vk::ACCESS_INPUT_ATTACHMENT_READ_BIT,
    shader_read => vk::ACCESS_SHADER_READ_BIT,
    shader_write => vk::ACCESS_SHADER_WRITE_BIT,
    color_attachment_read => vk::ACCESS_COLOR_ATTACHMENT_READ_BIT,
    color_attachment_write => vk::ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
    depth_stencil_attachment_read => vk::ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT,
    depth_stencil_attachment_write => vk::ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
    transfer_read => vk::ACCESS_TRANSFER_READ_BIT,
    transfer_write => vk::ACCESS_TRANSFER_WRITE_BIT,
    host_read => vk::ACCESS_HOST_READ_BIT,
    host_write => vk::ACCESS_HOST_WRITE_BIT,
    memory_read => vk::ACCESS_MEMORY_READ_BIT,
    memory_write => vk::ACCESS_MEMORY_WRITE_BIT,
}
<|endoftext|>


<commit_before>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use std::mem;
use std::default::Default;

use token::Token;
use token::Data;
use logger::Logger;

pub fn traverse_token<F1, F2, R>(token: &Token, f: &F1, add: &F2) -> R
    where F1: Fn(&str) -> R,
          R: Default,
          F2: Fn(R, R) -> R
{
    match *token {
        Token::Str(ref s) => f(s),

        Token::Rule |
        Token::SoftBreak |
        Token::HardBreak => f("\n"),

        Token::Image(..) |
        Token::StandaloneImage(..) |
        Token::Footnote(..) |
        Token::Table(..) |
        Token::TableHead(..) |
        Token::TableRow(..) |
        Token::TableCell(..) => f(""),

        _ => traverse_vec(token.inner().unwrap(), f, add),
    }
}

/// Traverse a vector of tokens
#[doc(hidden)]
pub fn traverse_vec<F1, F2, R>(tokens: &[Token], f: &F1, add: &F2) -> R
    where F1: Fn(&str) -> R,
          F2: Fn(R, R) -> R,
          R: Default
{
    tokens.iter()
        .map(|t| traverse_token(t, f, add))
        .fold(R::default(), add)
}

/// Returns the content of an AST as raw text, without any formatting
/// Useful for tools like grammar checks
pub fn view_as_text(tokens: &[Token]) -> String {
    traverse_vec(tokens, &|s| s.to_owned(), &|s1, s2| s1 + &s2)
}

pub fn count_length(tokens: &[Token]) -> usize {
    traverse_vec(tokens, &|s| s.chars().count(), &|s1, s2| s1 + s2)
}

/// Insert an annotation at begin and end pos begin+len in the text_view
#[doc(hidden)]
pub fn insert_annotation(tokens: &mut Vec<Token>,
                         annotation: &Data,
                         pos: usize,
                         length: usize)
                         -> Option<usize> {
    let mut pos = pos;
    let mut found_left = None;
    let mut found_right = None;
    for i in 0..tokens.len() {
        let recurse = match tokens[i] {
            Token::Str(ref s) => {
                let len = s.chars().count();
                if pos < len || (pos == len && found_left.is_some()) {
                    // We found the first element already, so now it's the right
                    if found_left.is_some() {
                        found_right = Some((i, pos));
                        break;
                    }
                    found_left = Some((i, pos));
                    pos += length;
                    if pos <= len {
                        found_right = Some((i, pos));
                        break;
                    }
                }
                pos -= len;
                false
            }

            Token::Rule | Token::SoftBreak | Token::HardBreak => {
                if pos < 1 {
                    if found_left.is_some() {
                        found_right = Some((i, pos));
                        break;
                    }
                    found_left = Some((i, pos));
                    pos += length;
                    if pos <= 1 {
                        found_right = Some((i, pos));
                        break;
                    }
                }
                pos -= 1;
                false
            }

            _ => {
                if let Some(inner) = tokens[i].inner() {
                    let len = count_length(inner);
                    // Only recurse if the two is in this subtree
                    if pos < len {
                        if found_left.is_none() {
                            true
                        } else {
                            Logger::display_warning(lformat!("ignored annotation {:?} as it \
                                                              wasn't compatible with the \
                                                              Markdown structure",
                                                             annotation));
                            return None;
                        }
                    } else {
                        pos -= len;
                        false
                    }
                } else {
                    false
                }

            }
        };

        // Moved out of the match 'thanks' to borrowcheck
        if recurse {
            if let Some(ref mut inner) = tokens[i].inner_mut() {
                if let Some(new_pos) = insert_annotation(inner, annotation, pos, length) {
                    pos = new_pos;
                } else {
                    return None;
                }
            }
        }
    }

    if let (Some((i, pos_left)), Some((j, pos_right))) = (found_left, found_right) {
        let pos_right = pos_right;
        let mut vec = vec![];

        // Beginning token: keep the left part in the str, put the right one in our vec
        if !tokens[i].is_str() || pos_left == 0 {
            // do nothing
        } else {
            let old_token = mem::replace(&mut tokens[i], Token::Str(String::new()));
            if let Token::Str(old_str) = old_token {
                let mut chars_left: Vec<char> = old_str.chars().collect();
                let mut chars_right = chars_left.split_off(pos_left);

                let str_left: String = chars_left.into_iter().collect();
                tokens[i] = Token::Str(str_left);

                if i == j {
                    // i and j are in same str, so split again
                    if length != chars_right.len() {
                        let inline_token = chars_right.split_off(length);
                        let inline_token = Token::Str(inline_token.into_iter().collect());
                        if pos_left == 0 {
                            tokens.insert(i, inline_token)
                        } else if i == tokens.len() {
                            tokens.push(inline_token);
                        } else {
                            tokens.insert(i + 1, inline_token);
                        }
                    }
                    let annot = Token::Annotation(annotation.clone(),
                                                  vec![Token::Str(chars_right.into_iter()
                                                           .collect())]);
                    if pos_left == 0 {
                        tokens.insert(i, annot)
                    } else if i == tokens.len() {
                        tokens.push(annot);
                    } else {
                        tokens.insert(i + 1, annot);
                    }
                    return None;
                }

                let str_right: String = chars_right.into_iter().collect();
                vec.push(Token::Str(str_right));
            } else {
                unreachable!();
            }
        }

        // Middle tokens: remove them entirely and put them in our vec
        for _ in i + 1..j {
            vec.push(tokens.remove(i + 1));
        }

        // End token: keep the right part in the str, put the left one in our vec
        // j is now i + 1 because all tokens in between have been removed
        // unless j was equal to i to begin with
        let j = if i == j || i >= tokens.len() - 1 {
            i
        } else {
            i + 1
        };

        if !tokens[j].is_str() {
            // do nothing
        } else {
            let count = if let Token::Str(ref s) = tokens[j] {
                s.chars().count()
            } else {
                unreachable!()
            };
            if count != pos_right {
                let old_token = mem::replace(&mut tokens[j], Token::Rule);
                if let Token::Str(old_str) = old_token {
                    let mut chars_left: Vec<char> = old_str.chars().collect();
                    let chars_right = chars_left.split_off(pos_right);
                    let str_left: String = chars_left.into_iter().collect();
                    let str_right: String = chars_right.into_iter().collect();
                    tokens[j] = Token::Str(str_right);
                    // todo: if only one token, maybe concat the strings
                    vec.push(Token::Str(str_left));
                } else {
                    unreachable!();
                }
            }
        }
        let new_token = Token::Annotation(annotation.clone(), vec);
        if pos_left == 0 {
            tokens.insert(i, new_token);
        } else if i >= tokens.len() - 1 {
            tokens.push(new_token);
        } else {
            tokens.insert(i + 1, new_token);
        }
        return None;
    } else if found_left.is_none() && found_right.is_none() {
        return Some(pos);
    } else {
        Logger::display_warning(lformat!("ignored annotation {:?} as it wasn't compatible \
                                          with the Markdown structure",
                                         annotation));
        return None;
    }
}

#[test]
fn test_text_view() {
    let ast = vec![Token::Str("123".to_owned()),
                   Token::Emphasis(vec![Token::Str("456".to_owned())]),
                   Token::Str("789".to_owned())];
    assert_eq!(view_as_text(&ast), "123456789");
}

// #[test]
// fn test_text_insert() {
//     let mut ast = vec!(Token::Str("123".to_owned()),
//                        Token::Emphasis(vec!(Token::Str("456".to_owned()))),
//                        Token::Str("789".to_owned()));
//     insert_at(&mut ast, "!!!", 5);
//     let expected = vec!(Token::Str("123".to_owned()),
//                         Token::Emphasis(vec!(Token::Str("45".to_owned()),
//                                              Token::Comment("!!!".to_owned()),
//                                              Token::Str("6".to_owned()))),
//                         Token::Str("789".to_owned()));
//     assert_eq!(expected, ast);
// }
<commit_msg>See Softbreaks as spaces in text_view<commit_after>// Copyright (C) 2016 Élisabeth HENRY.
//
// This file is part of Crowbook.
//
// Crowbook is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published
// by the Free Software Foundation, either version 2.1 of the License, or
// (at your option) any later version.
//
// Caribon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Lesser General Public License for more details.
//
// You should have received ba copy of the GNU Lesser General Public License
// along with Crowbook.  If not, see <http://www.gnu.org/licenses/>.

use std::mem;
use std::default::Default;

use token::Token;
use token::Data;
use logger::Logger;

pub fn traverse_token<F1, F2, R>(token: &Token, f: &F1, add: &F2) -> R
    where F1: Fn(&str) -> R,
          R: Default,
          F2: Fn(R, R) -> R
{
    match *token {
        Token::Str(ref s) => f(s),

        Token::SoftBreak => f(" "),

        Token::Rule |
        Token::HardBreak => f("\n"),

        Token::Image(..) |
        Token::StandaloneImage(..) |
        Token::Footnote(..) |
        Token::Table(..) |
        Token::TableHead(..) |
        Token::TableRow(..) |
        Token::TableCell(..) => f(""),

        _ => traverse_vec(token.inner().unwrap(), f, add),
    }
}

/// Traverse a vector of tokens
#[doc(hidden)]
pub fn traverse_vec<F1, F2, R>(tokens: &[Token], f: &F1, add: &F2) -> R
    where F1: Fn(&str) -> R,
          F2: Fn(R, R) -> R,
          R: Default
{
    tokens.iter()
        .map(|t| traverse_token(t, f, add))
        .fold(R::default(), add)
}

/// Returns the content of an AST as raw text, without any formatting
/// Useful for tools like grammar checks
pub fn view_as_text(tokens: &[Token]) -> String {
    traverse_vec(tokens, &|s| s.to_owned(), &|s1, s2| s1 + &s2)
}

pub fn count_length(tokens: &[Token]) -> usize {
    traverse_vec(tokens, &|s| s.chars().count(), &|s1, s2| s1 + s2)
}

/// Insert an annotation at begin and end pos begin+len in the text_view
#[doc(hidden)]
pub fn insert_annotation(tokens: &mut Vec<Token>,
                         annotation: &Data,
                         pos: usize,
                         length: usize)
                         -> Option<usize> {
    let mut pos = pos;
    let mut found_left = None;
    let mut found_right = None;
    for i in 0..tokens.len() {
        let recurse = match tokens[i] {
            Token::Str(ref s) => {
                let len = s.chars().count();
                if pos < len || (pos == len && found_left.is_some()) {
                    // We found the first element already, so now it's the right
                    if found_left.is_some() {
                        found_right = Some((i, pos));
                        break;
                    }
                    found_left = Some((i, pos));
                    pos += length;
                    if pos <= len {
                        found_right = Some((i, pos));
                        break;
                    }
                }
                pos -= len;
                false
            }

            Token::Rule | Token::SoftBreak | Token::HardBreak => {
                if pos < 1 {
                    if found_left.is_some() {
                        found_right = Some((i, pos));
                        break;
                    }
                    found_left = Some((i, pos));
                    pos += length;
                    if pos <= 1 {
                        found_right = Some((i, pos));
                        break;
                    }
                }
                pos -= 1;
                false
            }

            _ => {
                if let Some(inner) = tokens[i].inner() {
                    let len = count_length(inner);
                    // Only recurse if the two is in this subtree
                    if pos < len {
                        if found_left.is_none() {
                            true
                        } else {
                            Logger::display_warning(lformat!("ignored annotation {:?} as it \
                                                              wasn't compatible with the \
                                                              Markdown structure",
                                                             annotation));
                            return None;
                        }
                    } else {
                        pos -= len;
                        false
                    }
                } else {
                    false
                }

            }
        };

        // Moved out of the match 'thanks' to borrowcheck
        if recurse {
            if let Some(ref mut inner) = tokens[i].inner_mut() {
                if let Some(new_pos) = insert_annotation(inner, annotation, pos, length) {
                    pos = new_pos;
                } else {
                    return None;
                }
            }
        }
    }

    if let (Some((i, pos_left)), Some((j, pos_right))) = (found_left, found_right) {
        let pos_right = pos_right;
        let mut vec = vec![];

        // Beginning token: keep the left part in the str, put the right one in our vec
        if !tokens[i].is_str() || pos_left == 0 {
            // do nothing
        } else {
            let old_token = mem::replace(&mut tokens[i], Token::Str(String::new()));
            if let Token::Str(old_str) = old_token {
                let mut chars_left: Vec<char> = old_str.chars().collect();
                let mut chars_right = chars_left.split_off(pos_left);

                let str_left: String = chars_left.into_iter().collect();
                tokens[i] = Token::Str(str_left);

                if i == j {
                    // i and j are in same str, so split again
                    if length != chars_right.len() {
                        let inline_token = chars_right.split_off(length);
                        let inline_token = Token::Str(inline_token.into_iter().collect());
                        if pos_left == 0 {
                            tokens.insert(i, inline_token)
                        } else if i == tokens.len() {
                            tokens.push(inline_token);
                        } else {
                            tokens.insert(i + 1, inline_token);
                        }
                    }
                    let annot = Token::Annotation(annotation.clone(),
                                                  vec![Token::Str(chars_right.into_iter()
                                                           .collect())]);
                    if pos_left == 0 {
                        tokens.insert(i, annot)
                    } else if i == tokens.len() {
                        tokens.push(annot);
                    } else {
                        tokens.insert(i + 1, annot);
                    }
                    return None;
                }

                let str_right: String = chars_right.into_iter().collect();
                vec.push(Token::Str(str_right));
            } else {
                unreachable!();
            }
        }

        // Middle tokens: remove them entirely and put them in our vec
        for _ in i + 1..j {
            vec.push(tokens.remove(i + 1));
        }

        // End token: keep the right part in the str, put the left one in our vec
        // j is now i + 1 because all tokens in between have been removed
        // unless j was equal to i to begin with
        let j = if i == j || i >= tokens.len() - 1 {
            i
        } else {
            i + 1
        };

        if !tokens[j].is_str() {
            // do nothing
        } else {
            let count = if let Token::Str(ref s) = tokens[j] {
                s.chars().count()
            } else {
                unreachable!()
            };
            if count != pos_right {
                let old_token = mem::replace(&mut tokens[j], Token::Rule);
                if let Token::Str(old_str) = old_token {
                    let mut chars_left: Vec<char> = old_str.chars().collect();
                    let chars_right = chars_left.split_off(pos_right);
                    let str_left: String = chars_left.into_iter().collect();
                    let str_right: String = chars_right.into_iter().collect();
                    tokens[j] = Token::Str(str_right);
                    // todo: if only one token, maybe concat the strings
                    vec.push(Token::Str(str_left));
                } else {
                    unreachable!();
                }
            }
        }
        let new_token = Token::Annotation(annotation.clone(), vec);
        if pos_left == 0 {
            tokens.insert(i, new_token);
        } else if i >= tokens.len() - 1 {
            tokens.push(new_token);
        } else {
            tokens.insert(i + 1, new_token);
        }
        return None;
    } else if found_left.is_none() && found_right.is_none() {
        return Some(pos);
    } else {
        Logger::display_warning(lformat!("ignored annotation {:?} as it wasn't compatible \
                                          with the Markdown structure",
                                         annotation));
        return None;
    }
}

#[test]
fn test_text_view() {
    let ast = vec![Token::Str("123".to_owned()),
                   Token::Emphasis(vec![Token::Str("456".to_owned())]),
                   Token::Str("789".to_owned())];
    assert_eq!(view_as_text(&ast), "123456789");
}

// #[test]
// fn test_text_insert() {
//     let mut ast = vec!(Token::Str("123".to_owned()),
//                        Token::Emphasis(vec!(Token::Str("456".to_owned()))),
//                        Token::Str("789".to_owned()));
//     insert_at(&mut ast, "!!!", 5);
//     let expected = vec!(Token::Str("123".to_owned()),
//                         Token::Emphasis(vec!(Token::Str("45".to_owned()),
//                                              Token::Comment("!!!".to_owned()),
//                                              Token::Str("6".to_owned()))),
//                         Token::Str("789".to_owned()));
//     assert_eq!(expected, ast);
// }
<|endoftext|>


<commit_before>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Operations on tuples
//!
//! To access a single element of a tuple one can use the following
//! methods:
//!
//! * `valN` - returns a value of _N_-th element
//! * `refN` - returns a reference to _N_-th element
//! * `mutN` - returns a mutable reference to _N_-th element
//!
//! Indexing starts from zero, so `val0` returns first value, `val1`
//! returns second value, and so on. In general, a tuple with _S_
//! elements provides aforementioned methods suffixed with numbers
//! from `0` to `S-1`. Traits which contain these methods are
//! implemented for tuples with up to 12 elements.
//!
//! If every type inside a tuple implements one of the following
//! traits, then a tuple itself also implements it.
//!
//! * `Clone`
//! * `PartialEq`
//! * `Eq`
//! * `PartialOrd`
//! * `Ord`
//! * `Default`

#![stable(feature = "rust1", since = "1.0.0")]
#![doc(primitive = "tuple")]

use clone::Clone;
use cmp::*;
use cmp::Ordering::*;
use default::Default;
use option::Option;
use option::Option::Some;

// FIXME(#19630) Remove this work-around
macro_rules! e {
    ($e:expr) => { $e }
}

// macro for implementing n-ary tuple functions and operations
macro_rules! tuple_impls {
    ($(
        $Tuple:ident {
            $(($valN:ident, $refN:ident, $mutN:ident, $idx:tt) -> $T:ident)+
        }
    )+) => {
        $(
            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:Clone),+> Clone for ($($T,)+) {
                fn clone(&self) -> ($($T,)+) {
                    ($(e!(self.$idx.clone()),)+)
                }
            }

            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:PartialEq),+> PartialEq for ($($T,)+) {
                #[inline]
                fn eq(&self, other: &($($T,)+)) -> bool {
                    e!($(self.$idx == other.$idx)&&+)
                }
                #[inline]
                fn ne(&self, other: &($($T,)+)) -> bool {
                    e!($(self.$idx != other.$idx)||+)
                }
            }

            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:Eq),+> Eq for ($($T,)+) {}

            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:PartialOrd + PartialEq),+> PartialOrd for ($($T,)+) {
                #[inline]
                fn partial_cmp(&self, other: &($($T,)+)) -> Option<Ordering> {
                    lexical_partial_cmp!($(self.$idx, other.$idx),+)
                }
                #[inline]
                fn lt(&self, other: &($($T,)+)) -> bool {
                    lexical_ord!(lt, $(self.$idx, other.$idx),+)
                }
                #[inline]
                fn le(&self, other: &($($T,)+)) -> bool {
                    lexical_ord!(le, $(self.$idx, other.$idx),+)
                }
                #[inline]
                fn ge(&self, other: &($($T,)+)) -> bool {
                    lexical_ord!(ge, $(self.$idx, other.$idx),+)
                }
                #[inline]
                fn gt(&self, other: &($($T,)+)) -> bool {
                    lexical_ord!(gt, $(self.$idx, other.$idx),+)
                }
            }

            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:Ord),+> Ord for ($($T,)+) {
                #[inline]
                fn cmp(&self, other: &($($T,)+)) -> Ordering {
                    lexical_cmp!($(self.$idx, other.$idx),+)
                }
            }

            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:Default),+> Default for ($($T,)+) {
                #[stable(feature = "rust1", since = "1.0.0")]
                #[inline]
                fn default() -> ($($T,)+) {
                    ($({ let x: $T = Default::default(); x},)+)
                }
            }
        )+
    }
}

// Constructs an expression that performs a lexical ordering using method $rel.
// The values are interleaved, so the macro invocation for
// `(a1, a2, a3) < (b1, b2, b3)` would be `lexical_ord!(lt, a1, b1, a2, b2,
// a3, b3)` (and similarly for `lexical_cmp`)
macro_rules! lexical_ord {
    ($rel: ident, $a:expr, $b:expr, $($rest_a:expr, $rest_b:expr),+) => {
        if $a != $b { lexical_ord!($rel, $a, $b) }
        else { lexical_ord!($rel, $($rest_a, $rest_b),+) }
    };
    ($rel: ident, $a:expr, $b:expr) => { ($a) . $rel (& $b) };
}

macro_rules! lexical_partial_cmp {
    ($a:expr, $b:expr, $($rest_a:expr, $rest_b:expr),+) => {
        match ($a).partial_cmp(&$b) {
            Some(Equal) => lexical_partial_cmp!($($rest_a, $rest_b),+),
            ordering   => ordering
        }
    };
    ($a:expr, $b:expr) => { ($a).partial_cmp(&$b) };
}

macro_rules! lexical_cmp {
    ($a:expr, $b:expr, $($rest_a:expr, $rest_b:expr),+) => {
        match ($a).cmp(&$b) {
            Equal => lexical_cmp!($($rest_a, $rest_b),+),
            ordering   => ordering
        }
    };
    ($a:expr, $b:expr) => { ($a).cmp(&$b) };
}

tuple_impls! {
    Tuple1 {
        (val0, ref0, mut0, 0) -> A
    }
    Tuple2 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
    }
    Tuple3 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
        (val2, ref2, mut2, 2) -> C
    }
    Tuple4 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
        (val2, ref2, mut2, 2) -> C
        (val3, ref3, mut3, 3) -> D
    }
    Tuple5 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
        (val2, ref2, mut2, 2) -> C
        (val3, ref3, mut3, 3) -> D
        (val4, ref4, mut4, 4) -> E
    }
    Tuple6 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
        (val2, ref2, mut2, 2) -> C
        (val3, ref3, mut3, 3) -> D
        (val4, ref4, mut4, 4) -> E
        (val5, ref5, mut5, 5) -> F
    }
    Tuple7 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
        (val2, ref2, mut2, 2) -> C
        (val3, ref3, mut3, 3) -> D
        (val4, ref4, mut4, 4) -> E
        (val5, ref5, mut5, 5) -> F
        (val6, ref6, mut6, 6) -> G
    }
    Tuple8 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
        (val2, ref2, mut2, 2) -> C
        (val3, ref3, mut3, 3) -> D
        (val4, ref4, mut4, 4) -> E
        (val5, ref5, mut5, 5) -> F
        (val6, ref6, mut6, 6) -> G
        (val7, ref7, mut7, 7) -> H
    }
    Tuple9 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
        (val2, ref2, mut2, 2) -> C
        (val3, ref3, mut3, 3) -> D
        (val4, ref4, mut4, 4) -> E
        (val5, ref5, mut5, 5) -> F
        (val6, ref6, mut6, 6) -> G
        (val7, ref7, mut7, 7) -> H
        (val8, ref8, mut8, 8) -> I
    }
    Tuple10 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
        (val2, ref2, mut2, 2) -> C
        (val3, ref3, mut3, 3) -> D
        (val4, ref4, mut4, 4) -> E
        (val5, ref5, mut5, 5) -> F
        (val6, ref6, mut6, 6) -> G
        (val7, ref7, mut7, 7) -> H
        (val8, ref8, mut8, 8) -> I
        (val9, ref9, mut9, 9) -> J
    }
    Tuple11 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
        (val2, ref2, mut2, 2) -> C
        (val3, ref3, mut3, 3) -> D
        (val4, ref4, mut4, 4) -> E
        (val5, ref5, mut5, 5) -> F
        (val6, ref6, mut6, 6) -> G
        (val7, ref7, mut7, 7) -> H
        (val8, ref8, mut8, 8) -> I
        (val9, ref9, mut9, 9) -> J
        (val10, ref10, mut10, 10) -> K
    }
    Tuple12 {
        (val0, ref0, mut0, 0) -> A
        (val1, ref1, mut1, 1) -> B
        (val2, ref2, mut2, 2) -> C
        (val3, ref3, mut3, 3) -> D
        (val4, ref4, mut4, 4) -> E
        (val5, ref5, mut5, 5) -> F
        (val6, ref6, mut6, 6) -> G
        (val7, ref7, mut7, 7) -> H
        (val8, ref8, mut8, 8) -> I
        (val9, ref9, mut9, 9) -> J
        (val10, ref10, mut10, 10) -> K
        (val11, ref11, mut11, 11) -> L
    }
}
<commit_msg>Auto merge of #23395 - bluss:tuple-macro, r=alexcrichton<commit_after>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Operations on tuples
//!
//! To access a single element of a tuple one can use the `.0`
//! field access syntax.
//!
//! Indexing starts from zero, so `.0` returns first value, `.1`
//! returns second value, and so on. In general, a tuple with *N*
//! elements has field accessors from 0 to *N* - 1.
//!
//! If every type inside a tuple implements one of the following
//! traits, then a tuple itself also implements it.
//!
//! * `Clone`
//! * `PartialEq`
//! * `Eq`
//! * `PartialOrd`
//! * `Ord`
//! * `Default`

#![stable(feature = "rust1", since = "1.0.0")]
#![doc(primitive = "tuple")]

use clone::Clone;
use cmp::*;
use cmp::Ordering::*;
use default::Default;
use option::Option;
use option::Option::Some;

// FIXME(#19630) Remove this work-around
macro_rules! e {
    ($e:expr) => { $e }
}

// macro for implementing n-ary tuple functions and operations
macro_rules! tuple_impls {
    ($(
        $Tuple:ident {
            $(($idx:tt) -> $T:ident)+
        }
    )+) => {
        $(
            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:Clone),+> Clone for ($($T,)+) {
                fn clone(&self) -> ($($T,)+) {
                    ($(e!(self.$idx.clone()),)+)
                }
            }

            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:PartialEq),+> PartialEq for ($($T,)+) {
                #[inline]
                fn eq(&self, other: &($($T,)+)) -> bool {
                    e!($(self.$idx == other.$idx)&&+)
                }
                #[inline]
                fn ne(&self, other: &($($T,)+)) -> bool {
                    e!($(self.$idx != other.$idx)||+)
                }
            }

            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:Eq),+> Eq for ($($T,)+) {}

            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:PartialOrd + PartialEq),+> PartialOrd for ($($T,)+) {
                #[inline]
                fn partial_cmp(&self, other: &($($T,)+)) -> Option<Ordering> {
                    lexical_partial_cmp!($(self.$idx, other.$idx),+)
                }
                #[inline]
                fn lt(&self, other: &($($T,)+)) -> bool {
                    lexical_ord!(lt, $(self.$idx, other.$idx),+)
                }
                #[inline]
                fn le(&self, other: &($($T,)+)) -> bool {
                    lexical_ord!(le, $(self.$idx, other.$idx),+)
                }
                #[inline]
                fn ge(&self, other: &($($T,)+)) -> bool {
                    lexical_ord!(ge, $(self.$idx, other.$idx),+)
                }
                #[inline]
                fn gt(&self, other: &($($T,)+)) -> bool {
                    lexical_ord!(gt, $(self.$idx, other.$idx),+)
                }
            }

            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:Ord),+> Ord for ($($T,)+) {
                #[inline]
                fn cmp(&self, other: &($($T,)+)) -> Ordering {
                    lexical_cmp!($(self.$idx, other.$idx),+)
                }
            }

            #[stable(feature = "rust1", since = "1.0.0")]
            impl<$($T:Default),+> Default for ($($T,)+) {
                #[stable(feature = "rust1", since = "1.0.0")]
                #[inline]
                fn default() -> ($($T,)+) {
                    ($({ let x: $T = Default::default(); x},)+)
                }
            }
        )+
    }
}

// Constructs an expression that performs a lexical ordering using method $rel.
// The values are interleaved, so the macro invocation for
// `(a1, a2, a3) < (b1, b2, b3)` would be `lexical_ord!(lt, a1, b1, a2, b2,
// a3, b3)` (and similarly for `lexical_cmp`)
macro_rules! lexical_ord {
    ($rel: ident, $a:expr, $b:expr, $($rest_a:expr, $rest_b:expr),+) => {
        if $a != $b { lexical_ord!($rel, $a, $b) }
        else { lexical_ord!($rel, $($rest_a, $rest_b),+) }
    };
    ($rel: ident, $a:expr, $b:expr) => { ($a) . $rel (& $b) };
}

macro_rules! lexical_partial_cmp {
    ($a:expr, $b:expr, $($rest_a:expr, $rest_b:expr),+) => {
        match ($a).partial_cmp(&$b) {
            Some(Equal) => lexical_partial_cmp!($($rest_a, $rest_b),+),
            ordering   => ordering
        }
    };
    ($a:expr, $b:expr) => { ($a).partial_cmp(&$b) };
}

macro_rules! lexical_cmp {
    ($a:expr, $b:expr, $($rest_a:expr, $rest_b:expr),+) => {
        match ($a).cmp(&$b) {
            Equal => lexical_cmp!($($rest_a, $rest_b),+),
            ordering   => ordering
        }
    };
    ($a:expr, $b:expr) => { ($a).cmp(&$b) };
}

tuple_impls! {
    Tuple1 {
        (0) -> A
    }
    Tuple2 {
        (0) -> A
        (1) -> B
    }
    Tuple3 {
        (0) -> A
        (1) -> B
        (2) -> C
    }
    Tuple4 {
        (0) -> A
        (1) -> B
        (2) -> C
        (3) -> D
    }
    Tuple5 {
        (0) -> A
        (1) -> B
        (2) -> C
        (3) -> D
        (4) -> E
    }
    Tuple6 {
        (0) -> A
        (1) -> B
        (2) -> C
        (3) -> D
        (4) -> E
        (5) -> F
    }
    Tuple7 {
        (0) -> A
        (1) -> B
        (2) -> C
        (3) -> D
        (4) -> E
        (5) -> F
        (6) -> G
    }
    Tuple8 {
        (0) -> A
        (1) -> B
        (2) -> C
        (3) -> D
        (4) -> E
        (5) -> F
        (6) -> G
        (7) -> H
    }
    Tuple9 {
        (0) -> A
        (1) -> B
        (2) -> C
        (3) -> D
        (4) -> E
        (5) -> F
        (6) -> G
        (7) -> H
        (8) -> I
    }
    Tuple10 {
        (0) -> A
        (1) -> B
        (2) -> C
        (3) -> D
        (4) -> E
        (5) -> F
        (6) -> G
        (7) -> H
        (8) -> I
        (9) -> J
    }
    Tuple11 {
        (0) -> A
        (1) -> B
        (2) -> C
        (3) -> D
        (4) -> E
        (5) -> F
        (6) -> G
        (7) -> H
        (8) -> I
        (9) -> J
        (10) -> K
    }
    Tuple12 {
        (0) -> A
        (1) -> B
        (2) -> C
        (3) -> D
        (4) -> E
        (5) -> F
        (6) -> G
        (7) -> H
        (8) -> I
        (9) -> J
        (10) -> K
        (11) -> L
    }
}
<|endoftext|>


<commit_before>use rdkafka::Message;
use rdkafka::message::BorrowedMessage;
use rdkafka::consumer::{BaseConsumer, EmptyConsumerContext};
use rdkafka::config::ClientConfig;
use rocket::State;
use rocket::http::RawStr;
use scheduled_executor::ThreadPoolExecutor;

use config::{ClusterConfig, Config};
use metadata::ClusterId;
use error::*;

use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};

pub struct LiveConsumer {
    id: u64,
    cluster_id: ClusterId,
    topic: String,
    last_poll: RwLock<Instant>,
    consumer: BaseConsumer<EmptyConsumerContext>,
    active: AtomicBool,
}

impl LiveConsumer {
    fn new(id: u64, cluster_config: &ClusterConfig, topic: &str) -> Result<LiveConsumer> {
        let consumer = ClientConfig::new()
            .set("bootstrap.servers", &cluster_config.bootstrap_servers())
            .set("group.id", &format!("kafka_view_live_consumer_{}", id))
            .set("enable.partition.eof", "false")
            .set("api.version.request", "false")
            .set("enable.auto.commit", "false")
            .set("queued.max.messages.kbytes", "100") // Reduce memory usage
            .set("fetch.message.max.bytes", "102400")
            .create::<BaseConsumer<_>>()
            .chain_err(|| "Failed to create rdkafka consumer")?;

        debug!("Creating live consumer for {}", topic);

        Ok(LiveConsumer {
            id: id,
            cluster_id: cluster_config.cluster_id.clone().unwrap(),
            consumer: consumer,
            active: AtomicBool::new(false),
            last_poll: RwLock::new(Instant::now()),
            topic: topic.to_owned(),
        })
    }

    fn activate(&self) -> Result<()> {
        // TODO: start from the past
        debug!("Activating live consumer for {}", self.topic);

        // TODO: use assign instead
        self.consumer.subscribe(vec![self.topic.as_str()].as_slice())
            .chain_err(|| "Can't subscribe to specified topics")?;
        self.active.store(true, Ordering::Relaxed);
        Ok(())
    }

    pub fn is_active(&self) -> bool {
        self.active.load(Ordering::Relaxed)
    }

    pub fn last_poll(&self) -> Instant {
        *self.last_poll.read().unwrap()
    }

    pub fn id(&self) -> u64 {
        self.id
    }

    pub fn cluster_id(&self) -> &ClusterId {
        &self.cluster_id
    }

    pub fn topic(&self) -> &str {
        &self.topic
    }

    fn poll(&self, max_msg: usize, timeout: Duration) -> Vec<BorrowedMessage> {
        let start_time = Instant::now();
        let mut buffer = Vec::new();
        *self.last_poll.write().unwrap() = Instant::now();

        while Instant::elapsed(&start_time) < timeout && buffer.len() < max_msg {
            match self.consumer.poll(100) {
                Ok(None) => {},
                Ok(Some(m)) => buffer.push(m),
                Err(e) => {
                    error!("Error while receiving message {:?}", e);
                },
            };
        }

        debug!("{} messages received in {:?}", buffer.len(), Instant::elapsed(&start_time));
        buffer
    }
}

impl Drop for LiveConsumer {
    fn drop(&mut self) {
        debug!("Dropping consumer {}", self.id);
    }
}

type LiveConsumerMap = HashMap<u64, Arc<LiveConsumer>>;

fn remove_idle_consumers(consumers: &mut LiveConsumerMap) {
    consumers.retain(|_, ref consumer| consumer.last_poll().elapsed() < Duration::from_secs(20));
}

pub struct LiveConsumerStore {
    consumers: Arc<RwLock<LiveConsumerMap>>,
    _executor: ThreadPoolExecutor,
}

impl LiveConsumerStore {
    pub fn new(executor: ThreadPoolExecutor) -> LiveConsumerStore {
        let consumers = Arc::new(RwLock::new(HashMap::new()));
        let consumers_clone = Arc::clone(&consumers);
        executor.schedule_fixed_rate(
            Duration::from_secs(10),
            Duration::from_secs(10),
            move |_handle| {
                let mut consumers = consumers_clone.write().unwrap();
                remove_idle_consumers(&mut *consumers);
            }
        );
        LiveConsumerStore {
            consumers: consumers,
            _executor: executor,
        }
    }

    fn get_consumer(&self, id: u64) -> Option<Arc<LiveConsumer>> {
        let consumers = self.consumers.read().expect("Poison error");
        (*consumers).get(&id).cloned()
    }

    fn add_consumer(&self, id: u64, cluster_config: &ClusterConfig, topic: &str) -> Result<Arc<LiveConsumer>> {
        let live_consumer = LiveConsumer::new(id, cluster_config, topic)
            .chain_err(|| "Failed to create live consumer")?;

        let live_consumer_arc = Arc::new(live_consumer);

        // Add consumer immediately to the store, to prevent other threads from adding it again.
        match self.consumers.write() {
            Ok(mut consumers) => (*consumers).insert(id, live_consumer_arc.clone()),
            Err(_) => panic!("Poison error while writing consumer to cache")
        };

        live_consumer_arc.activate()
            .chain_err(|| "Failed to activate live consumer")?;

        Ok(live_consumer_arc)
    }

    pub fn consumers(&self) -> Vec<Arc<LiveConsumer>> {
        self.consumers.read().unwrap().iter()
            .map(|(_, consumer)| consumer.clone())
            .collect::<Vec<_>>()
    }
}

// TODO: check log in case of error

#[get("/api/tailer/<cluster_id>/<topic>/<id>")]
pub fn test_live_consumer_api(
    cluster_id: ClusterId,
    topic: &RawStr,
    id: u64,
    config: State<Config>,
    live_consumers_store: State<LiveConsumerStore>,
) -> Result<String> {
    let cluster_config = config.clusters.get(&cluster_id);

    if cluster_config.is_none() {
        return Ok("[]".to_owned());
    }
    let cluster_config = cluster_config.unwrap();

    let consumer = match live_consumers_store.get_consumer(id) {
        Some(consumer) => consumer,
        None => live_consumers_store.add_consumer(id, cluster_config, topic)
            .chain_err(|| format!("Error while creating live consumer for {} {}", cluster_id, topic))?,
    };

    if !consumer.is_active() {
        // Consumer is still being activated, no results for now.
        return Ok("[]".to_owned());
    }

    let mut output = Vec::new();
    for message in consumer.poll(100, Duration::from_secs(3)) {
        let payload = message.payload()
            .map(|bytes| String::from_utf8_lossy(bytes).to_string())
            .unwrap_or("".to_owned());
        // TODO: allocate less
        let truncated = if payload.len() > 1024 {
            format!("{}...", &payload[..1024])  // TODO: fix handling of utf8
        } else {
            payload.clone()
        };
        output.push(json!{(message.partition(), message.offset(), truncated)});
    }

    Ok(json!(output).to_string())
}

<commit_msg>Fix live consumer API panic with unicode handling<commit_after>use rdkafka::Message;
use rdkafka::message::BorrowedMessage;
use rdkafka::consumer::{BaseConsumer, EmptyConsumerContext};
use rdkafka::config::ClientConfig;
use rocket::State;
use rocket::http::RawStr;
use scheduled_executor::ThreadPoolExecutor;

use config::{ClusterConfig, Config};
use metadata::ClusterId;
use error::*;

use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use std::borrow::Cow;

pub struct LiveConsumer {
    id: u64,
    cluster_id: ClusterId,
    topic: String,
    last_poll: RwLock<Instant>,
    consumer: BaseConsumer<EmptyConsumerContext>,
    active: AtomicBool,
}

impl LiveConsumer {
    fn new(id: u64, cluster_config: &ClusterConfig, topic: &str) -> Result<LiveConsumer> {
        let consumer = ClientConfig::new()
            .set("bootstrap.servers", &cluster_config.bootstrap_servers())
            .set("group.id", &format!("kafka_view_live_consumer_{}", id))
            .set("enable.partition.eof", "false")
            .set("api.version.request", "false")
            .set("enable.auto.commit", "false")
            .set("queued.max.messages.kbytes", "100") // Reduce memory usage
            .set("fetch.message.max.bytes", "102400")
            .create::<BaseConsumer<_>>()
            .chain_err(|| "Failed to create rdkafka consumer")?;

        debug!("Creating live consumer for {}", topic);

        Ok(LiveConsumer {
            id: id,
            cluster_id: cluster_config.cluster_id.clone().unwrap(),
            consumer: consumer,
            active: AtomicBool::new(false),
            last_poll: RwLock::new(Instant::now()),
            topic: topic.to_owned(),
        })
    }

    fn activate(&self) -> Result<()> {
        // TODO: start from the past
        debug!("Activating live consumer for {}", self.topic);

        // TODO: use assign instead
        self.consumer.subscribe(vec![self.topic.as_str()].as_slice())
            .chain_err(|| "Can't subscribe to specified topics")?;
        self.active.store(true, Ordering::Relaxed);
        Ok(())
    }

    pub fn is_active(&self) -> bool {
        self.active.load(Ordering::Relaxed)
    }

    pub fn last_poll(&self) -> Instant {
        *self.last_poll.read().unwrap()
    }

    pub fn id(&self) -> u64 {
        self.id
    }

    pub fn cluster_id(&self) -> &ClusterId {
        &self.cluster_id
    }

    pub fn topic(&self) -> &str {
        &self.topic
    }

    fn poll(&self, max_msg: usize, timeout: Duration) -> Vec<BorrowedMessage> {
        let start_time = Instant::now();
        let mut buffer = Vec::new();
        *self.last_poll.write().unwrap() = Instant::now();

        while Instant::elapsed(&start_time) < timeout && buffer.len() < max_msg {
            match self.consumer.poll(100) {
                Ok(None) => {},
                Ok(Some(m)) => buffer.push(m),
                Err(e) => {
                    error!("Error while receiving message {:?}", e);
                },
            };
        }

        debug!("{} messages received in {:?}", buffer.len(), Instant::elapsed(&start_time));
        buffer
    }
}

impl Drop for LiveConsumer {
    fn drop(&mut self) {
        debug!("Dropping consumer {}", self.id);
    }
}

type LiveConsumerMap = HashMap<u64, Arc<LiveConsumer>>;

fn remove_idle_consumers(consumers: &mut LiveConsumerMap) {
    consumers.retain(|_, ref consumer| consumer.last_poll().elapsed() < Duration::from_secs(20));
}

pub struct LiveConsumerStore {
    consumers: Arc<RwLock<LiveConsumerMap>>,
    _executor: ThreadPoolExecutor,
}

impl LiveConsumerStore {
    pub fn new(executor: ThreadPoolExecutor) -> LiveConsumerStore {
        let consumers = Arc::new(RwLock::new(HashMap::new()));
        let consumers_clone = Arc::clone(&consumers);
        executor.schedule_fixed_rate(
            Duration::from_secs(10),
            Duration::from_secs(10),
            move |_handle| {
                let mut consumers = consumers_clone.write().unwrap();
                remove_idle_consumers(&mut *consumers);
            }
        );
        LiveConsumerStore {
            consumers: consumers,
            _executor: executor,
        }
    }

    fn get_consumer(&self, id: u64) -> Option<Arc<LiveConsumer>> {
        let consumers = self.consumers.read().expect("Poison error");
        (*consumers).get(&id).cloned()
    }

    fn add_consumer(&self, id: u64, cluster_config: &ClusterConfig, topic: &str) -> Result<Arc<LiveConsumer>> {
        let live_consumer = LiveConsumer::new(id, cluster_config, topic)
            .chain_err(|| "Failed to create live consumer")?;

        let live_consumer_arc = Arc::new(live_consumer);

        // Add consumer immediately to the store, to prevent other threads from adding it again.
        match self.consumers.write() {
            Ok(mut consumers) => (*consumers).insert(id, live_consumer_arc.clone()),
            Err(_) => panic!("Poison error while writing consumer to cache")
        };

        live_consumer_arc.activate()
            .chain_err(|| "Failed to activate live consumer")?;

        Ok(live_consumer_arc)
    }

    pub fn consumers(&self) -> Vec<Arc<LiveConsumer>> {
        self.consumers.read().unwrap().iter()
            .map(|(_, consumer)| consumer.clone())
            .collect::<Vec<_>>()
    }
}

// TODO: check log in case of error

#[get("/api/tailer/<cluster_id>/<topic>/<id>")]
pub fn test_live_consumer_api(
    cluster_id: ClusterId,
    topic: &RawStr,
    id: u64,
    config: State<Config>,
    live_consumers_store: State<LiveConsumerStore>,
) -> Result<String> {
    let cluster_config = config.clusters.get(&cluster_id);

    if cluster_config.is_none() {
        return Ok("[]".to_owned());
    }
    let cluster_config = cluster_config.unwrap();

    let consumer = match live_consumers_store.get_consumer(id) {
        Some(consumer) => consumer,
        None => live_consumers_store.add_consumer(id, cluster_config, topic)
            .chain_err(|| format!("Error while creating live consumer for {} {}", cluster_id, topic))?,
    };

    if !consumer.is_active() {
        // Consumer is still being activated, no results for now.
        return Ok("[]".to_owned());
    }

    let mut output = Vec::new();
    for message in consumer.poll(100, Duration::from_secs(3)) {
        let payload = message.payload()
            .map(|bytes| String::from_utf8_lossy(bytes))
            .unwrap_or(Cow::Borrowed(""));
        if payload.len() > 1024 {
            let truncated = format!("{}...", payload.chars().take(1024).collect::<String>());
            output.push(json!{(message.partition(), message.offset(), truncated)});
        } else {
            output.push(json!{(message.partition(), message.offset(), payload)});
        };
    }

    Ok(json!(output).to_string())
}

<|endoftext|>


<commit_before>use std::path::Path;

use gfx;
use image;
use image::{
    DynamicImage,
    GenericImage,
    RgbaImage,
};
use texture_lib::ImageSize;

/// Represents a texture.
#[derive(Clone, Debug)]
pub struct Texture<R: gfx::Resources> {
    /// A handle to the Gfx texture.
    pub handle: gfx::TextureHandle<R>,
}

impl<R: gfx::Resources> Texture<R> {
    /// Creates a texture from path.
    pub fn from_path<D: gfx::Factory<R>>(
        device: &mut D,
        path: &Path,
        settings: &::Settings,
    ) -> Result<Self, String> {
        let img = match image::open(path) {
            Ok(img) => img,
            Err(e)  => return Err(format!("Could not load '{:?}': {:?}",
                path.file_name().unwrap(), e)),
        };

        //if settings.force_alpha //TODO
        let img = match img {
            DynamicImage::ImageRgba8(img) => img,
            img => img.to_rgba(),
        };

        let img = if settings.flip_vertical {
            image::imageops::flip_vertical(&img)
        } else {
            img
        };

        let texture = Texture::from_image(device, &img);

        if settings.generate_mipmap {
            device.generate_mipmap(&texture.handle);
        }
        Ok(texture)
    }

    /// Creates a texture from image.
    pub fn from_image<D: gfx::Factory<R>>(
        device: &mut D,
        image: &RgbaImage
    ) -> Self {
        let (width, height) = image.dimensions();
        let texture_info = gfx::tex::TextureInfo {
            width: width as u16,
            height: height as u16,
            depth: 1,
            levels: 1,
            kind: gfx::tex::TextureKind::Texture2D,
            format: gfx::tex::RGBA8,
        };
        let image_info = texture_info.to_image_info();
        let texture = device.create_texture(texture_info).unwrap();
        device.update_texture(&texture, &image_info, &image,
                              Some(gfx::tex::TextureKind::Texture2D)).unwrap();
        Texture {
            handle: texture
        }
    }

    /// Creates texture from memory alpha.
    pub fn from_memory_alpha<D: gfx::Factory<R>>(
        device: &mut D,
        buffer: &[u8],
        width: u32,
        height: u32,
    ) -> Self {
        use std::cmp::max;

        let width = max(width, 1);
        let height = max(height, 1);

        let mut pixels = Vec::new();
        for alpha in buffer.iter() {
            pixels.push(255);
            pixels.push(255);
            pixels.push(255);
            pixels.push(*alpha);
        }

        let texture_info = gfx::tex::TextureInfo {
            width: width as u16,
            height: height as u16,
            depth: 1,
            levels: 1,
            kind: gfx::tex::TextureKind::Texture2D,
            format: gfx::tex::RGBA8,
        };

        let image_info = texture_info.to_image_info();
        let texture = device.create_texture(texture_info).unwrap();
        device.update_texture(&texture, &image_info, &pixels,
                              Some(gfx::tex::TextureKind::Texture2D)).unwrap();
        Texture {
            handle: texture
        }
    }

    /// Updates the texture with an image.
    pub fn update<D: gfx::Factory<R>>(&mut self, device: &mut D, image: &RgbaImage) {
        device.update_texture(&self.handle,
            &self.handle.get_info().to_image_info(),
            &image,
            Some(gfx::tex::TextureKind::Texture2D)
        ).unwrap();
    }
}

impl<R: gfx::Resources> ImageSize for Texture<R> {
    #[inline(always)]
    fn get_size(&self) -> (u32, u32) {
        let info = self.handle.get_info();
        (info.width as u32, info.height as u32)
    }
}
<commit_msg>Move compress and generate_mipmap to from_image<commit_after>use std::path::Path;

use gfx;
use image;
use image::{
    DynamicImage,
    GenericImage,
    RgbaImage,
};
use texture_lib::ImageSize;

/// Represents a texture.
#[derive(Clone, Debug)]
pub struct Texture<R: gfx::Resources> {
    /// A handle to the Gfx texture.
    pub handle: gfx::TextureHandle<R>,
}

impl<R: gfx::Resources> Texture<R> {
    /// Creates a texture from path.
    pub fn from_path<D: gfx::Factory<R>>(
        device: &mut D,
        path: &Path,
        settings: &::Settings,
    ) -> Result<Self, String> {
        let img = match image::open(path) {
            Ok(img) => img,
            Err(e)  => return Err(format!("Could not load '{:?}': {:?}",
                path.file_name().unwrap(), e)),
        };

        //if settings.force_alpha //TODO
        let img = match img {
            DynamicImage::ImageRgba8(img) => img,
            img => img.to_rgba(),
        };

        let img = if settings.flip_vertical {
            image::imageops::flip_vertical(&img)
        } else {
            img
        };

        let texture = Texture::from_image(device, &img,
                                          settings.compress,
                                          settings.generate_mipmap);
        Ok(texture)
    }

    /// Creates a texture from image.
    pub fn from_image<D: gfx::Factory<R>>(
        device: &mut D,
        image: &RgbaImage,
        _compress: bool,
        generate_mipmap: bool
    ) -> Self {
        let (width, height) = image.dimensions();
        let texture_info = gfx::tex::TextureInfo {
            width: width as u16,
            height: height as u16,
            depth: 1,
            levels: 1,
            kind: gfx::tex::TextureKind::Texture2D,
            format: gfx::tex::RGBA8,
        };
        let image_info = texture_info.to_image_info();
        let texture = device.create_texture(texture_info).unwrap();
        device.update_texture(&texture, &image_info, &image,
                              Some(gfx::tex::TextureKind::Texture2D)).unwrap();
        if generate_mipmap {
            device.generate_mipmap(&texture);
        }
        Texture {
            handle: texture
        }
    }

    /// Creates texture from memory alpha.
    pub fn from_memory_alpha<D: gfx::Factory<R>>(
        device: &mut D,
        buffer: &[u8],
        width: u32,
        height: u32,
    ) -> Self {
        use std::cmp::max;

        let width = max(width, 1);
        let height = max(height, 1);

        let mut pixels = Vec::new();
        for alpha in buffer.iter() {
            pixels.push(255);
            pixels.push(255);
            pixels.push(255);
            pixels.push(*alpha);
        }

        let texture_info = gfx::tex::TextureInfo {
            width: width as u16,
            height: height as u16,
            depth: 1,
            levels: 1,
            kind: gfx::tex::TextureKind::Texture2D,
            format: gfx::tex::RGBA8,
        };

        let image_info = texture_info.to_image_info();
        let texture = device.create_texture(texture_info).unwrap();
        device.update_texture(&texture, &image_info, &pixels,
                              Some(gfx::tex::TextureKind::Texture2D)).unwrap();
        Texture {
            handle: texture
        }
    }

    /// Updates the texture with an image.
    pub fn update<D: gfx::Factory<R>>(&mut self, device: &mut D, image: &RgbaImage) {
        device.update_texture(&self.handle,
            &self.handle.get_info().to_image_info(),
            &image,
            Some(gfx::tex::TextureKind::Texture2D)
        ).unwrap();
    }
}

impl<R: gfx::Resources> ImageSize for Texture<R> {
    #[inline(always)]
    fn get_size(&self) -> (u32, u32) {
        let info = self.handle.get_info();
        (info.width as u32, info.height as u32)
    }
}
<|endoftext|>


<commit_before>use std::ops::Range;

// A type `F` constrained to `Fn(&'a G, usize) -> P` could be used here, but it
// would not be possible to name that type for anything but functions (not
// closures).  Instead of a limited and somewhat redundant type `F`, just use
// `fn(&'a G, usize) -> P` for the member `f`.
pub struct Generate<'a, G, P>
where
    G: 'a,
{
    generator: &'a G,
    range: Range<usize>,
    f: fn(&'a G, usize) -> P,
}

impl<'a, G, P> Generate<'a, G, P>
where
    G: 'a,
{
    pub(super) fn new(generator: &'a G, range: Range<usize>, f: fn(&'a G, usize) -> P) -> Self {
        Generate {
            generator: generator,
            range: range,
            f: f,
        }
    }
}

impl<'a, G, P> Iterator for Generate<'a, G, P>
where
    G: 'a,
{
    type Item = P;

    fn next(&mut self) -> Option<Self::Item> {
        self.range
            .next()
            .map(|index| (self.f)(self.generator, index))
    }
}

pub trait ConjointPointGenerator<P> {
    fn conjoint_point(&self, index: usize) -> P;
    fn conjoint_point_count(&self) -> usize;
}

pub trait ConjointPoints<P>: Sized {
    fn conjoint_points(&self) -> Generate<Self, P>;
}

impl<G, P> ConjointPoints<P> for G
where
    G: ConjointPointGenerator<P>,
{
    fn conjoint_points(&self) -> Generate<Self, P> {
        Generate::new(self, 0..self.conjoint_point_count(), map_conjoint_point)
    }
}

pub trait PolygonGenerator {
    fn polygon_count(&self) -> usize;
}

pub trait IndexPolygonGenerator<P>: PolygonGenerator {
    fn index_polygon(&self, index: usize) -> P;
}

pub trait IndexPolygons<P>: Sized {
    fn index_polygons(&self) -> Generate<Self, P>;
}

impl<G, P> IndexPolygons<P> for G
where
    G: IndexPolygonGenerator<P> + PolygonGenerator,
{
    fn index_polygons(&self) -> Generate<Self, P> {
        Generate::new(self, 0..self.polygon_count(), map_index_polygon)
    }
}

pub trait TexturePolygonGenerator<P>: PolygonGenerator {
    fn texture_polygon(&self, index: usize) -> P;
}

pub trait TexturePolygons<P>: Sized {
    fn texture_polygons(&self) -> Generate<Self, P>;
}

impl<G, P> TexturePolygons<P> for G
where
    G: PolygonGenerator + TexturePolygonGenerator<P>,
{
    fn texture_polygons(&self) -> Generate<Self, P> {
        Generate::new(self, 0..self.polygon_count(), map_texture_polygon)
    }
}

fn map_conjoint_point<G, P>(generator: &G, index: usize) -> P
where
    G: ConjointPointGenerator<P>,
{
    generator.conjoint_point(index)
}

fn map_index_polygon<G, P>(generator: &G, index: usize) -> P
where
    G: IndexPolygonGenerator<P>,
{
    generator.index_polygon(index)
}

fn map_texture_polygon<G, P>(generator: &G, index: usize) -> P
where
    G: TexturePolygonGenerator<P>,
{
    generator.texture_polygon(index)
}
<commit_msg>Remove unnecessary generator mapping functions.<commit_after>use std::ops::Range;

// A type `F` constrained to `Fn(&'a G, usize) -> P` could be used here, but it
// would not be possible to name that type for anything but functions (not
// closures).  Instead of a limited and somewhat redundant type `F`, just use
// `fn(&'a G, usize) -> P` for the member `f`.
pub struct Generate<'a, G, P>
where
    G: 'a,
{
    generator: &'a G,
    range: Range<usize>,
    f: fn(&'a G, usize) -> P,
}

impl<'a, G, P> Generate<'a, G, P>
where
    G: 'a,
{
    pub(super) fn new(generator: &'a G, range: Range<usize>, f: fn(&'a G, usize) -> P) -> Self {
        Generate {
            generator: generator,
            range: range,
            f: f,
        }
    }
}

impl<'a, G, P> Iterator for Generate<'a, G, P>
where
    G: 'a,
{
    type Item = P;

    fn next(&mut self) -> Option<Self::Item> {
        self.range
            .next()
            .map(|index| (self.f)(self.generator, index))
    }
}

pub trait ConjointPointGenerator<P> {
    fn conjoint_point(&self, index: usize) -> P;
    fn conjoint_point_count(&self) -> usize;
}

pub trait ConjointPoints<P>: Sized {
    fn conjoint_points(&self) -> Generate<Self, P>;
}

impl<G, P> ConjointPoints<P> for G
where
    G: ConjointPointGenerator<P>,
{
    fn conjoint_points(&self) -> Generate<Self, P> {
        Generate::new(self, 0..self.conjoint_point_count(), G::conjoint_point)
    }
}

pub trait PolygonGenerator {
    fn polygon_count(&self) -> usize;
}

pub trait IndexPolygonGenerator<P>: PolygonGenerator {
    fn index_polygon(&self, index: usize) -> P;
}

pub trait IndexPolygons<P>: Sized {
    fn index_polygons(&self) -> Generate<Self, P>;
}

impl<G, P> IndexPolygons<P> for G
where
    G: IndexPolygonGenerator<P> + PolygonGenerator,
{
    fn index_polygons(&self) -> Generate<Self, P> {
        Generate::new(self, 0..self.polygon_count(), G::index_polygon)
    }
}

pub trait TexturePolygonGenerator<P>: PolygonGenerator {
    fn texture_polygon(&self, index: usize) -> P;
}

pub trait TexturePolygons<P>: Sized {
    fn texture_polygons(&self) -> Generate<Self, P>;
}

impl<G, P> TexturePolygons<P> for G
where
    G: PolygonGenerator + TexturePolygonGenerator<P>,
{
    fn texture_polygons(&self) -> Generate<Self, P> {
        Generate::new(self, 0..self.polygon_count(), G::texture_polygon)
    }
}
<|endoftext|>