<commit_before>#![feature(test)]
#![cfg(feature = "gpu_cache")]

extern crate rusttype;
extern crate test;
#[macro_use]
extern crate lazy_static;
extern crate unicode_normalization;

use rusttype::gpu_cache::*;
use rusttype::*;

/// Busy wait 2us
fn mock_gpu_upload(_region: Rect<u32>, _bytes: &[u8]) {
    use std::time::{Duration, Instant};

    let now = Instant::now();
    while now.elapsed() < Duration::from_micros(2) {}
}

fn test_glyphs<'a>(font: &Font<'a>, string: &str) -> Vec<PositionedGlyph<'a>> {
    let mut glyphs = vec![];
    // Set of scales, found through brute force, to reproduce GlyphNotCached issue
    // Cache settings also affect this, it occurs when position_tolerance is < 1.0
    for scale in &[25_f32, 24.5, 25.01, 24.7, 24.99] {
        for glyph in layout_paragraph(font, Scale::uniform(*scale), 500, string) {
            glyphs.push(glyph);
        }
    }
    glyphs
}

fn layout_paragraph<'a>(
    font: &Font<'a>,
    scale: Scale,
    width: u32,
    text: &str,
) -> Vec<PositionedGlyph<'a>> {
    use unicode_normalization::UnicodeNormalization;
    let mut result = Vec::new();
    let v_metrics = font.v_metrics(scale);
    let advance_height = v_metrics.ascent - v_metrics.descent + v_metrics.line_gap;
    let mut caret = point(0.0, v_metrics.ascent);
    let mut last_glyph_id = None;
    for c in text.nfc() {
        if c.is_control() {
            if c == '\n' {
                caret = point(0.0, caret.y + advance_height)
            }
            continue;
        }
        let base_glyph = font.glyph(c);
        if let Some(id) = last_glyph_id.take() {
            caret.x += font.pair_kerning(scale, id, base_glyph.id());
        }
        last_glyph_id = Some(base_glyph.id());
        let mut glyph = base_glyph.scaled(scale).positioned(caret);
        if let Some(bb) = glyph.pixel_bounding_box() {
            if bb.max.x > width as i32 {
                caret = point(0.0, caret.y + advance_height);
                glyph = glyph.into_unpositioned().positioned(caret);
                last_glyph_id = None;
            }
        }
        caret.x += glyph.unpositioned().h_metrics().advance_width;
        result.push(glyph);
    }
    result
}

lazy_static! {
    static ref FONTS: Vec<Font<'static>> = vec![
        include_bytes!("../fonts/wqy-microhei/WenQuanYiMicroHei.ttf") as &[u8],
        include_bytes!("../fonts/dejavu/DejaVuSansMono.ttf") as &[u8],
        include_bytes!("../fonts/opensans/OpenSans-Italic.ttf") as &[u8],
    ].into_iter()
    .map(|bytes| Font::from_bytes(bytes).unwrap())
    .collect();
}

const TEST_STR: &str = include_str!("../tests/lipsum.txt");

/// General use benchmarks.
mod cache {
    use super::*;

    /// Benchmark using a single font at "don't care" position tolerance
    #[bench]
    fn high_position_tolerance(b: &mut ::test::Bencher) {
        let font_id = 0;
        let glyphs = test_glyphs(&FONTS[font_id], TEST_STR);
        let mut cache = CacheBuilder {
            width: 1024,
            height: 1024,
            scale_tolerance: 0.1,
            position_tolerance: 1.0,
            ..CacheBuilder::default()
        }.build();

        b.iter(|| {
            for glyph in &glyphs {
                cache.queue_glyph(font_id, glyph.clone());
            }

            cache.cache_queued(|_, _| {}).expect("cache_queued");

            for (index, glyph) in glyphs.iter().enumerate() {
                let rect = cache.rect_for(font_id, glyph);
                assert!(
                    rect.is_ok(),
                    "Gpu cache rect lookup failed ({:?}) for glyph index {}, id {}",
                    rect,
                    index,
                    glyph.id().0
                );
            }
        });
    }

    /// Benchmark using a single font with default tolerances
    #[bench]
    fn single_font(b: &mut ::test::Bencher) {
        let font_id = 0;
        let glyphs = test_glyphs(&FONTS[font_id], TEST_STR);
        let mut cache = CacheBuilder {
            width: 1024,
            height: 1024,
            ..CacheBuilder::default()
        }.build();

        b.iter(|| {
            for glyph in &glyphs {
                cache.queue_glyph(font_id, glyph.clone());
            }

            cache.cache_queued(|_, _| {}).expect("cache_queued");

            for (index, glyph) in glyphs.iter().enumerate() {
                let rect = cache.rect_for(font_id, glyph);
                assert!(
                    rect.is_ok(),
                    "Gpu cache rect lookup failed ({:?}) for glyph index {}, id {}",
                    rect,
                    index,
                    glyph.id().0
                );
            }
        });
    }

    /// Benchmark using multiple fonts with default tolerances
    #[bench]
    fn multi_font(b: &mut ::test::Bencher) {
        // Use a smaller amount of the test string, to offset the extra font-glyph
        // bench load
        let up_to_index = TEST_STR
            .char_indices()
            .nth(TEST_STR.chars().count() / FONTS.len())
            .unwrap()
            .0;
        let string = &TEST_STR[..up_to_index];

        let font_glyphs: Vec<_> = FONTS
            .iter()
            .enumerate()
            .map(|(id, font)| (id, test_glyphs(font, string)))
            .collect();
        let mut cache = CacheBuilder {
            width: 1024,
            height: 1024,
            ..CacheBuilder::default()
        }.build();

        b.iter(|| {
            for &(font_id, ref glyphs) in &font_glyphs {
                for glyph in glyphs {
                    cache.queue_glyph(font_id, glyph.clone());
                }
            }

            cache.cache_queued(|_, _| {}).expect("cache_queued");

            for &(font_id, ref glyphs) in &font_glyphs {
                for (index, glyph) in glyphs.iter().enumerate() {
                    let rect = cache.rect_for(font_id, glyph);
                    assert!(
                        rect.is_ok(),
                        "Gpu cache rect lookup failed ({:?}) for font {} glyph index {}, id {}",
                        rect,
                        font_id,
                        index,
                        glyph.id().0
                    );
                }
            }
        });
    }

    /// Benchmark using multiple fonts with default tolerances, clears the
    /// cache each run to test the population "first run" performance
    #[bench]
    fn multi_font_population(b: &mut ::test::Bencher) {
        // Use a much smaller amount of the test string, to offset the extra font-glyph
        // bench load & much slower performance of fresh population each run
        let up_to_index = TEST_STR.char_indices().nth(70).unwrap().0;
        let string = &TEST_STR[..up_to_index];

        let font_glyphs: Vec<_> = FONTS
            .iter()
            .enumerate()
            .map(|(id, font)| (id, test_glyphs(font, string)))
            .collect();

        b.iter(|| {
            let mut cache = CacheBuilder {
                width: 1024,
                height: 1024,
                ..CacheBuilder::default()
            }.build();

            for &(font_id, ref glyphs) in &font_glyphs {
                for glyph in glyphs {
                    cache.queue_glyph(font_id, glyph.clone());
                }
            }

            cache.cache_queued(|_, _| {}).expect("cache_queued");

            for &(font_id, ref glyphs) in &font_glyphs {
                for (index, glyph) in glyphs.iter().enumerate() {
                    let rect = cache.rect_for(font_id, glyph);
                    assert!(
                        rect.is_ok(),
                        "Gpu cache rect lookup failed ({:?}) for font {} glyph index {}, id {}",
                        rect,
                        font_id,
                        index,
                        glyph.id().0
                    );
                }
            }
        });
    }

    /// Benchmark using multiple fonts and a different text group of glyphs
    /// each run
    #[bench]
    fn moving_text(b: &mut ::test::Bencher) {
        let chars: Vec<_> = TEST_STR.chars().collect();
        let subsection_len = chars.len() / FONTS.len();
        let distinct_subsection: Vec<_> = chars.windows(subsection_len).collect();

        let mut first_glyphs = vec![];
        let mut middle_glyphs = vec![];
        let mut last_glyphs = vec![];

        for (id, font) in FONTS.iter().enumerate() {
            let first_str: String = distinct_subsection[0].iter().collect();
            first_glyphs.push((id, test_glyphs(font, &first_str)));

            let middle_str: String = distinct_subsection[distinct_subsection.len() / 2]
                .iter()
                .collect();
            middle_glyphs.push((id, test_glyphs(font, &middle_str)));

            let last_str: String = distinct_subsection[distinct_subsection.len() - 1]
                .iter()
                .collect();
            last_glyphs.push((id, test_glyphs(font, &last_str)));
        }

        let test_variants = [first_glyphs, middle_glyphs, last_glyphs];
        let mut test_variants = test_variants.iter().cycle();

        let mut cache = CacheBuilder {
            width: 1500,
            height: 1500,
            scale_tolerance: 0.1,
            position_tolerance: 0.1,
            ..CacheBuilder::default()
        }.build();

        b.iter(|| {
            // switch text variant each run to force cache to deal with moving text
            // requirements
            let glyphs = test_variants.next().unwrap();
            for &(font_id, ref glyphs) in glyphs {
                for glyph in glyphs {
                    cache.queue_glyph(font_id, glyph.clone());
                }
            }

            cache.cache_queued(|_, _| {}).expect("cache_queued");

            for &(font_id, ref glyphs) in glyphs {
                for (index, glyph) in glyphs.iter().enumerate() {
                    let rect = cache.rect_for(font_id, glyph);
                    assert!(
                        rect.is_ok(),
                        "Gpu cache rect lookup failed ({:?}) for font {} glyph index {}, id {}",
                        rect,
                        font_id,
                        index,
                        glyph.id().0
                    );
                }
            }
        });
    }
}

/// Benchmarks for cases that should generally be avoided by the cache user if
/// at all possible (ie by picking a better initial cache size).
mod cache_bad_cases {
    use super::*;

    /// Cache isn't large enough for a queue so a new cache is created to hold
    /// the queue.
    #[bench]
    fn resizing(b: &mut ::test::Bencher) {
        let up_to_index = TEST_STR
            .char_indices()
            .nth(TEST_STR.chars().count() / FONTS.len())
            .unwrap()
            .0;
        let string = &TEST_STR[..up_to_index];

        let font_glyphs: Vec<_> = FONTS
            .iter()
            .enumerate()
            .map(|(id, font)| (id, test_glyphs(font, string)))
            .collect();

        b.iter(|| {
            let mut cache = CacheBuilder {
                width: 512,
                height: 512,
                ..CacheBuilder::default()
            }.build();

            for &(font_id, ref glyphs) in &font_glyphs {
                for glyph in glyphs {
                    cache.queue_glyph(font_id, glyph.clone());
                }
            }

            cache
                .cache_queued(mock_gpu_upload)
                .expect_err("shouldn't fit");

            CacheBuilder {
                width: 1024,
                height: 1024,
                ..cache.to_builder()
            }.rebuild(&mut cache);

            cache.cache_queued(mock_gpu_upload).expect("should fit now");

            for &(font_id, ref glyphs) in &font_glyphs {
                for (index, glyph) in glyphs.iter().enumerate() {
                    let rect = cache.rect_for(font_id, glyph);
                    assert!(
                        rect.is_ok(),
                        "Gpu cache rect lookup failed ({:?}) for font {} glyph index {}, id {}",
                        rect,
                        font_id,
                        index,
                        glyph.id().0
                    );
                }
            }
        });
    }

    /// Benchmark using multiple fonts and a different text group of glyphs
    /// each run. The cache is only large enough to fit each run if it is
    /// cleared and re-built.
    #[bench]
    fn moving_text_thrashing(b: &mut ::test::Bencher) {
        let chars: Vec<_> = TEST_STR.chars().collect();
        let subsection_len = chars.len() / FONTS.len();
        let distinct_subsection: Vec<_> = chars.windows(subsection_len).collect();

        let mut first_glyphs = vec![];
        let mut middle_glyphs = vec![];
        let mut last_glyphs = vec![];

        for (id, font) in FONTS.iter().enumerate() {
            let first_str: String = distinct_subsection[0].iter().collect();
            first_glyphs.push((id, test_glyphs(font, &first_str)));

            let middle_str: String = distinct_subsection[distinct_subsection.len() / 2]
                .iter()
                .collect();
            middle_glyphs.push((id, test_glyphs(font, &middle_str)));

            let last_str: String = distinct_subsection[distinct_subsection.len() - 1]
                .iter()
                .collect();
            last_glyphs.push((id, test_glyphs(font, &last_str)));
        }

        let test_variants = [first_glyphs, middle_glyphs, last_glyphs];
        let mut test_variants = test_variants.iter().cycle();

        let mut cache = CacheBuilder {
            width: 1024,
            height: 1024,
            scale_tolerance: 0.1,
            position_tolerance: 0.1,
            ..CacheBuilder::default()
        }.build();

        b.iter(|| {
            // switch text variant each run to force cache to deal with moving text
            // requirements
            let glyphs = test_variants.next().unwrap();
            for &(font_id, ref glyphs) in glyphs {
                for glyph in glyphs {
                    cache.queue_glyph(font_id, glyph.clone());
                }
            }

            cache.cache_queued(mock_gpu_upload).expect("cache_queued");

            for &(font_id, ref glyphs) in glyphs {
                for (index, glyph) in glyphs.iter().enumerate() {
                    let rect = cache.rect_for(font_id, glyph);
                    assert!(
                        rect.is_ok(),
                        "Gpu cache rect lookup failed ({:?}) for font {} glyph index {}, id {}",
                        rect,
                        font_id,
                        index,
                        glyph.id().0
                    );
                }
            }
        });
    }
}
<commit_msg>Rework cache_bad_cases benchmarks reduce runtime & variance<commit_after>#![feature(test)]
#![cfg(feature = "gpu_cache")]

extern crate rusttype;
extern crate test;
#[macro_use]
extern crate lazy_static;
extern crate unicode_normalization;

use rusttype::gpu_cache::*;
use rusttype::*;

/// Busy wait 2us
fn mock_gpu_upload(_region: Rect<u32>, _bytes: &[u8]) {
    use std::time::{Duration, Instant};

    let now = Instant::now();
    while now.elapsed() < Duration::from_micros(2) {}
}

fn test_glyphs<'a>(font: &Font<'a>, string: &str) -> Vec<PositionedGlyph<'a>> {
    let mut glyphs = vec![];
    // Set of scales, found through brute force, to reproduce GlyphNotCached issue
    // Cache settings also affect this, it occurs when position_tolerance is < 1.0
    for scale in &[25_f32, 24.5, 25.01, 24.7, 24.99] {
        for glyph in layout_paragraph(font, Scale::uniform(*scale), 500, string) {
            glyphs.push(glyph);
        }
    }
    glyphs
}

fn layout_paragraph<'a>(
    font: &Font<'a>,
    scale: Scale,
    width: u32,
    text: &str,
) -> Vec<PositionedGlyph<'a>> {
    use unicode_normalization::UnicodeNormalization;
    let mut result = Vec::new();
    let v_metrics = font.v_metrics(scale);
    let advance_height = v_metrics.ascent - v_metrics.descent + v_metrics.line_gap;
    let mut caret = point(0.0, v_metrics.ascent);
    let mut last_glyph_id = None;
    for c in text.nfc() {
        if c.is_control() {
            if c == '\n' {
                caret = point(0.0, caret.y + advance_height)
            }
            continue;
        }
        let base_glyph = font.glyph(c);
        if let Some(id) = last_glyph_id.take() {
            caret.x += font.pair_kerning(scale, id, base_glyph.id());
        }
        last_glyph_id = Some(base_glyph.id());
        let mut glyph = base_glyph.scaled(scale).positioned(caret);
        if let Some(bb) = glyph.pixel_bounding_box() {
            if bb.max.x > width as i32 {
                caret = point(0.0, caret.y + advance_height);
                glyph = glyph.into_unpositioned().positioned(caret);
                last_glyph_id = None;
            }
        }
        caret.x += glyph.unpositioned().h_metrics().advance_width;
        result.push(glyph);
    }
    result
}

lazy_static! {
    static ref FONTS: Vec<Font<'static>> = vec![
        include_bytes!("../fonts/wqy-microhei/WenQuanYiMicroHei.ttf") as &[u8],
        include_bytes!("../fonts/dejavu/DejaVuSansMono.ttf") as &[u8],
        include_bytes!("../fonts/opensans/OpenSans-Italic.ttf") as &[u8],
    ].into_iter()
    .map(|bytes| Font::from_bytes(bytes).unwrap())
    .collect();
}

const TEST_STR: &str = include_str!("../tests/lipsum.txt");

/// General use benchmarks.
mod cache {
    use super::*;

    /// Benchmark using a single font at "don't care" position tolerance
    #[bench]
    fn high_position_tolerance(b: &mut ::test::Bencher) {
        let font_id = 0;
        let glyphs = test_glyphs(&FONTS[font_id], TEST_STR);
        let mut cache = CacheBuilder {
            width: 1024,
            height: 1024,
            scale_tolerance: 0.1,
            position_tolerance: 1.0,
            ..CacheBuilder::default()
        }.build();

        b.iter(|| {
            for glyph in &glyphs {
                cache.queue_glyph(font_id, glyph.clone());
            }

            cache.cache_queued(|_, _| {}).expect("cache_queued");

            for (index, glyph) in glyphs.iter().enumerate() {
                let rect = cache.rect_for(font_id, glyph);
                assert!(
                    rect.is_ok(),
                    "Gpu cache rect lookup failed ({:?}) for glyph index {}, id {}",
                    rect,
                    index,
                    glyph.id().0
                );
            }
        });
    }

    /// Benchmark using a single font with default tolerances
    #[bench]
    fn single_font(b: &mut ::test::Bencher) {
        let font_id = 0;
        let glyphs = test_glyphs(&FONTS[font_id], TEST_STR);
        let mut cache = CacheBuilder {
            width: 1024,
            height: 1024,
            ..CacheBuilder::default()
        }.build();

        b.iter(|| {
            for glyph in &glyphs {
                cache.queue_glyph(font_id, glyph.clone());
            }

            cache.cache_queued(|_, _| {}).expect("cache_queued");

            for (index, glyph) in glyphs.iter().enumerate() {
                let rect = cache.rect_for(font_id, glyph);
                assert!(
                    rect.is_ok(),
                    "Gpu cache rect lookup failed ({:?}) for glyph index {}, id {}",
                    rect,
                    index,
                    glyph.id().0
                );
            }
        });
    }

    /// Benchmark using multiple fonts with default tolerances
    #[bench]
    fn multi_font(b: &mut ::test::Bencher) {
        // Use a smaller amount of the test string, to offset the extra font-glyph
        // bench load
        let up_to_index = TEST_STR
            .char_indices()
            .nth(TEST_STR.chars().count() / FONTS.len())
            .unwrap()
            .0;
        let string = &TEST_STR[..up_to_index];

        let font_glyphs: Vec<_> = FONTS
            .iter()
            .enumerate()
            .map(|(id, font)| (id, test_glyphs(font, string)))
            .collect();
        let mut cache = CacheBuilder {
            width: 1024,
            height: 1024,
            ..CacheBuilder::default()
        }.build();

        b.iter(|| {
            for &(font_id, ref glyphs) in &font_glyphs {
                for glyph in glyphs {
                    cache.queue_glyph(font_id, glyph.clone());
                }
            }

            cache.cache_queued(|_, _| {}).expect("cache_queued");

            for &(font_id, ref glyphs) in &font_glyphs {
                for (index, glyph) in glyphs.iter().enumerate() {
                    let rect = cache.rect_for(font_id, glyph);
                    assert!(
                        rect.is_ok(),
                        "Gpu cache rect lookup failed ({:?}) for font {} glyph index {}, id {}",
                        rect,
                        font_id,
                        index,
                        glyph.id().0
                    );
                }
            }
        });
    }

    /// Benchmark using multiple fonts with default tolerances, clears the
    /// cache each run to test the population "first run" performance
    #[bench]
    fn multi_font_population(b: &mut ::test::Bencher) {
        // Use a much smaller amount of the test string, to offset the extra font-glyph
        // bench load & much slower performance of fresh population each run
        let up_to_index = TEST_STR.char_indices().nth(70).unwrap().0;
        let string = &TEST_STR[..up_to_index];

        let font_glyphs: Vec<_> = FONTS
            .iter()
            .enumerate()
            .map(|(id, font)| (id, test_glyphs(font, string)))
            .collect();

        b.iter(|| {
            let mut cache = CacheBuilder {
                width: 1024,
                height: 1024,
                ..CacheBuilder::default()
            }.build();

            for &(font_id, ref glyphs) in &font_glyphs {
                for glyph in glyphs {
                    cache.queue_glyph(font_id, glyph.clone());
                }
            }

            cache.cache_queued(|_, _| {}).expect("cache_queued");

            for &(font_id, ref glyphs) in &font_glyphs {
                for (index, glyph) in glyphs.iter().enumerate() {
                    let rect = cache.rect_for(font_id, glyph);
                    assert!(
                        rect.is_ok(),
                        "Gpu cache rect lookup failed ({:?}) for font {} glyph index {}, id {}",
                        rect,
                        font_id,
                        index,
                        glyph.id().0
                    );
                }
            }
        });
    }

    /// Benchmark using multiple fonts and a different text group of glyphs
    /// each run
    #[bench]
    fn moving_text(b: &mut ::test::Bencher) {
        let chars: Vec<_> = TEST_STR.chars().collect();
        let subsection_len = chars.len() / FONTS.len();
        let distinct_subsection: Vec<_> = chars.windows(subsection_len).collect();

        let mut first_glyphs = vec![];
        let mut middle_glyphs = vec![];
        let mut last_glyphs = vec![];

        for (id, font) in FONTS.iter().enumerate() {
            let first_str: String = distinct_subsection[0].iter().collect();
            first_glyphs.push((id, test_glyphs(font, &first_str)));

            let middle_str: String = distinct_subsection[distinct_subsection.len() / 2]
                .iter()
                .collect();
            middle_glyphs.push((id, test_glyphs(font, &middle_str)));

            let last_str: String = distinct_subsection[distinct_subsection.len() - 1]
                .iter()
                .collect();
            last_glyphs.push((id, test_glyphs(font, &last_str)));
        }

        let test_variants = [first_glyphs, middle_glyphs, last_glyphs];
        let mut test_variants = test_variants.iter().cycle();

        let mut cache = CacheBuilder {
            width: 1500,
            height: 1500,
            scale_tolerance: 0.1,
            position_tolerance: 0.1,
            ..CacheBuilder::default()
        }.build();

        b.iter(|| {
            // switch text variant each run to force cache to deal with moving text
            // requirements
            let glyphs = test_variants.next().unwrap();
            for &(font_id, ref glyphs) in glyphs {
                for glyph in glyphs {
                    cache.queue_glyph(font_id, glyph.clone());
                }
            }

            cache.cache_queued(|_, _| {}).expect("cache_queued");

            for &(font_id, ref glyphs) in glyphs {
                for (index, glyph) in glyphs.iter().enumerate() {
                    let rect = cache.rect_for(font_id, glyph);
                    assert!(
                        rect.is_ok(),
                        "Gpu cache rect lookup failed ({:?}) for font {} glyph index {}, id {}",
                        rect,
                        font_id,
                        index,
                        glyph.id().0
                    );
                }
            }
        });
    }
}

/// Benchmarks for cases that should generally be avoided by the cache user if
/// at all possible (ie by picking a better initial cache size).
mod cache_bad_cases {
    use super::*;

    /// Cache isn't large enough for a queue so a new cache is created to hold
    /// the queue.
    #[bench]
    fn resizing(b: &mut ::test::Bencher) {
        let up_to_index = TEST_STR.char_indices().nth(120).unwrap().0;
        let string = &TEST_STR[..up_to_index];

        let font_glyphs: Vec<_> = FONTS
            .iter()
            .enumerate()
            .map(|(id, font)| (id, test_glyphs(font, string)))
            .collect();

        b.iter(|| {
            let mut cache = CacheBuilder {
                width: 256,
                height: 256,
                ..CacheBuilder::default()
            }.build();

            for &(font_id, ref glyphs) in &font_glyphs {
                for glyph in glyphs {
                    cache.queue_glyph(font_id, glyph.clone());
                }
            }

            cache
                .cache_queued(mock_gpu_upload)
                .expect_err("shouldn't fit");

            CacheBuilder {
                width: 512,
                height: 512,
                ..cache.to_builder()
            }.rebuild(&mut cache);

            cache.cache_queued(mock_gpu_upload).expect("should fit now");

            for &(font_id, ref glyphs) in &font_glyphs {
                for (index, glyph) in glyphs.iter().enumerate() {
                    let rect = cache.rect_for(font_id, glyph);
                    assert!(
                        rect.is_ok(),
                        "Gpu cache rect lookup failed ({:?}) for font {} glyph index {}, id {}",
                        rect,
                        font_id,
                        index,
                        glyph.id().0
                    );
                }
            }
        });
    }

    /// Benchmark using multiple fonts and a different text group of glyphs
    /// each run. The cache is only large enough to fit each run if it is
    /// cleared and re-built.
    #[bench]
    fn moving_text_thrashing(b: &mut ::test::Bencher) {
        let chars: Vec<_> = TEST_STR.chars().collect();
        let subsection_len = 60;
        let distinct_subsection: Vec<_> = chars.windows(subsection_len).collect();

        let mut first_glyphs = vec![];
        let mut middle_glyphs = vec![];
        let mut last_glyphs = vec![];

        for (id, font) in FONTS.iter().enumerate() {
            let first_str: String = distinct_subsection[0].iter().collect();
            first_glyphs.push((id, test_glyphs(font, &first_str)));

            let middle_str: String = distinct_subsection[distinct_subsection.len() / 2]
                .iter()
                .collect();
            middle_glyphs.push((id, test_glyphs(font, &middle_str)));

            let last_str: String = distinct_subsection[distinct_subsection.len() - 1]
                .iter()
                .collect();
            last_glyphs.push((id, test_glyphs(font, &last_str)));
        }

        let test_variants = [first_glyphs, middle_glyphs, last_glyphs];

        // Cache is only a little larger than each variants size meaning a lot of
        // re-ordering, re-rasterization & re-uploading has to occur.
        let mut cache = CacheBuilder {
            width: 450,
            height: 450,
            scale_tolerance: 0.1,
            position_tolerance: 0.1,
            ..CacheBuilder::default()
        }.build();

        b.iter(|| {
            // switch text variant each run to force cache to deal with moving text
            // requirements
            for glyphs in &test_variants {
                for &(font_id, ref glyphs) in glyphs {
                    for glyph in glyphs {
                        cache.queue_glyph(font_id, glyph.clone());
                    }
                }

                cache.cache_queued(mock_gpu_upload).expect("cache_queued");

                for &(font_id, ref glyphs) in glyphs {
                    for (index, glyph) in glyphs.iter().enumerate() {
                        let rect = cache.rect_for(font_id, glyph);
                        assert!(
                            rect.is_ok(),
                            "Gpu cache rect lookup failed ({:?}) for font {} glyph index {}, id {}",
                            rect,
                            font_id,
                            index,
                            glyph.id().0
                        );
                    }
                }
            }
        });
    }
}
<|endoftext|>


<commit_before>#![feature(test)]

extern crate pretty;
extern crate tempfile;
extern crate test;
extern crate typed_arena;

use std::io;

use pretty::BoxAllocator;
use typed_arena::Arena;
use trees::Tree;

#[path = "../examples/trees.rs"]
mod trees;

macro_rules! bench_trees {
    ($b: expr, $out: expr, $allocator: expr, $size: expr) => {{
        let arena = Arena::new();
        let b = $b;
        let mut out = $out;
        let size = $size;

        let mut example = Tree::node("aaaaaaaaaaaaaaaaaaaaaaaaaaaaa");

        for _ in 0..size {
            let bbbbbbs = arena.alloc_extend([
                example,
                Tree::node("dd"),
            ].iter().cloned());

            let ffffs = arena.alloc_extend([
                Tree::node("gg"),
                Tree::node("hhh"),
                Tree::node("ii"),
            ].iter().cloned());

            let aaas = arena.alloc_extend([
                Tree::node_with_forest("bbbbbb", bbbbbbs),
                Tree::node("eee"),
                Tree::node_with_forest("ffff", ffffs),
            ].iter().cloned());

            example = Tree::node_with_forest("aaa", aaas);
        }

        let allocator = $allocator;

        b.iter(|| {
            example.pretty::<(), _>(&allocator).1.render(70, &mut out).unwrap();
        });
    }}
}

#[bench]
fn bench_sink_box(b: &mut test::Bencher) -> () {
    bench_trees!(b, io::sink(), BoxAllocator, 1)
}

#[bench]
fn bench_sink_arena(b: &mut test::Bencher) -> () {
    bench_trees!(b, io::sink(), Arena::new(), 1)
}

#[bench]
fn bench_vec_box(b: &mut test::Bencher) -> () {
    bench_trees!(b, Vec::new(), BoxAllocator, 1)
}

#[bench]
fn bench_vec_arena(b: &mut test::Bencher) -> () {
    bench_trees!(b, Vec::new(), Arena::new(), 1)
}

#[bench]
fn bench_io_box(b: &mut test::Bencher) -> () {
    let out = tempfile::tempfile().unwrap();
    bench_trees!(b, io::BufWriter::new(out), BoxAllocator, 1)
}

#[bench]
fn bench_io_arena(b: &mut test::Bencher) -> () {
    let out = tempfile::tempfile().unwrap();
    bench_trees!(b, io::BufWriter::new(out), Arena::new(), 1)
}

#[bench]
fn bench_large_sink_box(b: &mut test::Bencher) -> () {
    bench_trees!(b, io::sink(), BoxAllocator, 50)
}

#[bench]
fn bench_large_sink_arena(b: &mut test::Bencher) -> () {
    bench_trees!(b, io::sink(), Arena::new(), 50)
}

#[bench]
fn bench_large_vec_box(b: &mut test::Bencher) -> () {
    bench_trees!(b, Vec::new(), BoxAllocator, 50)
}

#[bench]
fn bench_large_vec_arena(b: &mut test::Bencher) -> () {
    bench_trees!(b, Vec::new(), Arena::new(), 50)
}

#[bench]
fn bench_large_io_box(b: &mut test::Bencher) -> () {
    let out = tempfile::tempfile().unwrap();
    bench_trees!(b, io::BufWriter::new(out), BoxAllocator, 50)
}

#[bench]
fn bench_large_io_arena(b: &mut test::Bencher) -> () {
    let out = tempfile::tempfile().unwrap();
    bench_trees!(b, io::BufWriter::new(out), Arena::new(), 50)
}
<commit_msg>Fix benchmarks<commit_after>#![feature(test)]

extern crate pretty;
extern crate tempfile;
extern crate test;
extern crate typed_arena;

use std::io;

use pretty::BoxAllocator;
use trees::Tree;
use typed_arena::Arena;

#[path = "../examples/trees.rs"]
mod trees;

macro_rules! bench_trees {
    ($b:expr, $out:expr, $allocator:expr, $size:expr) => {{
        let arena = Arena::new();
        let b = $b;
        let mut out = $out;
        let size = $size;

        let mut example = Tree::node("aaaaaaaaaaaaaaaaaaaaaaaaaaaaa");

        for _ in 0..size {
            let bbbbbbs = arena.alloc_extend([example, Tree::node("dd")].iter().cloned());

            let ffffs = arena.alloc_extend(
                [Tree::node("gg"), Tree::node("hhh"), Tree::node("ii")]
                    .iter()
                    .cloned(),
            );

            let aaas = arena.alloc_extend(
                [
                    Tree::node_with_forest("bbbbbb", bbbbbbs),
                    Tree::node("eee"),
                    Tree::node_with_forest("ffff", ffffs),
                ].iter()
                    .cloned(),
            );

            example = Tree::node_with_forest("aaa", aaas);
        }

        let allocator = $allocator;

        b.iter(|| {
            example
                .pretty::<_, ()>(&allocator)
                .1
                .render(70, &mut out)
                .unwrap();
        });
    }};
}

#[bench]
fn bench_sink_box(b: &mut test::Bencher) -> () {
    bench_trees!(b, io::sink(), BoxAllocator, 1)
}

#[bench]
fn bench_sink_arena(b: &mut test::Bencher) -> () {
    bench_trees!(b, io::sink(), Arena::new(), 1)
}

#[bench]
fn bench_vec_box(b: &mut test::Bencher) -> () {
    bench_trees!(b, Vec::new(), BoxAllocator, 1)
}

#[bench]
fn bench_vec_arena(b: &mut test::Bencher) -> () {
    bench_trees!(b, Vec::new(), Arena::new(), 1)
}

#[bench]
fn bench_io_box(b: &mut test::Bencher) -> () {
    let out = tempfile::tempfile().unwrap();
    bench_trees!(b, io::BufWriter::new(out), BoxAllocator, 1)
}

#[bench]
fn bench_io_arena(b: &mut test::Bencher) -> () {
    let out = tempfile::tempfile().unwrap();
    bench_trees!(b, io::BufWriter::new(out), Arena::new(), 1)
}

#[bench]
fn bench_large_sink_box(b: &mut test::Bencher) -> () {
    bench_trees!(b, io::sink(), BoxAllocator, 50)
}

#[bench]
fn bench_large_sink_arena(b: &mut test::Bencher) -> () {
    bench_trees!(b, io::sink(), Arena::new(), 50)
}

#[bench]
fn bench_large_vec_box(b: &mut test::Bencher) -> () {
    bench_trees!(b, Vec::new(), BoxAllocator, 50)
}

#[bench]
fn bench_large_vec_arena(b: &mut test::Bencher) -> () {
    bench_trees!(b, Vec::new(), Arena::new(), 50)
}

#[bench]
fn bench_large_io_box(b: &mut test::Bencher) -> () {
    let out = tempfile::tempfile().unwrap();
    bench_trees!(b, io::BufWriter::new(out), BoxAllocator, 50)
}

#[bench]
fn bench_large_io_arena(b: &mut test::Bencher) -> () {
    let out = tempfile::tempfile().unwrap();
    bench_trees!(b, io::BufWriter::new(out), Arena::new(), 50)
}
<|endoftext|>


<commit_before>// Verified: http://judge.u-aizu.ac.jp/onlinejudge/review.jsp?rid=3429517#1

#![allow(non_snake_case)]

use std;

pub struct StrongComponentDecomposition<'a> {
    /// Number of vertices.
    N: usize,
    /// The graph.
    G: &'a [Vec<usize>],
    /// Dual of the graph.
    H: Vec<Vec<usize>>,
    /// gray[u] = true if the vertex u is visited.
    gray: Vec<bool>,
    /// Topological order index for each vertex.
    top_ord: Vec<usize>,
    /// List of vertices in topological order.
    top_seq: Vec<usize>,
    /// Next index of topological order.
    next_ord: usize,
    /// Output of the process.
    components: Vec<Vec<usize>>,
}

impl<'a> StrongComponentDecomposition<'a> {
    pub fn run(G: &'a [Vec<usize>]) -> Vec<Vec<usize>> {
        let N = G.len();

        let mut H = vec![vec![]; N];
        for u in 0..N {
            for &v in &G[u] {
                H[v].push(u);
            }
        }

        let it = StrongComponentDecomposition {
            N: N,
            G: G,
            H: H,
            gray: vec![],
            top_ord: vec![N; N],
            top_seq: vec![],
            next_ord: 0,
            components: vec![],
        };
        it.start()
    }

    fn start(mut self) -> Vec<Vec<usize>> {
        self.gray = vec![false; self.N];
        for u in 0..self.N {
            self.top(u);
        }

        self.gray = vec![false; self.N];
        let mut bot_seq = std::mem::replace(&mut self.top_seq, vec![]);
        bot_seq.reverse();
        for u in bot_seq {
            if self.gray[u] {
                continue;
            }

            // Here u is the last vertex in topological order.

            let mut component = vec![];
            self.bot(&mut component, u);
            self.components.push(component);
        }

        self.components
    }

    /// Topological sort.
    fn top(&mut self, u: usize) {
        if self.gray[u] {
            return;
        }
        self.gray[u] = true;

        for i in 0..self.G[u].len() {
            let v = self.G[u][i];
            self.top(v);
        }

        self.top_seq.push(u);
        self.top_ord[u] = self.next_ord;
        self.next_ord += 1;
    }

    /// DFS over the dual.
    fn bot(&mut self, c: &mut Vec<usize>, u: usize) {
        if self.gray[u] {
            return;
        }

        self.gray[u] = true;
        c.push(u);

        for i in 0..self.H[u].len() {
            let v = self.H[u][i];
            self.bot(c, v);
        }
    }
}
<commit_msg>refactor: なるべく snake_case を使う<commit_after>// Verified: http://judge.u-aizu.ac.jp/onlinejudge/review.jsp?rid=3429517#1

use std;

pub struct StrongComponentDecomposition<'a> {
    /// Number of vertices.
    n: usize,
    /// The graph.
    g: &'a [Vec<usize>],
    /// Dual of the graph.
    h: Vec<Vec<usize>>,
    /// gray[u] = true if the vertex u is visited.
    gray: Vec<bool>,
    /// Topological order index for each vertex.
    top_ord: Vec<usize>,
    /// List of vertices in topological order.
    top_seq: Vec<usize>,
    /// Next index of topological order.
    next_ord: usize,
    /// Output of the process.
    components: Vec<Vec<usize>>,
}

impl<'a> StrongComponentDecomposition<'a> {
    pub fn run(g: &'a [Vec<usize>]) -> Vec<Vec<usize>> {
        let n = g.len();

        let mut h = vec![vec![]; n];
        for u in 0..n {
            for &v in &g[u] {
                h[v].push(u);
            }
        }

        let it = StrongComponentDecomposition {
            n: n,
            g: g,
            h: h,
            gray: vec![],
            top_ord: vec![n; n],
            top_seq: vec![],
            next_ord: 0,
            components: vec![],
        };
        it.start()
    }

    fn start(mut self) -> Vec<Vec<usize>> {
        self.gray = vec![false; self.n];
        for u in 0..self.n {
            self.top(u);
        }

        self.gray = vec![false; self.n];
        let mut bot_seq = std::mem::replace(&mut self.top_seq, vec![]);
        bot_seq.reverse();
        for u in bot_seq {
            if self.gray[u] {
                continue;
            }

            // Here u is the last vertex in topological order.

            let mut component = vec![];
            self.bot(&mut component, u);
            self.components.push(component);
        }

        self.components
    }

    /// Topological sort.
    fn top(&mut self, u: usize) {
        if self.gray[u] {
            return;
        }
        self.gray[u] = true;

        for i in 0..self.g[u].len() {
            let v = self.g[u][i];
            self.top(v);
        }

        self.top_seq.push(u);
        self.top_ord[u] = self.next_ord;
        self.next_ord += 1;
    }

    /// DFS over the dual.
    fn bot(&mut self, c: &mut Vec<usize>, u: usize) {
        if self.gray[u] {
            return;
        }

        self.gray[u] = true;
        c.push(u);

        for i in 0..self.h[u].len() {
            let v = self.h[u][i];
            self.bot(c, v);
        }
    }
}
<|endoftext|>


<commit_before>use std::ops::{Mul, Add};

use rand::{Rand, Rng, Closed01};
use num::{Float, Zero};

use rblas::{Gemm, Matrix};
use rblas::attribute::Transpose;

use ndarray::{ArrayBase, DataOwned, DataMut};
use ndarray::blas::{BlasArrayViewMut, AsBlas};

use helpers::random01;

pub trait ShapeAsTuple<T> {
    fn shape_as_tuple(&self) -> T;
}

pub type FloatT = f32;

impl<T> ShapeAsTuple<(usize, usize)> for ArrayBase<Vec<T>, (usize, usize)> {
    #[inline]
    fn shape_as_tuple(&self) -> (usize, usize) {
        (self.shape()[0], self.shape()[1])
    }
}

pub type Ix = (usize, usize);
pub type MyMatrix = ArrayBase<Vec<FloatT>, Ix>;

pub struct OrthogonalNMFBlas {
    pub hidden: MyMatrix,
    pub weights: MyMatrix,

    // these hold temporary results during an iteration.
    // kept in this struct to prevent unnecessary memory allocations.
    pub weights_multiplier: MyMatrix,
    pub weights_divisor: MyMatrix,
    pub weights_divisor_reconstruction: MyMatrix,

    pub hidden_multiplier: MyMatrix,
    pub hidden_divisor: MyMatrix,
    pub hidden_divisor_partial: MyMatrix,

    pub gamma: MyMatrix,
}

impl OrthogonalNMFBlas
    // where FloatT: 'a + Float + Mul + Zero + Clone + Gemm,
          // ArrayBase<Vec<FloatT>, (usize, usize)>: 'a + AsBlas<FloatT, ArrayBase<Vec<FloatT>, (usize, usize)>, (usize, usize)> + DataOwned<Elem=FloatT> + DataMut<Elem=FloatT>,
          // BlasArrayViewMut<'a, FloatT, (usize, usize)>: Matrix<FloatT>,
          // Closed01<FloatT>: Rand
{
    pub fn new_random01<R: Rng>(nhidden: usize, nobserved: usize, nsamples: usize, rng: &mut R) -> OrthogonalNMFBlas {
        let mut hidden = MyMatrix::zeros((nhidden, nobserved));
        for x in hidden.iter_mut() {
            *x = random01(rng);
        }

        let mut weights = MyMatrix::zeros((nsamples, nhidden));
        for x in weights.iter_mut() {
            *x = random01(rng);
        }

        Self::new(hidden, weights)
    }

    pub fn new(hidden: MyMatrix, weights: MyMatrix) -> OrthogonalNMFBlas {
        let weights_shape = weights.shape_as_tuple();
        let hidden_shape = hidden.shape_as_tuple();

        let nhidden = hidden_shape.0;
        let nobserved = hidden_shape.1;
        let nsamples = weights_shape.0;

        let samples_shape = (nsamples, nobserved);

        OrthogonalNMFBlas {
            hidden: hidden,
            weights: weights,

            weights_multiplier: MyMatrix::zeros(weights_shape),
            weights_divisor: MyMatrix::zeros(weights_shape),
            weights_divisor_reconstruction: MyMatrix::zeros(samples_shape),

            hidden_multiplier: MyMatrix::zeros(hidden_shape),
            hidden_divisor: MyMatrix::zeros(hidden_shape),
            hidden_divisor_partial: MyMatrix::zeros((nhidden, nhidden)),

            gamma: OrthogonalNMFBlas::gamma(nhidden),
        }
    }

    /// returns the number of observed variables
    #[inline]
    pub fn nobserved(&self) -> usize {
        self.hidden.shape()[1]
    }

    /// returns the number of hidden variables
    #[inline]
    pub fn nhidden(&self) -> usize {
        self.hidden.shape()[0]
    }

    /// returns the number of data points
    #[inline]
    pub fn nsamples(&self) -> usize {
        self.weights.shape()[0]
    }

    pub fn samples_shape(&self) -> (usize, usize) {
        (self.nsamples(), self.nobserved())
    }

    /// gamma is a symetric matrix with diagonal elements equal to zero
    /// and other elements equal to 1
    pub fn gamma(size: usize) -> MyMatrix {
        let mut gamma = MyMatrix::from_elem((size, size), 1.);
        for x in gamma.diag_mut().iter_mut() {
            *x = 0.
        }
        gamma
    }

    /// `samples * hidden.transpose()`
    #[inline]
    pub fn update_weights_multiplier(&mut self, samples: &mut MyMatrix) {
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &samples.blas(),
            Transpose::Trans, &self.hidden.blas(),
            &0.,
            &mut self.weights_multiplier.blas());

    }

    // TODO this is the slowest
    /// `weights * hidden * hidden.transpose()`
    #[inline]
    pub fn update_weights_divisor(&mut self) {
        // TODO weights * hidden which we compute first is really large
        // we could first compute partial = hidden * hidden.transpose()
        // partial.shape() = (nhidden, nhidden)
        // and then in the second step compute
        // weights * partial.
        // however since we can't alias hidden we need to copy
        // it for this.
        // the question is whether the copying is faster than
        // the larger matrix multiplication. i guess it is.
        // TODO multiplying a matrix by its transpose is symetric
        // can we exploit that ?
        // reconstruction <- weights * hidden
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.weights.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            &0.,
            &mut self.weights_divisor_reconstruction.blas());
        // divisor <- reconstruction * hidden.transpose()
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.weights_divisor_reconstruction.blas(),
            Transpose::Trans, &self.hidden.blas(),
            &0.,
            &mut self.weights_divisor.blas());
    }

    /// `weights.transpose() * samples`
    #[inline]
    pub fn update_hidden_multiplier(&mut self, samples: &mut MyMatrix) {
        Gemm::gemm(
            &1.,
            Transpose::Trans, &self.weights.blas(),
            Transpose::NoTrans, &samples.blas(),
            &0.,
            &mut self.hidden_multiplier.blas());
    }

    // surprisingly this is the fastest
    /// `weights.transpose() * weights * hidden + alpha * gamma * hidden`
    #[inline]
    pub fn update_hidden_divisor(&mut self, alpha: FloatT) {
        // TODO benchmark whether using clone_from is faster
        let mut weights_copy = self.weights.clone();
        // partial <- weights.transpose() * weights
        Gemm::gemm(
            &1.,
            Transpose::Trans, &self.weights.blas(),
            Transpose::NoTrans, &weights_copy.blas(),
            &0.,
            &mut self.hidden_divisor_partial.blas());
        // partial.shape() = (nhidden, nhidden) very small
        // divisor <- alpha * gamma * hidden
        Gemm::gemm(
            &alpha,
            Transpose::NoTrans, &self.gamma.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            &0.,
            &mut self.hidden_divisor.blas());
        // divisor <- divisor + partial * hidden
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.hidden_divisor_partial.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            // add to previous contents of self.hidden_divisor
            &1.,
            &mut self.hidden_divisor.blas());
    }

    /// `result(i,j) <- result(i,j) * multiplier(i,j) / divisor(i,j)`
    #[inline]
    pub fn update_from_multiplier_and_divisor(
        multiplier: &MyMatrix,
        divisor: &MyMatrix,
        result: &mut MyMatrix,
    ) {
        let shape = result.shape_as_tuple();
        assert_eq!(shape, multiplier.shape_as_tuple());
        assert_eq!(shape, divisor.shape_as_tuple());

        for row in 0..shape.0 {
            for col in 0..shape.1 {
                let index = (row, col);
                let mut div = unsafe { divisor.uget(index).clone() };
                // if we have any 0 in any of the matrixes
                // self.weights or self.hidden then
                // divisor will be 0.
                // we can't divide by 0.
                // so we change them to the min positive value.
                if FloatT::zero() == div {
                    div = FloatT::min_positive_value();
                }
                unsafe {
                    *result.uget_mut(index) *= multiplier.uget(index) / div;
                }
            }
        }
    }

    // TODO consider calling this something like iteration_step
    // TODO how many iterations ?
    // TODO compare this to the seoung solution
    /// does one iteration step.
    /// `weights` and `hidden` get better and better with each iteration.
    /// usually around `10000` iterations are required.
    /// `samples` contains one observed per column, one sample per row.
    pub fn iterate(&mut self, alpha: FloatT, samples: &mut MyMatrix) {
        assert_eq!(samples.shape_as_tuple(), self.samples_shape());

        // weights_multiplier <- samples * hidden.transpose()
        self.update_weights_multiplier(samples);
        // weights_divisor <- weights * hidden * hidden.transpose()
        self.update_weights_divisor();
        // hidden_multiplier <- weights.transpose() * samples
        self.update_hidden_multiplier(samples);
        // hidden_divisor <-
        // weights.transpose() * weights * hidden + alpha * gamma * hidden
        self.update_hidden_divisor(alpha);

        // weights(i,j) <-
        // weights(i,j) * weights_multiplier(i,j) / weights_divisor(i,j)
        OrthogonalNMFBlas::update_from_multiplier_and_divisor(
            &self.weights_multiplier,
            &self.weights_divisor,
            &mut self.weights);
        // hidden(i,j) <-
        // hidden(i,j) * hidden_multiplier(i,j) / hidden_divisor(i,j)
        OrthogonalNMFBlas::update_from_multiplier_and_divisor(
            &self.hidden_multiplier,
            &self.hidden_divisor,
            &mut self.hidden);
    }
}
<commit_msg>add TODO<commit_after>use std::ops::{Mul, Add};

use rand::{Rand, Rng, Closed01};
use num::{Float, Zero};

use rblas::{Gemm, Matrix};
use rblas::attribute::Transpose;

use ndarray::{ArrayBase, DataOwned, DataMut};
use ndarray::blas::{BlasArrayViewMut, AsBlas};

use helpers::random01;

pub trait ShapeAsTuple<T> {
    fn shape_as_tuple(&self) -> T;
}

pub type FloatT = f32;

impl<T> ShapeAsTuple<(usize, usize)> for ArrayBase<Vec<T>, (usize, usize)> {
    #[inline]
    fn shape_as_tuple(&self) -> (usize, usize) {
        (self.shape()[0], self.shape()[1])
    }
}

pub type Ix = (usize, usize);
pub type MyMatrix = ArrayBase<Vec<FloatT>, Ix>;

pub struct OrthogonalNMFBlas {
    pub hidden: MyMatrix,
    pub weights: MyMatrix,

    // these hold temporary results during an iteration.
    // kept in this struct to prevent unnecessary memory allocations.
    pub weights_multiplier: MyMatrix,
    pub weights_divisor: MyMatrix,
    pub weights_divisor_reconstruction: MyMatrix,

    pub hidden_multiplier: MyMatrix,
    pub hidden_divisor: MyMatrix,
    pub hidden_divisor_partial: MyMatrix,

    pub gamma: MyMatrix,
}

impl OrthogonalNMFBlas
    // where FloatT: 'a + Float + Mul + Zero + Clone + Gemm,
          // ArrayBase<Vec<FloatT>, (usize, usize)>: 'a + AsBlas<FloatT, ArrayBase<Vec<FloatT>, (usize, usize)>, (usize, usize)> + DataOwned<Elem=FloatT> + DataMut<Elem=FloatT>,
          // BlasArrayViewMut<'a, FloatT, (usize, usize)>: Matrix<FloatT>,
          // Closed01<FloatT>: Rand
{
    pub fn new_random01<R: Rng>(nhidden: usize, nobserved: usize, nsamples: usize, rng: &mut R) -> OrthogonalNMFBlas {
        let mut hidden = MyMatrix::zeros((nhidden, nobserved));
        for x in hidden.iter_mut() {
            *x = random01(rng);
        }

        let mut weights = MyMatrix::zeros((nsamples, nhidden));
        for x in weights.iter_mut() {
            *x = random01(rng);
        }

        Self::new(hidden, weights)
    }

    pub fn new(hidden: MyMatrix, weights: MyMatrix) -> OrthogonalNMFBlas {
        let weights_shape = weights.shape_as_tuple();
        let hidden_shape = hidden.shape_as_tuple();

        let nhidden = hidden_shape.0;
        let nobserved = hidden_shape.1;
        let nsamples = weights_shape.0;

        let samples_shape = (nsamples, nobserved);

        OrthogonalNMFBlas {
            hidden: hidden,
            weights: weights,

            weights_multiplier: MyMatrix::zeros(weights_shape),
            weights_divisor: MyMatrix::zeros(weights_shape),
            weights_divisor_reconstruction: MyMatrix::zeros(samples_shape),

            hidden_multiplier: MyMatrix::zeros(hidden_shape),
            hidden_divisor: MyMatrix::zeros(hidden_shape),
            hidden_divisor_partial: MyMatrix::zeros((nhidden, nhidden)),

            gamma: OrthogonalNMFBlas::gamma(nhidden),
        }
    }

    /// returns the number of observed variables
    #[inline]
    pub fn nobserved(&self) -> usize {
        self.hidden.shape()[1]
    }

    /// returns the number of hidden variables
    #[inline]
    pub fn nhidden(&self) -> usize {
        self.hidden.shape()[0]
    }

    /// returns the number of data points
    #[inline]
    pub fn nsamples(&self) -> usize {
        self.weights.shape()[0]
    }

    pub fn samples_shape(&self) -> (usize, usize) {
        (self.nsamples(), self.nobserved())
    }

    /// gamma is a symetric matrix with diagonal elements equal to zero
    /// and other elements equal to 1
    pub fn gamma(size: usize) -> MyMatrix {
        let mut gamma = MyMatrix::from_elem((size, size), 1.);
        for x in gamma.diag_mut().iter_mut() {
            *x = 0.
        }
        gamma
    }

    // TODO call these `set` or something similar because
    // opposed to `update_from_multiplier_and_divisor` these
    // don't adjust but change entirely

    /// `samples * hidden.transpose()`
    #[inline]
    pub fn update_weights_multiplier(&mut self, samples: &mut MyMatrix) {
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &samples.blas(),
            Transpose::Trans, &self.hidden.blas(),
            &0.,
            &mut self.weights_multiplier.blas());

    }

    // TODO this is the slowest
    /// `weights * hidden * hidden.transpose()`
    #[inline]
    pub fn update_weights_divisor(&mut self) {
        // TODO weights * hidden which we compute first is really large
        // we could first compute partial = hidden * hidden.transpose()
        // partial.shape() = (nhidden, nhidden)
        // and then in the second step compute
        // weights * partial.
        // however since we can't alias hidden we need to copy
        // it for this.
        // the question is whether the copying is faster than
        // the larger matrix multiplication. i guess it is.
        // TODO multiplying a matrix by its transpose is symetric
        // can we exploit that ?
        // reconstruction <- weights * hidden
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.weights.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            &0.,
            &mut self.weights_divisor_reconstruction.blas());
        // divisor <- reconstruction * hidden.transpose()
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.weights_divisor_reconstruction.blas(),
            Transpose::Trans, &self.hidden.blas(),
            &0.,
            &mut self.weights_divisor.blas());
    }

    /// `weights.transpose() * samples`
    #[inline]
    pub fn update_hidden_multiplier(&mut self, samples: &mut MyMatrix) {
        Gemm::gemm(
            &1.,
            Transpose::Trans, &self.weights.blas(),
            Transpose::NoTrans, &samples.blas(),
            &0.,
            &mut self.hidden_multiplier.blas());
    }

    // surprisingly this is the fastest
    /// `weights.transpose() * weights * hidden + alpha * gamma * hidden`
    #[inline]
    pub fn update_hidden_divisor(&mut self, alpha: FloatT) {
        // TODO benchmark whether using clone_from is faster
        let mut weights_copy = self.weights.clone();
        // partial <- weights.transpose() * weights
        Gemm::gemm(
            &1.,
            Transpose::Trans, &self.weights.blas(),
            Transpose::NoTrans, &weights_copy.blas(),
            &0.,
            &mut self.hidden_divisor_partial.blas());
        // partial.shape() = (nhidden, nhidden) very small
        // divisor <- alpha * gamma * hidden
        Gemm::gemm(
            &alpha,
            Transpose::NoTrans, &self.gamma.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            &0.,
            &mut self.hidden_divisor.blas());
        // divisor <- divisor + partial * hidden
        Gemm::gemm(
            &1.,
            Transpose::NoTrans, &self.hidden_divisor_partial.blas(),
            Transpose::NoTrans, &self.hidden.blas(),
            // add to previous contents of self.hidden_divisor
            &1.,
            &mut self.hidden_divisor.blas());
    }

    /// `result(i,j) <- result(i,j) * multiplier(i,j) / divisor(i,j)`
    #[inline]
    pub fn update_from_multiplier_and_divisor(
        multiplier: &MyMatrix,
        divisor: &MyMatrix,
        result: &mut MyMatrix,
    ) {
        let shape = result.shape_as_tuple();
        assert_eq!(shape, multiplier.shape_as_tuple());
        assert_eq!(shape, divisor.shape_as_tuple());

        for row in 0..shape.0 {
            for col in 0..shape.1 {
                let index = (row, col);
                let mut div = unsafe { divisor.uget(index).clone() };
                // if we have any 0 in any of the matrixes
                // self.weights or self.hidden then
                // divisor will be 0.
                // we can't divide by 0.
                // so we change them to the min positive value.
                if FloatT::zero() == div {
                    div = FloatT::min_positive_value();
                }
                unsafe {
                    *result.uget_mut(index) *= multiplier.uget(index) / div;
                }
            }
        }
    }

    // TODO consider calling this something like iteration_step
    // TODO how many iterations ?
    // TODO compare this to the seoung solution
    /// does one iteration step.
    /// `weights` and `hidden` get better and better with each iteration.
    /// usually around `10000` iterations are required.
    /// `samples` contains one observed per column, one sample per row.
    pub fn iterate(&mut self, alpha: FloatT, samples: &mut MyMatrix) {
        assert_eq!(samples.shape_as_tuple(), self.samples_shape());

        // weights_multiplier <- samples * hidden.transpose()
        self.update_weights_multiplier(samples);
        // weights_divisor <- weights * hidden * hidden.transpose()
        self.update_weights_divisor();
        // hidden_multiplier <- weights.transpose() * samples
        self.update_hidden_multiplier(samples);
        // hidden_divisor <-
        // weights.transpose() * weights * hidden + alpha * gamma * hidden
        self.update_hidden_divisor(alpha);

        // weights(i,j) <-
        // weights(i,j) * weights_multiplier(i,j) / weights_divisor(i,j)
        OrthogonalNMFBlas::update_from_multiplier_and_divisor(
            &self.weights_multiplier,
            &self.weights_divisor,
            &mut self.weights);
        // hidden(i,j) <-
        // hidden(i,j) * hidden_multiplier(i,j) / hidden_divisor(i,j)
        OrthogonalNMFBlas::update_from_multiplier_and_divisor(
            &self.hidden_multiplier,
            &self.hidden_divisor,
            &mut self.hidden);
    }
}
<|endoftext|>


<commit_before>use std::sync::mpsc::Sender;
use std::cell::RefCell;

use sys::win32::ffi;

use event::Event;

thread_local!(
   pub static SENDER: RefCell<Option<Sender<Event>>> = RefCell::new(None)
);

fn send(event: Event) {
   SENDER.with(|sender_cell| {
      let sender_option = sender_cell.borrow();
      let sender = match *sender_option {
         Some(ref sender) => sender,
         None => return
      };

      sender.send(event).ok();
   });
}

#[allow(unused_variables)]
pub unsafe extern "system" fn wnd_proc(
   hwnd: ffi::HWND,
   msg: ffi::c_uint,
   wparam: ffi::WPARAM,
   lparam: ffi::LPARAM
) -> ffi::LRESULT {
   match msg {
      ffi::WM_CLOSE => {
         ffi::PostQuitMessage(0);
         0
      },
      ffi::WM_SIZE => {
         let width = ffi::LOWORD(lparam as ffi::DWORD) as u32;
         let height = ffi::HIWORD(lparam as ffi::DWORD) as u32;
         send(Event::Resize(width, height));
         0
      },
      ffi::WM_CREATE => {
         0
      },
      _ => {
         ffi::DefWindowProcW(hwnd, msg, wparam, lparam)
      }
   }
}
<commit_msg>Handling WM_ERASEBKGND on Windows by skipping it<commit_after>use std::sync::mpsc::Sender;
use std::cell::RefCell;

use sys::win32::ffi;

use event::Event;

thread_local!(
   pub static SENDER: RefCell<Option<Sender<Event>>> = RefCell::new(None)
);

fn send(event: Event) {
   SENDER.with(|sender_cell| {
      let sender_option = sender_cell.borrow();
      let sender = match *sender_option {
         Some(ref sender) => sender,
         None => return
      };

      sender.send(event).ok();
   });
}

#[allow(unused_variables)]
pub unsafe extern "system" fn wnd_proc(
   hwnd: ffi::HWND,
   msg: ffi::c_uint,
   wparam: ffi::WPARAM,
   lparam: ffi::LPARAM
) -> ffi::LRESULT {
   match msg {
      ffi::WM_CLOSE => {
         ffi::PostQuitMessage(0);
         0
      },
      ffi::WM_ERASEBKGND => {
         1
      },
      ffi::WM_SIZE => {
         let width = ffi::LOWORD(lparam as ffi::DWORD) as u32;
         let height = ffi::HIWORD(lparam as ffi::DWORD) as u32;
         send(Event::Resize(width, height));
         0
      },
      ffi::WM_CREATE => {
         0
      },
      _ => {
         ffi::DefWindowProcW(hwnd, msg, wparam, lparam)
      }
   }
}
<|endoftext|>


<commit_before>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

use std::cmp::Ordering;

use super::symbol_range::*;

///
/// A symbol map maps from one set of symbol ranges to another
///
pub struct SymbolMap<Symbol: PartialOrd+Clone> {
    // Ranges in this symbol map
    ranges: Vec<SymbolRange<Symbol>>,
}

impl<Symbol: PartialOrd+Clone> SymbolMap<Symbol> {
    ///
    /// Creates a new symbol map
    ///
    pub fn new() -> SymbolMap<Symbol> {
        SymbolMap { ranges: vec![] }
    }

    ///
    /// Orders two symbols (makes unordered symbols equal)
    ///
    #[inline]
    fn order_symbols(a: &Symbol, b: &Symbol) -> Ordering {
        if let Some(order) = a.partial_cmp(b) {
            order
        } else {
            Ordering::Equal
        }
    }

    ///
    /// Orders two symbol ranges
    ///
    #[inline]
    fn order_ranges(a: &SymbolRange<Symbol>, b: &SymbolRange<Symbol>) -> Ordering {
        let ordering = SymbolMap::order_symbols(&a.lowest, &b.lowest);

        if ordering == Ordering::Equal {
            SymbolMap::order_symbols(&a.highest, &b.highest)
        } else {
            ordering
        }
    }

    ///
    /// Adds a range to those that are known about by this object
    ///
    pub fn add_range(&mut self, range: &SymbolRange<Symbol>) {
        let existing = self.ranges.binary_search_by(|test_range| { SymbolMap::order_ranges(test_range, range) });

        // Insert the range if it is not already in the map
        if let Err(insertion_pos) = existing {
            self.ranges.insert(insertion_pos, range.clone());
        }
    }

    ///
    /// Finds the ranges in this map that overlap the target ranges
    ///
    pub fn find_overlapping_ranges(&self, range: &SymbolRange<Symbol>) -> Vec<&SymbolRange<Symbol>> {
        let mut result = vec![];

        // Find the first range that matches (or the insertion position, which should be the first range wit a lowest value higher than the target range)
        let existing = self.ranges.binary_search_by(|test_range| { SymbolMap::order_ranges(test_range, range) });

        // Start returning values from here
        let mut pos = match existing {
            Ok(found_position) => found_position,
            Err(insert_position) => insert_position
        };

        // Move backwards if the previous position overlaps this one
        // NOT if only by the highest value; see the adjacency rule in symbol_range.rs
        if pos > 0 && self.ranges[pos-1].highest > range.lowest {
            pos -= 1;
        }

        // TODO: can we construct a set of ranges such that one is missed here? Think maybe we can
        while pos < self.ranges.len() && self.ranges[pos].lowest <= range.highest {
            result.push(&self.ranges[pos]);
            pos += 1;
        }

        result
    }

    ///
    /// Creates a non-overlapping range from an overlapping one
    ///
    pub fn to_non_overlapping_map(&self) -> SymbolMap<Symbol> {
        // Create a vector of all the range symbols
        let mut range_symbols: Vec<Symbol> = self.ranges.iter()
            .map(|r| r.lowest.clone())
            .chain(self.ranges.iter().map(|r| r.highest.clone()))
            .collect();

        // Sort and dedupe (we'll get pairs of ranges)
        range_symbols.sort_by(SymbolMap::order_symbols);
        range_symbols.dedup();

        // Generate a new symbol map with these ranges
        // They'll overlap with the last value, which should be OK if following the rules for adjacent ranges in symbol_range.rs
        let mut result = vec![];
        for index in 0..range_symbols.len()-1 {
            result.push(SymbolRange::new(range_symbols[index].clone(), range_symbols[index+1].clone()));
        }

        // We already sorted everything, so bypass the usual 'add' method (which sorts as it goes)
        SymbolMap { ranges: result }
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use super::super::symbol_range::*;

    #[test]
    fn can_lookup_overlapping_ranges() {
        let mut map = SymbolMap::new();

        map.add_range(&SymbolRange::new(0, 4));
        map.add_range(&SymbolRange::new(2, 5));
        map.add_range(&SymbolRange::new(3, 6));

        let bottom = map.find_overlapping_ranges(&SymbolRange::new(0, 1));
        let all    = map.find_overlapping_ranges(&SymbolRange::new(1, 3));
        let top    = map.find_overlapping_ranges(&SymbolRange::new(6, 6));

        println!("All is: {:?}", all);

        assert!(bottom == vec![&SymbolRange::new(0, 4)]);
        assert!(all == vec![&SymbolRange::new(0, 4), &SymbolRange::new(2, 5), &SymbolRange::new(3, 6)]);
        assert!(top == vec![&SymbolRange::new(6, 6)]);
    }

    #[test]
    fn can_lookup_mid_range() {
        let mut map = SymbolMap::new();

        map.add_range(&SymbolRange::new(0, 4));
        map.add_range(&SymbolRange::new(5, 10));
        map.add_range(&SymbolRange::new(11, 15));

        let bottom = map.find_overlapping_ranges(&SymbolRange::new(1, 3));

        assert!(bottom == vec![&SymbolRange::new(0, 4)]);
    }

    #[test]
    fn works_with_duplicate_lower_values() {
        let mut map = SymbolMap::new();

        map.add_range(&SymbolRange::new(0, 2));
        map.add_range(&SymbolRange::new(0, 3));

        let all    = map.find_overlapping_ranges(&SymbolRange::new(0, 1));

        println!("With duplicate lower values is: {:?}", all);

        assert!(all == vec![&SymbolRange::new(0, 2), &SymbolRange::new(0, 3)]);
    }

    #[test]
    fn obeys_adjacency_rule() {
        let mut map = SymbolMap::new();

        // By the adjacency rule for symbol ranges (in symbol_range.rs), '4' here is only in the upper range
        map.add_range(&SymbolRange::new(0, 4));
        map.add_range(&SymbolRange::new(4, 8));

        let top = map.find_overlapping_ranges(&SymbolRange::new(4, 4));

        assert!(top == vec![&SymbolRange::new(4, 8)]);
    }

    #[test]
    fn can_get_non_overlapping_map() {
        let mut map = SymbolMap::new();

        map.add_range(&SymbolRange::new(0, 4));
        map.add_range(&SymbolRange::new(2, 5));
        map.add_range(&SymbolRange::new(3, 6));

        let non_overlapping = map.to_non_overlapping_map();

        let all = non_overlapping.find_overlapping_ranges(&SymbolRange::new(0, 6));

        assert!(all == vec![&SymbolRange::new(0, 2), &SymbolRange::new(2, 3), &SymbolRange::new(3, 4), &SymbolRange::new(4, 5), &SymbolRange::new(5, 6)]);
    }
}
<commit_msg>Oops, was testing for the wrong match when looking for the top part of overlapping ranges<commit_after>//
//   Copyright 2016 Andrew Hunter
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

use std::cmp::Ordering;

use super::symbol_range::*;

///
/// A symbol map maps from one set of symbol ranges to another
///
pub struct SymbolMap<Symbol: PartialOrd+Clone> {
    // Ranges in this symbol map
    ranges: Vec<SymbolRange<Symbol>>,
}

impl<Symbol: PartialOrd+Clone> SymbolMap<Symbol> {
    ///
    /// Creates a new symbol map
    ///
    pub fn new() -> SymbolMap<Symbol> {
        SymbolMap { ranges: vec![] }
    }

    ///
    /// Orders two symbols (makes unordered symbols equal)
    ///
    #[inline]
    fn order_symbols(a: &Symbol, b: &Symbol) -> Ordering {
        if let Some(order) = a.partial_cmp(b) {
            order
        } else {
            Ordering::Equal
        }
    }

    ///
    /// Orders two symbol ranges
    ///
    #[inline]
    fn order_ranges(a: &SymbolRange<Symbol>, b: &SymbolRange<Symbol>) -> Ordering {
        let ordering = SymbolMap::order_symbols(&a.lowest, &b.lowest);

        if ordering == Ordering::Equal {
            SymbolMap::order_symbols(&a.highest, &b.highest)
        } else {
            ordering
        }
    }

    ///
    /// Adds a range to those that are known about by this object
    ///
    pub fn add_range(&mut self, range: &SymbolRange<Symbol>) {
        let existing = self.ranges.binary_search_by(|test_range| { SymbolMap::order_ranges(test_range, range) });

        // Insert the range if it is not already in the map
        if let Err(insertion_pos) = existing {
            self.ranges.insert(insertion_pos, range.clone());
        }
    }

    ///
    /// Finds the ranges in this map that overlap the target ranges
    ///
    pub fn find_overlapping_ranges(&self, range: &SymbolRange<Symbol>) -> Vec<&SymbolRange<Symbol>> {
        let mut result = vec![];

        // Find the first range that matches (or the insertion position, which should be the first range wit a lowest value higher than the target range)
        let existing = self.ranges.binary_search_by(|test_range| { SymbolMap::order_ranges(test_range, range) });

        // Start returning values from here
        let mut pos = match existing {
            Ok(found_position) => found_position,
            Err(insert_position) => insert_position
        };

        // Move backwards if the previous position overlaps this one
        // NOT if only by the highest value; see the adjacency rule in symbol_range.rs
        if pos > 0 && self.ranges[pos-1].highest > range.lowest {
            pos -= 1;
        }

        // TODO: can we construct a set of ranges such that one is missed here? Think maybe we can
        while pos < self.ranges.len() && self.ranges[pos].lowest <= range.highest {
            result.push(&self.ranges[pos]);
            pos += 1;
        }

        result
    }

    ///
    /// Creates a non-overlapping range from an overlapping one
    ///
    pub fn to_non_overlapping_map(&self) -> SymbolMap<Symbol> {
        // Create a vector of all the range symbols
        let mut range_symbols: Vec<Symbol> = self.ranges.iter()
            .map(|r| r.lowest.clone())
            .chain(self.ranges.iter().map(|r| r.highest.clone()))
            .collect();

        // Sort and dedupe (we'll get pairs of ranges)
        range_symbols.sort_by(SymbolMap::order_symbols);
        range_symbols.dedup();

        // Generate a new symbol map with these ranges
        // They'll overlap with the last value, which should be OK if following the rules for adjacent ranges in symbol_range.rs
        let mut result = vec![];
        for index in 0..range_symbols.len()-1 {
            result.push(SymbolRange::new(range_symbols[index].clone(), range_symbols[index+1].clone()));
        }

        // We already sorted everything, so bypass the usual 'add' method (which sorts as it goes)
        SymbolMap { ranges: result }
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use super::super::symbol_range::*;

    #[test]
    fn can_lookup_overlapping_ranges() {
        let mut map = SymbolMap::new();

        map.add_range(&SymbolRange::new(0, 4));
        map.add_range(&SymbolRange::new(2, 5));
        map.add_range(&SymbolRange::new(3, 6));

        let bottom = map.find_overlapping_ranges(&SymbolRange::new(0, 1));
        let all    = map.find_overlapping_ranges(&SymbolRange::new(1, 3));
        let top    = map.find_overlapping_ranges(&SymbolRange::new(6, 6));

        assert!(bottom == vec![&SymbolRange::new(0, 4)]);
        assert!(all == vec![&SymbolRange::new(0, 4), &SymbolRange::new(2, 5), &SymbolRange::new(3, 6)]);
        assert!(top == vec![&SymbolRange::new(3, 6)]);
    }

    #[test]
    fn can_lookup_mid_range() {
        let mut map = SymbolMap::new();

        map.add_range(&SymbolRange::new(0, 4));
        map.add_range(&SymbolRange::new(5, 10));
        map.add_range(&SymbolRange::new(11, 15));

        let mid = map.find_overlapping_ranges(&SymbolRange::new(1, 3));

        assert!(mid == vec![&SymbolRange::new(0, 4)]);
    }

    #[test]
    fn works_with_duplicate_lower_values() {
        let mut map = SymbolMap::new();

        map.add_range(&SymbolRange::new(0, 2));
        map.add_range(&SymbolRange::new(0, 3));

        let all    = map.find_overlapping_ranges(&SymbolRange::new(0, 1));

        println!("With duplicate lower values is: {:?}", all);

        assert!(all == vec![&SymbolRange::new(0, 2), &SymbolRange::new(0, 3)]);
    }

    #[test]
    fn obeys_adjacency_rule() {
        let mut map = SymbolMap::new();

        // By the adjacency rule for symbol ranges (in symbol_range.rs), '4' here is only in the upper range
        map.add_range(&SymbolRange::new(0, 4));
        map.add_range(&SymbolRange::new(4, 8));

        let top = map.find_overlapping_ranges(&SymbolRange::new(4, 4));

        assert!(top == vec![&SymbolRange::new(4, 8)]);
    }

    #[test]
    fn can_get_non_overlapping_map() {
        let mut map = SymbolMap::new();

        map.add_range(&SymbolRange::new(0, 4));
        map.add_range(&SymbolRange::new(2, 5));
        map.add_range(&SymbolRange::new(3, 6));

        let non_overlapping = map.to_non_overlapping_map();

        let all = non_overlapping.find_overlapping_ranges(&SymbolRange::new(0, 6));

        assert!(all == vec![&SymbolRange::new(0, 2), &SymbolRange::new(2, 3), &SymbolRange::new(3, 4), &SymbolRange::new(4, 5), &SymbolRange::new(5, 6)]);
    }
}
<|endoftext|>


<commit_before>use pest::Parser;

use parser::parse;

fn assert_err_msg(input: &str, needles: &[&str]) {
    let res = parse(input);
    assert!(res.is_err());
    let err = res.unwrap_err();
    let err_msg = err.description();
    println!("{}", err_msg);
    for needle in needles {
        assert!(err_msg.contains(needle));
    }

}

#[test]
fn invalid_number() {
    assert_err_msg(
        "{{ 1.2.2 }}",
        &[
            "1:7",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ]
    );
}

#[test]
fn invalid_op() {
    assert_err_msg(
        "{{ 1.2 >+ 3 }}",
        &[
            "1:9",
            "expected an expression or a comparison value"
        ]
    );
}

#[test]
fn wrong_start_block() {
    assert_err_msg(
        "{{ if true %}",
        &[
            "1:7",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ]
    );
}

#[test]
fn wrong_end_block() {
    assert_err_msg(
        "{{ hey %}",
        &[
            "1:9",
            "expected an integer, a float, a string, `true` or `false`, an identifier, a dotted identifier (identifiers separated by `.`), or an expression"
        ]
    );
}

#[test]
fn unterminated_variable_block() {
    assert_err_msg(
        "{{ hey",
        &[
            "1:7",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ]
    );
}

#[test]
fn unterminated_string() {
    assert_err_msg(
        r#"{{ "hey }}"#,
        &[
            "1:4",
            "expected any expressions"
        ]
    );
}

#[test]
fn unterminated_if_tag() {
    assert_err_msg(
        r#"{% if true %}sd"#,
        &[
            "1:16",
            r#"expected tag, an include tag (`{% include "..." %}`), a comment tag (`{#...#}`), a variable tag (`{{ ... }}`), or some text"#
        ],
    );
}

#[test]
fn unterminated_filter_section() {
    assert_err_msg(
        r#"{% filter uppercase %}sd"#,
        &[
            "1:25",
            r#"expected tag, an include tag (`{% include "..." %}`), a comment tag (`{#...#}`), a variable tag (`{{ ... }}`), or some text"#
        ],
    );
}

#[test]
fn invalid_filter_section_missing_name() {
    assert_err_msg(
        r#"{% filter %}sd{% endfilter %}"#,
        &[
            "1:11",
            "expected an identifier or a function call"
        ],
    );
}

#[test]
fn invalid_macro_content() {
    assert_err_msg(
        r#"
{% macro input(label, type) %}
    {% macro nested() %}
    {% endmacro nested %}
{% endmacro input %}
    "#,
        &[
            "3:5",
            "unexpected tag; expected `{% endmacro %}` or the macro content"
        ]
    );
}

#[test]
fn invalid_macro_default_arg_value() {
    assert_err_msg(
        r#"
{% macro input(label=something) %}
{% endmacro input %}
    "#,
        &[
            "2:22",
            "expected an integer, a float, a string, or `true` or `false`"
        ]
    );
}

#[test]
fn invalid_elif() {
    assert_err_msg(
        r#"
{% if true %}
{% else %}
{% elif false %}
{% endif %}
    "#,
        &[
            "4:1",
            "unexpected tag; expected a `endif` tag` or some content"
        ]
    );
}

#[test]
fn invalid_else() {
        assert_err_msg(
        r#"
{% if true %}
{% else %}
{% else %}
{% endif %}
    "#,
        &[
            "4:1",
            "unexpected tag; expected a `endif` tag` or some content"
        ]
    );
}

#[test]
fn invalid_extends_position() {
        assert_err_msg(
        r#"
hello
{% extends "hey.html" %}
    "#,
        &[
            "3:1",
            "unexpected tag; expected some content"
        ]
    );
}

#[test]
fn invalid_operator() {
    assert_err_msg(
        "{{ hey =! }}",
        &[
            "1:8",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ]
    );
}

#[test]
fn missing_expression_with_not() {
        assert_err_msg(
        "{% if not %}",
        &[
            "1:11",
            "expected an expression"
        ]
    );
}

#[test]
fn missing_expression_in_if() {
        assert_err_msg(
        "{% if %}",
        &[
            "1:7",
            "expected any expression"
        ]
    );
}

#[test]
fn missing_container_name_in_forloop() {
        assert_err_msg(
        "{% for i in %}",
        &[
            "1:13",
            "expected an expression with an optional filter"
        ]
    );
}

#[test]
fn missing_variable_name_in_set() {
    assert_err_msg(
        "{% set = 1 %}",
        &[
            "1:8",
            "expected an identifier"
        ]
    );
}

#[test]
fn missing_value_in_set() {
    assert_err_msg(
        "{% set a =  %}",
        &[
            "1:13",
            "expected any expressions"
        ],
    );
}

#[test]
fn unterminated_fn_call() {
    assert_err_msg(
        "{{ a | slice( }}",
        &[
            "1:15",
            "expected a keyword argument: `key=value` where `value` can be any expression"
        ],
    );
}

#[test]
fn invalid_fn_call_missing_value() {
    assert_err_msg(
        "{{ a | slice(start=) }}",
        &[
            "1:20",
            "expected any expressions"
        ],
    );
}

#[test]
fn unterminated_macro_call() {
    assert_err_msg(
        "{{ my::macro( }}",
        &[
            "1:15",
            "expected a keyword argument: `key=value` where `value` can be any expression"
        ],
    );
}

#[test]
fn invalid_macro_call() {
    assert_err_msg(
        "{{ my:macro() }}",
        &[
            "1:6",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ],
    );
}

#[test]
fn unterminated_include() {
    assert_err_msg(
        "{% include %}",
        &[
            "1:12",
            "expected a string"
        ],
    );
}

#[test]
fn invalid_include_no_string() {
    assert_err_msg(
        "{% include 1 %}",
        &[
            "1:12",
            "expected a string"
        ],
    );
}

#[test]
fn unterminated_extends() {
    assert_err_msg(
        "{% extends %}",
        &[
            "1:12",
            "expected a string"
        ],
    );
}

#[test]
fn invalid_extends_no_string() {
    assert_err_msg(
        "{% extends 1 %}",
        &[
            "1:12",
            "expected a string"
        ],
    );
}

#[test]
fn invalid_import_macros_missing_filename() {
    assert_err_msg(
        "{% import as macros %}",
        &[
            "1:11",
            "expected a string"
        ],
    );
}

#[test]
fn invalid_import_macros_missing_namespace() {
    assert_err_msg(
        r#"{% import "hello" as %}"#,
        &[
            "1:22",
            "expected an identifier"
        ],
    );
}

#[test]
fn invalid_block_missing_name() {
    assert_err_msg(
        r#"{% block %}"#,
        &[
            "1:10",
            "expected an identifier"
        ],
    );
}

#[test]
fn unterminated_test() {
    assert_err_msg(
        r#"{% if a is odd( %}"#,
        &[
            "1:17",
            "expected a list of test arguments (any expressions)"
        ],
    );
}

#[test]
fn invalid_test_argument() {
    assert_err_msg(
        r#"{% if a is odd(key=1) %}"#,
        &[
            "1:19",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ],
    );
}

#[test]
fn unterminated_raw_tag() {
    assert_err_msg(
        r#"{% raw %}sd"#,
        &[
            "1:12",
            "expected `{% endraw %}`"
        ],
    );
}
<commit_msg>Fix test after pest update<commit_after>use pest::Parser;

use parser::parse;

fn assert_err_msg(input: &str, needles: &[&str]) {
    let res = parse(input);
    assert!(res.is_err());
    let err = res.unwrap_err();
    let err_msg = err.description();
    println!("{}", err_msg);
    for needle in needles {
        assert!(err_msg.contains(needle));
    }

}

#[test]
fn invalid_number() {
    assert_err_msg(
        "{{ 1.2.2 }}",
        &[
            "1:7",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ]
    );
}

#[test]
fn invalid_op() {
    assert_err_msg(
        "{{ 1.2 >+ 3 }}",
        &[
            "1:9",
            "expected a comparison value"
        ]
    );
}

#[test]
fn wrong_start_block() {
    assert_err_msg(
        "{{ if true %}",
        &[
            "1:7",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ]
    );
}

#[test]
fn wrong_end_block() {
    assert_err_msg(
        "{{ hey %}",
        &[
            "1:9",
            "expected an integer, a float, a string, `true` or `false`, an identifier, a dotted identifier (identifiers separated by `.`), or an expression"
        ]
    );
}

#[test]
fn unterminated_variable_block() {
    assert_err_msg(
        "{{ hey",
        &[
            "1:7",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ]
    );
}

#[test]
fn unterminated_string() {
    assert_err_msg(
        r#"{{ "hey }}"#,
        &[
            "1:4",
            "expected any expressions"
        ]
    );
}

#[test]
fn unterminated_if_tag() {
    assert_err_msg(
        r#"{% if true %}sd"#,
        &[
            "1:16",
            r#"expected tag, an include tag (`{% include "..." %}`), a comment tag (`{#...#}`), a variable tag (`{{ ... }}`), or some text"#
        ],
    );
}

#[test]
fn unterminated_filter_section() {
    assert_err_msg(
        r#"{% filter uppercase %}sd"#,
        &[
            "1:25",
            r#"expected tag, an include tag (`{% include "..." %}`), a comment tag (`{#...#}`), a variable tag (`{{ ... }}`), or some text"#
        ],
    );
}

#[test]
fn invalid_filter_section_missing_name() {
    assert_err_msg(
        r#"{% filter %}sd{% endfilter %}"#,
        &[
            "1:11",
            "expected an identifier or a function call"
        ],
    );
}

#[test]
fn invalid_macro_content() {
    assert_err_msg(
        r#"
{% macro input(label, type) %}
    {% macro nested() %}
    {% endmacro nested %}
{% endmacro input %}
    "#,
        &[
            "3:5",
            "unexpected tag; expected `{% endmacro %}` or the macro content"
        ]
    );
}

#[test]
fn invalid_macro_default_arg_value() {
    assert_err_msg(
        r#"
{% macro input(label=something) %}
{% endmacro input %}
    "#,
        &[
            "2:22",
            "expected an integer, a float, a string, or `true` or `false`"
        ]
    );
}

#[test]
fn invalid_elif() {
    assert_err_msg(
        r#"
{% if true %}
{% else %}
{% elif false %}
{% endif %}
    "#,
        &[
            "4:1",
            "unexpected tag; expected a `endif` tag` or some content"
        ]
    );
}

#[test]
fn invalid_else() {
        assert_err_msg(
        r#"
{% if true %}
{% else %}
{% else %}
{% endif %}
    "#,
        &[
            "4:1",
            "unexpected tag; expected a `endif` tag` or some content"
        ]
    );
}

#[test]
fn invalid_extends_position() {
        assert_err_msg(
        r#"
hello
{% extends "hey.html" %}
    "#,
        &[
            "3:1",
            "unexpected tag; expected some content"
        ]
    );
}

#[test]
fn invalid_operator() {
    assert_err_msg(
        "{{ hey =! }}",
        &[
            "1:8",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ]
    );
}

#[test]
fn missing_expression_with_not() {
        assert_err_msg(
        "{% if not %}",
        &[
            "1:11",
            "expected an expression"
        ]
    );
}

#[test]
fn missing_expression_in_if() {
        assert_err_msg(
        "{% if %}",
        &[
            "1:7",
            "expected any expression"
        ]
    );
}

#[test]
fn missing_container_name_in_forloop() {
        assert_err_msg(
        "{% for i in %}",
        &[
            "1:13",
            "expected an expression with an optional filter"
        ]
    );
}

#[test]
fn missing_variable_name_in_set() {
    assert_err_msg(
        "{% set = 1 %}",
        &[
            "1:8",
            "expected an identifier"
        ]
    );
}

#[test]
fn missing_value_in_set() {
    assert_err_msg(
        "{% set a =  %}",
        &[
            "1:13",
            "expected any expressions"
        ],
    );
}

#[test]
fn unterminated_fn_call() {
    assert_err_msg(
        "{{ a | slice( }}",
        &[
            "1:15",
            "expected a keyword argument: `key=value` where `value` can be any expression"
        ],
    );
}

#[test]
fn invalid_fn_call_missing_value() {
    assert_err_msg(
        "{{ a | slice(start=) }}",
        &[
            "1:20",
            "expected any expressions"
        ],
    );
}

#[test]
fn unterminated_macro_call() {
    assert_err_msg(
        "{{ my::macro( }}",
        &[
            "1:15",
            "expected a keyword argument: `key=value` where `value` can be any expression"
        ],
    );
}

#[test]
fn invalid_macro_call() {
    assert_err_msg(
        "{{ my:macro() }}",
        &[
            "1:6",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ],
    );
}

#[test]
fn unterminated_include() {
    assert_err_msg(
        "{% include %}",
        &[
            "1:12",
            "expected a string"
        ],
    );
}

#[test]
fn invalid_include_no_string() {
    assert_err_msg(
        "{% include 1 %}",
        &[
            "1:12",
            "expected a string"
        ],
    );
}

#[test]
fn unterminated_extends() {
    assert_err_msg(
        "{% extends %}",
        &[
            "1:12",
            "expected a string"
        ],
    );
}

#[test]
fn invalid_extends_no_string() {
    assert_err_msg(
        "{% extends 1 %}",
        &[
            "1:12",
            "expected a string"
        ],
    );
}

#[test]
fn invalid_import_macros_missing_filename() {
    assert_err_msg(
        "{% import as macros %}",
        &[
            "1:11",
            "expected a string"
        ],
    );
}

#[test]
fn invalid_import_macros_missing_namespace() {
    assert_err_msg(
        r#"{% import "hello" as %}"#,
        &[
            "1:22",
            "expected an identifier"
        ],
    );
}

#[test]
fn invalid_block_missing_name() {
    assert_err_msg(
        r#"{% block %}"#,
        &[
            "1:10",
            "expected an identifier"
        ],
    );
}

#[test]
fn unterminated_test() {
    assert_err_msg(
        r#"{% if a is odd( %}"#,
        &[
            "1:17",
            "expected a list of test arguments (any expressions)"
        ],
    );
}

#[test]
fn invalid_test_argument() {
    assert_err_msg(
        r#"{% if a is odd(key=1) %}"#,
        &[
            "1:19",
            "expected `or`, `and`, `<=`, `>=`, `<`, `>`, `==`, `!=`, `+`, `-`, `*`, `/`, `%`, or a filter"
        ],
    );
}

#[test]
fn unterminated_raw_tag() {
    assert_err_msg(
        r#"{% raw %}sd"#,
        &[
            "1:12",
            "expected `{% endraw %}`"
        ],
    );
}
<|endoftext|>