<commit_before>use std::ops::{Add, Sub, Div};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::collections::VecDeque;
use std::iter::repeat;
use std::fmt::{Debug, Formatter, Error};
use std::cmp::Ordering::Equal;
use std::f32;

#[derive(Clone,)]
pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Vec<f32>,
}

impl PartialEq for WordVec {
    fn eq(&self, other: &WordVec) -> bool {
        self.vec == other.vec
    }
}

impl Debug for WordVec {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        try!(self.word.fmt(fmt));
        try!(fmt.write_str(": "));
        try!(self.count.fmt(fmt));
        try!(fmt.write_str(", "));
        self.vec.iter().take(5).collect::<Vec<_>>().fmt(fmt)
    }
}

impl WordVec {

    fn dot_prod(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| x * y)
                .fold(0.0, |x, y| x + y)
    }

    fn magnitude(&self) -> f32 {
        self.vec.iter()
                .map(|x| x * x)
                .fold(0.0, |x, y| x + y)
                .sqrt()
    }

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter().zip(other.vec.iter())
                .map(|(x, y)| (x * x - y * y).abs().sqrt())
                .fold(0.0, |x, y| x + y)
    }

    fn new(word: String, num_words: usize) -> WordVec {
        WordVec {
            word: word,
            count: 0,
            vec: repeat(0.0).take(num_words).collect(),
        }
    }

    fn normalize(&mut self) {
        let count = self.count as f32;
        for f in &mut self.vec {
            *f /= count;
        }
    }

    fn rescale(&mut self, vec_scales: &Vec<(f32, f32)>) {
        for i in 0..self.vec.len() {
            let (min, max) = vec_scales[i];
            self.vec[i] = 2.0 * (self.vec[i] - min) / (max - min) - 1.0;
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }

}

impl Div<i32> for WordVec {
    type Output = WordVec;

    fn div(mut self, i: i32) -> WordVec {
        for f in &mut self.vec {
            *f /= i as f32;
        }

        self
    }
}

impl<'a> Add<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..newvec.vec.len() {
            newvec.vec[i] += other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} - {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..(newvec.vec.len()) {
            newvec.vec[i] -= other.vec[i];
        }

        newvec
    }
}

pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_width: usize,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct WordAcceptor<'a> {
    window: VecDeque<(usize, String)>,
    builder: &'a mut LanguageModelBuilder,
}

impl LanguageModelBuilder {
    pub fn new(window_radius: usize, words: Vec<String>) -> LanguageModelBuilder {

        let word_vecs = words.iter()
                             .map(|s| WordVec::new(s.clone(), words.len()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_width: 2 * window_radius + 1,
            words: words,
            word_vecs: word_vecs,
        }
    }

    pub fn build(mut self) -> LanguageModel {
        let len = self.word_vecs[0].vec.len();
        let mut vec_scales: Vec<(f32, f32)> = repeat((0.0, f32::MAX)).take(len).collect();

        for vec in self.word_vecs.iter_mut() {
            vec.normalize();
        }

        for (i, vec) in self.word_vecs.iter().map(|w| &w.vec).enumerate() {
            for j in (0..len).filter(|j| i != *j) {
                let (min, max) = vec_scales[j];
                vec_scales[j] = (min.min(vec[j]), max.max(vec[j]))
            }
        }

        for vec in self.word_vecs.iter_mut() {
            vec.rescale(&vec_scales);
        }

        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs,
        }
    }

    pub fn new_file<'a>(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            window: VecDeque::new(),
            builder: self,
        }
    }

    fn get_vec(&mut self, word: &str) -> &mut WordVec {
        let i = self.words[word];
        &mut self.word_vecs[i]
    }

    fn add_word(&mut self, from: usize, to: usize) {
        self.word_vecs[from].inc(to);
    }

    fn word_seen(&mut self, word: &str) {
        self.get_vec(word).count += 1;
    }
}

impl<'a> WordAcceptor<'a> {
    pub fn add_word(&mut self, word: String) {
        let ww = self.builder.window_width;
        let idx_opt = self.builder.words.get(&word).map(|w| *w);

        if let Some(next_idx) = idx_opt {
            self.window.push_back((next_idx, word));
            if self.window.len() > ww {
                self.window.pop_front();
            }

            if self.window.len() == ww {
                let (center_idx, ref center_word) = self.window[ww / 2];
                self.builder.word_seen(center_word);
                for i in (0..ww).filter(|a| *a != ww / 2) {
                    self.builder.add_word(center_idx, self.window[i].0);
                }
            }
        }
    }
}

impl LanguageModel {
    pub fn get(&self, word: &str) -> Option<&WordVec> {
        self.words.get(word).map(|i| &self.word_vecs[*i])
    }

    pub fn nearest_words(&self, word: &WordVec) -> Vec<&WordVec> {
        let mut vec_refs = self.word_vecs.iter().filter_map(|w| {
            let dist = w.distance(word);
            if w.word != word.word && !dist.is_nan() {
                Some((dist, w))
            } else {
                None
            }
        }).collect::<Vec<_>>();
        vec_refs.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or_else(|| {
            println!("bad vector: {:?}, {:?}", a, b);
            Equal
        }));
        vec_refs.into_iter().map(|x| x.1).collect()
    }
}<commit_msg>Some small bugfixes<commit_after>use std::ops::{Add, Sub, Div};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::collections::VecDeque;
use std::iter::repeat;
use std::fmt::{Debug, Formatter, Error};
use std::cmp::Ordering::Equal;
use std::f32;

#[derive(Clone,)]
pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Vec<f32>,
}

impl PartialEq for WordVec {
    fn eq(&self, other: &WordVec) -> bool {
        self.vec == other.vec
    }
}

impl Debug for WordVec {
    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
        try!(self.word.fmt(fmt));
        try!(fmt.write_str(": "));
        try!(self.count.fmt(fmt));
        try!(fmt.write_str(", "));
        self.vec.iter().take(5).collect::<Vec<_>>().fmt(fmt)
    }
}

impl WordVec {

    fn dot_prod(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| x * y)
                .fold(0.0, |x, y| x + y)
    }

    fn magnitude(&self) -> f32 {
        self.vec.iter()
                .map(|x| x * x)
                .fold(0.0, |x, y| x + y)
                .sqrt()
    }

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter().zip(other.vec.iter())
                .map(|(x, y)| (x * x - y * y).abs().sqrt())
                .fold(0.0, |x, y| x + y)
    }

    fn new(word: String, num_words: usize) -> WordVec {
        WordVec {
            word: word,
            count: 0,
            vec: repeat(0.0).take(num_words).collect(),
        }
    }

    fn normalize(&mut self) {
        let count = self.count as f32;
        for f in &mut self.vec {
            *f /= count;
        }
    }

    fn rescale(&mut self, vec_scales: &Vec<(f32, f32)>) {
        for i in 0..self.vec.len() {
            let (min, max) = vec_scales[i];
            if max != min {
                self.vec[i] = 2.0 * (self.vec[i] - min) / (max - min) - 1.0;
            }
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }

}

impl Div<i32> for WordVec {
    type Output = WordVec;

    fn div(mut self, i: i32) -> WordVec {
        for f in &mut self.vec {
            *f /= i as f32;
        }

        self
    }
}

impl<'a> Add<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..newvec.vec.len() {
            newvec.vec[i] += other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub<&'a WordVec> for WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        debug_assert!(self.vec.len() == other.vec.len());
        let mut newvec = WordVec {
            word: format!("{} - {}", &self.word, &other.word),
            count: 0,
            vec: self.vec,
        };

        for i in 0..(newvec.vec.len()) {
            newvec.vec[i] -= other.vec[i];
        }

        newvec
    }
}

pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_width: usize,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct WordAcceptor<'a> {
    window: VecDeque<(usize, String)>,
    builder: &'a mut LanguageModelBuilder,
}

impl LanguageModelBuilder {
    pub fn new(window_radius: usize, words: Vec<String>) -> LanguageModelBuilder {

        let word_vecs = words.iter()
                             .map(|s| WordVec::new(s.clone(), words.len()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_width: 2 * window_radius + 1,
            words: words,
            word_vecs: word_vecs,
        }
    }

    pub fn build(mut self) -> LanguageModel {
        let len = self.word_vecs[0].vec.len();
        let mut vec_scales: Vec<(f32, f32)> = repeat((f32::MAX, 0.0)).take(len).collect();

        for vec in self.word_vecs.iter_mut() {
            vec.normalize();
        }

        for (i, vec) in self.word_vecs.iter().map(|w| &w.vec).enumerate() {
            for j in (0..len).filter(|j| i != *j) {
                let (min, max) = vec_scales[j];
                vec_scales[j] = (min.min(vec[j]), max.max(vec[j]))
            }
        }

        for vec in self.word_vecs.iter_mut() {
            vec.rescale(&vec_scales);
        }

        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs,
        }
    }

    pub fn new_file<'a>(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            window: VecDeque::new(),
            builder: self,
        }
    }

    fn get_vec(&mut self, word: &str) -> &mut WordVec {
        let i = self.words[word];
        &mut self.word_vecs[i]
    }

    fn add_word(&mut self, from: usize, to: usize) {
        self.word_vecs[from].inc(to);
    }

    fn word_seen(&mut self, word: &str) {
        self.get_vec(word).count += 1;
    }
}

impl<'a> WordAcceptor<'a> {
    pub fn add_word(&mut self, word: String) {
        let ww = self.builder.window_width;
        let idx_opt = self.builder.words.get(&word).map(|w| *w);

        if let Some(next_idx) = idx_opt {
            self.window.push_back((next_idx, word));
            if self.window.len() > ww {
                self.window.pop_front();
            }

            if self.window.len() == ww {
                let (center_idx, ref center_word) = self.window[ww / 2];
                self.builder.word_seen(center_word);
                for i in (0..ww).filter(|a| *a != ww / 2) {
                    self.builder.add_word(center_idx, self.window[i].0);
                }
            }
        }
    }
}

impl LanguageModel {
    pub fn get(&self, word: &str) -> Option<&WordVec> {
        self.words.get(word).map(|i| &self.word_vecs[*i])
    }

    pub fn nearest_words(&self, word: &WordVec) -> Vec<&WordVec> {
        let mut vec_refs = self.word_vecs.iter().filter_map(|w| {
            let dist = w.distance(word);
            if w.word != word.word {
                Some((dist, w))
            } else {
                None
            }
        }).collect::<Vec<_>>();
        vec_refs.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or_else(|| {
            println!("bad vector: {:?}, {:?}", a, b);
            Equal
        }));
        vec_refs.into_iter().map(|x| x.1).collect()
    }
}<|endoftext|>


<commit_before>use std::ops::{Add, Sub};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::collections::VecDeque;
use std::cmp::Ordering::Equal;

struct WordVecBuilder {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

impl WordVecBuilder {
    fn new(word: String) -> WordVecBuilder {
        WordVecBuilder {
            word: word,
            count: 0,
            vec: Box::new([0f32; 10_000]),
        }
    }

    fn normalize(mut self, word_freqs: &Vec<f32>) -> WordVec {
        let count = self.count as f32;
        for i in 0..word_freqs.len() {
            self.vec[i] = self.vec[i] / count /  word_freqs[i];
        }

        WordVec {
            word: self.word,
            count: self.count,
            vec: self.vec,
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }
}

impl WordVec {

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| (x*x - y*y).abs().sqrt())
                .fold(0.0, |x, y| x + y)
    }

}

impl<'a> Add for &'a WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] + other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub for &'a WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] - other.vec[i];
        }

        newvec
    }
}

pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_width: u32,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVecBuilder>,
}

pub struct WordAcceptor<'a> {
    window: VecDeque<String>,
    focus: usize,
    builder: &'a mut LanguageModelBuilder,
}

impl<'a> LanguageModelBuilder {
    pub fn new(words: Vec<String>) -> LanguageModelBuilder {
        assert_eq!(words.len(), 10_000);

        let word_vecs = words.iter()
                             .map(|s| WordVecBuilder::new(s.clone()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_width: 21,
            words: words,
            word_vecs: word_vecs,
        }
    }

    pub fn build(self) -> LanguageModel {
        let total_words = self.word_vecs.iter().fold(0.0, |a, b| a + b.count as f32);
        let word_freqs = self.word_vecs.iter()
                                       .map(|v| v.count as f32 / total_words)
                                       .collect::<Vec<_>>();
        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs.into_iter()
                                     .map(|b| b.normalize(&word_freqs))
                                     .collect(),
        }
    }

    pub fn new_file(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            window: VecDeque::new(),
            focus: 0,
            builder: self,
        }
    }

    fn add_word(&mut self, word: &str) {
        let idx = self.words[word];
        let word_vec = &mut self.word_vecs[idx];
        word_vec.inc(idx);
    }
}

impl<'a> WordAcceptor<'a> {
    pub fn add_word(&mut self, word: String) {
        let allow_word = self.builder.words.contains_key(&word);

        if allow_word {
            self.window.push_back(word);
            if (self.window.len() > 21) {
                self.window.pop_front();
            }
            if (self.window.len() == 21) {
                for i in (0..21).filter(|a| *a != 10) {
                    self.builder.add_word(&self.window[i]);
                }
            }
        }
    }
}

impl LanguageModel {
    pub fn get(&self, word: &str) -> Option<&WordVec> {
        if let Some(i) = self.words.get(word) {
            Some(&self.word_vecs[*i])
        } else {
            None
        }
    }

    pub fn nearest_words(&self, word: &WordVec) -> Vec<&WordVec> {
        let mut vec_refs = self.word_vecs.iter().map(|v| v).collect::<Vec<_>>();
        vec_refs.sort_by(|a, b| a.distance(word).partial_cmp(&b.distance(word)).unwrap_or(Equal));
        vec_refs
    }
}<commit_msg>Slight simplification<commit_after>use std::ops::{Add, Sub};
use std::collections::HashMap;
use std::iter::FromIterator;
use std::collections::VecDeque;
use std::cmp::Ordering::Equal;

struct WordVecBuilder {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

pub struct WordVec {
    pub word: String,
    pub count: u64,
    vec: Box<[f32; 10_000]>,
}

impl WordVecBuilder {
    fn new(word: String) -> WordVecBuilder {
        WordVecBuilder {
            word: word,
            count: 0,
            vec: Box::new([0f32; 10_000]),
        }
    }

    fn normalize(mut self, word_freqs: &Vec<f32>) -> WordVec {
        let count = self.count as f32;
        for i in 0..word_freqs.len() {
            self.vec[i] = self.vec[i] / count /  word_freqs[i];
        }

        WordVec {
            word: self.word,
            count: self.count,
            vec: self.vec,
        }
    }

    fn inc(&mut self, i: usize) {
        self.vec[i] += 1.0;
    }
}

impl WordVec {

    pub fn distance(&self, other: &WordVec) -> f32 {
        self.vec.iter()
                .zip(other.vec.iter())
                .map(|(x, y)| (x*x - y*y).abs().sqrt())
                .fold(0.0, |x, y| x + y)
    }

}

impl<'a> Add for &'a WordVec {
    type Output = WordVec;

    fn add(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] + other.vec[i];
        }

        newvec
    }
}

impl<'a> Sub for &'a WordVec {
    type Output = WordVec;

    fn sub(self, other: &WordVec) -> WordVec {
        let mut newvec = WordVec {
            word: format!("{} + {}", &self.word, &other.word),
            count: 0,
            vec: Box::new([0f32; 10_000]),
        };

        for i in 0..10_000 {
            newvec.vec[i] = self.vec[i] - other.vec[i];
        }

        newvec
    }
}

pub struct LanguageModel {
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVec>,
}

pub struct LanguageModelBuilder {
    window_width: u32,
    words: HashMap<String, usize>,
    word_vecs: Vec<WordVecBuilder>,
}

pub struct WordAcceptor<'a> {
    window: VecDeque<String>,
    focus: usize,
    builder: &'a mut LanguageModelBuilder,
}

impl<'a> LanguageModelBuilder {
    pub fn new(words: Vec<String>) -> LanguageModelBuilder {
        assert_eq!(words.len(), 10_000);

        let word_vecs = words.iter()
                             .map(|s| WordVecBuilder::new(s.clone()))
                             .collect();

        let words = HashMap::from_iter(
                        words.into_iter()
                             .enumerate()
                             .map(|(a, b)| (b, a)));

        LanguageModelBuilder {
            window_width: 21,
            words: words,
            word_vecs: word_vecs,
        }
    }

    pub fn build(self) -> LanguageModel {
        let total_words = self.word_vecs.iter().fold(0.0, |a, b| a + b.count as f32);
        let word_freqs = self.word_vecs.iter()
                                       .map(|v| v.count as f32 / total_words)
                                       .collect::<Vec<_>>();
        LanguageModel {
            words: self.words,
            word_vecs: self.word_vecs.into_iter()
                                     .map(|b| b.normalize(&word_freqs))
                                     .collect(),
        }
    }

    pub fn new_file(&'a mut self) -> WordAcceptor<'a> {
        WordAcceptor {
            window: VecDeque::new(),
            focus: 0,
            builder: self,
        }
    }

    fn add_word(&mut self, word: &str) {
        let idx = self.words[word];
        let word_vec = &mut self.word_vecs[idx];
        word_vec.inc(idx);
    }
}

impl<'a> WordAcceptor<'a> {
    pub fn add_word(&mut self, word: String) {
        let allow_word = self.builder.words.contains_key(&word);

        if allow_word {
            self.window.push_back(word);
            if (self.window.len() > 21) {
                self.window.pop_front();
            }
            if (self.window.len() == 21) {
                for i in (0..21).filter(|a| *a != 10) {
                    self.builder.add_word(&self.window[i]);
                }
            }
        }
    }
}

impl LanguageModel {
    pub fn get(&self, word: &str) -> Option<&WordVec> {
        if let Some(i) = self.words.get(word) {
            Some(&self.word_vecs[*i])
        } else {
            None
        }
    }

    pub fn nearest_words(&self, word: &WordVec) -> Vec<&WordVec> {
        let mut vec_refs = self.word_vecs.iter().collect::<Vec<_>>();
        vec_refs.sort_by(|a, b| a.distance(word).partial_cmp(&b.distance(word)).unwrap_or(Equal));
        vec_refs
    }
}<|endoftext|>


<commit_before>use std::collections::HashMap;
// use num::traits::Num;

use core::{Block , BlockType , MutBlock , BlockId};
use graph::{Graph , NodeIndex};

// pub struct Buffer
// {
//
// }

// we could pull the vector graph out .. eg NodeData<T> and have module hold a graph ..

pub struct Module<'b>
{
    graph: Graph<& 'b BlockType<'b>>,
    buffers: Vec<Vec<u8>>,   ///TODO try a single buffer .
    buffers_for_node: HashMap< usize, Vec<usize>>

}

impl<'b> Module<'b>
{
    pub fn new() -> Module<'b> {
        Module { graph: Graph::<& 'b BlockType<'b>>::new() , buffers: Vec::new() , buffers_for_node: HashMap::new() }
    }

    //TODO NodeIndex replace with nodeid IF unique
    pub fn add_box_block(& mut self, block: & 'b mut BlockType<'b>) -> NodeIndex {

        self.graph.add_node(block)
    }

    pub fn add_link(& mut self, from: NodeIndex , to: NodeIndex) {  self.graph.add_edge(from, to); }

    /// this will get WAY more complicated ..
    /// we need to handle loops which never finish , re run on completion , threading , timed and priority etc
    pub fn process_blocks(& 'b mut self)
    {
        self.process_rec(0);

    }

    pub fn process_rec(&mut self , index: NodeIndex )
    {
        let successor_ids: Vec<NodeIndex> = { self.graph.successors(index).collect()};

        {
            let node: & BlockType = { self.graph.get_node(index)};
            self.process( node);

        }

        for i in successor_ids {
            self.process_rec ( i);
        }

    }

    fn process(& self ,  block: & 'b BlockType)
    {
        match *block {
            BlockType::Block(b) => self.process_block (b ),
            BlockType::MutBlock(ref b) => process_mut_block( *b ),
            BlockType::FunctionBlock(_) => {}
        }
    }

    // fn get_buffer_ids(&self ,  block_id: BlockId) -> Vec< (usize , usize , bool)>
    // {
    //
    //     self.buffers_for_node.get(block_id).map( )
    // }

    // fn get_buffers(& self ,  block_id: BlockId) -> Vec<& 'b [u8]>
    // {
    //
    //     let bufs = self.buffers_for_node.get(& (block_id as usize) ).iter( )
    //         .map(|x| & (self.buffers[block_id as usize] ) [..] );
    //
    //     bufs.collect::<Vec<_>>()
    //     //a.iter().map(|&x| 2 * x);
    //     //buffer_for_node: HashMap< usize, Vec< (usize , usize , bool)>>
    // }

    fn get_buffers(&self,  block_id: BlockId) -> Vec<&[u8]>
     {

         let bufs = self.buffers_for_node[&(block_id as usize)].iter( )
             .map(|x| &self.buffers[block_id as usize][..] );

         bufs.collect::<Vec<_>>()
     }

    //    Block (Box<Block + 'b> ),
    fn process_block(& self ,  blockType: & 'b Block)
    {
        let buffers = self.get_buffers(blockType.get_id());
        //let mut_buffers = get_mut_buffers(blockType.get_id());

        //  process(&mut self , data: & [u8] , inputs: & [u8] , outputs: & mut [u8])
        //*blockType.
    }

}

fn process_mut_block( blockType: &MutBlock)
{

}

// we dont really need an interface , maybe for network
pub trait IModule
{
    //fn process(<Vec<O>>) -> Vec<O>;
    //(args: &[&str])
    //fn process(&[ &[O] ]) -> Vec<O>;
    fn process_blocks(&mut self) ; // or return slice
    // void LoadInputsIntoModules();
    //  void ProcessFirstBlock();
    //  INeuronBlock GetEntryBlock();
    //  INeuronBlock GetBlock(uint id);
    //  void AddRunningTasks(Task task);
    //  void RemoveRunningTasks(Task task);

}
<commit_msg>change readme<commit_after>use std::collections::HashMap;
// use num::traits::Num;

use core::{Block , BlockType , MutBlock , BlockId};
use graph::{Graph , NodeIndex};

// pub struct Buffer
// {
//
// }

// we could pull the vector graph out .. eg NodeData<T> and have module hold a graph ..

pub struct Module<'b>
{
    graph: Graph<& 'b BlockType<'b>>,
    buffers: Vec<Vec<u8>>,   ///TODO try a single buffer .
    buffers_for_node: HashMap< usize, Vec<usize>>

}

impl<'b> Module<'b>
{
    pub fn new() -> Module<'b> {
        Module { graph: Graph::<& 'b BlockType<'b>>::new() , buffers: Vec::new() , buffers_for_node: HashMap::new() }
    }

    //TODO NodeIndex replace with nodeid IF unique
    pub fn add_box_block(& mut self, block: & 'b mut BlockType<'b>) -> NodeIndex {

        self.graph.add_node(block)
    }

    pub fn add_link(& mut self, from: NodeIndex , to: NodeIndex) {  self.graph.add_edge(from, to); }

    /// this will get WAY more complicated ..
    /// we need to handle loops which never finish , re run on completion , threading , timed and priority etc  1
    pub fn process_blocks(& 'b mut self)
    {
        self.process_rec(0);

    }

    pub fn process_rec(&mut self , index: NodeIndex )
    {
        let successor_ids: Vec<NodeIndex> = { self.graph.successors(index).collect()};

        {
            let node: & BlockType = { self.graph.get_node(index)};
            self.process( node);

        }

        for i in successor_ids {
            self.process_rec ( i);
        }

    }

    fn process(& self ,  block: & 'b BlockType)
    {
        match *block {
            BlockType::Block(b) => self.process_block (b ),
            BlockType::MutBlock(ref b) => process_mut_block( *b ),
            BlockType::FunctionBlock(_) => {}
        }
    }

    // fn get_buffer_ids(&self ,  block_id: BlockId) -> Vec< (usize , usize , bool)>
    // {
    //
    //     self.buffers_for_node.get(block_id).map( )
    // }

    // fn get_buffers(& self ,  block_id: BlockId) -> Vec<& 'b [u8]>
    // {
    //
    //     let bufs = self.buffers_for_node.get(& (block_id as usize) ).iter( )
    //         .map(|x| & (self.buffers[block_id as usize] ) [..] );
    //
    //     bufs.collect::<Vec<_>>()
    //     //a.iter().map(|&x| 2 * x);
    //     //buffer_for_node: HashMap< usize, Vec< (usize , usize , bool)>>
    // }

    fn get_buffers(&self,  block_id: BlockId) -> Vec<&[u8]>
     {

         let bufs = self.buffers_for_node[&(block_id as usize)].iter( )
             .map(|x| &self.buffers[block_id as usize][..] );

         bufs.collect::<Vec<_>>()
     }

    //    Block (Box<Block + 'b> ),
    fn process_block(& self ,  blockType: & 'b Block)
    {
        let buffers = self.get_buffers(blockType.get_id());
        //let mut_buffers = get_mut_buffers(blockType.get_id());

        //  process(&mut self , data: & [u8] , inputs: & [u8] , outputs: & mut [u8])
        //*blockType.
    }

}

fn process_mut_block( blockType: &MutBlock)
{

}

// we dont really need an interface , maybe for network
pub trait IModule
{
    //fn process(<Vec<O>>) -> Vec<O>;
    //(args: &[&str])
    //fn process(&[ &[O] ]) -> Vec<O>;
    fn process_blocks(&mut self) ; // or return slice
    // void LoadInputsIntoModules();
    //  void ProcessFirstBlock();
    //  INeuronBlock GetEntryBlock();
    //  INeuronBlock GetBlock(uint id);
    //  void AddRunningTasks(Task task);
    //  void RemoveRunningTasks(Task task);

}
<|endoftext|>


<commit_before>#![feature(test)]

extern crate test;
extern crate rustc_serialize;
extern crate data_encoding;

use test::Bencher;
use rustc_serialize::base64::{FromBase64, ToBase64, STANDARD};
use data_encoding::base64;

fn encode<F: FnMut(&[u8]) -> Vec<u8>>(b: &mut Bencher, mut f: F) {
    let input = &[0u8; 4096];
    b.iter(|| f(input));
}

fn decode<E, F: FnMut(&[u8]) -> Result<Vec<u8>, E>>(b: &mut Bencher, mut f: F) {
    let input = &[b'A'; 4096];
    b.iter(|| f(input));
}

#[bench]
fn data_encode(b: &mut Bencher) {
    encode(b, |x| base64::encode(x).into_bytes());
}

#[bench]
fn rustc_encode(b: &mut Bencher) {
    encode(b, |x| x.to_base64(STANDARD).into_bytes());
}

#[bench]
fn data_decode(b: &mut Bencher) {
    decode(b, base64::decode);
}

#[bench]
fn rustc_decode(b: &mut Bencher) {
    decode(b, |x| x.from_base64());
}
<commit_msg>benches/diff.rs: Rename functions<commit_after>#![feature(test)]

extern crate test;
extern crate rustc_serialize;
extern crate data_encoding;

use test::Bencher;
use rustc_serialize::base64::{FromBase64, ToBase64, STANDARD};
use data_encoding::base64;

fn encode<F: FnMut(&[u8]) -> String>(b: &mut Bencher, mut f: F) {
    let input = &[0u8; 4096];
    b.iter(|| f(input));
}

fn decode<E, F: FnMut(&[u8]) -> Result<Vec<u8>, E>>(b: &mut Bencher, mut f: F) {
    let input = &[b'A'; 4096];
    b.iter(|| f(input));
}

#[bench]
fn encode_crate(b: &mut Bencher) {
    encode(b, base64::encode);
}

#[bench]
fn encode_rustc(b: &mut Bencher) {
    encode(b, |x| x.to_base64(STANDARD));
}

#[bench]
fn decode_crate(b: &mut Bencher) {
    decode(b, base64::decode);
}

#[bench]
fn decode_rustc(b: &mut Bencher) {
    decode(b, |x| x.from_base64());
}
<|endoftext|>


<commit_before>use std::iter;
use super::QueueOfStrings;

const INITIAL_QUEUE_CAPACITY: usize = 2;

pub struct ResizingArrayQueueOfStrings {
    q: Vec<Option<String>>,
    head: usize,
    tail: usize
}

impl ResizingArrayQueueOfStrings {
    pub fn with_capacity(capacity: usize) -> ResizingArrayQueueOfStrings {
        let storage = iter::repeat(None).take(capacity).collect();

        ResizingArrayQueueOfStrings {
            q: storage,
            head: 0,
            tail: 0
        }
    }

    fn resize(&mut self, capacity: usize) {
        let cap = self.q.len();
        let mut new_storage: Vec<Option<String>> = iter::repeat(None).take(capacity).collect();
        let tail = if self.tail > self.head {
            self.tail
        } else {
            self.tail + cap
        };
        for i in self.head .. tail{
            new_storage[i] = self.q[i % cap].take();
        }
        self.q = new_storage;
        // self.head = self.head
        self.tail = tail
    }
}

impl QueueOfStrings for ResizingArrayQueueOfStrings {
    fn new() -> ResizingArrayQueueOfStrings {
        let storage = iter::repeat(None).take(INITIAL_QUEUE_CAPACITY).collect();

        ResizingArrayQueueOfStrings {
            q: storage,
            head: 0,
            tail: 0
        }
    }

    fn is_empty(&self) -> bool {
        self.head == self.tail
    }

    fn enqueue(&mut self, item: String) {
        let mut cap = self.q.len();
        if self.q[self.tail % cap].is_some() {
            cap = 2 * cap;
            self.resize(cap);
        }
        self.q[self.tail % cap] = Some(item);
        self.tail = (self.tail + 1) % cap
    }

    fn dequeue(&mut self) -> String {
        let cap = self.q.len();
        let item = self.q[self.head % cap].take();
        self.head = (self.head + 1) % cap;
        item.unwrap()
    }
}

#[test]
fn test_resizing_array_queue() {
    let mut queue: ResizingArrayQueueOfStrings = QueueOfStrings::new();

    let mut result = "to be or not to be".split(' ');

    for s in "to be or not to - be - - that - - - is".split(' ') {
        if s == "-" {
            assert_eq!(&queue.dequeue(), result.next().unwrap())
        } else {
            queue.enqueue(s.into())
        }
    }
}
<commit_msg>stacks_and_queues/resizing_array_queue: add generic ResizingArrayQueue<commit_after>use std::iter;
use super::{QueueOfStrings, Queue};

const INITIAL_QUEUE_CAPACITY: usize = 2;

pub struct ResizingArrayQueueOfStrings {
    q: Vec<Option<String>>,
    head: usize,
    tail: usize
}

impl ResizingArrayQueueOfStrings {
    pub fn with_capacity(capacity: usize) -> ResizingArrayQueueOfStrings {
        let storage = iter::repeat(None).take(capacity).collect();

        ResizingArrayQueueOfStrings {
            q: storage,
            head: 0,
            tail: 0
        }
    }

    fn resize(&mut self, capacity: usize) {
        let cap = self.q.len();
        let mut new_storage: Vec<Option<String>> = iter::repeat(None).take(capacity).collect();
        let tail = if self.tail > self.head {
            self.tail
        } else {
            self.tail + cap
        };
        for i in self.head .. tail{
            new_storage[i] = self.q[i % cap].take();
        }
        self.q = new_storage;
        // self.head = self.head
        self.tail = tail
    }
}

impl QueueOfStrings for ResizingArrayQueueOfStrings {
    fn new() -> ResizingArrayQueueOfStrings {
        let storage = iter::repeat(None).take(INITIAL_QUEUE_CAPACITY).collect();

        ResizingArrayQueueOfStrings {
            q: storage,
            head: 0,
            tail: 0
        }
    }

    fn is_empty(&self) -> bool {
        self.head == self.tail
    }

    fn enqueue(&mut self, item: String) {
        let mut cap = self.q.len();
        if self.q[self.tail % cap].is_some() {
            cap = 2 * cap;
            self.resize(cap);
        }
        self.q[self.tail % cap] = Some(item);
        self.tail = (self.tail + 1) % cap
    }

    fn dequeue(&mut self) -> String {
        let cap = self.q.len();
        let item = self.q[self.head % cap].take();
        self.head = (self.head + 1) % cap;
        item.unwrap()
    }
}

// generic ResizingArrayQueue
pub struct ResizingArrayQueue<T> {
    q: Vec<Option<T>>,
    head: usize,
    tail: usize
}

impl<T> ResizingArrayQueue<T> {
    pub fn with_capacity(capacity: usize) -> ResizingArrayQueue<T> {
        let mut storage = Vec::with_capacity(capacity);
        unsafe { storage.set_len(capacity) };

        ResizingArrayQueue {
            q: storage,
            head: 0,
            tail: 0
        }
    }

    fn resize(&mut self, capacity: usize) {
        let cap = self.q.len();
        let mut new_storage: Vec<Option<T>> = Vec::with_capacity(capacity);
        unsafe { new_storage.set_len(capacity) };

        let tail = if self.tail > self.head {
            self.tail
        } else {
            self.tail + cap
        };
        for i in self.head .. tail{
            new_storage[i] = self.q[i % cap].take();
        }
        self.q = new_storage;
        self.tail = tail
    }
}

impl<T> Queue<T> for ResizingArrayQueue<T> {
    fn new() -> ResizingArrayQueue<T> {
        ResizingArrayQueue::with_capacity(INITIAL_QUEUE_CAPACITY)
    }

    fn is_empty(&self) -> bool {
        self.head == self.tail
    }

    fn enqueue(&mut self, item: T) {
        let mut cap = self.q.len();
        if self.q[self.tail % cap].is_some() {
            cap = 2 * cap;
            self.resize(cap);
        }
        self.q[self.tail % cap] = Some(item);
        self.tail = (self.tail + 1) % cap
    }

    fn dequeue(&mut self) -> T {
        let cap = self.q.len();
        let item = self.q[self.head % cap].take();
        self.head = (self.head + 1) % cap;
        item.unwrap()
    }
}

#[test]
fn test_resizing_array_queue_of_strings() {
    let mut queue: ResizingArrayQueueOfStrings = QueueOfStrings::new();

    let mut result = "to be or not to be".split(' ');

    for s in "to be or not to - be - - that - - - is".split(' ') {
        if s == "-" {
            assert_eq!(&queue.dequeue(), result.next().unwrap())
        } else {
            queue.enqueue(s.into())
        }
    }
}

#[test]
fn test_resizing_array_queue() {
    let mut queue: ResizingArrayQueue<String> = Queue::new();

    let mut result = "to be or not to be".split(' ');

    for s in "to be or not to - be - - that - - - is".split(' ') {
        if s == "-" {
            assert_eq!(&queue.dequeue(), result.next().unwrap())
        } else {
            queue.enqueue(s.into())
        }
    }
}
<|endoftext|>


<commit_before>use std::str;
use std::thread;
use std::sync::Mutex;
use std::cell::RefCell;
use std::time::{Duration,Instant};
use std::iter::repeat;
use std::collections::BTreeMap;
use std::collections::VecDeque;
use std::fmt::Arguments;
use std::net::SocketAddr;
use mio::net::UdpSocket;
use std::io::{self,BufWriter,Write,Error,ErrorKind};
use nom::HexDisplay;
use hdrhistogram::Histogram;
use sozu_command::buffer::Buffer;
use sozu_command::messages::{FilteredData,MetricsData,Percentiles,BackendMetricsData,FilteredTimeSerie};

use super::{Subscriber,MetricData,StoredMetricData};

const VERSION: &'static str = env!("CARGO_PKG_VERSION");

#[derive(Debug,Clone,PartialEq)]
pub struct MetricLine {
  label:      &'static str,
  app_id:     Option<String>,
  backend_id: Option<String>,
  /// in milliseconds
  duration:   usize,
}

pub struct MetricSocket {
  pub addr:   SocketAddr,
  pub socket: UdpSocket,
}

impl Write for MetricSocket {
  fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
    self.socket.send_to(buf, &self.addr)
  }

  fn flush(&mut self) -> io::Result<()> {
    Ok(())
  }
}

pub struct NetworkDrain {
  queue:              VecDeque<MetricLine>,
  buffer:             Buffer,
  pub prefix:         String,
  pub remote:         BufWriter<MetricSocket>,
  is_writable:        bool,
  data:               BTreeMap<String, StoredMetricData>,
  /// (app_id, key) -> metric
  app_data:           BTreeMap<(String, String), StoredMetricData>,
  /// (app_id, backend_id, key) -> metric
  backend_data:       BTreeMap<(String, String, String), StoredMetricData>,
  pub use_tagged_metrics: bool,
  pub origin:         String,
  created:            Instant,
}

impl NetworkDrain {
  pub fn new(prefix: String, socket: UdpSocket, addr: SocketAddr) -> Self {
    NetworkDrain {
      queue:  VecDeque::new(),
      buffer: Buffer::with_capacity(2048),
      prefix: prefix,
      remote: BufWriter::new(MetricSocket {
        addr, socket
      }),
      is_writable: true,
      data: BTreeMap::new(),
      app_data: BTreeMap::new(),
      backend_data: BTreeMap::new(),
      use_tagged_metrics: false,
      origin: String::from("x"),
      created: Instant::now(),
    }
  }

  pub fn writable(&mut self) {
    self.is_writable = true;
  }

  pub fn send_data(&mut self) {
    let now  = Instant::now();
    let secs = Duration::new(2, 0);

    if self.is_writable {
    for (ref key, ref mut stored_metric) in self.data.iter_mut().filter(|&(_, ref value)| now.duration_since(value.last_sent) > secs) {
      //info!("will write {} -> {:#?}", key, stored_metric);
      let res = match stored_metric.data {
        MetricData::Gauge(value) => {
          if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.{},origin={},version={}:{}|g\n", self.prefix, key, self.origin, VERSION, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.{}:{}|g\n", self.prefix, self.origin, key, value))
          }
        },
        MetricData::Count(value) => {
          if value == 0 {
            stored_metric.last_sent = now;
          }

          let res = if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.{},origin={},version={}:{}|c\n", self.prefix, key, self.origin, VERSION, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.{}:{}|c\n", self.prefix, self.origin, key, value))
          };

          if res.is_ok() {
            stored_metric.data = MetricData::Count(0);
          }

          res
        },
        _ => {Ok(())}
      };

      match res {
        Ok(()) => {
          stored_metric.last_sent = now;
        },
        Err(e) => match e.kind() {
          ErrorKind::WriteZero => {
            self.remote.flush();
          },
          ErrorKind::WouldBlock => {
            error!("WouldBlock while writing global metrics to socket");
            self.is_writable = false;
            break;
          },
          e => {
            error!("metrics socket write error={:?}", e);
            break;
          },
        }
      }
    }
    }

    self.remote.flush();

    if self.is_writable {
    for (ref key, ref mut stored_metric) in self.app_data.iter_mut().filter(|&(_, ref value)| now.duration_since(value.last_sent) > secs) {
      //info!("will write {} -> {:#?}", key, stored_metric);
      let res = match stored_metric.data {
        MetricData::Gauge(value) => {
          if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.app.{},origin={},version={},app_id={}:{}|g\n",
              self.prefix, key.1, self.origin, VERSION, key.0, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.app.{}.{}:{}|g\n", self.prefix, self.origin, key.0, key.1, value))
          }
        },
        MetricData::Count(value) => {
          if value == 0 {
            stored_metric.last_sent = now;
          }

          let res = if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.app.{},origin={},version={},app_id={}:{}|c\n",
              self.prefix, key.1, self.origin, VERSION, key.0, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.app.{}.{}:{}|c\n", self.prefix, self.origin, key.0, key.1, value))
          };

          if res.is_ok() {
            stored_metric.data = MetricData::Count(0);
          }

          res
        },
        _ => {Ok(())}
      };

      match res {
        Ok(()) => {
          stored_metric.last_sent = now;
        },
        Err(e) => match e.kind() {
          ErrorKind::WriteZero => {
            self.remote.flush();
          },
          ErrorKind::WouldBlock => {
            error!("WouldBlock while writing app metrics to socket");
            self.is_writable = false;
            break;
          },
          e => {
            error!("metrics socket write error={:?}", e);
            break;
          },
        }
      }
    }
    }

    self.remote.flush();

    if self.is_writable {
    for (ref key, ref mut stored_metric) in self.backend_data.iter_mut().filter(|&(_, ref value)| now.duration_since(value.last_sent) > secs) {
      //info!("will write {} -> {:#?}", key, stored_metric);
      let res = match stored_metric.data {
        MetricData::Gauge(value) => {
          if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.backend.{},origin={},version={},app_id={},backend_id={}:{}|g\n",
              self.prefix, key.2, self.origin, VERSION, key.0, key.1, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.app.{}.backend.{}.{}:{}|g\n", self.prefix, self.origin, key.0, key.1, key.2, value))
          }
        },
        MetricData::Count(value) => {
          if value == 0 {
            stored_metric.last_sent = now;
          }

          let res = if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.app.{},origin={},version={},app_id={},backend_id={}:{}|c\n",
              self.prefix, key.2, self.origin, VERSION, key.0, key.1, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.app.{}.backend.{}.{}:{}|c\n", self.prefix, self.origin, key.0, key.1, key.2, value))
          };

          if res.is_ok() {
            stored_metric.data = MetricData::Count(0);
          }

          res
        },
        _ => {Ok(())}
      };

      match res {
        Ok(()) => {
          stored_metric.last_sent = now;
        },
        Err(e) => match e.kind() {
          ErrorKind::WriteZero => {
            self.remote.flush();
          },
          ErrorKind::WouldBlock => {
            error!("WouldBlock while writing backend metrics to socket");
            self.is_writable = false;
            break;
          },
          e => {
            error!("metrics socket write error={:?}", e);
            break;
          },
        }
      }
    }
    }
    self.remote.flush();

    if self.is_writable {
    for metric in self.queue.drain(..) {
       let res = match (metric.app_id, metric.backend_id) {
        (Some(app_id), Some(backend_id)) => {
          if self.use_tagged_metrics {
            self.buffer.write_fmt(format_args!("{}.backend.{},origin={},version={},app_id={},backend_id={}:{}|ms\n",
              self.prefix, metric.label, self.origin, VERSION, app_id, backend_id, metric.duration))
          } else {
            self.buffer.write_fmt(format_args!("{}.{}.app.{}.backend.{}.{}:{}|ms\n", self.prefix, self.origin, app_id, backend_id,
              metric.label, metric.duration))
          }
        },
        (Some(app_id), None) => {
          if self.use_tagged_metrics {
            self.buffer.write_fmt(format_args!("{}.app.{},origin={},version={},app_id={}:{}|ms\n",
              self.prefix, metric.label, self.origin, VERSION, app_id, metric.duration))
          } else {
            self.buffer.write_fmt(format_args!("{}.{}.app.{}.{}:{}|ms\n", self.prefix, self.origin, app_id,
              metric.label, metric.duration))
          }
        },
        (None, None) => {
          if self.use_tagged_metrics {
            self.buffer.write_fmt(format_args!("{}.{},origin={},version={}:{}|ms\n",
              self.prefix, metric.label, self.origin, VERSION, metric.duration))
          } else {
            self.buffer.write_fmt(format_args!("{}.{}.{}:{}|ms\n", self.prefix, self.origin,
              metric.label, metric.duration))
          }
        },
        _ => { Ok(()) }
      };

      match res {
        Ok(()) => {
        },
        Err(e) => match e.kind() {
          ErrorKind::WriteZero => {
            self.remote.flush();
          },
          ErrorKind::WouldBlock => {
            error!("WouldBlock while writing timing metrics to socket");
            self.is_writable = false;
            break;
          },
          e => {
            error!("metrics socket write error={:?}", e);
            break;
          },
        }
      }
    }
    self.remote.flush();
    }
  }
}

impl Subscriber for NetworkDrain {
  fn receive_metric(&mut self, key: &'static str, app_id: Option<&str>, backend_id: Option<&str>, metric: MetricData) {
    if metric.is_time() {
      if let MetricData::Time(millis) = metric {
        self.queue.push_back(MetricLine {
          label: key,
          app_id: app_id.map(|s| s.to_string()),
          backend_id: backend_id.map(|s| s.to_string()),
          duration: millis,
        });
      }
    } else {
      if let Some(id) = app_id {
        if let Some(bid) = backend_id {
          let k = (String::from(id), String::from(bid), String::from(key));
          if !self.backend_data.contains_key(&k) {
            self.backend_data.insert(
              k,
              StoredMetricData::new(self.created, metric)
            );
          } else {
            self.backend_data.get_mut(&k).map(|stored_metric| {
              stored_metric.data.update(key, metric);
            });
          }
        } else {
          let k = (String::from(id), String::from(key));
          if !self.app_data.contains_key(&k) {
            self.app_data.insert(
              k,
              StoredMetricData::new(self.created, metric)
            );
          } else {
            self.app_data.get_mut(&k).map(|stored_metric| {
              stored_metric.data.update(key, metric);
            });
          }
        }
      } else {
        if !self.data.contains_key(key) {
          self.data.insert(
            String::from(key),
            StoredMetricData::new(self.created, metric)
          );
        } else {
          self.data.get_mut(key).map(|stored_metric| {
            stored_metric.data.update(key, metric);
          });
        }
      }
    }
  }
}

<commit_msg>send the timing metrics to UDP instead of the local buffer<commit_after>use std::str;
use std::thread;
use std::sync::Mutex;
use std::cell::RefCell;
use std::time::{Duration,Instant};
use std::iter::repeat;
use std::collections::BTreeMap;
use std::collections::VecDeque;
use std::fmt::Arguments;
use std::net::SocketAddr;
use mio::net::UdpSocket;
use std::io::{self,BufWriter,Write,Error,ErrorKind};
use nom::HexDisplay;
use hdrhistogram::Histogram;
use sozu_command::messages::{FilteredData,MetricsData,Percentiles,BackendMetricsData,FilteredTimeSerie};

use super::{Subscriber,MetricData,StoredMetricData};

const VERSION: &'static str = env!("CARGO_PKG_VERSION");

#[derive(Debug,Clone,PartialEq)]
pub struct MetricLine {
  label:      &'static str,
  app_id:     Option<String>,
  backend_id: Option<String>,
  /// in milliseconds
  duration:   usize,
}

pub struct MetricSocket {
  pub addr:   SocketAddr,
  pub socket: UdpSocket,
}

impl Write for MetricSocket {
  fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
    self.socket.send_to(buf, &self.addr)
  }

  fn flush(&mut self) -> io::Result<()> {
    Ok(())
  }
}

pub struct NetworkDrain {
  queue:              VecDeque<MetricLine>,
  pub prefix:         String,
  pub remote:         BufWriter<MetricSocket>,
  is_writable:        bool,
  data:               BTreeMap<String, StoredMetricData>,
  /// (app_id, key) -> metric
  app_data:           BTreeMap<(String, String), StoredMetricData>,
  /// (app_id, backend_id, key) -> metric
  backend_data:       BTreeMap<(String, String, String), StoredMetricData>,
  pub use_tagged_metrics: bool,
  pub origin:         String,
  created:            Instant,
}

impl NetworkDrain {
  pub fn new(prefix: String, socket: UdpSocket, addr: SocketAddr) -> Self {
    NetworkDrain {
      queue:  VecDeque::new(),
      prefix: prefix,
      remote: BufWriter::new(MetricSocket {
        addr, socket
      }),
      is_writable: true,
      data: BTreeMap::new(),
      app_data: BTreeMap::new(),
      backend_data: BTreeMap::new(),
      use_tagged_metrics: false,
      origin: String::from("x"),
      created: Instant::now(),
    }
  }

  pub fn writable(&mut self) {
    self.is_writable = true;
  }

  pub fn send_data(&mut self) {
    let now  = Instant::now();
    let secs = Duration::new(2, 0);

    if self.is_writable {
    for (ref key, ref mut stored_metric) in self.data.iter_mut().filter(|&(_, ref value)| now.duration_since(value.last_sent) > secs) {
      //info!("will write {} -> {:#?}", key, stored_metric);
      let res = match stored_metric.data {
        MetricData::Gauge(value) => {
          if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.{},origin={},version={}:{}|g\n", self.prefix, key, self.origin, VERSION, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.{}:{}|g\n", self.prefix, self.origin, key, value))
          }
        },
        MetricData::Count(value) => {
          if value == 0 {
            stored_metric.last_sent = now;
          }

          let res = if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.{},origin={},version={}:{}|c\n", self.prefix, key, self.origin, VERSION, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.{}:{}|c\n", self.prefix, self.origin, key, value))
          };

          if res.is_ok() {
            stored_metric.data = MetricData::Count(0);
          }

          res
        },
        _ => {Ok(())}
      };

      match res {
        Ok(()) => {
          stored_metric.last_sent = now;
        },
        Err(e) => match e.kind() {
          ErrorKind::WriteZero => {
            self.remote.flush();
          },
          ErrorKind::WouldBlock => {
            error!("WouldBlock while writing global metrics to socket");
            self.is_writable = false;
            break;
          },
          e => {
            error!("metrics socket write error={:?}", e);
            break;
          },
        }
      }
    }
    }

    self.remote.flush();

    if self.is_writable {
    for (ref key, ref mut stored_metric) in self.app_data.iter_mut().filter(|&(_, ref value)| now.duration_since(value.last_sent) > secs) {
      //info!("will write {} -> {:#?}", key, stored_metric);
      let res = match stored_metric.data {
        MetricData::Gauge(value) => {
          if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.app.{},origin={},version={},app_id={}:{}|g\n",
              self.prefix, key.1, self.origin, VERSION, key.0, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.app.{}.{}:{}|g\n", self.prefix, self.origin, key.0, key.1, value))
          }
        },
        MetricData::Count(value) => {
          if value == 0 {
            stored_metric.last_sent = now;
          }

          let res = if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.app.{},origin={},version={},app_id={}:{}|c\n",
              self.prefix, key.1, self.origin, VERSION, key.0, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.app.{}.{}:{}|c\n", self.prefix, self.origin, key.0, key.1, value))
          };

          if res.is_ok() {
            stored_metric.data = MetricData::Count(0);
          }

          res
        },
        _ => {Ok(())}
      };

      match res {
        Ok(()) => {
          stored_metric.last_sent = now;
        },
        Err(e) => match e.kind() {
          ErrorKind::WriteZero => {
            self.remote.flush();
          },
          ErrorKind::WouldBlock => {
            error!("WouldBlock while writing app metrics to socket");
            self.is_writable = false;
            break;
          },
          e => {
            error!("metrics socket write error={:?}", e);
            break;
          },
        }
      }
    }
    }

    self.remote.flush();

    if self.is_writable {
    for (ref key, ref mut stored_metric) in self.backend_data.iter_mut().filter(|&(_, ref value)| now.duration_since(value.last_sent) > secs) {
      //info!("will write {} -> {:#?}", key, stored_metric);
      let res = match stored_metric.data {
        MetricData::Gauge(value) => {
          if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.backend.{},origin={},version={},app_id={},backend_id={}:{}|g\n",
              self.prefix, key.2, self.origin, VERSION, key.0, key.1, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.app.{}.backend.{}.{}:{}|g\n", self.prefix, self.origin, key.0, key.1, key.2, value))
          }
        },
        MetricData::Count(value) => {
          if value == 0 {
            stored_metric.last_sent = now;
          }

          let res = if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.app.{},origin={},version={},app_id={},backend_id={}:{}|c\n",
              self.prefix, key.2, self.origin, VERSION, key.0, key.1, value))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.app.{}.backend.{}.{}:{}|c\n", self.prefix, self.origin, key.0, key.1, key.2, value))
          };

          if res.is_ok() {
            stored_metric.data = MetricData::Count(0);
          }

          res
        },
        _ => {Ok(())}
      };

      match res {
        Ok(()) => {
          stored_metric.last_sent = now;
        },
        Err(e) => match e.kind() {
          ErrorKind::WriteZero => {
            self.remote.flush();
          },
          ErrorKind::WouldBlock => {
            error!("WouldBlock while writing backend metrics to socket");
            self.is_writable = false;
            break;
          },
          e => {
            error!("metrics socket write error={:?}", e);
            break;
          },
        }
      }
    }
    }
    self.remote.flush();

    if self.is_writable {
    for metric in self.queue.drain(..) {
       let res = match (metric.app_id, metric.backend_id) {
        (Some(app_id), Some(backend_id)) => {
          if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.backend.{},origin={},version={},app_id={},backend_id={}:{}|ms\n",
              self.prefix, metric.label, self.origin, VERSION, app_id, backend_id, metric.duration))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.app.{}.backend.{}.{}:{}|ms\n", self.prefix, self.origin, app_id, backend_id,
              metric.label, metric.duration))
          }
        },
        (Some(app_id), None) => {
          if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.app.{},origin={},version={},app_id={}:{}|ms\n",
              self.prefix, metric.label, self.origin, VERSION, app_id, metric.duration))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.app.{}.{}:{}|ms\n", self.prefix, self.origin, app_id,
              metric.label, metric.duration))
          }
        },
        (None, None) => {
          if self.use_tagged_metrics {
            self.remote.write_fmt(format_args!("{}.{},origin={},version={}:{}|ms\n",
              self.prefix, metric.label, self.origin, VERSION, metric.duration))
          } else {
            self.remote.write_fmt(format_args!("{}.{}.{}:{}|ms\n", self.prefix, self.origin,
              metric.label, metric.duration))
          }
        },
        _ => { Ok(()) }
      };

      match res {
        Ok(()) => {
        },
        Err(e) => match e.kind() {
          ErrorKind::WriteZero => {
            self.remote.flush();
          },
          ErrorKind::WouldBlock => {
            error!("WouldBlock while writing timing metrics to socket");
            self.is_writable = false;
            break;
          },
          e => {
            error!("metrics socket write error={:?}", e);
            break;
          },
        }
      }
    }
    self.remote.flush();
    }
  }
}

impl Subscriber for NetworkDrain {
  fn receive_metric(&mut self, key: &'static str, app_id: Option<&str>, backend_id: Option<&str>, metric: MetricData) {
    if metric.is_time() {
      if let MetricData::Time(millis) = metric {
        self.queue.push_back(MetricLine {
          label: key,
          app_id: app_id.map(|s| s.to_string()),
          backend_id: backend_id.map(|s| s.to_string()),
          duration: millis,
        });
      }
    } else {
      if let Some(id) = app_id {
        if let Some(bid) = backend_id {
          let k = (String::from(id), String::from(bid), String::from(key));
          if !self.backend_data.contains_key(&k) {
            self.backend_data.insert(
              k,
              StoredMetricData::new(self.created, metric)
            );
          } else {
            self.backend_data.get_mut(&k).map(|stored_metric| {
              stored_metric.data.update(key, metric);
            });
          }
        } else {
          let k = (String::from(id), String::from(key));
          if !self.app_data.contains_key(&k) {
            self.app_data.insert(
              k,
              StoredMetricData::new(self.created, metric)
            );
          } else {
            self.app_data.get_mut(&k).map(|stored_metric| {
              stored_metric.data.update(key, metric);
            });
          }
        }
      } else {
        if !self.data.contains_key(key) {
          self.data.insert(
            String::from(key),
            StoredMetricData::new(self.created, metric)
          );
        } else {
          self.data.get_mut(key).map(|stored_metric| {
            stored_metric.data.update(key, metric);
          });
        }
      }
    }
  }
}

<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate webrender_build;

use std::borrow::Cow;
use std::env;
use std::fs::{canonicalize, read_dir, File};
use std::io::prelude::*;
use std::path::{Path, PathBuf};
use std::collections::hash_map::DefaultHasher;
use std::hash::Hasher;
use webrender_build::shader::*;
use webrender_build::shader_features::{ShaderFeatureFlags, get_shader_features};

/// Compute the shader path for insertion into the include_str!() macro.
/// This makes for more compact generated code than inserting the literal
/// shader source into the generated file.
///
/// If someone is building on a network share, I'm sorry.
fn escape_include_path(path: &Path) -> String {
    let full_path = canonicalize(path).unwrap();
    let full_name = full_path.as_os_str().to_str().unwrap();
    let full_name = full_name.replace("\\\\?\\", "");
    let full_name = full_name.replace("\\", "/");

    full_name
}

fn write_unoptimized_shaders(mut glsl_files: Vec<PathBuf>, shader_file: &mut File) -> Result<(), std::io::Error> {
    writeln!(
        shader_file,
        "  pub static ref UNOPTIMIZED_SHADERS: HashMap<&'static str, SourceWithDigest> = {{"
    )?;
    writeln!(shader_file, "    let mut shaders = HashMap::new();")?;

    // Sort the file list so that the shaders.rs file is filled
    // deterministically.
    glsl_files.sort_by(|a, b| a.file_name().cmp(&b.file_name()));

    for glsl in glsl_files {
        // Compute the shader name.
        assert!(glsl.is_file());
        let shader_name = glsl.file_name().unwrap().to_str().unwrap();
        let shader_name = shader_name.replace(".glsl", "");

        // Compute a digest of the #include-expanded shader source. We store
        // this as a literal alongside the source string so that we don't need
        // to hash large strings at runtime.
        let mut hasher = DefaultHasher::new();
        let base = glsl.parent().unwrap();
        assert!(base.is_dir());
        ShaderSourceParser::new().parse(
            Cow::Owned(shader_source_from_file(&glsl)),
            &|f| Cow::Owned(shader_source_from_file(&base.join(&format!("{}.glsl", f)))),
            &mut |s| hasher.write(s.as_bytes()),
        );
        let digest: ProgramSourceDigest = hasher.into();

        writeln!(
            shader_file,
            "    shaders.insert(\"{}\", SourceWithDigest {{ source: include_str!(\"{}\"), digest: \"{}\"}});",
            shader_name,
            escape_include_path(&glsl),
            digest,
        )?;
    }
    writeln!(shader_file, "    shaders")?;
    writeln!(shader_file, "  }};")?;

    Ok(())
}

#[derive(Clone, Debug)]
struct ShaderOptimizationInput {
    shader_name: &'static str,
    config: String,
    gl_version: ShaderVersion,
}

#[derive(Debug)]
struct ShaderOptimizationOutput {
    full_shader_name: String,
    gl_version: ShaderVersion,
    vert_file_path: PathBuf,
    frag_file_path: PathBuf,
    digest: ProgramSourceDigest,
}

#[derive(Debug)]
struct ShaderOptimizationError {
    shader: ShaderOptimizationInput,
    message: String,
}

fn write_optimized_shaders(shader_dir: &Path, shader_file: &mut File, out_dir: &str) -> Result<(), std::io::Error> {
    writeln!(
        shader_file,
        "  pub static ref OPTIMIZED_SHADERS: HashMap<(ShaderVersion, &'static str), OptimizedSourceWithDigest> = {{"
    )?;
    writeln!(shader_file, "    let mut shaders = HashMap::new();")?;

    // The full set of optimized shaders can be quite large, so only optimize
    // for the GL version we expect to be used on the target platform. If a different GL
    // version is used we will simply fall back to the unoptimized shaders.
    let shader_versions = match env::var("CARGO_CFG_TARGET_OS").as_ref().map(|s| &**s) {
        Ok("android") | Ok("windows") => [ShaderVersion::Gles],
        _ => [ShaderVersion::Gl],
    };

    let mut shaders = Vec::default();
    for &gl_version in &shader_versions {
        let mut flags = ShaderFeatureFlags::all();
        if gl_version != ShaderVersion::Gl {
            flags.remove(ShaderFeatureFlags::GL);
        }
        if gl_version != ShaderVersion::Gles {
            flags.remove(ShaderFeatureFlags::GLES);
            flags.remove(ShaderFeatureFlags::TEXTURE_EXTERNAL);
        }
        flags.remove(ShaderFeatureFlags::DITHERING);
        flags.remove(ShaderFeatureFlags::PIXEL_LOCAL_STORAGE);

        for (shader_name, configs) in get_shader_features(flags) {
            for config in configs {
                shaders.push(ShaderOptimizationInput {
                    shader_name,
                    config,
                    gl_version,
                });
            }
        }
    }

    let outputs = build_parallel::compile_objects(&|shader: &ShaderOptimizationInput| {
        println!("Optimizing shader {:?}", shader);
        let target = match shader.gl_version {
            ShaderVersion::Gl => glslopt::Target::OpenGl,
            ShaderVersion::Gles => glslopt::Target::OpenGles30,
        };
        let glslopt_ctx = glslopt::Context::new(target);

        let features = shader.config.split(",").filter(|f| !f.is_empty()).collect::<Vec<_>>();

        let (vert_src, frag_src) = build_shader_strings(
            shader.gl_version,
            &features,
            shader.shader_name,
            &|f| Cow::Owned(shader_source_from_file(&shader_dir.join(&format!("{}.glsl", f)))),
        );

        let full_shader_name = if shader.config.is_empty() {
            shader.shader_name.to_string()
        } else {
            format!("{}_{}", shader.shader_name, shader.config.replace(",", "_"))
        };

        let vert = glslopt_ctx.optimize(glslopt::ShaderType::Vertex, vert_src);
        if !vert.get_status() {
            return Err(ShaderOptimizationError {
                shader: shader.clone(),
                message: vert.get_log().to_string(),
            });
        }
        let frag = glslopt_ctx.optimize(glslopt::ShaderType::Fragment, frag_src);
        if !frag.get_status() {
            return Err(ShaderOptimizationError {
                shader: shader.clone(),
                message: frag.get_log().to_string(),
            });
        }

        let vert_source = vert.get_output().unwrap();
        let frag_source = frag.get_output().unwrap();

        // Compute a digest of the optimized shader sources. We store this
        // as a literal alongside the source string so that we don't need
        // to hash large strings at runtime.
        let mut hasher = DefaultHasher::new();
        hasher.write(vert_source.as_bytes());
        hasher.write(frag_source.as_bytes());
        let digest: ProgramSourceDigest = hasher.into();

        let vert_file_path = Path::new(out_dir)
            .join(format!("{}_{:?}.vert", full_shader_name, shader.gl_version));
        let mut vert_file = File::create(&vert_file_path).unwrap();
        vert_file.write_all(vert_source.as_bytes()).unwrap();
        let frag_file_path = vert_file_path.with_extension("frag");
        let mut frag_file = File::create(&frag_file_path).unwrap();
        frag_file.write_all(frag_source.as_bytes()).unwrap();

        println!("Finished optimizing shader {:?}", shader);

        Ok(ShaderOptimizationOutput {
            full_shader_name,
            gl_version: shader.gl_version,
            vert_file_path,
            frag_file_path,
            digest,
        })
    }, &shaders);

    match outputs {
        Ok(mut outputs) => {
            // Sort the shader list so that the shaders.rs file is filled
            // deterministically.
            outputs.sort_by(|a, b| {
                (a.gl_version, a.full_shader_name.clone()).cmp(&(b.gl_version, b.full_shader_name.clone()))
            });

            for shader in outputs {
                writeln!(
                    shader_file,
                    "    shaders.insert(({}, \"{}\"), OptimizedSourceWithDigest {{",
                    shader.gl_version.variant_name(),
                    shader.full_shader_name,
                )?;
                writeln!(
                    shader_file,
                    "        vert_source: include_str!(\"{}\"),",
                    escape_include_path(&shader.vert_file_path),
                )?;
                writeln!(
                    shader_file,
                    "        frag_source: include_str!(\"{}\"),",
                    escape_include_path(&shader.frag_file_path),
                )?;
                writeln!(shader_file, "        digest: \"{}\",", shader.digest)?;
                writeln!(shader_file, "    }});")?;
            }
        }
        Err(err) => match err {
            build_parallel::Error::BuildError(err) => {
                panic!("Error optimizing shader {:?}: {}", err.shader, err.message)
            }
            _ => panic!("Error optimizing shaders."),
        }
    }

    writeln!(shader_file, "    shaders")?;
    writeln!(shader_file, "  }};")?;

    Ok(())
}

fn main() -> Result<(), std::io::Error> {
    let out_dir = env::var("OUT_DIR").unwrap_or("out".to_owned());

    let shaders_file_path = Path::new(&out_dir).join("shaders.rs");
    let mut glsl_files = vec![];

    println!("cargo:rerun-if-changed=res");
    let res_dir = Path::new("res");
    for entry in read_dir(res_dir)? {
        let entry = entry?;
        let path = entry.path();

        if entry.file_name().to_str().unwrap().ends_with(".glsl") {
            println!("cargo:rerun-if-changed={}", path.display());
            glsl_files.push(path.to_owned());
        }
    }

    let mut shader_file = File::create(shaders_file_path)?;

    writeln!(shader_file, "/// AUTO GENERATED BY build.rs\n")?;
    writeln!(shader_file, "use std::collections::HashMap;\n")?;
    writeln!(shader_file, "use webrender_build::shader::ShaderVersion;\n")?;
    writeln!(shader_file, "pub struct SourceWithDigest {{")?;
    writeln!(shader_file, "    pub source: &'static str,")?;
    writeln!(shader_file, "    pub digest: &'static str,")?;
    writeln!(shader_file, "}}\n")?;
    writeln!(shader_file, "pub struct OptimizedSourceWithDigest {{")?;
    writeln!(shader_file, "    pub vert_source: &'static str,")?;
    writeln!(shader_file, "    pub frag_source: &'static str,")?;
    writeln!(shader_file, "    pub digest: &'static str,")?;
    writeln!(shader_file, "}}\n")?;
    writeln!(shader_file, "lazy_static! {{")?;

    write_unoptimized_shaders(glsl_files, &mut shader_file)?;
    writeln!(shader_file, "")?;
    write_optimized_shaders(&res_dir, &mut shader_file, &out_dir)?;
    writeln!(shader_file, "}}")?;

    Ok(())
}
<commit_msg>Bug 1653544 - Add usueful information in the generated shaders. r=jnicol<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate webrender_build;

use std::borrow::Cow;
use std::env;
use std::fs::{canonicalize, read_dir, File};
use std::io::prelude::*;
use std::path::{Path, PathBuf};
use std::collections::hash_map::DefaultHasher;
use std::hash::Hasher;
use webrender_build::shader::*;
use webrender_build::shader_features::{ShaderFeatureFlags, get_shader_features};

/// Compute the shader path for insertion into the include_str!() macro.
/// This makes for more compact generated code than inserting the literal
/// shader source into the generated file.
///
/// If someone is building on a network share, I'm sorry.
fn escape_include_path(path: &Path) -> String {
    let full_path = canonicalize(path).unwrap();
    let full_name = full_path.as_os_str().to_str().unwrap();
    let full_name = full_name.replace("\\\\?\\", "");
    let full_name = full_name.replace("\\", "/");

    full_name
}

fn write_unoptimized_shaders(mut glsl_files: Vec<PathBuf>, shader_file: &mut File) -> Result<(), std::io::Error> {
    writeln!(
        shader_file,
        "  pub static ref UNOPTIMIZED_SHADERS: HashMap<&'static str, SourceWithDigest> = {{"
    )?;
    writeln!(shader_file, "    let mut shaders = HashMap::new();")?;

    // Sort the file list so that the shaders.rs file is filled
    // deterministically.
    glsl_files.sort_by(|a, b| a.file_name().cmp(&b.file_name()));

    for glsl in glsl_files {
        // Compute the shader name.
        assert!(glsl.is_file());
        let shader_name = glsl.file_name().unwrap().to_str().unwrap();
        let shader_name = shader_name.replace(".glsl", "");

        // Compute a digest of the #include-expanded shader source. We store
        // this as a literal alongside the source string so that we don't need
        // to hash large strings at runtime.
        let mut hasher = DefaultHasher::new();
        let base = glsl.parent().unwrap();
        assert!(base.is_dir());
        ShaderSourceParser::new().parse(
            Cow::Owned(shader_source_from_file(&glsl)),
            &|f| Cow::Owned(shader_source_from_file(&base.join(&format!("{}.glsl", f)))),
            &mut |s| hasher.write(s.as_bytes()),
        );
        let digest: ProgramSourceDigest = hasher.into();

        writeln!(
            shader_file,
            "    shaders.insert(\"{}\", SourceWithDigest {{ source: include_str!(\"{}\"), digest: \"{}\"}});",
            shader_name,
            escape_include_path(&glsl),
            digest,
        )?;
    }
    writeln!(shader_file, "    shaders")?;
    writeln!(shader_file, "  }};")?;

    Ok(())
}

#[derive(Clone, Debug)]
struct ShaderOptimizationInput {
    shader_name: &'static str,
    config: String,
    gl_version: ShaderVersion,
}

#[derive(Debug)]
struct ShaderOptimizationOutput {
    full_shader_name: String,
    gl_version: ShaderVersion,
    vert_file_path: PathBuf,
    frag_file_path: PathBuf,
    digest: ProgramSourceDigest,
}

#[derive(Debug)]
struct ShaderOptimizationError {
    shader: ShaderOptimizationInput,
    message: String,
}

fn write_optimized_shaders(shader_dir: &Path, shader_file: &mut File, out_dir: &str) -> Result<(), std::io::Error> {
    writeln!(
        shader_file,
        "  pub static ref OPTIMIZED_SHADERS: HashMap<(ShaderVersion, &'static str), OptimizedSourceWithDigest> = {{"
    )?;
    writeln!(shader_file, "    let mut shaders = HashMap::new();")?;

    // The full set of optimized shaders can be quite large, so only optimize
    // for the GL version we expect to be used on the target platform. If a different GL
    // version is used we will simply fall back to the unoptimized shaders.
    let shader_versions = match env::var("CARGO_CFG_TARGET_OS").as_ref().map(|s| &**s) {
        Ok("android") | Ok("windows") => [ShaderVersion::Gles],
        _ => [ShaderVersion::Gl],
    };

    let mut shaders = Vec::default();
    for &gl_version in &shader_versions {
        let mut flags = ShaderFeatureFlags::all();
        if gl_version != ShaderVersion::Gl {
            flags.remove(ShaderFeatureFlags::GL);
        }
        if gl_version != ShaderVersion::Gles {
            flags.remove(ShaderFeatureFlags::GLES);
            flags.remove(ShaderFeatureFlags::TEXTURE_EXTERNAL);
        }
        flags.remove(ShaderFeatureFlags::DITHERING);
        flags.remove(ShaderFeatureFlags::PIXEL_LOCAL_STORAGE);

        for (shader_name, configs) in get_shader_features(flags) {
            for config in configs {
                shaders.push(ShaderOptimizationInput {
                    shader_name,
                    config,
                    gl_version,
                });
            }
        }
    }

    let outputs = build_parallel::compile_objects(&|shader: &ShaderOptimizationInput| {
        println!("Optimizing shader {:?}", shader);
        let target = match shader.gl_version {
            ShaderVersion::Gl => glslopt::Target::OpenGl,
            ShaderVersion::Gles => glslopt::Target::OpenGles30,
        };
        let glslopt_ctx = glslopt::Context::new(target);

        let features = shader.config.split(",").filter(|f| !f.is_empty()).collect::<Vec<_>>();

        let (vert_src, frag_src) = build_shader_strings(
            shader.gl_version,
            &features,
            shader.shader_name,
            &|f| Cow::Owned(shader_source_from_file(&shader_dir.join(&format!("{}.glsl", f)))),
        );

        let full_shader_name = if shader.config.is_empty() {
            shader.shader_name.to_string()
        } else {
            format!("{}_{}", shader.shader_name, shader.config.replace(",", "_"))
        };

        let vert = glslopt_ctx.optimize(glslopt::ShaderType::Vertex, vert_src);
        if !vert.get_status() {
            return Err(ShaderOptimizationError {
                shader: shader.clone(),
                message: vert.get_log().to_string(),
            });
        }
        let frag = glslopt_ctx.optimize(glslopt::ShaderType::Fragment, frag_src);
        if !frag.get_status() {
            return Err(ShaderOptimizationError {
                shader: shader.clone(),
                message: frag.get_log().to_string(),
            });
        }

        let vert_source = vert.get_output().unwrap();
        let frag_source = frag.get_output().unwrap();

        // Compute a digest of the optimized shader sources. We store this
        // as a literal alongside the source string so that we don't need
        // to hash large strings at runtime.
        let mut hasher = DefaultHasher::new();

        let vert_file_path = Path::new(out_dir)
            .join(format!("{}_{:?}.vert", full_shader_name, shader.gl_version));
        write_optimized_shader_file(&vert_file_path, vert_source, &shader.shader_name, &features, &mut hasher);

        let frag_file_path = vert_file_path.with_extension("frag");
        write_optimized_shader_file(&frag_file_path, frag_source, &shader.shader_name, &features, &mut hasher);

        let digest: ProgramSourceDigest = hasher.into();

        println!("Finished optimizing shader {:?}", shader);

        Ok(ShaderOptimizationOutput {
            full_shader_name,
            gl_version: shader.gl_version,
            vert_file_path,
            frag_file_path,
            digest,
        })
    }, &shaders);

    match outputs {
        Ok(mut outputs) => {
            // Sort the shader list so that the shaders.rs file is filled
            // deterministically.
            outputs.sort_by(|a, b| {
                (a.gl_version, a.full_shader_name.clone()).cmp(&(b.gl_version, b.full_shader_name.clone()))
            });

            for shader in outputs {
                writeln!(
                    shader_file,
                    "    shaders.insert(({}, \"{}\"), OptimizedSourceWithDigest {{",
                    shader.gl_version.variant_name(),
                    shader.full_shader_name,
                )?;
                writeln!(
                    shader_file,
                    "        vert_source: include_str!(\"{}\"),",
                    escape_include_path(&shader.vert_file_path),
                )?;
                writeln!(
                    shader_file,
                    "        frag_source: include_str!(\"{}\"),",
                    escape_include_path(&shader.frag_file_path),
                )?;
                writeln!(shader_file, "        digest: \"{}\",", shader.digest)?;
                writeln!(shader_file, "    }});")?;
            }
        }
        Err(err) => match err {
            build_parallel::Error::BuildError(err) => {
                panic!("Error optimizing shader {:?}: {}", err.shader, err.message)
            }
            _ => panic!("Error optimizing shaders."),
        }
    }

    writeln!(shader_file, "    shaders")?;
    writeln!(shader_file, "  }};")?;

    Ok(())
}

fn write_optimized_shader_file(
    path: &Path,
    source: &str,
    shader_name: &str,
    features: &[&str],
    hasher: &mut DefaultHasher,
) {
    let mut file = File::create(&path).unwrap();
    for (line_number, line) in source.lines().enumerate() {
        // We embed the shader name and features as a comment in the
        // source to make debugging easier.
        // The #version directive must be on the first line so we insert
        // the extra information on the next line.
        if line_number == 1 {
            let prelude = format!(
                "// {}\n// features: {:?}\n\n",
                shader_name, features
            );
            file.write_all(prelude.as_bytes()).unwrap();
            hasher.write(prelude.as_bytes());
        }
        file.write_all(line.as_bytes()).unwrap();
        file.write_all("\n".as_bytes()).unwrap();
        hasher.write(line.as_bytes());
        hasher.write("\n".as_bytes());
    }
}

fn main() -> Result<(), std::io::Error> {
    let out_dir = env::var("OUT_DIR").unwrap_or("out".to_owned());

    let shaders_file_path = Path::new(&out_dir).join("shaders.rs");
    let mut glsl_files = vec![];

    println!("cargo:rerun-if-changed=res");
    let res_dir = Path::new("res");
    for entry in read_dir(res_dir)? {
        let entry = entry?;
        let path = entry.path();

        if entry.file_name().to_str().unwrap().ends_with(".glsl") {
            println!("cargo:rerun-if-changed={}", path.display());
            glsl_files.push(path.to_owned());
        }
    }

    let mut shader_file = File::create(shaders_file_path)?;

    writeln!(shader_file, "/// AUTO GENERATED BY build.rs\n")?;
    writeln!(shader_file, "use std::collections::HashMap;\n")?;
    writeln!(shader_file, "use webrender_build::shader::ShaderVersion;\n")?;
    writeln!(shader_file, "pub struct SourceWithDigest {{")?;
    writeln!(shader_file, "    pub source: &'static str,")?;
    writeln!(shader_file, "    pub digest: &'static str,")?;
    writeln!(shader_file, "}}\n")?;
    writeln!(shader_file, "pub struct OptimizedSourceWithDigest {{")?;
    writeln!(shader_file, "    pub vert_source: &'static str,")?;
    writeln!(shader_file, "    pub frag_source: &'static str,")?;
    writeln!(shader_file, "    pub digest: &'static str,")?;
    writeln!(shader_file, "}}\n")?;
    writeln!(shader_file, "lazy_static! {{")?;

    write_unoptimized_shaders(glsl_files, &mut shader_file)?;
    writeln!(shader_file, "")?;
    write_optimized_shaders(&res_dir, &mut shader_file, &out_dir)?;
    writeln!(shader_file, "}}")?;

    Ok(())
}
<|endoftext|>


<commit_before>use noise::{Seed, Brownian2, Brownian3, perlin2, perlin3};

use field::Field;

pub struct HeightMap {
  pub height: Brownian2<f64, fn (&Seed, &[f64; 2]) -> f64>,
  pub features: Brownian3<f64, fn (&Seed, &[f64; 3]) -> f64>,
  pub seed: Seed,
}

impl HeightMap {
  pub fn new(seed: Seed) -> HeightMap {
    let perlin2: fn(&Seed, &[f64; 2]) -> f64 = perlin2;
    let perlin3: fn(&Seed, &[f64; 3]) -> f64 = perlin3;
    HeightMap {
      seed: seed,
      height:
        Brownian2::new(perlin2, 5)
        .frequency(1.0 / 8.0)
        .persistence(4.0)
        .lacunarity(1.0 / 3.0)
      ,
      features:
        Brownian3::new(perlin3, 2)
        .frequency(1.0 / 64.0)
        .persistence(1.0 / 16.0)
        .lacunarity(8.0)
      ,
    }
  }
}

impl Field for HeightMap {
  /// The height of the field at a given x,y,z.
  fn density_at(&self, x: f32, y: f32, z: f32) -> f32 {
    let coords = [x as f64, z as f64];
    let height = self.height.apply(&self.seed, &coords);
    let height = height as f32;
    height - y
  }
}
<commit_msg>Incorporate 3D perlin features again<commit_after>use noise::{Seed, Brownian2, Brownian3, perlin2, perlin3};

use field::Field;

pub struct HeightMap {
  pub height: Brownian2<f64, fn (&Seed, &[f64; 2]) -> f64>,
  pub features: Brownian3<f64, fn (&Seed, &[f64; 3]) -> f64>,
  pub seed: Seed,
}

impl HeightMap {
  pub fn new(seed: Seed) -> HeightMap {
    let perlin2: fn(&Seed, &[f64; 2]) -> f64 = perlin2;
    let perlin3: fn(&Seed, &[f64; 3]) -> f64 = perlin3;
    HeightMap {
      seed: seed,
      height:
        Brownian2::new(perlin2, 5)
        .frequency(1.0 / 8.0)
        .persistence(4.0)
        .lacunarity(1.0 / 3.0)
      ,
      features:
        Brownian3::new(perlin3, 2)
        .frequency(1.0 / 32.0)
        .persistence(8.0)
        .lacunarity(1.0 / 4.0)
      ,
    }
  }
}

impl Field for HeightMap {
  /// The height of the field at a given x,y,z.
  fn density_at(&self, x: f32, y: f32, z: f32) -> f32 {
    let height = self.height.apply(&self.seed, &[x as f64, z as f64]);
    let height = height as f32;
    let heightmap_density = height - y;

    let feature_density = self.features.apply(&self.seed, &[x as f64, y as f64, z as f64]) * 8.0;
    let feature_density = feature_density as f32;

    heightmap_density + feature_density
  }
}
<|endoftext|>