<commit_before>use configuration::Configuration;
use score;
use sorted_result_set::SortedResultSet;
use std::slice::SliceExt;
use std::cmp::min;
use std::ascii::AsciiExt;

#[derive(Debug)]
pub struct Search<'s> {
    pub config: &'s Configuration,
    pub current: usize,
    pub query: String,
    pub result: Vec<String>,
    choice_stack: ChoiceStack<'s>,
    done: bool,
}

#[derive(Debug)]
struct ChoiceStack<'s> {
    content: Vec<Vec<&'s String>>,
}

impl <'s>ChoiceStack<'s> {
    pub fn new(input: &'s Vec<String>) -> ChoiceStack<'s> {
        let initial_choices = input.iter().map(|x| x).collect();

        ChoiceStack { content: vec![initial_choices] }
    }

    pub fn push(&mut self, frame: Vec<&'s String>) {
        self.content.push(frame);
    }

    pub fn pop(&mut self) {
        if self.content.len() > 1 {
            self.content.pop();
        }
    }

    pub fn peek(&self) -> &Vec<&'s String> {
        self.content.last().unwrap()
    }

    pub fn last_size(&self) -> usize {
        self.peek().len()
    }
}

impl<'s> Search<'s> {
    pub fn blank(config: &'s Configuration) -> Search<'s> {
        let query = config.initial_search.clone();

        let choice_stack = ChoiceStack::new(&config.choices);
        let result = Search::copy_items(&config.choices, config.visible_limit);

        Search::new(config, query, choice_stack, result, 0, false)
    }

    fn copy_items(input: &Vec<String>, size: usize) -> Vec<String> {
        input.iter().take(size).map(|x| x.clone() ).collect()
    }

    fn new(config: &'s Configuration, query: String, choice_stack: ChoiceStack<'s>, result: Vec<String>, index: usize, done: bool) -> Search<'s> {
        Search { config: config,
                 current: index,
                 query: query,
                 result: result,
                 choice_stack: choice_stack,
                 done: done}
    }

    pub fn is_done(&self) -> bool {
        self.done
    }

    pub fn done(self) -> Search<'s> {
        Search::new(self.config, self.query, self.choice_stack, self.result, self.current, true)
    }

    pub fn selection(&self) -> Option<String> {
        if self.result.len() > 0 {
            Some(self.result[self.current].to_string())
        } else {
            None
        }
    }

    fn new_for_index(self, index: usize) -> Search<'s> {
        Search::new(self.config, self.query, self.choice_stack, self.result, index, self.done)
    }

    pub fn iter_matches<F: FnMut(&'s String, f32)>(query: &str, choices: &Vec<&'s String>, mut f: F) {
        let lower_query = query.to_ascii_lowercase();

        for choice in choices.iter() {
            let lower_choice = choice.to_ascii_lowercase();

            match score::score(&lower_choice, &lower_query) {
                0.0     => continue,
                quality => f(choice, quality),
            };
        }
    }

    pub fn down(self) -> Search<'s> {
        let next_index = self.next_index();
        self.new_for_index(next_index)
    }

    pub fn up(self) -> Search<'s> {
        let next_index = self.prev_index();
        self.new_for_index(next_index)
    }

    pub fn append_to_search(mut self, input: &str) -> Search<'s> {
        let mut new_query = self.query.clone();
        new_query.push_str(input.as_slice());

        let mut results = SortedResultSet::new(self.config.visible_limit);
        let mut filtered_choices: Vec<&String> = Vec::new();
        Search::iter_matches(new_query.as_slice(), &self.choice_stack.peek(),
                        |match_str, quality| {
                                               results.push(match_str, quality);
                                               filtered_choices.push(match_str)
                                             });

        self.choice_stack.push(filtered_choices);
        Search::new(self.config, new_query, self.choice_stack, results.as_sorted_vec(), 0, self.done)
    }

    pub fn backspace(mut self) -> Search<'s> {
        let mut new_query = self.query.clone();
        new_query.pop();

        self.choice_stack.pop();

        let mut results = SortedResultSet::new(self.config.visible_limit);
        Search::iter_matches(new_query.as_slice(), &self.choice_stack.peek(), |match_str, quality| results.push(match_str, quality) );

        Search::new(self.config, new_query, self.choice_stack, results.as_sorted_vec(), 0, self.done)
    }

    fn next_index(&self) -> usize {
        let next_index = self.current + 1;

        if next_index >= self.actual_limit() {
            0
        } else {
            next_index
        }
    }

    fn prev_index(&self) -> usize {
        if self.current == 0 {
            self.actual_limit() - 1
        } else {
            self.current - 1
        }
    }

    fn actual_limit(&self) -> usize {
        min(self.config.visible_limit, self.num_matches())
    }

    pub fn num_matches(&self) -> usize {
        self.choice_stack.last_size()
    }
}

#[cfg(test)]
mod tests {
    use configuration::Configuration;
    use super::*;

    fn one_two_three() -> Vec<String> {
        vec!["one".to_string(),
             "two".to_string(),
             "three".to_string()]
    }

    fn input_times(n: usize) ->Vec<String> {
        let mut result: Vec<String> = Vec::new();
        for thing in one_two_three().iter().cycle().take(n) {
            result.push(thing.clone());
        }
        result
    }

    fn basic_configuration() -> Configuration {
        let input = one_two_three();
        Configuration::from_inputs(input, None, None)
    }

    #[test]
    fn it_selects_the_first_choice_by_default() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.selection(), Some("one".to_string()));
    }

    #[test]
    fn it_selets_the_second_when_down_is_called() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.down().selection(), Some("two".to_string()));
    }

    #[test]
    fn it_loop_around_when_reaching_end_of_list() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.down().down().down().down().selection(), Some("two".to_string()));
    }

    #[test]
    fn it_loop_around_when_reaching_top_of_list() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.up().up().selection(), Some("two".to_string()));
    }

    #[test]
    fn it_loop_around_when_reaching_visible_limit() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.down().down().down().down().selection(), Some("two".to_string()));
    }

    #[test]
    fn it_moves_down_the_filtered_search_results() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.append_to_search("t").down().selection(), Some("three".to_string()));
    }

    #[test]
    fn it_moves_down_the_filtered_search_results_twice() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.append_to_search("t").append_to_search("w").selection(), Some("two".to_string()));
    }

    #[test]
    fn it_handles_not_matching_anything() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.append_to_search("x").selection(), None);
    }

    #[test]
    fn up_match_nothing_after_filtering_all_out() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("x");

        assert_eq!(search.up().selection(), None);
    }

    #[test]
    fn down_match_nothing_after_filtering_all_out() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("x");

        assert_eq!(search.down().selection(), None);
    }

    #[test]
    fn backspaces_over_characters() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("e");

        assert_eq!(search.query, "e");
        assert_eq!(search.backspace().query, "");
    }

    #[test]
    fn resets_the_index_when_removing_char_from_search() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("e");

        assert_eq!(search.down().backspace().current, 0);
    }

    #[test]
    fn resets_the_index_when_adding_char_to_seach() {
        let config = basic_configuration();
        let search = Search::blank(&config).down();

        assert_eq!(search.append_to_search("o").current, 0);
    }

    #[test]
    fn previous_results_appear_after_backspace() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("t");

        assert_eq!(search.backspace().num_matches(), 3);
    }

    #[test]
    fn initial_search_is_not_done() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert!(!search.is_done());
    }

    #[test]
    fn backspace_over_initial_doesnt_crash() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        search.backspace();
    }

    #[test]
    fn done_search_is_done() {
        let config = basic_configuration();
        let search = Search::blank(&config).done();

        assert!(search.is_done());
    }

    #[test]
    fn done_search_has_selection() {
        let config = basic_configuration();
        let search = Search::blank(&config).done();

        assert_eq!(search.selection(), Some("one".to_string()));
    }

    #[test]
    fn loop_around_when_reaching_bottom_of_choices() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("n").down();

        assert_eq!(search.selection(), Some("one".to_string()));
    }

    #[test]
    fn loop_around_when_reaching_top_of_choices() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("n").up();

        assert_eq!(search.selection(), Some("one".to_string()));
    }

    #[test]
    fn search_is_case_insensitive() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("T");

        assert_eq!(search.num_matches(), 2);
    }

    #[test]
    fn uses_configs_visible_limit_as_result_size() {
        let input = input_times(30);

        let mut bar = Vec::new();
        for f in input.iter() {
            bar.push((*f).clone());
        }

        let config = Configuration::from_inputs(bar, None, Some(20));
        let search = Search::blank(&config).append_to_search("T");

        assert_eq!(search.num_matches(), 20);
    }
}
<commit_msg>Remove pointless copy<commit_after>use configuration::Configuration;
use score;
use sorted_result_set::SortedResultSet;
use std::slice::SliceExt;
use std::cmp::min;
use std::ascii::AsciiExt;

#[derive(Debug)]
pub struct Search<'s> {
    pub config: &'s Configuration,
    pub current: usize,
    pub query: String,
    pub result: Vec<String>,
    choice_stack: ChoiceStack<'s>,
    done: bool,
}

#[derive(Debug)]
struct ChoiceStack<'s> {
    content: Vec<Vec<&'s String>>,
}

impl <'s>ChoiceStack<'s> {
    pub fn new(input: &'s Vec<String>) -> ChoiceStack<'s> {
        let initial_choices = input.iter().map(|x| x).collect();

        ChoiceStack { content: vec![initial_choices] }
    }

    pub fn push(&mut self, frame: Vec<&'s String>) {
        self.content.push(frame);
    }

    pub fn pop(&mut self) {
        if self.content.len() > 1 {
            self.content.pop();
        }
    }

    pub fn peek(&self) -> &Vec<&'s String> {
        self.content.last().unwrap()
    }

    pub fn last_size(&self) -> usize {
        self.peek().len()
    }
}

impl<'s> Search<'s> {
    pub fn blank(config: &'s Configuration) -> Search<'s> {
        let query = config.initial_search.clone();

        let choice_stack = ChoiceStack::new(&config.choices);
        let result = Search::copy_items(&config.choices, config.visible_limit);

        Search::new(config, query, choice_stack, result, 0, false)
    }

    fn copy_items(input: &Vec<String>, size: usize) -> Vec<String> {
        input.iter().take(size).map(|x| x.clone() ).collect()
    }

    fn new(config: &'s Configuration, query: String, choice_stack: ChoiceStack<'s>, result: Vec<String>, index: usize, done: bool) -> Search<'s> {
        Search { config: config,
                 current: index,
                 query: query,
                 result: result,
                 choice_stack: choice_stack,
                 done: done}
    }

    pub fn is_done(&self) -> bool {
        self.done
    }

    pub fn done(self) -> Search<'s> {
        Search::new(self.config, self.query, self.choice_stack, self.result, self.current, true)
    }

    pub fn selection(&self) -> Option<String> {
        if self.result.len() > 0 {
            Some(self.result[self.current].to_string())
        } else {
            None
        }
    }

    fn new_for_index(self, index: usize) -> Search<'s> {
        Search::new(self.config, self.query, self.choice_stack, self.result, index, self.done)
    }

    pub fn iter_matches<F: FnMut(&'s String, f32)>(query: &str, choices: &Vec<&'s String>, mut f: F) {
        let lower_query = query.to_ascii_lowercase();

        for choice in choices.iter() {
            let lower_choice = choice.to_ascii_lowercase();

            match score::score(&lower_choice, &lower_query) {
                0.0     => continue,
                quality => f(choice, quality),
            };
        }
    }

    pub fn down(self) -> Search<'s> {
        let next_index = self.next_index();
        self.new_for_index(next_index)
    }

    pub fn up(self) -> Search<'s> {
        let next_index = self.prev_index();
        self.new_for_index(next_index)
    }

    pub fn append_to_search(mut self, input: &str) -> Search<'s> {
        let mut new_query = self.query.clone();
        new_query.push_str(input.as_slice());

        let mut results = SortedResultSet::new(self.config.visible_limit);
        let mut filtered_choices: Vec<&String> = Vec::new();
        Search::iter_matches(new_query.as_slice(), &self.choice_stack.peek(),
                        |match_str, quality| {
                                               results.push(match_str, quality);
                                               filtered_choices.push(match_str)
                                             });

        self.choice_stack.push(filtered_choices);
        Search::new(self.config, new_query, self.choice_stack, results.as_sorted_vec(), 0, self.done)
    }

    pub fn backspace(mut self) -> Search<'s> {
        let mut new_query = self.query.clone();
        new_query.pop();

        self.choice_stack.pop();

        let mut results = SortedResultSet::new(self.config.visible_limit);
        Search::iter_matches(new_query.as_slice(), &self.choice_stack.peek(), |match_str, quality| results.push(match_str, quality) );

        Search::new(self.config, new_query, self.choice_stack, results.as_sorted_vec(), 0, self.done)
    }

    fn next_index(&self) -> usize {
        let next_index = self.current + 1;

        if next_index >= self.actual_limit() {
            0
        } else {
            next_index
        }
    }

    fn prev_index(&self) -> usize {
        if self.current == 0 {
            self.actual_limit() - 1
        } else {
            self.current - 1
        }
    }

    fn actual_limit(&self) -> usize {
        min(self.config.visible_limit, self.num_matches())
    }

    pub fn num_matches(&self) -> usize {
        self.choice_stack.last_size()
    }
}

#[cfg(test)]
mod tests {
    use configuration::Configuration;
    use super::*;

    fn one_two_three() -> Vec<String> {
        vec!["one".to_string(),
             "two".to_string(),
             "three".to_string()]
    }

    fn input_times(n: usize) ->Vec<String> {
        let mut result: Vec<String> = Vec::new();
        for thing in one_two_three().iter().cycle().take(n) {
            result.push(thing.clone());
        }
        result
    }

    fn basic_configuration() -> Configuration {
        let input = one_two_three();
        Configuration::from_inputs(input, None, None)
    }

    #[test]
    fn it_selects_the_first_choice_by_default() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.selection(), Some("one".to_string()));
    }

    #[test]
    fn it_selets_the_second_when_down_is_called() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.down().selection(), Some("two".to_string()));
    }

    #[test]
    fn it_loop_around_when_reaching_end_of_list() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.down().down().down().down().selection(), Some("two".to_string()));
    }

    #[test]
    fn it_loop_around_when_reaching_top_of_list() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.up().up().selection(), Some("two".to_string()));
    }

    #[test]
    fn it_loop_around_when_reaching_visible_limit() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.down().down().down().down().selection(), Some("two".to_string()));
    }

    #[test]
    fn it_moves_down_the_filtered_search_results() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.append_to_search("t").down().selection(), Some("three".to_string()));
    }

    #[test]
    fn it_moves_down_the_filtered_search_results_twice() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.append_to_search("t").append_to_search("w").selection(), Some("two".to_string()));
    }

    #[test]
    fn it_handles_not_matching_anything() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert_eq!(search.append_to_search("x").selection(), None);
    }

    #[test]
    fn up_match_nothing_after_filtering_all_out() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("x");

        assert_eq!(search.up().selection(), None);
    }

    #[test]
    fn down_match_nothing_after_filtering_all_out() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("x");

        assert_eq!(search.down().selection(), None);
    }

    #[test]
    fn backspaces_over_characters() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("e");

        assert_eq!(search.query, "e");
        assert_eq!(search.backspace().query, "");
    }

    #[test]
    fn resets_the_index_when_removing_char_from_search() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("e");

        assert_eq!(search.down().backspace().current, 0);
    }

    #[test]
    fn resets_the_index_when_adding_char_to_seach() {
        let config = basic_configuration();
        let search = Search::blank(&config).down();

        assert_eq!(search.append_to_search("o").current, 0);
    }

    #[test]
    fn previous_results_appear_after_backspace() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("t");

        assert_eq!(search.backspace().num_matches(), 3);
    }

    #[test]
    fn initial_search_is_not_done() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        assert!(!search.is_done());
    }

    #[test]
    fn backspace_over_initial_doesnt_crash() {
        let config = basic_configuration();
        let search = Search::blank(&config);

        search.backspace();
    }

    #[test]
    fn done_search_is_done() {
        let config = basic_configuration();
        let search = Search::blank(&config).done();

        assert!(search.is_done());
    }

    #[test]
    fn done_search_has_selection() {
        let config = basic_configuration();
        let search = Search::blank(&config).done();

        assert_eq!(search.selection(), Some("one".to_string()));
    }

    #[test]
    fn loop_around_when_reaching_bottom_of_choices() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("n").down();

        assert_eq!(search.selection(), Some("one".to_string()));
    }

    #[test]
    fn loop_around_when_reaching_top_of_choices() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("n").up();

        assert_eq!(search.selection(), Some("one".to_string()));
    }

    #[test]
    fn search_is_case_insensitive() {
        let config = basic_configuration();
        let search = Search::blank(&config).append_to_search("T");

        assert_eq!(search.num_matches(), 2);
    }

    #[test]
    fn uses_configs_visible_limit_as_result_size() {
        let input = input_times(30);
        let config = Configuration::from_inputs(input, None, Some(20));
        let search = Search::blank(&config).append_to_search("T");

        assert_eq!(search.num_matches(), 20);
    }
}
<|endoftext|>


<commit_before>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn transpose(filename: String, index: usize, peers: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let graph = GraphMMap::<u32>::new(&filename);

    let mut edges = Vec::new();
    let mut deg = vec![];

    for node in 0..graph.nodes() {
        if node % peers == index {
            deg.push(graph.edges(node).len() as u32);
            for &b in graph.edges(node) {
                edges.push((b as u32, node as u32));
            }
        }
    }

    println!("slice {} of {} extracted {} edges", index, peers, edges.len());

    edges.sort();

    let mut rev = vec![(0,0);0];
    let mut reversed = vec![];
    for (d, s) in edges.drain_temp() {
        let len = rev.len();
        if (len == 0) || (rev[len-1].0 < d) {
            rev.push((d, 0));
        }

        let len = rev.len();
        rev[len-1].1 += 1;
        reversed.push(s / peers as u32);
    }

    return (deg, rev, reversed);
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let mut start = time::precise_time_s();

        let (deg, rev, edges) = transpose(filename, index, peers);

        println!("sorted {} edges; {}s", edges.len(), time::precise_time_s() - start);
        println!("src: {}, rev: {}, edges: {}", deg.len(), rev.len(), edges.len());
        start = time::precise_time_s();

        let mut src = vec![0.0; deg.len()];

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node] / deg[node] as f32;
                    }

                    let mut index = 0;
                    let mut slice = &edges[..];
                    while index < rev.len() {

                        let mut session = output.session(&iter);

                        for _ in 0..std::cmp::min(1_000, rev.len() - index) {

                            let (dst, deg) = rev[index];

                            let mut accum = 0.0;
                            for &s in &slice[..deg as usize] {
                                accum += src[s as usize];
                            }
                            slice = &slice[deg as usize..];
                            session.give((dst, accum));

                            index += 1;
                        }
                    }

                    for s in &mut src { *s = 0.0; }

                    println!("iteration {:?}: {}s", iter, time::precise_time_s() - start);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        // let local_index = index as usize % workers;
        // let mut acc = vec![0.0; src.len()];

        ranks
        // .unary_notify(
        //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for (node, rank) in data.drain_temp() {
        //                 acc[node as usize / workers] += rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
        //
        //             for _ in 0..(1 + (nodes/workers)) { acc.push(0.0); }
        //             assert!(acc.len() == (1 + (nodes/workers)));
        //         }
        //     }
        // )
        .connect_loop(helper);
    }

    while root.step() { }
}
<commit_msg>updated transpose<commit_after>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn transpose(filename: String, index: usize, peers: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let graph = GraphMMap::<u32>::new(&filename);

    let mut edges = Vec::new();
    let mut deg = vec![];

    for node in 0..graph.nodes() {
        if node % peers == index {
            deg.push(graph.edges(node).len() as u32);
            for &b in graph.edges(node) {
                edges.push((b as u32, node as u32));
            }
        }
    }

    println!("slice {} of {} extracted {} edges", index, peers, edges.len());

    edges.sort();

    let mut rev = vec![(0,0);0];
    let mut reversed = vec![];
    for (d, s) in edges.drain_temp() {
        let len = rev.len();
        if (len == 0) || (rev[len-1].0 < d) {
            rev.push((d, 0));
        }

        let len = rev.len();
        rev[len-1].1 += 1;
        reversed.push(s / peers as u32);
    }

    return (deg, rev, reversed);
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let mut start = time::precise_time_s();

        let (deg, rev, edges) = transpose(filename, index, peers);

        println!("sorted {} edges; {}s", edges.len(), time::precise_time_s() - start);
        println!("src: {}, rev: {}, edges: {}", deg.len(), rev.len(), edges.len());
        start = time::precise_time_s();

        let mut src = vec![0.0; deg.len()];

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node] / deg[node] as f32;
                    }

                    let mut index = 0;
                    let mut slice = &edges[..];
                    while index < rev.len() {

                        let mut session = output.session(&iter);

                        for _ in 0..std::cmp::min(100_000, rev.len() - index) {

                            let (dst, deg) = rev[index];

                            let mut accum = 0.0;
                            for &s in &slice[..deg as usize] {
                                accum += src[s as usize];
                            }
                            slice = &slice[deg as usize..];
                            session.give((dst, accum));

                            index += 1;
                        }
                    }

                    for s in &mut src { *s = 0.0; }

                    println!("iteration {:?}: {}s", iter, time::precise_time_s() - start);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        // let local_index = index as usize % workers;
        // let mut acc = vec![0.0; src.len()];

        ranks
        // .unary_notify(
        //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for (node, rank) in data.drain_temp() {
        //                 acc[node as usize / workers] += rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
        //
        //             for _ in 0..(1 + (nodes/workers)) { acc.push(0.0); }
        //             assert!(acc.len() == (1 + (nodes/workers)));
        //         }
        //     }
        // )
        .connect_loop(helper);
    }

    while root.step() { }
}
<|endoftext|>


<commit_before>/*
  This Source Code Form is subject to the terms of the Mozilla Public
  License, v. 2.0. If a copy of the MPL was not distributed with this
  file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

use std::collections::BTreeMap;
use std::path::{Path, PathBuf};

use rusqlite::Connection;
use rusqlite;

use collections::Collection;
use folders::{Folder, Folders, RootFolder};
use fromdb::FromDb;
use images::Image;
use keywords::Keyword;
use keywordtree::KeywordTree;
use libraryfiles::LibraryFile;
use lrobject::{LrId, LrObject};

const LR3_VERSION: i32 = 3;
const LR4_VERSION: i32 = 4;
const LR6_VERSION: i32 = 6;

/// Catalog version.
#[derive(Debug, PartialEq)]
pub enum CatalogVersion {
    /// Unknown version
    Unknown,
    /// Lightroom 3.x catalog. Unhandled.
    Lr3,
    /// Lightroom 4.x catalog.
    Lr4,
    /// Lightroom 6.x catalog.
    Lr6,
}

/// A Lightroom Catalog.
pub struct Catalog {
    /// catalog path
    path: PathBuf,
    /// The version string
    pub version: String,
    /// The catalog version
    pub catalog_version: CatalogVersion,
    /// Id for the root (top level) keyword
    pub root_keyword_id: LrId,

    /// The keywords, mapped in the local `LrId`
    keywords: BTreeMap<LrId, Keyword>,
    /// The `Folders` container.
    folders: Folders,
    /// The `Image` container
    images: Vec<Image>,
    /// The `LibraryFile` container
    libfiles: Vec<LibraryFile>,
    /// The `Collection` container
    collections: Vec<Collection>,

    /// The sqlite connectio to the catalog
    dbconn: Option<Connection>,
}

impl Catalog {
    /// Create a new catalog.
    pub fn new<P>(path: P) -> Catalog
        where P: AsRef<Path>
    {
        Catalog {
            path: path.as_ref().to_path_buf(),
            version: "".to_string(),
            catalog_version: CatalogVersion::Unknown,
            root_keyword_id: 0,
            keywords: BTreeMap::new(),
            folders: Folders::new(),
            images: vec![],
            libfiles: vec![],
            collections: vec![],
            dbconn: None,
        }
    }

    /// Open catalog. Return false in failure.
    /// This doesn't check if the content is valid beyond the backing sqlite3.
    pub fn open(&mut self) -> bool {
        let conn_attempt = Connection::open(&self.path);
        if let Ok(conn) = conn_attempt {
            self.dbconn = Some(conn);

            return true;
        }

        false
    }

    /// Get a variable from the table.
    fn get_variable<T>(&self, name: &str) -> Option<T>
    where
        T: rusqlite::types::FromSql,
    {
        let conn = try_opt!(self.dbconn.as_ref());
        if let Ok(mut stmt) = conn.prepare("SELECT value FROM Adobe_variablesTable WHERE name=?1") {
            let mut rows = stmt.query(&[&name]).unwrap();
            if let Some(Ok(row)) = rows.next() {
                return Some(row.get(0));
            }
        }
        None
    }

    /// Pare the version string from the database.
    fn parse_version(mut v: String) -> i32 {
        v.truncate(2);
        if let Ok(version) = v.parse::<i32>() {
            version
        } else {
            0
        }
    }

    /// Load version info for the catalog.
    pub fn load_version(&mut self) {
        if let Some(version) = self.get_variable::<String>("Adobe_DBVersion") {
            self.version = version;
            let v = Catalog::parse_version(self.version.clone());
            self.catalog_version = match v {
                LR6_VERSION => CatalogVersion::Lr6,
                LR4_VERSION => CatalogVersion::Lr4,
                LR3_VERSION => CatalogVersion::Lr3,
                _ => CatalogVersion::Unknown,
            };
        }

        if let Some(root_keyword_id) = self.get_variable::<f64>("AgLibraryKeyword_rootTagID") {
            self.root_keyword_id = root_keyword_id.round() as LrId;
        }
    }

    /// Generic object loader leveraging the FromDb protocol
    fn load_objects<T: FromDb>(conn: &Connection) -> Vec<T> {
        let mut result: Vec<T> = vec![];
        let query = format!(
            "SELECT {} FROM {}",
            T::read_db_columns(),
            T::read_db_tables()
        );
        if let Ok(mut stmt) = conn.prepare(&query) {
            let mut rows = stmt.query(&[]).unwrap();
            while let Some(Ok(row)) = rows.next() {
                if let Some(object) = T::read_from(&row) {
                    result.push(object);
                }
            }
        }
        result
    }

    /// Load a keyword tree
    pub fn load_keywords_tree(&mut self) -> KeywordTree {
        let keywords = self.load_keywords();

        let mut tree = KeywordTree::new();
        tree.add_children(keywords);

        tree
    }

    /// Load keywords.
    pub fn load_keywords(&mut self) -> &BTreeMap<LrId, Keyword> {
        if self.keywords.is_empty() {
            if let Some(ref conn) = self.dbconn {
                let result = Catalog::load_objects::<Keyword>(&conn);
                for keyword in result {
                    self.keywords.insert(keyword.id(), keyword);
                }
            }
        }
        &self.keywords
    }

    /// Load folders.
    pub fn load_folders(&mut self) -> &Folders {
        if self.folders.is_empty() {
            if let Some(ref conn) = self.dbconn {
                let folders = Catalog::load_objects::<RootFolder>(&conn);
                self.folders.append_root_folders(folders);
                let mut folders = Catalog::load_objects::<Folder>(&conn);
                for folder in &mut folders {
                    folder.content = Some(folder.read_content(conn));
                }
                self.folders.append_folders(folders);
            }
        }
        &self.folders
    }

    /// Load library files (that back images)
    pub fn load_library_files(&mut self) -> &Vec<LibraryFile> {
        if self.libfiles.is_empty() {
            if let Some(ref conn) = self.dbconn {
                let mut result = Catalog::load_objects::<LibraryFile>(&conn);
                self.libfiles.append(&mut result);
            }
        }
        &self.libfiles
    }

    /// Load images.
    pub fn load_images(&mut self) -> &Vec<Image> {
        if self.images.is_empty() {
            if let Some(ref conn) = self.dbconn {
                let mut result = Catalog::load_objects::<Image>(&conn);
                self.images.append(&mut result);
            }
        }
        &self.images
    }

    /// Load collectons.
    pub fn load_collections(&mut self) -> &Vec<Collection> {
        if self.collections.is_empty() {
            if let Some(ref conn) = self.dbconn {
                let mut collections = Catalog::load_objects::<Collection>(&conn);
                for collection in &mut collections {
                    collection.content = Some(collection.read_content(conn));
                }
                self.collections.append(&mut collections);
            }
        }
        &self.collections
    }
}
<commit_msg>Minor documentation fixes<commit_after>/*
  This Source Code Form is subject to the terms of the Mozilla Public
  License, v. 2.0. If a copy of the MPL was not distributed with this
  file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

use std::collections::BTreeMap;
use std::path::{Path, PathBuf};

use rusqlite::Connection;
use rusqlite;

use collections::Collection;
use folders::{Folder, Folders, RootFolder};
use fromdb::FromDb;
use images::Image;
use keywords::Keyword;
use keywordtree::KeywordTree;
use libraryfiles::LibraryFile;
use lrobject::{LrId, LrObject};

const LR3_VERSION: i32 = 3;
const LR4_VERSION: i32 = 4;
const LR6_VERSION: i32 = 6;

/// Catalog version.
#[derive(Debug, PartialEq)]
pub enum CatalogVersion {
    /// Unknown version
    Unknown,
    /// Lightroom 3.x catalog. Unhandled.
    Lr3,
    /// Lightroom 4.x catalog.
    Lr4,
    /// Lightroom 6.x catalog.
    Lr6,
}

/// Catalog is the main container for Lightroom. It represents
/// the .lrcat database.
pub struct Catalog {
    /// Catalog path
    path: PathBuf,
    /// The version string
    pub version: String,
    /// The catalog version
    pub catalog_version: CatalogVersion,
    /// Id for the root (top level) keyword
    pub root_keyword_id: LrId,

    /// The keywords, mapped in the local `LrId`
    keywords: BTreeMap<LrId, Keyword>,
    /// The `Folders` container.
    folders: Folders,
    /// The `Image` container
    images: Vec<Image>,
    /// The `LibraryFile` container
    libfiles: Vec<LibraryFile>,
    /// The `Collection` container
    collections: Vec<Collection>,

    /// The sqlite connectio to the catalog
    dbconn: Option<Connection>,
}

impl Catalog {
    /// Create a new catalog.
    pub fn new<P>(path: P) -> Catalog
        where P: AsRef<Path>
    {
        Catalog {
            path: path.as_ref().to_path_buf(),
            version: "".to_string(),
            catalog_version: CatalogVersion::Unknown,
            root_keyword_id: 0,
            keywords: BTreeMap::new(),
            folders: Folders::new(),
            images: vec![],
            libfiles: vec![],
            collections: vec![],
            dbconn: None,
        }
    }

    /// Open catalog. Return false in failure.
    /// This doesn't check if the content is valid beyond the backing sqlite3.
    pub fn open(&mut self) -> bool {
        let conn_attempt = Connection::open(&self.path);
        if let Ok(conn) = conn_attempt {
            self.dbconn = Some(conn);

            return true;
        }

        false
    }

    /// Get a variable from the table.
    fn get_variable<T>(&self, name: &str) -> Option<T>
    where
        T: rusqlite::types::FromSql,
    {
        let conn = try_opt!(self.dbconn.as_ref());
        if let Ok(mut stmt) = conn.prepare("SELECT value FROM Adobe_variablesTable WHERE name=?1") {
            let mut rows = stmt.query(&[&name]).unwrap();
            if let Some(Ok(row)) = rows.next() {
                return Some(row.get(0));
            }
        }
        None
    }

    /// Pare the version string from the database.
    fn parse_version(mut v: String) -> i32 {
        v.truncate(2);
        if let Ok(version) = v.parse::<i32>() {
            version
        } else {
            0
        }
    }

    /// Load version info for the catalog.
    pub fn load_version(&mut self) {
        if let Some(version) = self.get_variable::<String>("Adobe_DBVersion") {
            self.version = version;
            let v = Catalog::parse_version(self.version.clone());
            self.catalog_version = match v {
                LR6_VERSION => CatalogVersion::Lr6,
                LR4_VERSION => CatalogVersion::Lr4,
                LR3_VERSION => CatalogVersion::Lr3,
                _ => CatalogVersion::Unknown,
            };
        }

        if let Some(root_keyword_id) = self.get_variable::<f64>("AgLibraryKeyword_rootTagID") {
            self.root_keyword_id = root_keyword_id.round() as LrId;
        }
    }

    /// Generic object loader leveraging the FromDb protocol
    fn load_objects<T: FromDb>(conn: &Connection) -> Vec<T> {
        let mut result: Vec<T> = vec![];
        let query = format!(
            "SELECT {} FROM {}",
            T::read_db_columns(),
            T::read_db_tables()
        );
        if let Ok(mut stmt) = conn.prepare(&query) {
            let mut rows = stmt.query(&[]).unwrap();
            while let Some(Ok(row)) = rows.next() {
                if let Some(object) = T::read_from(&row) {
                    result.push(object);
                }
            }
        }
        result
    }

    /// Load a keyword tree
    pub fn load_keywords_tree(&mut self) -> KeywordTree {
        let keywords = self.load_keywords();

        let mut tree = KeywordTree::new();
        tree.add_children(keywords);

        tree
    }

    /// Load keywords.
    pub fn load_keywords(&mut self) -> &BTreeMap<LrId, Keyword> {
        if self.keywords.is_empty() {
            if let Some(ref conn) = self.dbconn {
                let result = Catalog::load_objects::<Keyword>(&conn);
                for keyword in result {
                    self.keywords.insert(keyword.id(), keyword);
                }
            }
        }
        &self.keywords
    }

    /// Load folders.
    pub fn load_folders(&mut self) -> &Folders {
        if self.folders.is_empty() {
            if let Some(ref conn) = self.dbconn {
                let folders = Catalog::load_objects::<RootFolder>(&conn);
                self.folders.append_root_folders(folders);
                let mut folders = Catalog::load_objects::<Folder>(&conn);
                for folder in &mut folders {
                    folder.content = Some(folder.read_content(conn));
                }
                self.folders.append_folders(folders);
            }
        }
        &self.folders
    }

    /// Load library files (that back images)
    pub fn load_library_files(&mut self) -> &Vec<LibraryFile> {
        if self.libfiles.is_empty() {
            if let Some(ref conn) = self.dbconn {
                let mut result = Catalog::load_objects::<LibraryFile>(&conn);
                self.libfiles.append(&mut result);
            }
        }
        &self.libfiles
    }

    /// Load images.
    pub fn load_images(&mut self) -> &Vec<Image> {
        if self.images.is_empty() {
            if let Some(ref conn) = self.dbconn {
                let mut result = Catalog::load_objects::<Image>(&conn);
                self.images.append(&mut result);
            }
        }
        &self.images
    }

    /// Load collectons.
    pub fn load_collections(&mut self) -> &Vec<Collection> {
        if self.collections.is_empty() {
            if let Some(ref conn) = self.dbconn {
                let mut collections = Catalog::load_objects::<Collection>(&conn);
                for collection in &mut collections {
                    collection.content = Some(collection.read_content(conn));
                }
                self.collections.append(&mut collections);
            }
        }
        &self.collections
    }
}
<|endoftext|>


<commit_before>use element::Element;
use errors::Result;
use glium;
use pipeline::Pipeline;
use rand::{SeedableRng, StdRng, random};
use std::{fs, thread, time};

pub struct SketchCfg {
    pub size: u32,
    pub root_frame_filename: Option<String>,
    pub seed: Option<usize>,
}

pub struct SketchContext {
    pub cfg: SketchCfg,
    pub frame: usize,
    pub rng: StdRng,
    pub current_seed: usize,
}

pub trait Sketch: Seed {
    fn draw<'a>(&'a self, ctx: &SketchContext) -> Result<Box<Iterator<Item = Element<'a>> + 'a>>;
    fn step(self,
            _ctx: &SketchContext,
            _events: Vec<glium::glutin::WindowEvent>)
            -> Result<Option<Self>> {
        Ok(Some(self))
    }
}

pub trait Seed: Sized {
    fn seed(self, ctx: &SketchContext) -> Result<Self>;
}

impl<T: Default> Seed for T {
    fn seed(self, _ctx: &SketchContext) -> Result<Self> { Ok(Self::default()) }
}

pub fn sketch<S: Sketch>(cfg: SketchCfg, sketch: S) -> Result<()> {
    let pipeline = Pipeline::new(cfg.size)?;
    let current_seed = cfg.seed.unwrap_or(random());
    let rng = StdRng::from_seed(&[current_seed]);
    let mut context = SketchContext { cfg, frame: 0, rng, current_seed };
    let mut sketch_bin = Some(sketch);

    let mut cycle = pipeline.events();
    while let Ok(Some((mut pipeline, events))) = cycle {
        if events
               .iter()
               .find(|event| match **event {
                         glium::glutin::WindowEvent::ReceivedCharacter('r') => true,
                         _ => false,
                     })
               .is_some() {
            context.current_seed = random();
            context.rng = StdRng::from_seed(&[context.current_seed]);
            sketch_bin = Some(sketch_bin.unwrap().seed(&context)?);
        }
        pipeline
            .draw(sketch_bin.as_ref().unwrap().draw(&context)?)?;
        sketch_bin = sketch_bin.unwrap().step(&context, events)?;
        if let Some(ref root_frame_filename) = context.cfg.root_frame_filename {
            let saves_dir = format!("{}/{:14}/", root_frame_filename, context.current_seed);
            fs::create_dir_all(&saves_dir)?;
            pipeline.save_frame(&saves_dir, context.frame)?;
        }
        cycle = match sketch_bin {
            Some(_) => pipeline.events(),
            None => Ok(None),
        };
        thread::sleep(time::Duration::from_millis(16));
        context.frame += 1;
    }
    cycle.map(|_| ())
}
<commit_msg>Update frame number when reseeding.<commit_after>use element::Element;
use errors::Result;
use glium;
use pipeline::Pipeline;
use rand::{SeedableRng, StdRng, random};
use std::{fs, thread, time};

pub struct SketchCfg {
    pub size: u32,
    pub root_frame_filename: Option<String>,
    pub seed: Option<usize>,
}

pub struct SketchContext {
    pub cfg: SketchCfg,
    pub frame: usize,
    pub rng: StdRng,
    pub current_seed: usize,
}

pub trait Sketch: Seed {
    fn draw<'a>(&'a self, ctx: &SketchContext) -> Result<Box<Iterator<Item = Element<'a>> + 'a>>;
    fn step(self,
            _ctx: &SketchContext,
            _events: Vec<glium::glutin::WindowEvent>)
            -> Result<Option<Self>> {
        Ok(Some(self))
    }
}

pub trait Seed: Sized {
    fn seed(self, ctx: &SketchContext) -> Result<Self>;
}

impl<T: Default> Seed for T {
    fn seed(self, _ctx: &SketchContext) -> Result<Self> { Ok(Self::default()) }
}

pub fn sketch<S: Sketch>(cfg: SketchCfg, sketch: S) -> Result<()> {
    let pipeline = Pipeline::new(cfg.size)?;
    let current_seed = cfg.seed.unwrap_or(random());
    let rng = StdRng::from_seed(&[current_seed]);
    let mut context = SketchContext { cfg, frame: 0, rng, current_seed };
    let mut sketch_bin = Some(sketch);

    let mut cycle = pipeline.events();
    while let Ok(Some((mut pipeline, events))) = cycle {
        if events
               .iter()
               .find(|event| match **event {
                         glium::glutin::WindowEvent::ReceivedCharacter('r') => true,
                         _ => false,
                     })
               .is_some() {
            context.current_seed = random();
            context.rng = StdRng::from_seed(&[context.current_seed]);
            context.frame = 0;
            sketch_bin = Some(sketch_bin.unwrap().seed(&context)?);
        }
        pipeline
            .draw(sketch_bin.as_ref().unwrap().draw(&context)?)?;
        sketch_bin = sketch_bin.unwrap().step(&context, events)?;
        if let Some(ref root_frame_filename) = context.cfg.root_frame_filename {
            let saves_dir = format!("{}/{:14}/", root_frame_filename, context.current_seed);
            fs::create_dir_all(&saves_dir)?;
            pipeline.save_frame(&saves_dir, context.frame)?;
        }
        cycle = match sketch_bin {
            Some(_) => pipeline.events(),
            None => Ok(None),
        };
        thread::sleep(time::Duration::from_millis(16));
        context.frame += 1;
    }
    cycle.map(|_| ())
}
<|endoftext|>


<commit_before>// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::fmt;
use std::io;
use std::mem;
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
#[cfg(unix)]
use libc::c_int;
#[cfg(windows)]
use winapi::c_int;

use sys;
use sys::c;

pub struct Socket {
    inner: sys::Socket,
}

impl Socket {
    pub fn new(family: c_int, ty: c_int) -> io::Result<Socket> {
        Ok(Socket { inner: try!(sys::Socket::new(family, ty)) })
    }

    pub fn bind(&self, addr: &SocketAddr) -> io::Result<()> {
        let (addr, len) = addr2raw(addr);
        unsafe {
            ::cvt(c::bind(self.inner.raw(), addr, len)).map(|_| ())
        }
    }

    pub fn listen(&self, backlog: i32) -> io::Result<()> {
        unsafe {
            ::cvt(c::listen(self.inner.raw(), backlog)).map(|_| ())
        }
    }

    pub fn connect(&self, addr: &SocketAddr) -> io::Result<()> {
        let (addr, len) = addr2raw(addr);
        unsafe {
            ::cvt(c::connect(self.inner.raw(), addr, len)).map(|_| ())
        }
    }

    pub fn getsockname(&self) -> io::Result<SocketAddr> {
        unsafe {
            let mut storage: c::sockaddr_storage = mem::zeroed();
            let mut len = mem::size_of_val(&storage) as c::socklen_t;
            try!(::cvt(c::getsockname(self.inner.raw(),
                                      &mut storage as *mut _ as *mut _,
                                      &mut len)));
            raw2addr(&storage, len)
        }
    }
}

impl fmt::Debug for Socket {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.inner.raw().fmt(f)
    }
}

impl ::AsInner for Socket {
    type Inner = sys::Socket;
    fn as_inner(&self) -> &sys::Socket { &self.inner }
}

impl ::FromInner for Socket {
    type Inner = sys::Socket;
    fn from_inner(sock: sys::Socket) -> Socket {
        Socket { inner: sock }
    }
}

impl ::IntoInner for Socket {
    type Inner = sys::Socket;
    fn into_inner(self) -> sys::Socket { self.inner }
}

fn addr2raw(addr: &SocketAddr) -> (*const c::sockaddr, c::socklen_t) {
    match *addr {
        SocketAddr::V4(ref a) => {
            (a as *const _ as *const _, mem::size_of_val(a) as c::socklen_t)
        }
        SocketAddr::V6(ref a) => {
            (a as *const _ as *const _, mem::size_of_val(a) as c::socklen_t)
        }
    }
}

fn raw2addr(storage: &c::sockaddr_storage, len: c::socklen_t) -> io::Result<SocketAddr> {
    match storage.ss_family as c_int {
        c::AF_INET => {
            unsafe {
                assert!(len as usize >= mem::size_of::<c::sockaddr_in>());
                let sa = storage as *const _ as *const c::sockaddr_in;
                let bits = c::sockaddr_in_u32(&(*sa));
                let ip = Ipv4Addr::new((bits >> 24) as u8,
                                       (bits >> 16) as u8,
                                       (bits >> 8) as u8,
                                       bits as u8);
                Ok(SocketAddr::V4(SocketAddrV4::new(ip, ::ntoh((*sa).sin_port))))
            }
        }
        c::AF_INET6 => {
            unsafe {
                assert!(len as usize >= mem::size_of::<c::sockaddr_in6>());

                let sa = storage as *const _ as *const c::sockaddr_in6;
                let arr = (*sa).sin6_addr.s6_addr;

                let ip = Ipv6Addr::new(
                    (arr[0] as u16) << 8 | (arr[1] as u16),
                    (arr[2] as u16) << 8 | (arr[3] as u16),
                    (arr[4] as u16) << 8 | (arr[5] as u16),
                    (arr[6] as u16) << 8 | (arr[7] as u16),
                    (arr[8] as u16) << 8 | (arr[9] as u16),
                    (arr[10] as u16) << 8 | (arr[11] as u16),
                    (arr[12] as u16) << 8 | (arr[13] as u16),
                    (arr[14] as u16) << 8 | (arr[15] as u16),
                );

                Ok(SocketAddr::V6(SocketAddrV6::new(ip,
                                                    ::ntoh((*sa).sin6_port),
                                                    (*sa).sin6_flowinfo,
                                                    (*sa).sin6_scope_id)))
            }
        }
        _ => Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid argument")),
    }
}
<commit_msg>fix cross compile on target aarch64-linux-android<commit_after>// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::fmt;
use std::io;
use std::mem;
use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
#[cfg(unix)]
use libc::c_int;
#[cfg(windows)]
use winapi::c_int;

use sys;
use sys::c;

pub struct Socket {
    inner: sys::Socket,
}

impl Socket {
    pub fn new(family: c_int, ty: c_int) -> io::Result<Socket> {
        Ok(Socket { inner: try!(sys::Socket::new(family, ty)) })
    }

    #[cfg(not(all(target_arch = "aarch64",target_os = "android")))]
    pub fn bind(&self, addr: &SocketAddr) -> io::Result<()> {
        let (addr, len) = addr2raw(addr);
        unsafe {
            ::cvt(c::bind(self.inner.raw(), addr, len)).map(|_| ())
        }
    }

    #[cfg(all(target_arch = "aarch64",target_os = "android"))]
    pub fn bind(&self, addr: &SocketAddr) -> io::Result<()> {
        let (addr, len) = addr2raw(addr);
        let len_i32 = len as i32;
        unsafe {
            ::cvt(c::bind(self.inner.raw(), addr, len_i32)).map(|_| ())
        }
    }

    pub fn listen(&self, backlog: i32) -> io::Result<()> {
        unsafe {
            ::cvt(c::listen(self.inner.raw(), backlog)).map(|_| ())
        }
    }

    pub fn connect(&self, addr: &SocketAddr) -> io::Result<()> {
        let (addr, len) = addr2raw(addr);
        unsafe {
            ::cvt(c::connect(self.inner.raw(), addr, len)).map(|_| ())
        }
    }

    pub fn getsockname(&self) -> io::Result<SocketAddr> {
        unsafe {
            let mut storage: c::sockaddr_storage = mem::zeroed();
            let mut len = mem::size_of_val(&storage) as c::socklen_t;
            try!(::cvt(c::getsockname(self.inner.raw(),
                                      &mut storage as *mut _ as *mut _,
                                      &mut len)));
            raw2addr(&storage, len)
        }
    }
}

impl fmt::Debug for Socket {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.inner.raw().fmt(f)
    }
}

impl ::AsInner for Socket {
    type Inner = sys::Socket;
    fn as_inner(&self) -> &sys::Socket { &self.inner }
}

impl ::FromInner for Socket {
    type Inner = sys::Socket;
    fn from_inner(sock: sys::Socket) -> Socket {
        Socket { inner: sock }
    }
}

impl ::IntoInner for Socket {
    type Inner = sys::Socket;
    fn into_inner(self) -> sys::Socket { self.inner }
}

fn addr2raw(addr: &SocketAddr) -> (*const c::sockaddr, c::socklen_t) {
    match *addr {
        SocketAddr::V4(ref a) => {
            (a as *const _ as *const _, mem::size_of_val(a) as c::socklen_t)
        }
        SocketAddr::V6(ref a) => {
            (a as *const _ as *const _, mem::size_of_val(a) as c::socklen_t)
        }
    }
}

fn raw2addr(storage: &c::sockaddr_storage, len: c::socklen_t) -> io::Result<SocketAddr> {
    match storage.ss_family as c_int {
        c::AF_INET => {
            unsafe {
                assert!(len as usize >= mem::size_of::<c::sockaddr_in>());
                let sa = storage as *const _ as *const c::sockaddr_in;
                let bits = c::sockaddr_in_u32(&(*sa));
                let ip = Ipv4Addr::new((bits >> 24) as u8,
                                       (bits >> 16) as u8,
                                       (bits >> 8) as u8,
                                       bits as u8);
                Ok(SocketAddr::V4(SocketAddrV4::new(ip, ::ntoh((*sa).sin_port))))
            }
        }
        c::AF_INET6 => {
            unsafe {
                assert!(len as usize >= mem::size_of::<c::sockaddr_in6>());

                let sa = storage as *const _ as *const c::sockaddr_in6;
                let arr = (*sa).sin6_addr.s6_addr;

                let ip = Ipv6Addr::new(
                    (arr[0] as u16) << 8 | (arr[1] as u16),
                    (arr[2] as u16) << 8 | (arr[3] as u16),
                    (arr[4] as u16) << 8 | (arr[5] as u16),
                    (arr[6] as u16) << 8 | (arr[7] as u16),
                    (arr[8] as u16) << 8 | (arr[9] as u16),
                    (arr[10] as u16) << 8 | (arr[11] as u16),
                    (arr[12] as u16) << 8 | (arr[13] as u16),
                    (arr[14] as u16) << 8 | (arr[15] as u16),
                );

                Ok(SocketAddr::V6(SocketAddrV6::new(ip,
                                                    ::ntoh((*sa).sin6_port),
                                                    (*sa).sin6_flowinfo,
                                                    (*sa).sin6_scope_id)))
            }
        }
        _ => Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid argument")),
    }
}
<|endoftext|>


<commit_before>pub enum XdrOp {
    Encode,
    Decode,
    Free,
}

pub struct Xdr {
    op: XdrOp,
    /// Pointer to users' data
    public: usize,
    /// Pointer to private data
    private: usize,
    /// Pointer to private used for position info
    base: usize,
    /// extra private word
    handy: isize,
}

pub trait XdrOps {
    /// Get a long from underlying stream
    fn get_long(&mut self, l: &mut usize) -> bool;

    /// Put a long to underlying stream
    fn put_long(&mut self, l: &usize) -> bool;

    /// Get a i32 from underlying stream
    fn get_i32(&mut self, i: &mut i32) -> bool;

    /// Put a i32 to underlying stream
    fn put_i32(&mut self, i: &i32) -> bool;

    /// Get some bytes from the underlying stream
    fn get_bytes(&mut self, bytes: &mut [u8]) -> bool;

    /// Put some bytes into the underlying stream
    fn put_bytes(&mut self, bytes: &[u8]) -> bool;

    /// Returns bytes off from beginning
    fn get_pos(&self) -> usize;

    /// Lets you reposition the stream
    fn set_pos(&mut self, offset: usize) -> bool;

    // TODO: Not sure if we'll need this?
    // Buf quick ptr to buffered data
    //fn inline(&mut self, len: usize) -> *mut i32;

    /// Free privates of this xdr_stream
    fn destroy(&mut self);

    // TODO: Not sure if we'll need this?
    // Change, retrieve client info
    //fn control(&mut self, req: isize, op: void *);
}
<commit_msg>Removed xdr_rpc.rs<commit_after><|endoftext|>


<commit_before>// Copyright 2013-2015, The Gtk-rs Project Developers.
// See the COPYRIGHT file at the top-level directory of this distribution.
// Licensed under the MIT license, see the LICENSE file or <http://opensource.org/licenses/MIT>

use std::cell::RefCell;
use std::mem::transmute;
use std::process;
use std::thread;
use ffi as glib_ffi;
use ffi::{gboolean, gpointer};
use translate::{from_glib, FromGlib, ToGlib};

/// The id of a source that is returned by `idle_add` and `timeout_add`.
pub struct Id(u32);

impl ToGlib for Id {
    type GlibType = u32;

    #[inline]
    fn to_glib(&self) -> u32 {
        self.0
    }
}

impl FromGlib<u32> for Id {
    #[inline]
    fn from_glib(val: u32) -> Id {
        Id(val)
    }
}

/// Continue calling the closure in the future iterations or drop it.
///
/// This is the return type of `idle_add` and `timeout_add` closures.
///
/// `Continue(true)` keeps the closure assigned, to be rerun when appropriate.
///
/// `Continue(false)` disconnects and drops it.
pub struct Continue(pub bool);

impl ToGlib for Continue {
    type GlibType = gboolean;

    #[inline]
    fn to_glib(&self) -> gboolean {
        self.0.to_glib()
    }
}

/// Unwinding propagation guard. Aborts the process if destroyed while
/// panicking.
pub struct CallbackGuard(());

impl CallbackGuard {
    pub fn new() -> CallbackGuard {
        CallbackGuard(())
    }
}

impl Drop for CallbackGuard {
    fn drop(&mut self) {
        if thread::panicking() {
            process::exit(101);
        }
    }
}

unsafe extern "C" fn trampoline(func: gpointer) -> gboolean {
    let _guard = CallbackGuard::new();
    let func: &RefCell<Box<FnMut() -> Continue + 'static>> = transmute(func);
    (&mut *func.borrow_mut())().to_glib()
}

unsafe extern "C" fn destroy_closure(ptr: gpointer) {
    let _guard = CallbackGuard::new();
    Box::<RefCell<Box<FnMut() -> Continue + 'static>>>::from_raw(ptr as *mut _);
}

fn into_raw<F: FnMut() -> Continue + Send + 'static>(func: F) -> gpointer {
    let func: Box<RefCell<Box<FnMut() -> Continue + Send + 'static>>> =
        Box::new(RefCell::new(Box::new(func)));
    Box::into_raw(func) as gpointer
}

/// Adds a closure to be called by the default main loop when it's idle.
///
/// `func` will be called repeatedly until it returns `Continue(false)`.
///
/// The default main loop almost always is the main loop of the main thread.
/// Thus the closure is called on the main thread.
pub fn idle_add<F>(func: F) -> Id
where F: FnMut() -> Continue + Send + 'static {
    from_glib(unsafe {
        glib_ffi::g_idle_add_full(glib_ffi::G_PRIORITY_DEFAULT_IDLE, Some(trampoline),
            into_raw(func), Some(destroy_closure))
    })
}

/// Adds a closure to be called by the default main loop at regular intervals
/// with millisecond granularity.
///
/// `func` will be called repeatedly every `interval` milliseconds until it
/// returns `Continue(false)`. Precise timing is not guaranteed, the timeout may
/// be delayed by other events. Prefer `timeout_add_seconds` when millisecond
/// precision is not necessary.
///
/// The default main loop almost always is the main loop of the main thread.
/// Thus the closure is called on the main thread.
pub fn timeout_add<F>(interval: u32, func: F) -> Id
where F: FnMut() -> Continue + Send + 'static {
    from_glib(unsafe {
        glib_ffi::g_timeout_add_full(glib_ffi::G_PRIORITY_DEFAULT, interval, Some(trampoline),
            into_raw(func), Some(destroy_closure))
    })
}

/// Adds a closure to be called by the default main loop at regular intervals
/// with second granularity.
///
/// `func` will be called repeatedly every `interval` seconds until it
/// returns `Continue(false)`. Precise timing is not guaranteed, the timeout may
/// be delayed by other events.
///
/// The default main loop almost always is the main loop of the main thread.
/// Thus the closure is called on the main thread.
pub fn timeout_add_seconds<F>(interval: u32, func: F) -> Id
where F: FnMut() -> Continue + Send + 'static {
    from_glib(unsafe {
        glib_ffi::g_timeout_add_seconds_full(glib_ffi::G_PRIORITY_DEFAULT, interval,
            Some(trampoline), into_raw(func), Some(destroy_closure))
    })
}

/// Removes the source with the given id `source_id` from the default main context.
///
/// It is a programmer error to attempt to remove a non-existent source.
/// Note: source id are reused.
///
/// For historical reasons, the native function always returns true, so we
/// ignore it here.
pub fn source_remove(source_id: Id) {
    unsafe {
        glib_ffi::g_source_remove(source_id.to_glib());
    }
}
<commit_msg>Standard derive for SourceId.<commit_after>// Copyright 2013-2015, The Gtk-rs Project Developers.
// See the COPYRIGHT file at the top-level directory of this distribution.
// Licensed under the MIT license, see the LICENSE file or <http://opensource.org/licenses/MIT>

use std::cell::RefCell;
use std::mem::transmute;
use std::process;
use std::thread;
use ffi as glib_ffi;
use ffi::{gboolean, gpointer};
use translate::{from_glib, FromGlib, ToGlib};

/// The id of a source that is returned by `idle_add` and `timeout_add`.
///
/// A value of 0 is a good default as it is never a valid source ID.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct Id(u32);

impl ToGlib for Id {
    type GlibType = u32;

    #[inline]
    fn to_glib(&self) -> u32 {
        self.0
    }
}

impl FromGlib<u32> for Id {
    #[inline]
    fn from_glib(val: u32) -> Id {
        Id(val)
    }
}

/// Continue calling the closure in the future iterations or drop it.
///
/// This is the return type of `idle_add` and `timeout_add` closures.
///
/// `Continue(true)` keeps the closure assigned, to be rerun when appropriate.
///
/// `Continue(false)` disconnects and drops it.
pub struct Continue(pub bool);

impl ToGlib for Continue {
    type GlibType = gboolean;

    #[inline]
    fn to_glib(&self) -> gboolean {
        self.0.to_glib()
    }
}

/// Unwinding propagation guard. Aborts the process if destroyed while
/// panicking.
pub struct CallbackGuard(());

impl CallbackGuard {
    pub fn new() -> CallbackGuard {
        CallbackGuard(())
    }
}

impl Drop for CallbackGuard {
    fn drop(&mut self) {
        if thread::panicking() {
            process::exit(101);
        }
    }
}

unsafe extern "C" fn trampoline(func: gpointer) -> gboolean {
    let _guard = CallbackGuard::new();
    let func: &RefCell<Box<FnMut() -> Continue + 'static>> = transmute(func);
    (&mut *func.borrow_mut())().to_glib()
}

unsafe extern "C" fn destroy_closure(ptr: gpointer) {
    let _guard = CallbackGuard::new();
    Box::<RefCell<Box<FnMut() -> Continue + 'static>>>::from_raw(ptr as *mut _);
}

fn into_raw<F: FnMut() -> Continue + Send + 'static>(func: F) -> gpointer {
    let func: Box<RefCell<Box<FnMut() -> Continue + Send + 'static>>> =
        Box::new(RefCell::new(Box::new(func)));
    Box::into_raw(func) as gpointer
}

/// Adds a closure to be called by the default main loop when it's idle.
///
/// `func` will be called repeatedly until it returns `Continue(false)`.
///
/// The default main loop almost always is the main loop of the main thread.
/// Thus the closure is called on the main thread.
pub fn idle_add<F>(func: F) -> Id
where F: FnMut() -> Continue + Send + 'static {
    unsafe {
        from_glib(glib_ffi::g_idle_add_full(glib_ffi::G_PRIORITY_DEFAULT_IDLE, Some(trampoline),
            into_raw(func), Some(destroy_closure)))
    }
}

/// Adds a closure to be called by the default main loop at regular intervals
/// with millisecond granularity.
///
/// `func` will be called repeatedly every `interval` milliseconds until it
/// returns `Continue(false)`. Precise timing is not guaranteed, the timeout may
/// be delayed by other events. Prefer `timeout_add_seconds` when millisecond
/// precision is not necessary.
///
/// The default main loop almost always is the main loop of the main thread.
/// Thus the closure is called on the main thread.
pub fn timeout_add<F>(interval: u32, func: F) -> Id
where F: FnMut() -> Continue + Send + 'static {
    unsafe {
        from_glib(glib_ffi::g_timeout_add_full(glib_ffi::G_PRIORITY_DEFAULT, interval,
            Some(trampoline), into_raw(func), Some(destroy_closure)))
    }
}

/// Adds a closure to be called by the default main loop at regular intervals
/// with second granularity.
///
/// `func` will be called repeatedly every `interval` seconds until it
/// returns `Continue(false)`. Precise timing is not guaranteed, the timeout may
/// be delayed by other events.
///
/// The default main loop almost always is the main loop of the main thread.
/// Thus the closure is called on the main thread.
pub fn timeout_add_seconds<F>(interval: u32, func: F) -> Id
where F: FnMut() -> Continue + Send + 'static {
    unsafe {
        from_glib(glib_ffi::g_timeout_add_seconds_full(glib_ffi::G_PRIORITY_DEFAULT, interval,
            Some(trampoline), into_raw(func), Some(destroy_closure)))
    }
}

/// Removes the source with the given id `source_id` from the default main context.
///
/// It is a programmer error to attempt to remove a non-existent source.
/// Note: source id are reused.
///
/// For historical reasons, the native function always returns true, so we
/// ignore it here.
pub fn source_remove(source_id: Id) {
    unsafe {
        glib_ffi::g_source_remove(source_id.to_glib());
    }
}
<|endoftext|>


<commit_before>//! # Source module.
//!
//! The Source module fetch metrics to Prometheus.
use std::thread;
use std::time::Duration;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use time;
use std::cmp;
use hyper;
use std::io::prelude::*;
use std::collections::HashMap;
use std::fs;
use std::fs::File;
use std::error::Error;
use std::path::Path;

use config;

/// Thread sleeping time.
const REST_TIME: u64 = 10;

/// Source loop.
pub fn source(source: &config::Source,
              labels: &HashMap<String, String>,
              parameters: &config::Parameters,
              sigint: Arc<AtomicBool>) {
    let labels: String = labels.iter()
        .fold(String::new(), |acc, (k, v)| {
            let sep = if acc.is_empty() { "" } else { "," };
            acc + sep + k + "=" + v
        });

    loop {
        let start = time::now_utc();

        match fetch(source, &labels, parameters) {
            Err(err) => error!("fetch fail: {}", err),
            Ok(_) => info!("fetch success"),
        }

        let elapsed = (time::now_utc() - start).num_milliseconds() as u64;
        let sleep_time = if elapsed > source.period {
            REST_TIME
        } else {
            cmp::max(source.period - elapsed, REST_TIME)
        };
        for _ in 0..sleep_time / REST_TIME {
            thread::sleep(Duration::from_millis(REST_TIME));
            if sigint.load(Ordering::Relaxed) {
                return;
            }
        }
    }
}

/// Fetch retrieve metrics from Prometheus.
fn fetch(source: &config::Source,
         labels: &String,
         parameters: &config::Parameters)
         -> Result<(), Box<Error>> {
    debug!("fetch {}", &source.url);

    // Fetch metrics
    let client = hyper::Client::new();
    let mut res = try!(client.get(&source.url).send());
    if !res.status.is_success() {
        return Err(From::from("non 200 received"));
    }

    // Read body
    let mut body = String::new();
    try!(res.read_to_string(&mut body));
    trace!("data {}", &body);

    // Get now as millis
    let start = time::now_utc();
    let now = start.to_timespec().sec * 1000 * 1000 + (start.to_timespec().nsec as i64 / 1000);

    let dir = Path::new(&parameters.source_dir);
    let temp_file = dir.join(format!("{}.tmp", source.name));
    debug!("write to tmp file {}", format!("{:?}", temp_file));
    {
        // Open tmp file
        let mut file = try!(File::create(&temp_file));

        for line in body.lines() {
            let line = match source.format {
                config::SourceFormat::Sensision => {
                    match format_sensision(line.trim(), labels) {
                        Err(_) => {
                            warn!("bad row {}", &line);
                            continue;
                        }
                        Ok(v) => v,
                    }
                },
                config::SourceFormat::Prometheus => {
                    match format_prometheus(line.trim(), labels, now) {
                        Err(_) => {
                            warn!("bad row {}", &line);
                            continue;
                        }
                        Ok(v) => v,
                    }
                }
            };

            if line.is_empty() {
                continue;
            }

            if source.metrics.is_some() {
                if !source.metrics.as_ref().unwrap().is_match(&line) {
                    continue;
                }
            }

            try!(file.write(line.as_bytes()));
            try!(file.write(b"\n"));
        }

        try!(file.flush());
    }

    // Rotate source file
    let dest_file = dir.join(format!("{}-{}.metrics", source.name, now));
    debug!("rotate tmp file to {}", format!("{:?}", dest_file));
    try!(fs::rename(&temp_file, &dest_file));

    Ok(())
}

/// Format Warp10 metrics from Prometheus one.
fn format_prometheus(line: &str, labels: &String, now: i64) -> Result<String, Box<Error>> {
    // Skip comments
    if line.starts_with("#") {
        return Ok(String::new());
    }

    // Extract Prometheus metric
    let index = if line.contains("{") {
        try!(line.rfind('}').ok_or("bad class"))
    } else {
        try!(line.find(' ').ok_or("bad class"))
    };
    let (class, v) = line.split_at(index + 1);
    let mut tokens = v.split_whitespace();

    let value = try!(tokens.next().ok_or("no value"));
    let timestamp = tokens.next()
        .map(|v| {
            i64::from_str_radix(v, 10)
                .map(|v| v * 1000 * 1000)
                .unwrap_or(now)
        })
        .unwrap_or(now);

    // Format class
    let mut parts = class.splitn(2, "{");
    let class = String::from(try!(parts.next().ok_or("no_class")));
    let plabels = parts.next();
    let mut slabels = if plabels.is_some() {
        let mut labels = plabels.unwrap().split("\",")
            .map(|v| v.replace("=", "%3D")) // escape
            .map(|v| v.replace("%3D\"", "=")) // remove left double quote
            .map(|v| v.replace("\"}", "")) // remove right double quote
            .map(|v| v.replace(",", "%2C")) // escape
            .map(|v| v.replace("}", "%7D")) // escape
            .map(|v| v.replace(r"\\", r"\")) // unescape
            .map(|v| v.replace("\\\"", "\"")) // unescape
            .map(|v| v.replace(r"\n", "%0A")) // unescape
            .fold(String::new(), |acc, x| acc + &x + ",");
        labels.pop();
        labels
    } else {
        String::new()
    };

    if !labels.is_empty() {
        if !slabels.is_empty() {
            slabels += ",";
        }
        slabels += labels;
    }

    let class = format!("{}{{{}}}", class, slabels);

    Ok(format!("{}// {} {}", timestamp, class, value))
}

/// Format Warp10 metrics from Sensision one.
fn format_sensision(line: &str, labels: &String) -> Result<String, Box<Error>> {
    if labels.is_empty() {
        return Ok(String::from(line));
    }
    let mut parts = line.splitn(2, "{");

    let class = String::from(try!(parts.next().ok_or("no_class")));
    let plabels = String::from(try!(parts.next().ok_or("no_labels")));

    let slabels = labels.clone() + if plabels.trim().starts_with("}") {""} else {","} + &plabels;

    Ok(format!("{}{{{}", class, slabels))
}
<commit_msg>fix(sources): handle trailling comma<commit_after>//! # Source module.
//!
//! The Source module fetch metrics to Prometheus.
use std::thread;
use std::time::Duration;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use time;
use std::cmp;
use hyper;
use std::io::prelude::*;
use std::collections::HashMap;
use std::fs;
use std::fs::File;
use std::error::Error;
use std::path::Path;

use config;

/// Thread sleeping time.
const REST_TIME: u64 = 10;

/// Source loop.
pub fn source(source: &config::Source,
              labels: &HashMap<String, String>,
              parameters: &config::Parameters,
              sigint: Arc<AtomicBool>) {
    let labels: String = labels.iter()
        .fold(String::new(), |acc, (k, v)| {
            let sep = if acc.is_empty() { "" } else { "," };
            acc + sep + k + "=" + v
        });

    loop {
        let start = time::now_utc();

        match fetch(source, &labels, parameters) {
            Err(err) => error!("fetch fail: {}", err),
            Ok(_) => info!("fetch success"),
        }

        let elapsed = (time::now_utc() - start).num_milliseconds() as u64;
        let sleep_time = if elapsed > source.period {
            REST_TIME
        } else {
            cmp::max(source.period - elapsed, REST_TIME)
        };
        for _ in 0..sleep_time / REST_TIME {
            thread::sleep(Duration::from_millis(REST_TIME));
            if sigint.load(Ordering::Relaxed) {
                return;
            }
        }
    }
}

/// Fetch retrieve metrics from Prometheus.
fn fetch(source: &config::Source,
         labels: &String,
         parameters: &config::Parameters)
         -> Result<(), Box<Error>> {
    debug!("fetch {}", &source.url);

    // Fetch metrics
    let client = hyper::Client::new();
    let mut res = try!(client.get(&source.url).send());
    if !res.status.is_success() {
        return Err(From::from("non 200 received"));
    }

    // Read body
    let mut body = String::new();
    try!(res.read_to_string(&mut body));
    trace!("data {}", &body);

    // Get now as millis
    let start = time::now_utc();
    let now = start.to_timespec().sec * 1000 * 1000 + (start.to_timespec().nsec as i64 / 1000);

    let dir = Path::new(&parameters.source_dir);
    let temp_file = dir.join(format!("{}.tmp", source.name));
    debug!("write to tmp file {}", format!("{:?}", temp_file));
    {
        // Open tmp file
        let mut file = try!(File::create(&temp_file));

        for line in body.lines() {
            let line = match source.format {
                config::SourceFormat::Sensision => {
                    match format_sensision(line.trim(), labels) {
                        Err(_) => {
                            warn!("bad row {}", &line);
                            continue;
                        }
                        Ok(v) => v,
                    }
                },
                config::SourceFormat::Prometheus => {
                    match format_prometheus(line.trim(), labels, now) {
                        Err(_) => {
                            warn!("bad row {}", &line);
                            continue;
                        }
                        Ok(v) => v,
                    }
                }
            };

            if line.is_empty() {
                continue;
            }

            if source.metrics.is_some() {
                if !source.metrics.as_ref().unwrap().is_match(&line) {
                    continue;
                }
            }

            try!(file.write(line.as_bytes()));
            try!(file.write(b"\n"));
        }

        try!(file.flush());
    }

    // Rotate source file
    let dest_file = dir.join(format!("{}-{}.metrics", source.name, now));
    debug!("rotate tmp file to {}", format!("{:?}", dest_file));
    try!(fs::rename(&temp_file, &dest_file));

    Ok(())
}

/// Format Warp10 metrics from Prometheus one.
fn format_prometheus(line: &str, labels: &String, now: i64) -> Result<String, Box<Error>> {
    // Skip comments
    if line.starts_with("#") {
        return Ok(String::new());
    }

    // Extract Prometheus metric
    let index = if line.contains("{") {
        try!(line.rfind('}').ok_or("bad class"))
    } else {
        try!(line.find(' ').ok_or("bad class"))
    };
    let (class, v) = line.split_at(index + 1);
    let mut tokens = v.split_whitespace();

    let value = try!(tokens.next().ok_or("no value"));
    let timestamp = tokens.next()
        .map(|v| {
            i64::from_str_radix(v, 10)
                .map(|v| v * 1000 * 1000)
                .unwrap_or(now)
        })
        .unwrap_or(now);

    // Format class
    let mut parts = class.splitn(2, "{");
    let class = String::from(try!(parts.next().ok_or("no_class")));
    let plabels = parts.next();
    let mut slabels = if plabels.is_some() {
        let mut labels = plabels.unwrap().split("\",")
            .map(|v| v.replace("=", "%3D")) // escape
            .map(|v| v.replace("%3D\"", "=")) // remove left double quote
            .map(|v| v.replace("\"}", "")) // remove right double quote
            .map(|v| v.replace(",", "%2C")) // escape
            .map(|v| v.replace("}", "%7D")) // escape
            .map(|v| v.replace(r"\\", r"\")) // unescape
            .map(|v| v.replace("\\\"", "\"")) // unescape
            .map(|v| v.replace(r"\n", "%0A")) // unescape
            .fold(String::new(), |acc, x| {
                // skip invalid values
                if !x.contains("=") {
                    return acc
                }
                acc + &x + ","
            });
        labels.pop();
        labels
    } else {
        String::new()
    };

    if !labels.is_empty() {
        if !slabels.is_empty() {
            slabels += ",";
        }
        slabels += labels;
    }

    let class = format!("{}{{{}}}", class, slabels);

    Ok(format!("{}// {} {}", timestamp, class, value))
}

/// Format Warp10 metrics from Sensision one.
fn format_sensision(line: &str, labels: &String) -> Result<String, Box<Error>> {
    if labels.is_empty() {
        return Ok(String::from(line));
    }
    let mut parts = line.splitn(2, "{");

    let class = String::from(try!(parts.next().ok_or("no_class")));
    let plabels = String::from(try!(parts.next().ok_or("no_labels")));

    let slabels = labels.clone() + if plabels.trim().starts_with("}") {""} else {","} + &plabels;

    Ok(format!("{}{{{}", class, slabels))
}
<|endoftext|>