<commit_before>extern crate regex;
#[macro_use]
extern crate log;
extern crate tini;

pub mod parser;
pub mod base;
pub mod extra;
pub mod logger;

use std::fs::File;
use std::io::Read;
use std::env::args;
use std::process::exit;
use extra::*;
use regex::Regex;
use tini::Ini;

static USAGE_STRING: &'static str = "\
>> доступные команды:
 -{n}       -- номер серии -n { стандартное значение = 1 }
 +{n}       -- номер серии +n { стандартное значение = 1 }
 a          -- добавить элемент [ a/имя | a/\"имя\" ]
 d          -- удалить элементы { найденые элементы параметром f }
 i          -- вывести весь список
 f{??}      -- поиск по параметру
   f        -- поиск по названию [ f/\"имя или regex\" ]
   fs{??}   -- по статусу { ?? -- буква статуса }
     где ??: c -- complete, d -- drop, h -- hold, p -- plan, w -- watch
   fp{??}   -- по номеру серии { ?? -- номер серии }
   fr{??}   -- по оценке { ?? -- оценка }
 s{??}      -- установить параметр
  sn        -- изменить имя на новое [ sn/имя | sn/\"имя\" ]
  sm{число} -- изменить максимальный номер серии { ? в случае онгоинга }
  sp{число} -- изменить номер серии на { число }
  sr{число} -- изменить рейтинг на { число }
  ss{??}    -- изменить статуc на { ?? -- буква статуса }
    где ??: c -- complete, d -- drop, h -- hold, p -- plan, w -- watch
 w          -- записать изменения в базу
>> example: 'f/\"One Piece\"/sm?/+5/-/sr7/sp23/ssc/sn/d.gray-man/sm24/w'";

fn main() {
    let config = Ini::from_file("./config.ini").unwrap();
    let log_level: String = config.get("main", "log_level").unwrap();
    let filename: String = config.get("main", "open_file").unwrap();
    logger::init(&log_level).unwrap();

    let arg_line = match args().nth(1) {
        Some(value) => value,
        None => {
            let app_name = args().nth(0).unwrap();
            println!("usage: {} '<regex>'\n{}", app_name, USAGE_STRING);
            exit(0);
        }
    };

    debug!("read list from file `{}`", filename);
    let mut anime_base = AnimeBase::new();
    let mut file = File::open(filename).unwrap();
    let mut buffer = String::new();
    file.read_to_string(&mut buffer).unwrap();
    for string in buffer.lines() {
        anime_base.push(base::Item::new(string));
    }

    debug!("command list:");
    let mut iterator = parser::Splitter::new(&arg_line, parser::SplitFormat::Commands);
    let mut anime_list: Vec<usize> = Vec::new();
    while let Some(cmd) = iterator.next() {
        match ExecCmd::get(cmd, &mut iterator) {
            ExecCmd::Increment(value) => {
                debug!("command inc by `{}`", value);
                for index in &anime_list {
                    anime_base.progress_increment(*index).unwrap();
                    println!("> update: {}", anime_base.format_by_index(*index));
                }
            }
            ExecCmd::Decrement(value) => {
                debug!("command dec by `{}`", value);
                for index in &anime_list {
                    anime_base.progress_decrement(*index).unwrap();
                    println!("> update: {}", anime_base.format_by_index(*index));
                }
            }
            ExecCmd::Append(name) => {
                debug!("command new anime `{}`", name);
                let new_item = anime_base.append(&name);
                anime_list.push(new_item);
                println!("> append: {}", anime_base.format_by_index(new_item));
            }
            ExecCmd::Delete => {
                debug!("command delete item");
            }
            ExecCmd::Info => {
                debug!("command print list");
                println!("{}", anime_base);
            }
            ExecCmd::Find(regex) => {
                debug!("command find `{}`", regex);
                let re = Regex::new(&regex).unwrap();
                for (index, item) in anime_base.list.iter().enumerate() {
                    if re.is_match(&item.name) {
                        anime_list.push(index);
                        println!(">  found: {}", anime_base.format(item));
                    }
                }
            }
            ExecCmd::FindParam(param) => {
                debug!("command find by param `{:?}`", param);
            }
            ExecCmd::Maximum(value) => {
                debug!("command series limit to `{}`", value);
                for index in &anime_list {
                    anime_base.set_maximum(*index, value).unwrap();
                    println!("> update: {}", anime_base.format_by_index(*index));
                }
            }
            ExecCmd::Rename(new_name) => {
                debug!("command new name `{}`", new_name);
            }
            ExecCmd::Progress(value) => {
                debug!("command progress `{}`", value);
                for index in &anime_list {
                    anime_base.set_progress(*index, value).unwrap();
                    println!("> update: {}", anime_base.format_by_index(*index));
                }
            }
            ExecCmd::Status(status) => {
                debug!("command status `{:?}`", status);
            }
            ExecCmd::Rate(value) => {
                debug!("command rate `{}`", value);
            }
            ExecCmd::Write => {
                debug!("command write changes");
                let mut file = File::create("anime-list").unwrap();
                anime_base.write_to_file(&mut file).expect("Can't write to file!");
            }
            ExecCmd::Error(kind) => warn!("command unknown `{}`: {:?}", cmd, kind),
        };
    }
}
<commit_msg>work with config<commit_after>extern crate regex;
#[macro_use]
extern crate log;
extern crate tini;

pub mod parser;
pub mod base;
pub mod extra;
pub mod logger;

use std::fs::File;
use std::io::Read;
use std::env::{self, args};
use std::process::exit;
use extra::*;
use regex::Regex;
use tini::Ini;

static USAGE_STRING: &'static str = "\
>> доступные команды:
 -{n}       -- номер серии -n { стандартное значение = 1 }
 +{n}       -- номер серии +n { стандартное значение = 1 }
 a          -- добавить элемент [ a/имя | a/\"имя\" ]
 d          -- удалить элементы { найденые элементы параметром f }
 i          -- вывести весь список
 f{??}      -- поиск по параметру
   f        -- поиск по названию [ f/\"имя или regex\" ]
   fs{??}   -- по статусу { ?? -- буква статуса }
     где ??: c -- complete, d -- drop, h -- hold, p -- plan, w -- watch
   fp{??}   -- по номеру серии { ?? -- номер серии }
   fr{??}   -- по оценке { ?? -- оценка }
 s{??}      -- установить параметр
  sn        -- изменить имя на новое [ sn/имя | sn/\"имя\" ]
  sm{число} -- изменить максимальный номер серии { ? в случае онгоинга }
  sp{число} -- изменить номер серии на { число }
  sr{число} -- изменить рейтинг на { число }
  ss{??}    -- изменить статуc на { ?? -- буква статуса }
    где ??: c -- complete, d -- drop, h -- hold, p -- plan, w -- watch
 w          -- записать изменения в базу
>> example: 'f/\"One Piece\"/sm?/+5/-/sr7/sp23/ssc/sn/d.gray-man/sm24/w'";

fn main() {
    let mut config_file = env::home_dir().unwrap();
    config_file.push(".config/watch-list/config.ini");
    let config = Ini::from_file(config_file.as_path()).unwrap();
    let log_level: String = config.get("main", "log_level").unwrap();
    let filename: String = config.get("main", "open_file").unwrap();
    logger::init(&log_level).unwrap();

    let arg_line = match args().nth(1) {
        Some(value) => value,
        None => {
            let app_name = args().nth(0).unwrap();
            println!("usage: {} '<regex>'\n{}", app_name, USAGE_STRING);
            exit(0);
        }
    };

    debug!("read list from file `{}`", filename);
    let mut anime_base = AnimeBase::new();
    let mut file = File::open(&filename).unwrap();
    let mut buffer = String::new();
    file.read_to_string(&mut buffer).unwrap();
    for string in buffer.lines() {
        anime_base.push(base::Item::new(string));
    }

    debug!("command list:");
    let mut iterator = parser::Splitter::new(&arg_line, parser::SplitFormat::Commands);
    let mut anime_list: Vec<usize> = Vec::new();
    while let Some(cmd) = iterator.next() {
        match ExecCmd::get(cmd, &mut iterator) {
            ExecCmd::Increment(value) => {
                debug!("command inc by `{}`", value);
                for index in &anime_list {
                    anime_base.progress_increment(*index).unwrap();
                    println!("> update: {}", anime_base.format_by_index(*index));
                }
            }
            ExecCmd::Decrement(value) => {
                debug!("command dec by `{}`", value);
                for index in &anime_list {
                    anime_base.progress_decrement(*index).unwrap();
                    println!("> update: {}", anime_base.format_by_index(*index));
                }
            }
            ExecCmd::Append(name) => {
                debug!("command new anime `{}`", name);
                let new_item = anime_base.append(&name);
                anime_list.push(new_item);
                println!("> append: {}", anime_base.format_by_index(new_item));
            }
            ExecCmd::Delete => {
                debug!("command delete item");
            }
            ExecCmd::Info => {
                debug!("command print list");
                println!("{}", anime_base);
            }
            ExecCmd::Find(regex) => {
                debug!("command find `{}`", regex);
                let re = Regex::new(&regex).unwrap();
                for (index, item) in anime_base.list.iter().enumerate() {
                    if re.is_match(&item.name) {
                        anime_list.push(index);
                        println!(">  found: {}", anime_base.format(item));
                    }
                }
            }
            ExecCmd::FindParam(param) => {
                debug!("command find by param `{:?}`", param);
            }
            ExecCmd::Maximum(value) => {
                debug!("command series limit to `{}`", value);
                for index in &anime_list {
                    anime_base.set_maximum(*index, value).unwrap();
                    println!("> update: {}", anime_base.format_by_index(*index));
                }
            }
            ExecCmd::Rename(new_name) => {
                debug!("command new name `{}`", new_name);
            }
            ExecCmd::Progress(value) => {
                debug!("command progress `{}`", value);
                for index in &anime_list {
                    anime_base.set_progress(*index, value).unwrap();
                    println!("> update: {}", anime_base.format_by_index(*index));
                }
            }
            ExecCmd::Status(status) => {
                debug!("command status `{:?}`", status);
            }
            ExecCmd::Rate(value) => {
                debug!("command rate `{}`", value);
            }
            ExecCmd::Write => {
                debug!("command write changes");
                let mut file = File::create(&filename).unwrap();
                anime_base.write_to_file(&mut file).expect("Can't write to file!");
            }
            ExecCmd::Error(kind) => warn!("command unknown `{}`: {:?}", cmd, kind),
        };
    }
}
<|endoftext|>


<commit_before>use std::io;
use std::io::prelude::*;
use std::net::TcpStream;
use std::fs::File;
use std::env;

extern crate chrono;
use chrono::*;

#[macro_use]
extern crate clap;

extern crate env_logger;
#[macro_use]
extern crate log;

extern crate stats;

extern crate prometheus;
use prometheus::Encoder;
extern crate hyper;

mod analyzer;
mod args;
mod filter;
mod log_parser;
use log_parser::log_events::*;
mod render;
use render::Renderer;
mod request_response_matcher;
use request_response_matcher::*;

fn main() {
    env_logger::init().expect("Failed to initialize logging.");

    let args = args::parse_args(env::args()).expect("Failed to parse arguments.");

    fn run(args: &args::RequestLogAnalyzerArgs) -> Option<analyzer::RequestLogAnalyzerResult> {
        let input: Box<io::Read> = match args.filename.as_ref() {
            "-" => Box::new(io::stdin()),
            _ => Box::new(File::open(&args.filename).expect("Failed to open file.")),
        };

        let timings = extract_timings(input, &args.conditions);
        let result = analyzer::analyze(&timings);

        result
    };

    let result = run(&args);
    run(&args);

    let mut stream;
    let mut renderer: Box<render::Renderer>;

    renderer = match args.graphite_server {
        Some(ref graphite_server) => {
            stream = TcpStream::connect((graphite_server.as_ref(), args.graphite_port.unwrap()))
                .expect("Could not connect to the Graphite server");

            Box::new(render::GraphiteRenderer::new(UTC::now(),
                                                   args.graphite_prefix.clone(),
                                                   &mut stream))
        }
        None => Box::new(render::TerminalRenderer::new()),
    };

    match result {
        Some(result) => {
            renderer.render(result);
        }
        None => warn!("No matching log lines in file."),
    }

    let addr = "127.0.0.1:9898";
    println!("listening addr {:?}", addr);
    hyper::server::Server::http(addr)
        .unwrap()
        .handle(move |_: hyper::server::Request, mut res: hyper::server::Response| {

            let result = run(&args);

            let mut renderer = render::prometheus::PrometheusRenderer::new();
            renderer.render(result.unwrap());
            res.headers_mut()
                .set(hyper::header::ContentType(renderer.encoder
                    .format_type()
                    .parse::<hyper::mime::Mime>()
                    .unwrap()));
            res.send(&renderer.buffer).unwrap();
        })
        .unwrap();
}

fn extract_timings(input: Box<io::Read>, conditions: &filter::FilterConditions) -> Vec<i64> {
    let reader = io::BufReader::new(input);

    let mut requests: Vec<Request> = Vec::new();
    let mut responses: Vec<Response> = Vec::new();
    let mut timings: Vec<i64> = Vec::new();

    for line in reader.lines() {
        let parsed_line = log_parser::parse_line(&line.expect("Failed to read line."));

        match parsed_line {
            Ok(event) => {
                match event {
                    LogEvent::Request(request) => requests.push(request),
                    LogEvent::Response(response) => responses.push(response),
                }

                let pairs = extract_matching_request_response_pairs(&mut requests, &mut responses);

                let mut new_timings: Vec<i64> = pairs.iter()
                    .filter(|pair| filter::matches_filter(&pair, conditions))
                    .map(|pair| pair.response.response_time.num_milliseconds())
                    .collect();

                timings.append(&mut new_timings);
            }
            Err(err) => warn!("{}", err),
        }
    }
    timings
}

#[test]
fn test_extract_timings() {
    let conditions = filter::FilterConditions {
        include_terms: None,
        exclude_terms: None,
        latest_time: None,
    };

    let timings = extract_timings(Box::new(File::open("src/test/simple-1.log").unwrap()),
                                  &conditions);

    assert_eq!(timings, vec![7, 10]);
}

#[test]
fn test_extract_timings_ignore_broken_lines() {
    let conditions = filter::FilterConditions {
        include_terms: None,
        exclude_terms: None,
        latest_time: None,
    };

    let timings = extract_timings(Box::new(File::open("src/test/broken.log").unwrap()),
                                  &conditions);

    assert_eq!(timings, vec![7]);
}
<commit_msg>Keep main() small<commit_after>use std::io;
use std::io::prelude::*;
use std::net::TcpStream;
use std::fs::File;
use std::env;

extern crate chrono;
use chrono::*;

#[macro_use]
extern crate clap;

extern crate env_logger;
#[macro_use]
extern crate log;

extern crate stats;

extern crate prometheus;
use prometheus::Encoder;
extern crate hyper;

mod analyzer;
mod args;
mod filter;
mod log_parser;
use log_parser::log_events::*;
mod render;
use render::Renderer;
mod request_response_matcher;
use request_response_matcher::*;

fn main() {
    env_logger::init().expect("Failed to initialize logging.");

    let args = args::parse_args(env::args()).expect("Failed to parse arguments.");

    let result = run(&args);

    let mut stream;
    let mut renderer: Box<render::Renderer>;

    renderer = match args.graphite_server {
        Some(ref graphite_server) => {
            stream = TcpStream::connect((graphite_server.as_ref(), args.graphite_port.unwrap()))
                .expect("Could not connect to the Graphite server");

            Box::new(render::GraphiteRenderer::new(UTC::now(),
                                                   args.graphite_prefix.clone(),
                                                   &mut stream))
        }
        None => Box::new(render::TerminalRenderer::new()),
    };

    match result {
        Some(result) => {
            renderer.render(result);
        }
        None => warn!("No matching log lines in file."),
    }

    let addr = "127.0.0.1:9898";
    println!("listening addr {:?}", addr);
    hyper::server::Server::http(addr)
        .unwrap()
        .handle(move |_: hyper::server::Request, mut res: hyper::server::Response| {

            let result = run(&args);

            let mut renderer = render::prometheus::PrometheusRenderer::new();
            renderer.render(result.unwrap());
            res.headers_mut()
                .set(hyper::header::ContentType(renderer.encoder
                    .format_type()
                    .parse::<hyper::mime::Mime>()
                    .unwrap()));
            res.send(&renderer.buffer).unwrap();
        })
        .unwrap();
}

fn run(args: &args::RequestLogAnalyzerArgs) -> Option<analyzer::RequestLogAnalyzerResult> {
    let input: Box<io::Read> = match args.filename.as_ref() {
        "-" => Box::new(io::stdin()),
        _ => Box::new(File::open(&args.filename).expect("Failed to open file.")),
    };

    let timings = extract_timings(input, &args.conditions);
    let result = analyzer::analyze(&timings);

    result
}

fn extract_timings(input: Box<io::Read>, conditions: &filter::FilterConditions) -> Vec<i64> {
    let reader = io::BufReader::new(input);

    let mut requests: Vec<Request> = Vec::new();
    let mut responses: Vec<Response> = Vec::new();
    let mut timings: Vec<i64> = Vec::new();

    for line in reader.lines() {
        let parsed_line = log_parser::parse_line(&line.expect("Failed to read line."));

        match parsed_line {
            Ok(event) => {
                match event {
                    LogEvent::Request(request) => requests.push(request),
                    LogEvent::Response(response) => responses.push(response),
                }

                let pairs = extract_matching_request_response_pairs(&mut requests, &mut responses);

                let mut new_timings: Vec<i64> = pairs.iter()
                    .filter(|pair| filter::matches_filter(&pair, conditions))
                    .map(|pair| pair.response.response_time.num_milliseconds())
                    .collect();

                timings.append(&mut new_timings);
            }
            Err(err) => warn!("{}", err),
        }
    }
    timings
}

#[test]
fn test_extract_timings() {
    let conditions = filter::FilterConditions {
        include_terms: None,
        exclude_terms: None,
        latest_time: None,
    };

    let timings = extract_timings(Box::new(File::open("src/test/simple-1.log").unwrap()),
                                  &conditions);

    assert_eq!(timings, vec![7, 10]);
}

#[test]
fn test_extract_timings_ignore_broken_lines() {
    let conditions = filter::FilterConditions {
        include_terms: None,
        exclude_terms: None,
        latest_time: None,
    };

    let timings = extract_timings(Box::new(File::open("src/test/broken.log").unwrap()),
                                  &conditions);

    assert_eq!(timings, vec![7]);
}
<|endoftext|>


<commit_before>extern crate liner;

use liner::Context;
use liner::KeyBindings;
use std::collections::HashMap;
use std::io::ErrorKind;

fn main() {
    let mut cache: HashMap<String, String> = HashMap::new();

    // TODO: Implement actual op-parsing
    let dummy_key = String::from("some_key");
    let dummy_value = String::from("123");
    let empty_value = String::from("0");

    let mut con = Context::new();

    loop {
        let res = con.read_line("> ", &mut |_| {});

        match res {
            Ok(res) => {
                match res.as_str() {
                    "emacs" => {
                        con.key_bindings = KeyBindings::Emacs;
                        println!("emacs mode");
                    }
                    "vi" | "vim" =>  {
                        con.key_bindings = KeyBindings::Vi;
                        println!("vi mode");
                    }
                    s if s.starts_with("get") =>  {
                        println!("{}", cache.get(&dummy_key).unwrap_or(&empty_value));
                    }
                    s if s.starts_with("set") =>  {
                        cache.insert(dummy_key.clone(), dummy_value.clone());
                    }
                    "exit" =>  {
                        break;
                    }
                    _ => {}
                }

                con.history.push(res.into()).unwrap();
            }
            Err(e) => {
                match e.kind() {
                    // ctrl-c pressed
                    ErrorKind::Interrupted => {}
                    // ctrl-d pressed
                    ErrorKind::UnexpectedEof => {
                        break;
                    }
                    _ => panic!("error: {:?}", e),
                }
            }
        }
    }
}
<commit_msg>Implement get, set operations<commit_after>extern crate liner;

use liner::Context;
use liner::KeyBindings;
use std::collections::HashMap;
use std::io::ErrorKind;

fn main() {
    let mut cache: HashMap<String, String> = HashMap::new();

    let empty_value = String::from("");

    let mut con = Context::new();

    loop {
        let res = con.read_line("> ", &mut |_| {});

        match res {
            Ok(res) => {
                match res.as_str() {
                    "emacs" => {
                        con.key_bindings = KeyBindings::Emacs;
                        println!("emacs mode");
                    }
                    "vi" | "vim" =>  {
                        con.key_bindings = KeyBindings::Vi;
                        println!("vi mode");
                    }
                    s if s.starts_with("get ") =>  {
                        let (_, key) = s.split_at(4);
                        let key_trimmed = String::from(key.trim());
                        println!("{}", cache.get(&key_trimmed).unwrap_or(&empty_value));
                    }
                    s if s.starts_with("set ") =>  {
                        let (_, key_value) = s.split_at(4);
                        match key_value.find(' ') {
                            Some(i) => {
                                let (key, value) = key_value.split_at(i);
                                let key_trimmed = String::from(key.trim());
                                let value_trimmed = String::from(value.trim());
                                cache.insert(key_trimmed, value_trimmed);
                            }
                            None => println!("no value specified")
                        }
                    }
                    "exit" =>  {
                        break;
                    }
                    _ => {}
                }

                con.history.push(res.into()).unwrap();
            }
            Err(e) => {
                match e.kind() {
                    // ctrl-c pressed
                    ErrorKind::Interrupted => {}
                    // ctrl-d pressed
                    ErrorKind::UnexpectedEof => {
                        break;
                    }
                    _ => panic!("error: {:?}", e),
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>extern crate hyper;
extern crate toml;

extern crate rustc_serialize;
use rustc_serialize::json;

use std::io::{Read, Write};
use std::thread;

use hyper::Client;
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;

// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcDecodable)]
pub struct SymbolRequest {
    memoryMap: Vec<(String,String)>,
    // TODO check that key actually fits in 8-bit int
    stacks: Vec<Vec<(i8,i64)>>,
    symbolSources: Vec<String>,
    version: u8,
}

fn main() {
    let address = "0.0.0.0:8080";

    println!("Listening on {}", address);
    Server::http(address).unwrap().handle(server).unwrap();
}

/**
  * Receives single HTTP requests and demuxes to symbols file fetches from S3 bucket.
  */
fn server(mut req: Request, mut res: Response) {
    match req.method {
        hyper::Post => {
            let mut res = res.start().unwrap();

            let mut buffer = String::new();
            let _ = req.read_to_string(&mut buffer);
            println!("DEBUG raw POST: {:?}", &buffer);
            let decoded: SymbolRequest = json::decode(&buffer).unwrap();

            println!("DEBUG decoded memoryMap: {:?}", decoded.memoryMap);
            println!("DEBUG decoded stacks: {:?}", decoded.stacks);
            println!("DEBUG decoded symbolSources: {:?}", decoded.symbolSources);
            println!("DEBUG decoded version: {}", decoded.version);

            let symbol_url = get_config("symbol_urls.public");
            let symbols = client(symbol_url, decoded.memoryMap);
            let _ = res.write_all(symbols.as_bytes());

            res.end().unwrap();
        },
        _ => { *res.status_mut() = StatusCode::MethodNotAllowed },
    }
}

/**
  * Creates multiple client connections and aggregates result.
  */
fn client(url: String, memory_map: Vec<(String,String)>) -> String {
    let mut handles = vec![];
    for (debug_file, debug_id) in memory_map {
        let pdb = debug_file.find(".pdb").unwrap();
        let (symbol_name, _) = debug_file.split_at(pdb);
        let symbol_file = format!("{}.sym", symbol_name);
        let this_url = format!("{}/{}/{}/{}", url, debug_file, debug_id, symbol_file);

        // TODO decide min/max possible threads, possibly based on number of cores?
        // TODO most of the time is spent waiting on I/O, maybe async would be more appropriate?
        handles.push(thread::spawn(move || {
            let c = Client::new();
            let mut res = c.get(&this_url).send().unwrap();
            let mut body = String::new();
            let _ = res.read_to_string(&mut body);

            (symbol_file, body)
        }));
    }

    let mut result = String::new();

    for handle in handles {
        let (symbol_file, body) = handle.join().unwrap();
        for c in format!("{:?} {:?}\n", symbol_file, body).chars() {
            result.push(c);
        }
    }

    result
}

fn get_config(value_name: &str) -> String {
    // TODO move to actual file, static str for the moment
    let &'static toml = r#"
        [symbol_urls]
        public = "https://s3-us-west-2.amazonaws.com/org.mozilla.crash-stats.symbols-public/v1"
    "#;
    // TODO support multiple URLs
    let value: toml::Value = toml.parse().unwrap();

    value.lookup(value_name).unwrap().as_str().unwrap().to_string()
}
<commit_msg>specify static str<commit_after>extern crate hyper;
extern crate toml;

extern crate rustc_serialize;
use rustc_serialize::json;

use std::io::{Read, Write};
use std::thread;

use hyper::Client;
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;

// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcDecodable)]
pub struct SymbolRequest {
    memoryMap: Vec<(String,String)>,
    // TODO check that key actually fits in 8-bit int
    stacks: Vec<Vec<(i8,i64)>>,
    symbolSources: Vec<String>,
    version: u8,
}

fn main() {
    let address = "0.0.0.0:8080";

    println!("Listening on {}", address);
    Server::http(address).unwrap().handle(server).unwrap();
}

/**
  * Receives single HTTP requests and demuxes to symbols file fetches from S3 bucket.
  */
fn server(mut req: Request, mut res: Response) {
    match req.method {
        hyper::Post => {
            let mut res = res.start().unwrap();

            let mut buffer = String::new();
            let _ = req.read_to_string(&mut buffer);
            println!("DEBUG raw POST: {:?}", &buffer);
            let decoded: SymbolRequest = json::decode(&buffer).unwrap();

            println!("DEBUG decoded memoryMap: {:?}", decoded.memoryMap);
            println!("DEBUG decoded stacks: {:?}", decoded.stacks);
            println!("DEBUG decoded symbolSources: {:?}", decoded.symbolSources);
            println!("DEBUG decoded version: {}", decoded.version);

            let symbol_url = get_config("symbol_urls.public");
            let symbols = client(symbol_url, decoded.memoryMap);
            let _ = res.write_all(symbols.as_bytes());

            res.end().unwrap();
        },
        _ => { *res.status_mut() = StatusCode::MethodNotAllowed },
    }
}

/**
  * Creates multiple client connections and aggregates result.
  */
fn client(url: String, memory_map: Vec<(String,String)>) -> String {
    let mut handles = vec![];
    for (debug_file, debug_id) in memory_map {
        let pdb = debug_file.find(".pdb").unwrap();
        let (symbol_name, _) = debug_file.split_at(pdb);
        let symbol_file = format!("{}.sym", symbol_name);
        let this_url = format!("{}/{}/{}/{}", url, debug_file, debug_id, symbol_file);

        // TODO decide min/max possible threads, possibly based on number of cores?
        // TODO most of the time is spent waiting on I/O, maybe async would be more appropriate?
        handles.push(thread::spawn(move || {
            let c = Client::new();
            let mut res = c.get(&this_url).send().unwrap();
            let mut body = String::new();
            let _ = res.read_to_string(&mut body);

            (symbol_file, body)
        }));
    }

    let mut result = String::new();

    for handle in handles {
        let (symbol_file, body) = handle.join().unwrap();
        for c in format!("{:?} {:?}\n", symbol_file, body).chars() {
            result.push(c);
        }
    }

    result
}

fn get_config(value_name: &str) -> String {
    // TODO move to actual file, static str for the moment
    let toml: &'static str = r#"
        [symbol_urls]
        public = "https://s3-us-west-2.amazonaws.com/org.mozilla.crash-stats.symbols-public/v1"
    "#;
    // TODO support multiple URLs
    let value: toml::Value = toml.parse().unwrap();

    value.lookup(value_name).unwrap().as_str().unwrap().to_string()
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::ops::Index;
use std::ops::IndexMut;

#[derive(Copy, Clone)]
pub enum MemSection {
    Vram,
    RomBank0,
}

pub struct AddressSpace {
    data: [u8; 0x10000],
}

pub const IOREG_P1:     u16 = 0xFF00;
pub const IOREG_SB:     u16 = 0xFF01;
pub const IOREG_DIV:    u16 = 0xFF04;
pub const IOREG_TIMA:   u16 = 0xFF05;
pub const IOREG_TMA:    u16 = 0xFF06;
pub const IOREG_TAC:    u16 = 0xFF07;
pub const IOREG_IF:     u16 = 0xFF0F;
pub const IOREG_LCDC:   u16 = 0xFF40;
pub const IOREG_STAT:   u16 = 0xFF41;
pub const IOREG_SCY:    u16 = 0xFF42;
pub const IOREG_SCX:    u16 = 0xFF43;
pub const IOREG_LY:     u16 = 0xFF44;
pub const IOREG_LYC:    u16 = 0xFF45;
pub const IOREG_BGP:    u16 = 0xFF47;
pub const IOREG_OBP0:   u16 = 0xFF48;
pub const IOREG_OBP1:   u16 = 0xFF49;
pub const IOREG_WY:     u16 = 0xFF4A;
pub const IOREG_WX:     u16 = 0xFF4B;
pub const IOREG_IE:     u16 = 0xFFFF;

impl AddressSpace {

    pub fn new() -> AddressSpace {
        AddressSpace { data: [0; 0x10000] }
    }

    pub fn read(&self, addr: u16) -> u8 {
        self.data[addr as usize]
    }

    pub fn read_u16(&self, addr: u16) -> u16 {
        let lo = self.read(addr) as u16;
        let hi = (self.read(addr + 1) as u16) << 8;
        lo | hi
    }

    pub fn read_slice(&self, addr: u16, bytes: u16) -> &[u8] {
        &self.data[addr as usize .. (addr + bytes) as usize]
    }

    pub fn write(&mut self, addr: u16, data: u8) {
        let mut addr = addr;
        let mut data = data;
        let rw = match addr {
            // ROM Banks, read only
            0x0000...0x7FFF => false,
            // Switchable RAM bank
            // TODO: Make RAM switchable
            0xA000...0xBFFF => true,
            // Internal RAM echo
            0xE000...0xFDFF => {
                addr -= 0x2000;
                true
            },
            // I/O registers
            IOREG_DIV => {
                data = 0;
                true
            },
            IOREG_LY => {
                data = 0;
                true
            },
            // No special write rules
            _ => true,
        };
        if rw {
            self.data[addr as usize] = data;
        }
    }

    pub fn write_u16(&mut self, addr: u16, data: u16) {
        let lo = (data & 0xFF) as u8;
        let hi = ((data & 0xFF00) >> 8) as u8;
        self.write(addr, lo);
        self.write(addr + 1, hi);
    }

    pub fn load_bios(&mut self, bios: &mut File) -> ::std::io::Result<()> {
        try!(bios.read(&mut self.data[0x000..0x100]));
        Ok(())
    }

    pub fn load_rom(&mut self, rom: &mut File) -> ::std::io::Result<()> {
        // Read in header first
        try!(rom.seek(SeekFrom::Start(0x100)));
        try!(rom.read(&mut self.data[0x100..0x150]));
        // Then read in remaining cart data
        try!(rom.read(&mut self.data[0x0150..0x8000]));
        Ok(())
    }

}

impl Index<u16> for AddressSpace {
    type Output = u8;

    fn index(&self, idx: u16) -> &u8 {
        &self.data[idx as usize]
    }
}

impl IndexMut<u16> for AddressSpace {
    fn index_mut(&mut self, idx: u16) -> &mut u8 {
        &mut self.data[idx as usize]
    }
}

#[derive(Copy, Clone)]
pub enum RegFlag {
    Zero,
    Subtract,
    HalfCarry,
    Carry,
}

#[derive(Copy, Clone)]
pub enum Register {
    A,
    B,
    C,
    D,
    E,
    F,
    H,
    L,
    AF,
    BC,
    DE,
    HL,
    SP,
    PC,
    Flag,
}

pub struct RegData {
    a: u8,
    b: u8,
    c: u8,
    d: u8,
    e: u8,
    f: u8,
    h: u8,
    l: u8,
    sp: u16,
    pc: u16,
    flag: u8,
}

impl RegData {

    pub fn new() -> RegData {
        RegData {
            a: 0,
            b: 0,
            c: 0,
            d: 0,
            e: 0,
            f: 0,
            h: 0,
            l: 0,
            sp: 0xFFFE,
            pc: 0x100,
            flag: 0,
        }
    }

    pub fn read(&self, reg: Register) -> u8 {
        match reg {
            Register::A => self.a,
            Register::B => self.b,
            Register::C => self.c,
            Register::D => self.d,
            Register::E => self.e,
            Register::F => self.f,
            Register::H => self.h,
            Register::L => self.l,
            Register::Flag => self.flag,
            _ => panic!("Register not available for 8-bit read"),
        }
    }

    pub fn read_u16(&self, reg: Register) -> u16 {
        match reg {
            Register::AF => (self.a as u16) << 8 | self.f as u16,
            Register::BC => (self.b as u16) << 8 | self.c as u16,
            Register::DE => (self.d as u16) << 8 | self.e as u16,
            Register::HL => (self.h as u16) << 8 | self.l as u16,
            Register::SP => self.sp,
            Register::PC => self.pc,
            _ => panic!("Register not available for 16-bit read"),
        }
    }

    pub fn write(&mut self, reg: Register, data: u8) {
        match reg {
            Register::A => self.a = data,
            Register::B => self.b = data,
            Register::C => self.c = data,
            Register::D => self.d = data,
            Register::E => self.e = data,
            Register::F => self.f = data,
            Register::H => self.h = data,
            Register::L => self.l = data,
            _ => panic!("Register not available for 8-bit write"),
        }
    }

    pub fn write_u16(&mut self, reg: Register, data: u16) {
        let (hi, lo) = ((data & 0xFF00 >> 8) as u8, (data & 0xFF) as u8);
        match reg {
            Register::AF => {
                self.a = hi;
                self.f = lo;
            },
            Register::BC => {
                self.b = hi;
                self.c = lo;
            },
            Register::DE => {
                self.d = hi;
                self.c = lo;
            },
            Register::HL => {
                self.h = hi;
                self.l = lo;
            },
            Register::SP => self.sp = data,
            Register::PC => self.pc = data,
            _ => panic!("Register not available for 16-bit write"),
        }
    }

    pub fn copy(&mut self, dst: Register, src: Register) {
        let data = self.read(src);
        self.write(dst, data);
    }

    pub fn copy_u16(&mut self, dst: Register, src: Register) {
        let data = self.read_u16(src);
        self.write_u16(dst, data);
    }

    pub fn set_flag(&mut self, flag: RegFlag, on: bool) {
        let bit = match flag {
            RegFlag::Zero => 0x80,
            RegFlag::Subtract => 0x40,
            RegFlag::HalfCarry => 0x20,
            RegFlag::Carry => 0x10,
        };
        if (on) {
            self.flag |= bit;
        } else {
            self.flag &= bit ^ 0xFF;
        }
    }

    pub fn get_flag(&self, flag: RegFlag) -> bool {
        let bit = match flag {
            RegFlag::Zero => 0x80,
            RegFlag::Subtract => 0x40,
            RegFlag::HalfCarry => 0x20,
            RegFlag::Carry => 0x10,
        };
        self.flag & bit != 0
    }

    pub fn advance_pc(&mut self) -> u16 {
        let pc = self.pc;
        self.pc += 1;
        pc
    }

    pub fn set_pc(&mut self, addr: u16) -> u16 {
        let pc = self.pc;
        self.pc = addr;
        pc
    }

    pub fn add_pc(&mut self, n: i8) -> u16 {
        let pc = self.pc;
        self.pc = ((self.pc as i32) + (n as i32)) as u16;
        pc
    }

    pub fn get_pc(&self) -> u16 {
        self.pc
    }

}
<commit_msg>Fix parenthesis error in register write function<commit_after>use std::fs::File;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::ops::Index;
use std::ops::IndexMut;

#[derive(Copy, Clone)]
pub enum MemSection {
    Vram,
    RomBank0,
}

pub struct AddressSpace {
    data: [u8; 0x10000],
}

pub const IOREG_P1:     u16 = 0xFF00;
pub const IOREG_SB:     u16 = 0xFF01;
pub const IOREG_DIV:    u16 = 0xFF04;
pub const IOREG_TIMA:   u16 = 0xFF05;
pub const IOREG_TMA:    u16 = 0xFF06;
pub const IOREG_TAC:    u16 = 0xFF07;
pub const IOREG_IF:     u16 = 0xFF0F;
pub const IOREG_LCDC:   u16 = 0xFF40;
pub const IOREG_STAT:   u16 = 0xFF41;
pub const IOREG_SCY:    u16 = 0xFF42;
pub const IOREG_SCX:    u16 = 0xFF43;
pub const IOREG_LY:     u16 = 0xFF44;
pub const IOREG_LYC:    u16 = 0xFF45;
pub const IOREG_BGP:    u16 = 0xFF47;
pub const IOREG_OBP0:   u16 = 0xFF48;
pub const IOREG_OBP1:   u16 = 0xFF49;
pub const IOREG_WY:     u16 = 0xFF4A;
pub const IOREG_WX:     u16 = 0xFF4B;
pub const IOREG_IE:     u16 = 0xFFFF;

impl AddressSpace {

    pub fn new() -> AddressSpace {
        AddressSpace { data: [0; 0x10000] }
    }

    pub fn read(&self, addr: u16) -> u8 {
        self.data[addr as usize]
    }

    pub fn read_u16(&self, addr: u16) -> u16 {
        let lo = self.read(addr) as u16;
        let hi = (self.read(addr + 1) as u16) << 8;
        lo | hi
    }

    pub fn read_slice(&self, addr: u16, bytes: u16) -> &[u8] {
        &self.data[addr as usize .. (addr + bytes) as usize]
    }

    pub fn write(&mut self, addr: u16, data: u8) {
        let mut addr = addr;
        let mut data = data;
        let rw = match addr {
            // ROM Banks, read only
            0x0000...0x7FFF => false,
            // Switchable RAM bank
            // TODO: Make RAM switchable
            0xA000...0xBFFF => true,
            // Internal RAM echo
            0xE000...0xFDFF => {
                addr -= 0x2000;
                true
            },
            // I/O registers
            IOREG_DIV => {
                data = 0;
                true
            },
            IOREG_LY => {
                data = 0;
                true
            },
            // No special write rules
            _ => true,
        };
        if rw {
            self.data[addr as usize] = data;
        }
    }

    pub fn write_u16(&mut self, addr: u16, data: u16) {
        let lo = (data & 0xFF) as u8;
        let hi = ((data & 0xFF00) >> 8) as u8;
        self.write(addr, lo);
        self.write(addr + 1, hi);
    }

    pub fn load_bios(&mut self, bios: &mut File) -> ::std::io::Result<()> {
        try!(bios.read(&mut self.data[0x000..0x100]));
        Ok(())
    }

    pub fn load_rom(&mut self, rom: &mut File) -> ::std::io::Result<()> {
        // Read in header first
        try!(rom.seek(SeekFrom::Start(0x100)));
        try!(rom.read(&mut self.data[0x100..0x150]));
        // Then read in remaining cart data
        try!(rom.read(&mut self.data[0x0150..0x8000]));
        Ok(())
    }

}

impl Index<u16> for AddressSpace {
    type Output = u8;

    fn index(&self, idx: u16) -> &u8 {
        &self.data[idx as usize]
    }
}

impl IndexMut<u16> for AddressSpace {
    fn index_mut(&mut self, idx: u16) -> &mut u8 {
        &mut self.data[idx as usize]
    }
}

#[derive(Copy, Clone)]
pub enum RegFlag {
    Zero,
    Subtract,
    HalfCarry,
    Carry,
}

#[derive(Copy, Clone)]
pub enum Register {
    A,
    B,
    C,
    D,
    E,
    F,
    H,
    L,
    AF,
    BC,
    DE,
    HL,
    SP,
    PC,
    Flag,
}

pub struct RegData {
    a: u8,
    b: u8,
    c: u8,
    d: u8,
    e: u8,
    f: u8,
    h: u8,
    l: u8,
    sp: u16,
    pc: u16,
    flag: u8,
}

impl RegData {

    pub fn new() -> RegData {
        RegData {
            a: 0,
            b: 0,
            c: 0,
            d: 0,
            e: 0,
            f: 0,
            h: 0,
            l: 0,
            sp: 0xFFFE,
            pc: 0x100,
            flag: 0,
        }
    }

    pub fn read(&self, reg: Register) -> u8 {
        match reg {
            Register::A => self.a,
            Register::B => self.b,
            Register::C => self.c,
            Register::D => self.d,
            Register::E => self.e,
            Register::F => self.f,
            Register::H => self.h,
            Register::L => self.l,
            Register::Flag => self.flag,
            _ => panic!("Register not available for 8-bit read"),
        }
    }

    pub fn read_u16(&self, reg: Register) -> u16 {
        match reg {
            Register::AF => (self.a as u16) << 8 | self.f as u16,
            Register::BC => (self.b as u16) << 8 | self.c as u16,
            Register::DE => (self.d as u16) << 8 | self.e as u16,
            Register::HL => (self.h as u16) << 8 | self.l as u16,
            Register::SP => self.sp,
            Register::PC => self.pc,
            _ => panic!("Register not available for 16-bit read"),
        }
    }

    pub fn write(&mut self, reg: Register, data: u8) {
        match reg {
            Register::A => self.a = data,
            Register::B => self.b = data,
            Register::C => self.c = data,
            Register::D => self.d = data,
            Register::E => self.e = data,
            Register::F => self.f = data,
            Register::H => self.h = data,
            Register::L => self.l = data,
            _ => panic!("Register not available for 8-bit write"),
        }
    }

    pub fn write_u16(&mut self, reg: Register, data: u16) {
        let (hi, lo) = (((data & 0xFF00) >> 8) as u8, (data & 0xFF) as u8);
        match reg {
            Register::AF => {
                self.a = hi;
                self.f = lo;
            },
            Register::BC => {
                self.b = hi;
                self.c = lo;
            },
            Register::DE => {
                self.d = hi;
                self.c = lo;
            },
            Register::HL => {
                self.h = hi;
                self.l = lo;
            },
            Register::SP => self.sp = data,
            Register::PC => self.pc = data,
            _ => panic!("Register not available for 16-bit write"),
        }
    }

    pub fn copy(&mut self, dst: Register, src: Register) {
        let data = self.read(src);
        self.write(dst, data);
    }

    pub fn copy_u16(&mut self, dst: Register, src: Register) {
        let data = self.read_u16(src);
        self.write_u16(dst, data);
    }

    pub fn set_flag(&mut self, flag: RegFlag, on: bool) {
        let bit = match flag {
            RegFlag::Zero => 0x80,
            RegFlag::Subtract => 0x40,
            RegFlag::HalfCarry => 0x20,
            RegFlag::Carry => 0x10,
        };
        if (on) {
            self.flag |= bit;
        } else {
            self.flag &= bit ^ 0xFF;
        }
    }

    pub fn get_flag(&self, flag: RegFlag) -> bool {
        let bit = match flag {
            RegFlag::Zero => 0x80,
            RegFlag::Subtract => 0x40,
            RegFlag::HalfCarry => 0x20,
            RegFlag::Carry => 0x10,
        };
        self.flag & bit != 0
    }

    pub fn advance_pc(&mut self) -> u16 {
        let pc = self.pc;
        self.pc += 1;
        pc
    }

    pub fn set_pc(&mut self, addr: u16) -> u16 {
        let pc = self.pc;
        self.pc = addr;
        pc
    }

    pub fn add_pc(&mut self, n: i8) -> u16 {
        let pc = self.pc;
        self.pc = ((self.pc as i32) + (n as i32)) as u16;
        pc
    }

    pub fn get_pc(&self) -> u16 {
        self.pc
    }

}
<|endoftext|>


<commit_before>use std::task::TaskBuilder;
use std::io::{TcpListener, TcpStream, Acceptor, Listener};
use std::io::net::tcp::TcpAcceptor;
use std::collections::{HashSet, HashMap};
use std::io::MemWriter;

use uuid::Uuid;
use addr::Addr;
use broadcast::Broadcast;
use result::{GossipResult, GossipError, NotListening};

pub type Callback = (Broadcast, Response);
pub type BroadcastFrom = (Peer, Broadcast);

pub struct Response {
    id: Uuid,
    stream: Stream,
    wr: MemWriter
}

impl Response {
    pub fn new(id: Uuid, stream: Stream) -> Response {
        Response {
            id: id,
            stream: stream,
            wr: MemWriter::new()
        }
    }

    /// Acknowledge the incoming broadcast with a simple OK
    /// message back. Responses aren't always required, but it's
    /// often very useful to have a nice short way of saying
    /// "Got the message, it's all good!".
    ///
    /// This takes `self` as a value because we don't
    /// allow multiple responses. So the response will be moved and
    /// further responses won't be possible.
    pub fn ok(mut self) -> GossipResult<()> {
        write!(self.stream.stream, "{},OK", self.id);
        Ok(())
    }
}

#[unsafe_destructor]
impl Drop for Response {
    /// Handle the response on the drop call.
    fn drop(&mut self) {
    }
}

/// An iterator that receives new broadcasts and iterates over them.
pub struct Incoming {
    node_tx: Sender<BroadcastFrom>,
    tx: Sender<(Broadcast, Stream)>,
    rx: Receiver<(Broadcast, Stream)>,
    listening: bool
}

impl Incoming {
    pub fn new(node_tx: Sender<BroadcastFrom>, sender: Sender<Sender<(Broadcast, Stream)>>) -> Incoming {
        let (tx, rx) = channel();

        sender.send(tx.clone());

        Incoming {
            node_tx: node_tx,
            tx: tx,
            rx: rx,
            listening: true
        }
    }
}

impl Iterator<Callback> for Incoming {

    /// TODO: Handle the response with the Drop trait. This means
    ///       we'll need to have a reference to the `Stream`.
    fn next(&mut self) -> Option<Callback> {
        if self.listening {
            let (broadcast, stream) = self.rx.recv();
            let id = broadcast.id().clone();
            Some((broadcast, Response::new(id, stream)))
        } else {
            None
        }
    }
}

/// A dedicated Rust task to manage a single `TcpStream`. Each
/// stream is then associated to a given peer (although the peer
/// isn't always set right away).
///
/// This task then has channels to communicate with the main
/// AcceptorTask which can communicate with other StreamTasks.
struct StreamTask {
    peer: Option<Peer>,
    pub stream: Stream
}

impl StreamTask {
    pub fn new(stream: Stream) -> StreamTask {
        StreamTask {
            peer: None,
            stream: stream
        }
    }

    pub fn incoming(&mut self) {
    }
}

#[deriving(Clone)]
pub struct Stream {
    stream: TcpStream,
    peer: Option<Peer>
}

impl Stream {
    pub fn new(stream: TcpStream) -> Stream {
        Stream {
            stream: stream,
            peer: None
        }
    }
}

impl Iterator<Callback> for Stream {
    fn next(&mut self) -> Option<Callback> {
        None
    }
}

/// A dedicated task to handle the incoming connections
/// over a tcp socket (called streams). This task does
/// not need to communicate with the streams themselves, that's
/// done at another task that handles the core logic of the
/// protocol.
struct AcceptorTask {
    acceptor: TcpAcceptor,
    server_tx: Sender<Directive>,
    tx: Sender<Broadcast>,
    rx: Receiver<Broadcast>
}

enum Directive {
    StreamDirective(Stream),
    BroadcastDirective(Broadcast)
}

impl AcceptorTask {
    pub fn new(host: &str, port: u16, server_tx: Sender<Directive>,
               inter_tx: Sender<Sender<Broadcast>>) -> AcceptorTask {
        let listener = TcpListener::bind(host, port).unwrap();
        let (tx, rx) = channel();

        inter_tx.send(tx.clone());

        AcceptorTask {
            acceptor: listener.listen().unwrap(),
            server_tx: server_tx,
            tx: tx,
            rx: rx
        }
    }

    pub fn run(&mut self) {
        for stream in self.acceptor.incoming() {
            match stream {
                Ok(s) => {
                    // Handle the joining here...
                    let stream_send = s.clone();
                    let server = self.server_tx.clone();
                    spawn(proc() {
                        let stream = Stream::new(stream_send);
                        server.send(StreamDirective(stream.clone()));
                        StreamTask::new(stream).incoming();
                    });
                },
                Err(e) => println!("Error: {}", e)
            }
        }
    }
}

struct ServerTask {
    streams: HashMap<Peer, TcpStream>,
    acceptor_tx: Sender<Broadcast>,
    tx: Sender<Directive>,
    rx: Receiver<Directive>
}

impl ServerTask {
    pub fn new(host: String, port: u16) -> ServerTask {
        // Local channels that deal with broadcasts.
        let (tx, rx) = channel();

        // Intermediate channels for the acceptor task.
        // We'll use this to retrieve the sender of the
        // acceptor task.
        let (acceptor_tx, acceptor_rx) = channel();

        let server_tx = tx.clone();
        spawn(proc() {
            AcceptorTask::new(host.as_slice(), port, server_tx, acceptor_tx).run();
        });

        ServerTask {
            streams: HashMap::new(),
            acceptor_tx: acceptor_rx.recv(),
            tx: tx,
            rx: rx
        }
    }

    pub fn broadcast(&self, peer: &Peer) -> GossipResult<()> {
        Ok(())
    }

    pub fn run(&mut self) {
        for signal in self.rx.iter() {
            match signal {
                StreamDirective(stream) => {
                    // self.streams.insert(peer, stream);
                },
                BroadcastDirective(broadcast) => {}
            }
        }
    }
}

/// A peer describes a member within the cluster/network that
/// is not the current one.
#[deriving(Clone, Show, PartialEq, Hash, Eq)]
pub struct Peer {
    id: Uuid,
    addr: Addr
}

impl Peer {
    pub fn new(id: Uuid, host: &str, port: u16) -> Peer {
        Peer {
            id: id,
            addr: Addr::new(host, port)
        }
    }

    pub fn empty() -> Peer {
        Peer {
            id: Uuid::new_v4(),
            addr: Addr::new("localhost", 3444)
        }
    }
}

/// A `Node` is a single member within the gossip protocol. Nodes that
/// join together is called a cluster. This forms the distributed system
/// in which the gossip protocol runs within.
///
/// Each Node is an equal member in the cluster. That means there isn't a
/// single leader. This has a significant trade-off and one must understand
/// it before being able to use the system correctly.
///
/// Node: Handle the state.
/// Incoming: Handle incoming connections and broadcasts.
pub struct Node {
    /// Each node generates their own unique Uuid (v4) to uniquely
    /// identify each other within the cluster. Instead of saying
    /// "I'm node A", you would say "I'm node 123e4567-e89b-12d3-a456-426655440000".
    id: Uuid,

    /// A set of other members within the cluster. This forms the basic
    /// information about each Node. This doesn't, however, contain connection
    /// information and what not.
    members: Vec<Peer>,
    incoming_tx: Option<Sender<(Broadcast, Stream)>>,
    tx: Sender<(Peer, Broadcast)>,
    rx: Receiver<(Peer, Broadcast)>
}

impl Node {
    /// Usage:
    ///
    /// ```rust
    /// use gossip::Node;
    /// let mut node = Node::new();
    /// ```
    pub fn new() -> Node {
        let (tx, rx) = channel();

        Node {
            id: Uuid::new_v4(),
            members: Vec::new(),
            incoming_tx: None,
            tx: tx,
            rx: rx
        }
    }

    /// Initialize the Node to listen on the specified address/port
    /// combination. This will bootup the appropriate tasks to allow
    /// incoming connections and broadcasts.
    #[unimplemented]
    pub fn listen(&mut self, host: &str, port: u16) -> GossipResult<()> {
        let host = host.to_string();

        spawn(proc() {
            ServerTask::new(host, port).run();
        });

        Ok(())
    }

    /// Given a peer node, join it's existing cluster. Each node technically
    /// creates their own cluster automatically. Joining multiple nodes together
    /// is an explicit process. The peer node doesn't need to be the same one,
    /// but it's not a bad idea.
    #[unimplemented]
    pub fn join(&mut self, host: &str, port: u16) -> GossipResult<()> {
        Ok(())
    }

    /// Shutdown all the running tasks that are listening to new broadcasts
    /// and incoming connections. This will send one last broadcast
    /// to the current cluster notifying all other nodes of the shutdown.
    ///
    /// Afterwhich tasks will shutdown and the node will be terminated.
    #[unimplemented]
    pub fn shutdown(&mut self) {}

    /// Create a new `Incoming` iterator that iterates over newly received
    /// broadcasts that the user can handle.
    ///
    /// Usage:
    ///
    /// ```notrust
    /// use gossip::Node;
    /// spawn(proc() {
    ///     let mut node = Node::new();
    ///
    ///     node.listen("localhost", 4888).unwrap();
    ///
    ///     for (broadcast, mut res) in node.incoming() {
    ///         println!("Broadcast ...");
    ///         // ...
    ///     }
    /// });
    /// ```
    pub fn incoming(&mut self) -> Incoming {
        let (tx, rx) = channel();
        let incoming = Incoming::new(self.tx.clone(), tx);

        self.incoming_tx = Some(rx.recv());

        incoming
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn empty_member_set() {
        let mut node = Node::new();
        assert_eq!(node.members.len(), 0);
    }

    #[test]
    fn bind_listening() {
        let mut node = Node::new();
    }
}
<commit_msg>Fixed style<commit_after>use std::task::TaskBuilder;
use std::io::{TcpListener, TcpStream, Acceptor, Listener};
use std::io::net::tcp::TcpAcceptor;
use std::collections::{HashSet, HashMap};
use std::io::MemWriter;

use uuid::Uuid;
use addr::Addr;
use broadcast::Broadcast;
use result::{GossipResult, GossipError, NotListening};

pub type Callback = (Broadcast, Response);
pub type BroadcastFrom = (Peer, Broadcast);

pub struct Response {
    id: Uuid,
    stream: Stream,
    wr: MemWriter
}

impl Response {
    pub fn new(id: Uuid, stream: Stream) -> Response {
        Response {
            id: id,
            stream: stream,
            wr: MemWriter::new()
        }
    }

    /// Acknowledge the incoming broadcast with a simple OK
    /// message back. Responses aren't always required, but it's
    /// often very useful to have a nice short way of saying
    /// "Got the message, it's all good!".
    ///
    /// This takes `self` as a value because we don't
    /// allow multiple responses. So the response will be moved and
    /// further responses won't be possible.
    pub fn ok(mut self) -> GossipResult<()> {
        write!(self.stream.stream, "{},OK", self.id);
        Ok(())
    }
}

#[unsafe_destructor]
impl Drop for Response {
    /// Handle the response on the drop call.
    fn drop(&mut self) {
    }
}

/// An iterator that receives new broadcasts and iterates over them.
pub struct Incoming {
    node_tx: Sender<BroadcastFrom>,
    tx: Sender<(Broadcast, Stream)>,
    rx: Receiver<(Broadcast, Stream)>,
    listening: bool
}

impl Incoming {
    pub fn new(node_tx: Sender<BroadcastFrom>,
               sender: Sender<Sender<(Broadcast, Stream)>>) -> Incoming {
        let (tx, rx) = channel();

        sender.send(tx.clone());

        Incoming {
            node_tx: node_tx,
            tx: tx,
            rx: rx,
            listening: true
        }
    }
}

impl Iterator<Callback> for Incoming {

    /// TODO: Handle the response with the Drop trait. This means
    ///       we'll need to have a reference to the `Stream`.
    fn next(&mut self) -> Option<Callback> {
        if self.listening {
            let (broadcast, stream) = self.rx.recv();
            let id = broadcast.id().clone();
            Some((broadcast, Response::new(id, stream)))
        } else {
            None
        }
    }
}

/// A dedicated Rust task to manage a single `TcpStream`. Each
/// stream is then associated to a given peer (although the peer
/// isn't always set right away).
///
/// This task then has channels to communicate with the main
/// AcceptorTask which can communicate with other StreamTasks.
struct StreamTask {
    peer: Option<Peer>,
    pub stream: Stream
}

impl StreamTask {
    pub fn new(stream: Stream) -> StreamTask {
        StreamTask {
            peer: None,
            stream: stream
        }
    }

    pub fn incoming(&mut self) {
    }
}

#[deriving(Clone)]
pub struct Stream {
    stream: TcpStream,
    peer: Option<Peer>
}

impl Stream {
    pub fn new(stream: TcpStream) -> Stream {
        Stream {
            stream: stream,
            peer: None
        }
    }
}

impl Iterator<Callback> for Stream {
    fn next(&mut self) -> Option<Callback> {
        None
    }
}

/// A dedicated task to handle the incoming connections
/// over a tcp socket (called streams). This task does
/// not need to communicate with the streams themselves, that's
/// done at another task that handles the core logic of the
/// protocol.
struct AcceptorTask {
    acceptor: TcpAcceptor,
    server_tx: Sender<Directive>,
    tx: Sender<Broadcast>,
    rx: Receiver<Broadcast>
}

enum Directive {
    StreamDirective(Stream),
    BroadcastDirective(Broadcast)
}

impl AcceptorTask {
    pub fn new(host: &str, port: u16, server_tx: Sender<Directive>,
               inter_tx: Sender<Sender<Broadcast>>) -> AcceptorTask {
        let listener = TcpListener::bind(host, port).unwrap();
        let (tx, rx) = channel();

        inter_tx.send(tx.clone());

        AcceptorTask {
            acceptor: listener.listen().unwrap(),
            server_tx: server_tx,
            tx: tx,
            rx: rx
        }
    }

    pub fn run(&mut self) {
        for stream in self.acceptor.incoming() {
            match stream {
                Ok(s) => {
                    // Handle the joining here...
                    let stream_send = s.clone();
                    let server = self.server_tx.clone();
                    spawn(proc() {
                        let stream = Stream::new(stream_send);
                        server.send(StreamDirective(stream.clone()));
                        StreamTask::new(stream).incoming();
                    });
                },
                Err(e) => println!("Error: {}", e)
            }
        }
    }
}

struct ServerTask {
    streams: HashMap<Peer, TcpStream>,
    acceptor_tx: Sender<Broadcast>,
    tx: Sender<Directive>,
    rx: Receiver<Directive>
}

impl ServerTask {
    pub fn new(host: String, port: u16) -> ServerTask {
        // Local channels that deal with broadcasts.
        let (tx, rx) = channel();

        // Intermediate channels for the acceptor task.
        // We'll use this to retrieve the sender of the
        // acceptor task.
        let (acceptor_tx, acceptor_rx) = channel();

        let server_tx = tx.clone();
        spawn(proc() {
            AcceptorTask::new(host.as_slice(), port, server_tx, acceptor_tx).run();
        });

        ServerTask {
            streams: HashMap::new(),
            acceptor_tx: acceptor_rx.recv(),
            tx: tx,
            rx: rx
        }
    }

    pub fn broadcast(&self, peer: &Peer) -> GossipResult<()> {
        Ok(())
    }

    pub fn run(&mut self) {
        for signal in self.rx.iter() {
            match signal {
                StreamDirective(stream) => {
                    // self.streams.insert(peer, stream);
                },
                BroadcastDirective(broadcast) => {}
            }
        }
    }
}

/// A peer describes a member within the cluster/network that
/// is not the current one.
#[deriving(Clone, Show, PartialEq, Hash, Eq)]
pub struct Peer {
    id: Uuid,
    addr: Addr
}

impl Peer {
    pub fn new(id: Uuid, host: &str, port: u16) -> Peer {
        Peer {
            id: id,
            addr: Addr::new(host, port)
        }
    }

    pub fn empty() -> Peer {
        Peer {
            id: Uuid::new_v4(),
            addr: Addr::new("localhost", 3444)
        }
    }
}

/// A `Node` is a single member within the gossip protocol. Nodes that
/// join together is called a cluster. This forms the distributed system
/// in which the gossip protocol runs within.
///
/// Each Node is an equal member in the cluster. That means there isn't a
/// single leader. This has a significant trade-off and one must understand
/// it before being able to use the system correctly.
///
/// Node: Handle the state.
/// Incoming: Handle incoming connections and broadcasts.
pub struct Node {
    /// Each node generates their own unique Uuid (v4) to uniquely
    /// identify each other within the cluster. Instead of saying
    /// "I'm node A", you would say "I'm node 123e4567-e89b-12d3-a456-426655440000".
    id: Uuid,

    /// A set of other members within the cluster. This forms the basic
    /// information about each Node. This doesn't, however, contain connection
    /// information and what not.
    members: Vec<Peer>,
    incoming_tx: Option<Sender<(Broadcast, Stream)>>,
    tx: Sender<(Peer, Broadcast)>,
    rx: Receiver<(Peer, Broadcast)>
}

impl Node {
    /// Usage:
    ///
    /// ```rust
    /// use gossip::Node;
    /// let mut node = Node::new();
    /// ```
    pub fn new() -> Node {
        let (tx, rx) = channel();

        Node {
            id: Uuid::new_v4(),
            members: Vec::new(),
            incoming_tx: None,
            tx: tx,
            rx: rx
        }
    }

    /// Initialize the Node to listen on the specified address/port
    /// combination. This will bootup the appropriate tasks to allow
    /// incoming connections and broadcasts.
    #[unimplemented]
    pub fn listen(&mut self, host: &str, port: u16) -> GossipResult<()> {
        let host = host.to_string();

        spawn(proc() {
            ServerTask::new(host, port).run();
        });

        Ok(())
    }

    /// Given a peer node, join it's existing cluster. Each node technically
    /// creates their own cluster automatically. Joining multiple nodes together
    /// is an explicit process. The peer node doesn't need to be the same one,
    /// but it's not a bad idea.
    #[unimplemented]
    pub fn join(&mut self, host: &str, port: u16) -> GossipResult<()> {
        Ok(())
    }

    /// Shutdown all the running tasks that are listening to new broadcasts
    /// and incoming connections. This will send one last broadcast
    /// to the current cluster notifying all other nodes of the shutdown.
    ///
    /// Afterwhich tasks will shutdown and the node will be terminated.
    #[unimplemented]
    pub fn shutdown(&mut self) {}

    /// Create a new `Incoming` iterator that iterates over newly received
    /// broadcasts that the user can handle.
    ///
    /// Usage:
    ///
    /// ```notrust
    /// use gossip::Node;
    /// spawn(proc() {
    ///     let mut node = Node::new();
    ///
    ///     node.listen("localhost", 4888).unwrap();
    ///
    ///     for (broadcast, mut res) in node.incoming() {
    ///         println!("Broadcast ...");
    ///         // ...
    ///     }
    /// });
    /// ```
    pub fn incoming(&mut self) -> Incoming {
        let (tx, rx) = channel();
        let incoming = Incoming::new(self.tx.clone(), tx);

        self.incoming_tx = Some(rx.recv());

        incoming
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn empty_member_set() {
        let mut node = Node::new();
        assert_eq!(node.members.len(), 0);
    }

    #[test]
    fn bind_listening() {
        let mut node = Node::new();
    }
}
<|endoftext|>


<commit_before>//! Contains all HTML-related code. See the `Page` documentation for details.

// TODO: Why do I need these two lines "extern crate html5ever" and "extern crate
// html5ever_dom_sink" both here and in main.rs?
extern crate html5ever;
extern crate html5ever_dom_sink;
extern crate rand;
extern crate tendril;

use std::str::FromStr;
use std::sync::mpsc::{channel, Receiver, Sender};
use std::thread;

use html5ever::Attribute;
use html5ever::tree_builder::interface::TreeSink;
use html5ever_dom_sink::common::NodeEnum;
use html5ever_dom_sink::rcdom::Handle;
use html5ever_dom_sink::rcdom::RcDom;
use regex::Captures;
use regex::Regex;

use wiki::Wiki;

use ::START_MARKER;
use ::END_MARKER;

// TODO: massive cleanup, all over this file.

/// Represents, and owns all behavior related to, the contents of the HTML page shown to the
/// user. This includes fetching the rendered article from Wikipedia, replacing its contents with
/// the rendered wikitext, and processing/removing merge markers.
///
/// The API to this struct comprises two methods: Page::new() constructs a new Page. It should be
/// called early, so the Page can start fetching article HTML from Wikipedia.
/// Page:replace_body_and_remove_merge_markers() processes the merge markers in the rendered
/// wikitext and puts the header and footer around it.
pub struct Page {
    /// The string used as a placeholder for the article body in the page skeleton.
    placeholder: String,
    /// The Receiver that will receive the page skeleton when it's been fetched and processed.
    page_skeleton_receiver: Receiver<Result<String, String>>,
}

impl Page {
    /// Creates a new Page representing the article at `title`. This kicks off a background thread
    /// that fetches the current article HTML from Wikipedia. Because of that, it should be called
    /// as early as possible (as soon as the title being served is known), so that the page fetch
    /// stays off the critical path for page load.
    pub fn new(title: &str, wiki: Wiki) -> Page {
        let placeholder = format!("WMW_PLACEHOLDER_{}", rand::random::<u64>());
        let page_skeleton_receiver =
            Page::spawn_page_skeleton_fetch_thread(title, placeholder.clone(), wiki);
        Page {
            placeholder: placeholder,
            page_skeleton_receiver: page_skeleton_receiver,
        }
    }

    /// This finishes the HTML processing - it replaces the merge markers in `article_body` with
    /// HTML tags, and inserts the resulting HTML into the page skeleton.
    pub fn replace_body_and_remove_merge_markers(&self, article_body: String)
                                                 -> Result<String, String> {
        match self.page_skeleton_receiver.recv() {
            Ok(Ok(page_skeleton)) => {
                let finished_article_body = process_merge_markers(article_body);
                Ok(page_skeleton.replace(&self.placeholder, &finished_article_body))
            },
            Ok(Err(msg))=> Err(msg),
            Err(err) => Err(format!("error: {}", err)),
        }
    }

    fn spawn_page_skeleton_fetch_thread(title: &str, placeholder: String, wiki: Wiki)
                                  -> Receiver<Result<String, String>> {
        let (page_skeleton_sender, page_skeleton_receiver) = channel::<Result<String, String>>();
        let title = title.to_owned().clone();
        thread::spawn(move|| {
            page_skeleton_sender.send(
                match wiki.get_current_page_content(&title) {
                    Ok(content) =>
                        replace_node_with_placeholder(&content, "mw-content-text", &placeholder),
                    Err(msg) => Err(msg),
                }).unwrap();
        });
        page_skeleton_receiver
    }
}

fn replace_node_with_placeholder(original_html: &str, div_id: &str, placeholder: &str)
    -> Result<String, String> {
    let html = tendril::StrTendril::from_str(original_html).unwrap();
    let mut dom: RcDom = html5ever::parse(html5ever::one_input(html), Default::default());

    let handle = try!(find_node_by_id(&dom.get_document(), div_id));
    let child_handles =
        (&handle.borrow().children).into_iter().map(|child| child.clone()).collect::<Vec<_>>();
    for child_handle in child_handles {
        dom.remove_from_parent(child_handle);
    }
    dom.append(handle,
               html5ever::tree_builder::interface::NodeOrText::AppendText(
                   tendril::StrTendril::from_str(placeholder).unwrap()));
    let mut serialized: Vec<u8> = vec![];
    try_display!(
        html5ever::serialize::serialize(&mut serialized, &dom.document, Default::default()),
        "Failed to serialize modified HTML");
    Ok(try_display!(String::from_utf8(serialized),
                    "Error converting serialized HTML to UTF-8 string"))
}

fn find_node_by_id(handle: &Handle, id: &str) -> Result<Handle, String> {
    fn has_matching_id(attributes: &Vec<Attribute>, id: &str) -> bool {
        return attributes.into_iter().any(
            |attribute| attribute.name.local.as_slice() == "id" &&
                format!("{}", attribute.value) == id);
    }

    let node = handle.borrow();
    match node.node {
        NodeEnum::Element(_, ref attributes) if has_matching_id(attributes, id) => Ok(handle.clone()),
        _ => (&node.children).into_iter()
            .map(|child| find_node_by_id(child, id))
            .filter(|result| result.is_ok())
            .map(|result| result.unwrap())
            .next().ok_or(format!("No node with ID {} found", id)),
    }
}

/// Removes merge markers that are inside HTML tags, and replaces the others with  tags.
fn process_merge_markers(html: String) -> String {
    // TODO: Use the constants
    let start_regex = regex!("\u{E000}([0-9]+)\u{E000}");
    let end_regex = regex!("\u{E001}[0-9]+\u{E001}");

    let html = remove_merge_markers(html);
    let html = start_regex.replace_all(
        &html, |captures: &Captures| format!("",
                                             captures.at(1).unwrap()));
    end_regex.replace_all(&html, "")
}

fn remove_merge_markers(html: String) -> String {
    // Finds markers where the end, but not the start, is inside a tag.
    let regex1 = Regex::new(&format!(
        r"{}[0-9]+{}([^{}]*?)<([^>]*?){}[0-9]+{}([^>]*?)>",
        START_MARKER, START_MARKER, END_MARKER, END_MARKER, END_MARKER)).unwrap();
    // Finds markers where the start, but not the end, is inside a tag.
    let regex2 = Regex::new(&format!(
        r"<([^>]*?){}[0-9]+{}([^>]*?)>([^{}]*?){}[0-9]+{}",
        START_MARKER, START_MARKER, END_MARKER, END_MARKER, END_MARKER)).unwrap();
    // Finds markers where both the start and end are inside tags.
    let regex3 = Regex::new(&format!(
        r"<([^>]*?){}[0-9]+{}([^>]*?)>([^{}{}]*?)<([^>]*?){}[0-9]+{}([^>]*?)>",
        START_MARKER, START_MARKER, START_MARKER, END_MARKER, END_MARKER, END_MARKER)).unwrap();
    let html = regex1.replace_all(
        &html, |captures: &Captures|
        format!("{}<{}{}>", captures.at(1).unwrap(), captures.at(2).unwrap(),
                captures.at(3).unwrap()));
    let html = regex2.replace_all(
        &html, |captures: &Captures|
        format!("<{}{}>{}", captures.at(1).unwrap(), captures.at(2).unwrap(),
                captures.at(3).unwrap()));
    regex3.replace_all(
        &html, |captures: &Captures|
        format!("<{}{}>{}<{}{}>", captures.at(1).unwrap(), captures.at(2).unwrap(),
                captures.at(3).unwrap(), captures.at(4).unwrap(), captures.at(5).unwrap()))
}

#[cfg(test)]
mod tests {
    use super::{remove_merge_markers, replace_node_with_placeholder};

    #[test]
    fn test_remove_merge_markers_keep() {
        let html = format!("{}456{}{}456{}",
                           ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        assert_eq!(html.clone(), remove_merge_markers(html));
    }

    #[test]
    fn test_remove_merge_markers_keep_one_remove_one() {
        let html = format!(
            "{}234{}text{}234{}{}567{}",
            ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER, ::START_MARKER, ::START_MARKER,
            ::END_MARKER, ::END_MARKER);
        let expected = format!(
            "{}234{}text{}234{}",
            ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        assert_eq!(expected, remove_merge_markers(html));
    }

    #[test]
    fn test_remove_merge_markers_end_inside_tag() {
        let html = format!("{}123{}",
                           ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        let expected = "";
        assert_eq!(expected, remove_merge_markers(html));
    }

    #[test]
    fn test_remove_merge_markers_start_inside_tag() {
        let html = format!("{}123{}",
                           ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        let expected = "";
        assert_eq!(expected, remove_merge_markers(html));
    }

    #[test]
    fn test_remove_merge_markers_both_inside_tag() {
        let html = format!("text",
                           ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        let expected = "text";
        assert_eq!(expected, remove_merge_markers(html));
    }

    #[test]
    fn test_replace_html_content() {
        let original_html = "

original text

Other text

";
        let expected_html = "

replaced text

Other text

";
        let processed_html = replace_node_with_placeholder(original_html, "mw-content-text", "replaced text").unwrap();
        assert_eq!(expected_html, processed_html);
    }
}
<commit_msg>Use the *_MARKER constants in process_merge_markers(), and write a test for it.<commit_after>//! Contains all HTML-related code. See the `Page` documentation for details.

// TODO: Why do I need these two lines "extern crate html5ever" and "extern crate
// html5ever_dom_sink" both here and in main.rs?
extern crate html5ever;
extern crate html5ever_dom_sink;
extern crate rand;
extern crate tendril;

use std::str::FromStr;
use std::sync::mpsc::{channel, Receiver, Sender};
use std::thread;

use html5ever::Attribute;
use html5ever::tree_builder::interface::TreeSink;
use html5ever_dom_sink::common::NodeEnum;
use html5ever_dom_sink::rcdom::Handle;
use html5ever_dom_sink::rcdom::RcDom;
use regex::Captures;
use regex::Regex;

use wiki::Wiki;

use ::START_MARKER;
use ::END_MARKER;

// TODO: massive cleanup, all over this file.

/// Represents, and owns all behavior related to, the contents of the HTML page shown to the
/// user. This includes fetching the rendered article from Wikipedia, replacing its contents with
/// the rendered wikitext, and processing/removing merge markers.
///
/// The API to this struct comprises two methods: Page::new() constructs a new Page. It should be
/// called early, so the Page can start fetching article HTML from Wikipedia.
/// Page:replace_body_and_remove_merge_markers() processes the merge markers in the rendered
/// wikitext and puts the header and footer around it.
pub struct Page {
    /// The string used as a placeholder for the article body in the page skeleton.
    placeholder: String,
    /// The Receiver that will receive the page skeleton when it's been fetched and processed.
    page_skeleton_receiver: Receiver<Result<String, String>>,
}

impl Page {
    /// Creates a new Page representing the article at `title`. This kicks off a background thread
    /// that fetches the current article HTML from Wikipedia. Because of that, it should be called
    /// as early as possible (as soon as the title being served is known), so that the page fetch
    /// stays off the critical path for page load.
    pub fn new(title: &str, wiki: Wiki) -> Page {
        let placeholder = format!("WMW_PLACEHOLDER_{}", rand::random::<u64>());
        let page_skeleton_receiver =
            Page::spawn_page_skeleton_fetch_thread(title, placeholder.clone(), wiki);
        Page {
            placeholder: placeholder,
            page_skeleton_receiver: page_skeleton_receiver,
        }
    }

    /// This finishes the HTML processing - it replaces the merge markers in `article_body` with
    /// HTML tags, and inserts the resulting HTML into the page skeleton.
    pub fn replace_body_and_remove_merge_markers(&self, article_body: String)
                                                 -> Result<String, String> {
        match self.page_skeleton_receiver.recv() {
            Ok(Ok(page_skeleton)) => {
                let finished_article_body = process_merge_markers(article_body);
                Ok(page_skeleton.replace(&self.placeholder, &finished_article_body))
            },
            Ok(Err(msg))=> Err(msg),
            Err(err) => Err(format!("error: {}", err)),
        }
    }

    fn spawn_page_skeleton_fetch_thread(title: &str, placeholder: String, wiki: Wiki)
                                  -> Receiver<Result<String, String>> {
        let (page_skeleton_sender, page_skeleton_receiver) = channel::<Result<String, String>>();
        let title = title.to_owned().clone();
        thread::spawn(move|| {
            page_skeleton_sender.send(
                match wiki.get_current_page_content(&title) {
                    Ok(content) =>
                        replace_node_with_placeholder(&content, "mw-content-text", &placeholder),
                    Err(msg) => Err(msg),
                }).unwrap();
        });
        page_skeleton_receiver
    }
}

fn replace_node_with_placeholder(original_html: &str, div_id: &str, placeholder: &str)
    -> Result<String, String> {
    let html = tendril::StrTendril::from_str(original_html).unwrap();
    let mut dom: RcDom = html5ever::parse(html5ever::one_input(html), Default::default());

    let handle = try!(find_node_by_id(&dom.get_document(), div_id));
    let child_handles =
        (&handle.borrow().children).into_iter().map(|child| child.clone()).collect::<Vec<_>>();
    for child_handle in child_handles {
        dom.remove_from_parent(child_handle);
    }
    dom.append(handle,
               html5ever::tree_builder::interface::NodeOrText::AppendText(
                   tendril::StrTendril::from_str(placeholder).unwrap()));
    let mut serialized: Vec<u8> = vec![];
    try_display!(
        html5ever::serialize::serialize(&mut serialized, &dom.document, Default::default()),
        "Failed to serialize modified HTML");
    Ok(try_display!(String::from_utf8(serialized),
                    "Error converting serialized HTML to UTF-8 string"))
}

fn find_node_by_id(handle: &Handle, id: &str) -> Result<Handle, String> {
    fn has_matching_id(attributes: &Vec<Attribute>, id: &str) -> bool {
        return attributes.into_iter().any(
            |attribute| attribute.name.local.as_slice() == "id" &&
                format!("{}", attribute.value) == id);
    }

    let node = handle.borrow();
    match node.node {
        NodeEnum::Element(_, ref attributes) if has_matching_id(attributes, id) => Ok(handle.clone()),
        _ => (&node.children).into_iter()
            .map(|child| find_node_by_id(child, id))
            .filter(|result| result.is_ok())
            .map(|result| result.unwrap())
            .next().ok_or(format!("No node with ID {} found", id)),
    }
}

/// Removes merge markers that are inside HTML tags, and replaces the others with  tags.
fn process_merge_markers(html: String) -> String {
    let start_regex = Regex::new(&format!("{}([0-9]+){}", START_MARKER, START_MARKER)).unwrap();
    let end_regex = Regex::new(&format!("{}[0-9]+{}", END_MARKER, END_MARKER)).unwrap();

    let html = remove_merge_markers(html);
    let html = start_regex.replace_all(
        &html, |captures: &Captures| format!("",
                                             captures.at(1).unwrap()));
    end_regex.replace_all(&html, "")
}

fn remove_merge_markers(html: String) -> String {
    // Finds markers where the end, but not the start, is inside a tag.
    let regex1 = Regex::new(&format!(
        r"{}[0-9]+{}([^{}]*?)<([^>]*?){}[0-9]+{}([^>]*?)>",
        START_MARKER, START_MARKER, END_MARKER, END_MARKER, END_MARKER)).unwrap();
    // Finds markers where the start, but not the end, is inside a tag.
    let regex2 = Regex::new(&format!(
        r"<([^>]*?){}[0-9]+{}([^>]*?)>([^{}]*?){}[0-9]+{}",
        START_MARKER, START_MARKER, END_MARKER, END_MARKER, END_MARKER)).unwrap();
    // Finds markers where both the start and end are inside tags.
    let regex3 = Regex::new(&format!(
        r"<([^>]*?){}[0-9]+{}([^>]*?)>([^{}{}]*?)<([^>]*?){}[0-9]+{}([^>]*?)>",
        START_MARKER, START_MARKER, START_MARKER, END_MARKER, END_MARKER, END_MARKER)).unwrap();
    let html = regex1.replace_all(
        &html, |captures: &Captures|
        format!("{}<{}{}>", captures.at(1).unwrap(), captures.at(2).unwrap(),
                captures.at(3).unwrap()));
    let html = regex2.replace_all(
        &html, |captures: &Captures|
        format!("<{}{}>{}", captures.at(1).unwrap(), captures.at(2).unwrap(),
                captures.at(3).unwrap()));
    regex3.replace_all(
        &html, |captures: &Captures|
        format!("<{}{}>{}<{}{}>", captures.at(1).unwrap(), captures.at(2).unwrap(),
                captures.at(3).unwrap(), captures.at(4).unwrap(), captures.at(5).unwrap()))
}

#[cfg(test)]
mod tests {
    use super::{remove_merge_markers, replace_node_with_placeholder};

    fn test_process_merge_markers() {
        let html = format!(
            "{}456{}{}456{}{}123{}t",
            ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER,
            ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        let expected_regex =
            regex!("]*>");
        assert!(expected_regex.is_match(&html));
    }

    #[test]
    fn test_remove_merge_markers_keep() {
        let html = format!("{}456{}{}456{}",
                           ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        assert_eq!(html.clone(), remove_merge_markers(html));
    }

    #[test]
    fn test_remove_merge_markers_keep_one_remove_one() {
        let html = format!(
            "{}234{}text{}234{}{}567{}",
            ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER, ::START_MARKER, ::START_MARKER,
            ::END_MARKER, ::END_MARKER);
        let expected = format!(
            "{}234{}text{}234{}",
            ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        assert_eq!(expected, remove_merge_markers(html));
    }

    #[test]
    fn test_remove_merge_markers_end_inside_tag() {
        let html = format!("{}123{}",
                           ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        let expected = "";
        assert_eq!(expected, remove_merge_markers(html));
    }

    #[test]
    fn test_remove_merge_markers_start_inside_tag() {
        let html = format!("{}123{}",
                           ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        let expected = "";
        assert_eq!(expected, remove_merge_markers(html));
    }

    #[test]
    fn test_remove_merge_markers_both_inside_tag() {
        let html = format!("text",
                           ::START_MARKER, ::START_MARKER, ::END_MARKER, ::END_MARKER);
        let expected = "text";
        assert_eq!(expected, remove_merge_markers(html));
    }

    #[test]
    fn test_replace_html_content() {
        let original_html = "

original text

Other text

";
        let expected_html = "

replaced text

Other text

";
        let processed_html = replace_node_with_placeholder(original_html, "mw-content-text", "replaced text").unwrap();
        assert_eq!(expected_html, processed_html);
    }
}
<|endoftext|>


<commit_before>//! A collection of traits abstracting over Listeners and Streams.
use std::io::{IoResult, IoError, ConnectionAborted, InvalidInput, OtherIoError,
              Stream, Listener, Acceptor};
use std::io::net::ip::{SocketAddr, Port};
use std::io::net::tcp::{TcpStream, TcpListener, TcpAcceptor};
use std::sync::{Arc, Mutex};

use openssl::ssl::{SslStream, SslContext, Sslv23};
use openssl::ssl::error::{SslError, StreamError, OpenSslErrors, SslSessionClosed};

/// The write-status indicating headers have not been written.
pub struct Fresh;

/// The write-status indicating headers have been written.
pub struct Streaming;

/// An abstraction to listen for connections on a certain port.
pub trait NetworkListener<S: NetworkStream, A: NetworkAcceptor<S>>: Listener<S, A> {
    /// Bind to a socket.
    ///
    /// Note: This does not start listening for connections. You must call
    /// `listen()` to do that.
    fn bind(host: &str, port: Port) -> IoResult<Self>;

    /// Get the address this Listener ended up listening on.
    fn socket_name(&mut self) -> IoResult<SocketAddr>;
}

/// An abstraction to receive `NetworkStream`s.
pub trait NetworkAcceptor<S: NetworkStream>: Acceptor<S> + Clone + Send {
    /// Closes the Acceptor, so no more incoming connections will be handled.
    fn close(&mut self) -> IoResult<()>;
}

/// An abstraction over streams that a Server can utilize.
pub trait NetworkStream: Stream + Clone + Send {
    /// Get the remote address of the underlying connection.
    fn peer_name(&mut self) -> IoResult<SocketAddr>;

    /// Connect to a remote address.
    fn connect(host: &str, Port, scheme: &str) -> IoResult<Self>;

    /// Turn this into an appropriately typed trait object.
    #[inline]
    fn abstract(self) -> Box<NetworkStream + Send> {
        box self as Box<NetworkStream + Send>
    }

    #[doc(hidden)]
    #[inline]
    // Hack to work around lack of Clone impl for Box<Clone>
    fn clone_box(&self) -> Box<NetworkStream + Send> { self.clone().abstract() }
}

impl Clone for Box<NetworkStream + Send> {
    #[inline]
    fn clone(&self) -> Box<NetworkStream + Send> { self.clone_box() }
}

impl Reader for Box<NetworkStream + Send> {
    #[inline]
    fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> { self.read(buf) }
}

impl Writer for Box<NetworkStream + Send> {
    #[inline]
    fn write(&mut self, msg: &[u8]) -> IoResult<()> { self.write(msg) }

    #[inline]
    fn flush(&mut self) -> IoResult<()> { self.flush() }
}

/// A `NetworkListener` for `HttpStream`s.
pub struct HttpListener {
    inner: TcpListener
}

impl Listener<HttpStream, HttpAcceptor> for HttpListener {
    #[inline]
    fn listen(self) -> IoResult<HttpAcceptor> {
        Ok(HttpAcceptor {
            inner: try!(self.inner.listen())
        })
    }
}

impl NetworkListener<HttpStream, HttpAcceptor> for HttpListener {
    #[inline]
    fn bind(host: &str, port: Port) -> IoResult<HttpListener> {
        Ok(HttpListener {
            inner: try!(TcpListener::bind(host, port))
        })
    }

    #[inline]
    fn socket_name(&mut self) -> IoResult<SocketAddr> {
        self.inner.socket_name()
    }
}

/// A `NetworkAcceptor` for `HttpStream`s.
#[deriving(Clone)]
pub struct HttpAcceptor {
    inner: TcpAcceptor
}

impl Acceptor<HttpStream> for HttpAcceptor {
    #[inline]
    fn accept(&mut self) -> IoResult<HttpStream> {
        Ok(Http(try!(self.inner.accept())))
    }
}

impl NetworkAcceptor<HttpStream> for HttpAcceptor {
    #[inline]
    fn close(&mut self) -> IoResult<()> {
        self.inner.close_accept()
    }
}

/// A wrapper around a TcpStream.
#[deriving(Clone)]
pub enum HttpStream {
    /// A stream over the HTTP protocol.
    Http(TcpStream),
    /// A stream over the HTTP protocol, protected by SSL.
    // You may be asking wtf an Arc and Mutex? That's because SslStream
    // doesn't implement Clone, and we need Clone to use the stream for
    // both the Request and Response.
    // FIXME: https://github.com/sfackler/rust-openssl/issues/6
    Https(Arc<Mutex<SslStream<TcpStream>>>, SocketAddr),
}

impl Reader for HttpStream {
    #[inline]
    fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> {
        match *self {
            Http(ref mut inner) => inner.read(buf),
            Https(ref mut inner, _) => inner.lock().read(buf)
        }
    }
}

impl Writer for HttpStream {
    #[inline]
    fn write(&mut self, msg: &[u8]) -> IoResult<()> {
        match *self {
            Http(ref mut inner) => inner.write(msg),
            Https(ref mut inner, _) => inner.lock().write(msg)
        }
    }
    #[inline]
    fn flush(&mut self) -> IoResult<()> {
        match *self {
            Http(ref mut inner) => inner.flush(),
            Https(ref mut inner, _) => inner.lock().flush(),
        }
    }
}

impl NetworkStream for HttpStream {
    fn connect(host: &str, port: Port, scheme: &str) -> IoResult<HttpStream> {
        match scheme {
            "http" => {
                debug!("http scheme");
                Ok(Http(try!(TcpStream::connect(host, port))))
            },
            "https" => {
                debug!("https scheme");
                let mut stream = try!(TcpStream::connect(host, port));
                // we can't access the tcp stream once it's wrapped in an
                // SslStream, so grab the ip address now, just in case.
                let addr = try!(stream.peer_name());
                let context = try!(SslContext::new(Sslv23).map_err(lift_ssl_error));
                let stream = try!(SslStream::new(&context, stream).map_err(lift_ssl_error));
                Ok(Https(Arc::new(Mutex::new(stream)), addr))
            },
            _ => {
                Err(IoError {
                    kind: InvalidInput,
                    desc: "Invalid scheme for Http",
                    detail: None
                })
            }
        }
    }

    fn peer_name(&mut self) -> IoResult<SocketAddr> {
        match *self {
            Http(ref mut inner) => inner.peer_name(),
            Https(_, addr) => Ok(addr)
        }
    }
}

fn lift_ssl_error(ssl: SslError) -> IoError {
    match ssl {
        StreamError(err) => err,
        SslSessionClosed => IoError {
            kind: ConnectionAborted,
            desc: "SSL Connection Closed",
            detail: None
        },
        // Unfortunately throw this away. No way to support this
        // detail without a better Error abstraction.
        OpenSslErrors(errs) => IoError {
            kind: OtherIoError,
            desc: "Error in OpenSSL",
            detail: Some(format!("{}", errs))
        }
    }
}
<commit_msg>fix recursive overflow of read()<commit_after>//! A collection of traits abstracting over Listeners and Streams.
use std::io::{IoResult, IoError, ConnectionAborted, InvalidInput, OtherIoError,
              Stream, Listener, Acceptor};
use std::io::net::ip::{SocketAddr, Port};
use std::io::net::tcp::{TcpStream, TcpListener, TcpAcceptor};
use std::sync::{Arc, Mutex};

use openssl::ssl::{SslStream, SslContext, Sslv23};
use openssl::ssl::error::{SslError, StreamError, OpenSslErrors, SslSessionClosed};

/// The write-status indicating headers have not been written.
pub struct Fresh;

/// The write-status indicating headers have been written.
pub struct Streaming;

/// An abstraction to listen for connections on a certain port.
pub trait NetworkListener<S: NetworkStream, A: NetworkAcceptor<S>>: Listener<S, A> {
    /// Bind to a socket.
    ///
    /// Note: This does not start listening for connections. You must call
    /// `listen()` to do that.
    fn bind(host: &str, port: Port) -> IoResult<Self>;

    /// Get the address this Listener ended up listening on.
    fn socket_name(&mut self) -> IoResult<SocketAddr>;
}

/// An abstraction to receive `NetworkStream`s.
pub trait NetworkAcceptor<S: NetworkStream>: Acceptor<S> + Clone + Send {
    /// Closes the Acceptor, so no more incoming connections will be handled.
    fn close(&mut self) -> IoResult<()>;
}

/// An abstraction over streams that a Server can utilize.
pub trait NetworkStream: Stream + Clone + Send {
    /// Get the remote address of the underlying connection.
    fn peer_name(&mut self) -> IoResult<SocketAddr>;

    /// Connect to a remote address.
    fn connect(host: &str, Port, scheme: &str) -> IoResult<Self>;

    /// Turn this into an appropriately typed trait object.
    #[inline]
    fn abstract(self) -> Box<NetworkStream + Send> {
        box self as Box<NetworkStream + Send>
    }

    #[doc(hidden)]
    #[inline]
    // Hack to work around lack of Clone impl for Box<Clone>
    fn clone_box(&self) -> Box<NetworkStream + Send> { self.clone().abstract() }
}

impl Clone for Box<NetworkStream + Send> {
    #[inline]
    fn clone(&self) -> Box<NetworkStream + Send> { self.clone_box() }
}

impl Reader for Box<NetworkStream + Send> {
    #[inline]
    fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> { (**self).read(buf) }
}

impl Writer for Box<NetworkStream + Send> {
    #[inline]
    fn write(&mut self, msg: &[u8]) -> IoResult<()> { (**self).write(msg) }

    #[inline]
    fn flush(&mut self) -> IoResult<()> { (**self).flush() }
}

/// A `NetworkListener` for `HttpStream`s.
pub struct HttpListener {
    inner: TcpListener
}

impl Listener<HttpStream, HttpAcceptor> for HttpListener {
    #[inline]
    fn listen(self) -> IoResult<HttpAcceptor> {
        Ok(HttpAcceptor {
            inner: try!(self.inner.listen())
        })
    }
}

impl NetworkListener<HttpStream, HttpAcceptor> for HttpListener {
    #[inline]
    fn bind(host: &str, port: Port) -> IoResult<HttpListener> {
        Ok(HttpListener {
            inner: try!(TcpListener::bind(host, port))
        })
    }

    #[inline]
    fn socket_name(&mut self) -> IoResult<SocketAddr> {
        self.inner.socket_name()
    }
}

/// A `NetworkAcceptor` for `HttpStream`s.
#[deriving(Clone)]
pub struct HttpAcceptor {
    inner: TcpAcceptor
}

impl Acceptor<HttpStream> for HttpAcceptor {
    #[inline]
    fn accept(&mut self) -> IoResult<HttpStream> {
        Ok(Http(try!(self.inner.accept())))
    }
}

impl NetworkAcceptor<HttpStream> for HttpAcceptor {
    #[inline]
    fn close(&mut self) -> IoResult<()> {
        self.inner.close_accept()
    }
}

/// A wrapper around a TcpStream.
#[deriving(Clone)]
pub enum HttpStream {
    /// A stream over the HTTP protocol.
    Http(TcpStream),
    /// A stream over the HTTP protocol, protected by SSL.
    // You may be asking wtf an Arc and Mutex? That's because SslStream
    // doesn't implement Clone, and we need Clone to use the stream for
    // both the Request and Response.
    // FIXME: https://github.com/sfackler/rust-openssl/issues/6
    Https(Arc<Mutex<SslStream<TcpStream>>>, SocketAddr),
}

impl Reader for HttpStream {
    #[inline]
    fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> {
        match *self {
            Http(ref mut inner) => inner.read(buf),
            Https(ref mut inner, _) => inner.lock().read(buf)
        }
    }
}

impl Writer for HttpStream {
    #[inline]
    fn write(&mut self, msg: &[u8]) -> IoResult<()> {
        match *self {
            Http(ref mut inner) => inner.write(msg),
            Https(ref mut inner, _) => inner.lock().write(msg)
        }
    }
    #[inline]
    fn flush(&mut self) -> IoResult<()> {
        match *self {
            Http(ref mut inner) => inner.flush(),
            Https(ref mut inner, _) => inner.lock().flush(),
        }
    }
}

impl NetworkStream for HttpStream {
    fn connect(host: &str, port: Port, scheme: &str) -> IoResult<HttpStream> {
        match scheme {
            "http" => {
                debug!("http scheme");
                Ok(Http(try!(TcpStream::connect(host, port))))
            },
            "https" => {
                debug!("https scheme");
                let mut stream = try!(TcpStream::connect(host, port));
                // we can't access the tcp stream once it's wrapped in an
                // SslStream, so grab the ip address now, just in case.
                let addr = try!(stream.peer_name());
                let context = try!(SslContext::new(Sslv23).map_err(lift_ssl_error));
                let stream = try!(SslStream::new(&context, stream).map_err(lift_ssl_error));
                Ok(Https(Arc::new(Mutex::new(stream)), addr))
            },
            _ => {
                Err(IoError {
                    kind: InvalidInput,
                    desc: "Invalid scheme for Http",
                    detail: None
                })
            }
        }
    }

    fn peer_name(&mut self) -> IoResult<SocketAddr> {
        match *self {
            Http(ref mut inner) => inner.peer_name(),
            Https(_, addr) => Ok(addr)
        }
    }
}

fn lift_ssl_error(ssl: SslError) -> IoError {
    match ssl {
        StreamError(err) => err,
        SslSessionClosed => IoError {
            kind: ConnectionAborted,
            desc: "SSL Connection Closed",
            detail: None
        },
        // Unfortunately throw this away. No way to support this
        // detail without a better Error abstraction.
        OpenSslErrors(errs) => IoError {
            kind: OtherIoError,
            desc: "Error in OpenSSL",
            detail: Some(format!("{}", errs))
        }
    }
}
<|endoftext|>