<commit_before>#![feature(scoped)]

extern crate cucache;
use cucache::memcache;

extern crate getopts;
extern crate time;
extern crate byteorder;
use byteorder::{BigEndian, ByteOrder};

#[macro_use]
extern crate log;
use log::{LogRecord, LogLevel, LogLevelFilter, LogMetadata};

use std::io::Write;
use std::net;
use std::thread;
use std::env;
use std::str::FromStr;
use std::process;
use std::sync;
use std::sync::mpsc;
use std::io;
use std::str;

extern crate num;
use num::traits::FromPrimitive;

fn from_mctime(i : u32) -> i64 {
    if i > 0 && i < 30*24*60*60 {
        time::get_time().sec + i as i64
    } else {
        i as i64
    }
}

fn main() {
    let _ = log::set_logger(|max_log_level| {
        max_log_level.set(LogLevelFilter::Debug);
        Box::new(SimpleLogger)
    });

    let mut opts = getopts::Options::new();
    opts.optflagopt("p", "", "Listen on TCP port <num>, the default is port 11211.", "<num>");
    opts.optflagopt("U", "", "Listen on UDP port <num>, the default is port 11211, 0 is off.", "<num>");
    opts.optflag("h", "help", "display this help and exit");

    let args: Vec<String> = env::args().collect();
    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => {
            println!("{}", f);
            process::exit(1);
        }
    };

    if matches.opt_present("help") {
        let brief = format!("Usage: {} [options]", args[0]);
        println!("{}", opts.usage(&brief));
        return;
    }

    let host = "127.0.0.1";
    let mut tcp_port = 11211;
    if let Some(port) = matches.opt_str("p") {
        if let Ok(portn) = u16::from_str(&port) {
            tcp_port = portn;
        }
    }

    info!("listening on {}:tcp:{}", host, tcp_port);
    let listener = net::TcpListener::bind((host, tcp_port)).unwrap();

    info!("allocating map");
    let m = cucache::new(1 << 10);

    info!("spinning up worker pool");
    let (tx, rx) = mpsc::channel();
    let rxmx = sync::Mutex::new(rx);
    let pool : Vec<thread::JoinGuard<()>> = (1..100).map(|_| {
        let map = &m;
        let rxl = &rxmx;
        thread::scoped(move || {
            handle_clients(map, rxl);
        })
    }).collect();

    info!("accepting connections");
    for stream in listener.incoming() {
        match stream {
            Ok(stream) => {
                if let Err(e) = tx.send(stream) {
                    panic!("failed to queue incoming request {}", e);
                }
            }
            Err(e) => {
                warn!("{}", e);
            }
        }
    }

    drop(pool);
}

fn read_full(r : &mut io::Read, to : &mut [u8]) -> Result<(), io::Error> {
    let mut nread = 0 as usize;
    while nread < to.len() {
        match r.read(&mut to[nread..]) {
            Ok(n) if n == 0 => { return Err(std::io::Error::new(std::io::ErrorKind::ConnectionAborted, "")); }
            Ok(n) => { nread += n; }
            Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {},
            Err(e) => return Err(From::from(e))
        }
    }
    Ok(())
}

fn execute(m : &cucache::Map, req : &memcache::Request, c : &mut net::TcpStream) -> bool {
    let mut rh = memcache::ResponseHeader::from_req(req);
    match memcache::Command::from_u8(req.op) {
        Some(op) => {
            let key = req.key();
            let extras = req.extras();
            let body = req.body();

            match op.noq() {
                memcache::Command::FLUSH if extras.len() == 4 => {
                    let mut tm = from_mctime(BigEndian::read_u32(extras));
                    if tm == 0 {
                        tm = time::get_time().sec;
                    }
                    m.touchall(tm);

                    rh.status = memcache::Status::SUCCESS as u16;
                    rh.construct(&[], &[], &[]).transmit(c)
                }
                memcache::Command::SET
                | memcache::Command::ADD
                | memcache::Command::REPLACE
                if extras.len() == 8 => {
                    let flags = BigEndian::read_u32(&extras[0..4]);
                    let expires = from_mctime(BigEndian::read_u32(&extras[4..8]));

                    // XXX: unfortunate copy...
                    let b = body.to_vec();

                    let res = match op.noq() {
                        memcache::Command::SET if req.cas == 0 => m.set(key, b, flags, expires),
                        memcache::Command::REPLACE if req.cas == 0 => m.replace(key, b, flags, expires),
                        memcache::Command::ADD => m.add(key, b, flags, expires),
                        memcache::Command::SET | memcache::Command::REPLACE | _ => {
                            m.cas(key, b, flags, expires, req.cas)
                        }
                    };

                    rh.status = res.0 as u16;
                    if let (memcache::Status::SUCCESS, Ok(v)) = res {
                        rh.cas = v.val.casid;
                    }

                    rh.construct(&[], &[], &[]).transmit(c)
                }
                memcache::Command::DELETE => {
                    rh.status = m.delete(key, req.cas).0 as u16;
                    rh.construct(&[], &[], &[]).transmit(c)
                }
                memcache::Command::INCREMENT
                | memcache::Command::DECREMENT
                if extras.len() == 20 => {
                    let by = BigEndian::read_u64(&extras[0..8]);
                    let def = BigEndian::read_u64(&extras[8..16]);
                    let expires = from_mctime(BigEndian::read_u32(&extras[16..20]));

                    let v = match op.noq() {
                        memcache::Command::INCREMENT => m.incr(key, by, def, expires),
                        memcache::Command::DECREMENT | _ => m.decr(key, by, def, expires),
                    };

                    rh.status = v.0 as u16;
                    if let (memcache::Status::SUCCESS, Ok(v)) = v {
                        let extras = &mut [0u8; 8];
                        BigEndian::write_u64(&mut extras[..], u64::from_str(str::from_utf8(&v.val.bytes[..]).unwrap()).unwrap());

                        rh.cas = v.val.casid;
                        rh.construct(&extras[..], &[], &[]).transmit(c)
                    } else {
                        rh.construct(&[], &[], &[]).transmit(c)
                    }
                }
                memcache::Command::GET | memcache::Command::GETK => {
                    let v = m.get(key);
                    rh.status = v.0 as u16;

                    if let (memcache::Status::SUCCESS, Ok(v)) = v {
                        rh.cas = v.val.casid;
                        let extras = &mut [0u8; 4];
                        BigEndian::write_u32(&mut extras[..], v.val.flags);
                        rh.construct(
                            &extras[..],
                            if op.noq() == memcache::Command::GETK {key} else {&[]},
                            &v.val.bytes[..]
                        ).transmit(c)
                    } else {
                        rh.construct(&[], &[], &[]).transmit(c)
                    }
                }
                memcache::Command::APPEND | memcache::Command::PREPEND => {
                    // XXX: unfortunate copy...
                    let b = body.to_vec();

                    let res = match op.noq() {
                        memcache::Command::PREPEND => m.prepend(key, b, req.cas),
                        memcache::Command::APPEND | _ => m.append(key, b, req.cas),
                    };

                    rh.status = res.0 as u16;
                    if let (memcache::Status::SUCCESS, Ok(v)) = res {
                        rh.cas = v.val.casid;
                    }

                    rh.construct(&[], &[], &[]).transmit(c)
                }
                memcache::Command::VERSION => {
                    rh.status = memcache::Status::SUCCESS as u16;
                    rh.construct(&[], &[], "0.1.0".as_bytes()).transmit(c)
                }
                memcache::Command::NOOP => {
                    rh.status = memcache::Status::SUCCESS as u16;
                    rh.construct(&[], &[], &[]).transmit(c)
                }
                _ => {
                    warn!("client sent unhandled command: {:?}", op);
                    rh.status = memcache::Status::EINVAL as u16;
                    rh.construct(&[], &[], &[]).transmit(c)
                }
            }
        }
        None => {
            rh.status = memcache::Status::UNKNOWN_COMMAND as u16;
            rh.construct(&[], &[], &[]).transmit(c)
        }
    }
}

fn handle_client(m : &cucache::Map, mut c : net::TcpStream) {
    let mut magic = [0_u8; 1];
    let mut body = Vec::with_capacity(100);
    'outer: loop {
        if let Err(_) = read_full(&mut c, &mut magic) {
            break 'outer;
        }

        unsafe { body.set_len(100); }
        body.shrink_to_fit();
        body.clear();
        unsafe { body.set_len(24); }
        match magic[0] {
            0x80 => {
                // binary protocol
                // memcache request
                body[0] = magic[0];
                if let Err(_) = read_full(&mut c, &mut body[1..24]) {
                    break 'outer;
                }

                let blen = BigEndian::read_u32(&body[8..12]) as usize;
                body.reserve_exact(blen);
                unsafe { body.set_len(24 + blen); }

                if let Err(_) = read_full(&mut c, &mut body[24..(24+blen)]) {
                    break 'outer;
                }

                let req = memcache::Request::parse(&mut body[..]);
                if !execute(m, req, &mut c) {
                    break 'outer;
                }
            }
            0x81 => {
                warn!("expected request from client, got response magic");
                break 'outer;
            }
            _ => {
                // text protocol
            }
        }
    };

    info!("client {:?} hung up", c);
}

fn handle_clients(m : &cucache::Map, rxmx : &sync::Mutex<mpsc::Receiver<net::TcpStream>>) {
    loop {
        if let Ok(rx) = rxmx.lock() {
            if let Ok(c) = rx.recv() {
                drop(rx);
                handle_client(&m, c);
            } else {
                // channel receive error
                return;
            }
        } else {
            // lock poisoned by panic
            return;
        }
    }
}

struct SimpleLogger;
impl log::Log for SimpleLogger {
    fn enabled(&self, metadata: &LogMetadata) -> bool {
        metadata.level() <= LogLevel::Debug
    }

    #[allow(unused_must_use)]
    fn log(&self, record: &LogRecord) {
        if self.enabled(record.metadata()) {
            writeln!(&mut std::io::stderr(), "{} - {}", record.level(), record.args());
        }
    }
}

<commit_msg>Change default debug level to warn<commit_after>#![feature(scoped)]

extern crate cucache;
use cucache::memcache;

extern crate getopts;
extern crate time;
extern crate byteorder;
use byteorder::{BigEndian, ByteOrder};

#[macro_use]
extern crate log;
use log::{LogRecord, LogLevel, LogLevelFilter, LogMetadata};

use std::io::Write;
use std::net;
use std::thread;
use std::env;
use std::str::FromStr;
use std::process;
use std::sync;
use std::sync::mpsc;
use std::io;
use std::str;

extern crate num;
use num::traits::FromPrimitive;

fn from_mctime(i : u32) -> i64 {
    if i > 0 && i < 30*24*60*60 {
        time::get_time().sec + i as i64
    } else {
        i as i64
    }
}

fn main() {
    let _ = log::set_logger(|max_log_level| {
        max_log_level.set(LogLevelFilter::Warn);
        Box::new(SimpleLogger)
    });

    let mut opts = getopts::Options::new();
    opts.optflagopt("p", "", "Listen on TCP port <num>, the default is port 11211.", "<num>");
    opts.optflagopt("U", "", "Listen on UDP port <num>, the default is port 11211, 0 is off.", "<num>");
    opts.optflag("h", "help", "display this help and exit");

    let args: Vec<String> = env::args().collect();
    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => {
            println!("{}", f);
            process::exit(1);
        }
    };

    if matches.opt_present("help") {
        let brief = format!("Usage: {} [options]", args[0]);
        println!("{}", opts.usage(&brief));
        return;
    }

    let host = "127.0.0.1";
    let mut tcp_port = 11211;
    if let Some(port) = matches.opt_str("p") {
        if let Ok(portn) = u16::from_str(&port) {
            tcp_port = portn;
        }
    }

    info!("listening on {}:tcp:{}", host, tcp_port);
    let listener = net::TcpListener::bind((host, tcp_port)).unwrap();

    info!("allocating map");
    let m = cucache::new(1 << 10);

    info!("spinning up worker pool");
    let (tx, rx) = mpsc::channel();
    let rxmx = sync::Mutex::new(rx);
    let pool : Vec<thread::JoinGuard<()>> = (1..100).map(|_| {
        let map = &m;
        let rxl = &rxmx;
        thread::scoped(move || {
            handle_clients(map, rxl);
        })
    }).collect();

    info!("accepting connections");
    for stream in listener.incoming() {
        match stream {
            Ok(stream) => {
                if let Err(e) = tx.send(stream) {
                    panic!("failed to queue incoming request {}", e);
                }
            }
            Err(e) => {
                warn!("{}", e);
            }
        }
    }

    drop(pool);
}

fn read_full(r : &mut io::Read, to : &mut [u8]) -> Result<(), io::Error> {
    let mut nread = 0 as usize;
    while nread < to.len() {
        match r.read(&mut to[nread..]) {
            Ok(n) if n == 0 => { return Err(std::io::Error::new(std::io::ErrorKind::ConnectionAborted, "")); }
            Ok(n) => { nread += n; }
            Err(ref e) if e.kind() == io::ErrorKind::Interrupted => {},
            Err(e) => return Err(From::from(e))
        }
    }
    Ok(())
}

fn execute(m : &cucache::Map, req : &memcache::Request, c : &mut net::TcpStream) -> bool {
    let mut rh = memcache::ResponseHeader::from_req(req);
    match memcache::Command::from_u8(req.op) {
        Some(op) => {
            let key = req.key();
            let extras = req.extras();
            let body = req.body();

            match op.noq() {
                memcache::Command::FLUSH if extras.len() == 4 => {
                    let mut tm = from_mctime(BigEndian::read_u32(extras));
                    if tm == 0 {
                        tm = time::get_time().sec;
                    }
                    m.touchall(tm);

                    rh.status = memcache::Status::SUCCESS as u16;
                    rh.construct(&[], &[], &[]).transmit(c)
                }
                memcache::Command::SET
                | memcache::Command::ADD
                | memcache::Command::REPLACE
                if extras.len() == 8 => {
                    let flags = BigEndian::read_u32(&extras[0..4]);
                    let expires = from_mctime(BigEndian::read_u32(&extras[4..8]));

                    // XXX: unfortunate copy...
                    let b = body.to_vec();

                    let res = match op.noq() {
                        memcache::Command::SET if req.cas == 0 => m.set(key, b, flags, expires),
                        memcache::Command::REPLACE if req.cas == 0 => m.replace(key, b, flags, expires),
                        memcache::Command::ADD => m.add(key, b, flags, expires),
                        memcache::Command::SET | memcache::Command::REPLACE | _ => {
                            m.cas(key, b, flags, expires, req.cas)
                        }
                    };

                    rh.status = res.0 as u16;
                    if let (memcache::Status::SUCCESS, Ok(v)) = res {
                        rh.cas = v.val.casid;
                    }

                    rh.construct(&[], &[], &[]).transmit(c)
                }
                memcache::Command::DELETE => {
                    rh.status = m.delete(key, req.cas).0 as u16;
                    rh.construct(&[], &[], &[]).transmit(c)
                }
                memcache::Command::INCREMENT
                | memcache::Command::DECREMENT
                if extras.len() == 20 => {
                    let by = BigEndian::read_u64(&extras[0..8]);
                    let def = BigEndian::read_u64(&extras[8..16]);
                    let expires = from_mctime(BigEndian::read_u32(&extras[16..20]));

                    let v = match op.noq() {
                        memcache::Command::INCREMENT => m.incr(key, by, def, expires),
                        memcache::Command::DECREMENT | _ => m.decr(key, by, def, expires),
                    };

                    rh.status = v.0 as u16;
                    if let (memcache::Status::SUCCESS, Ok(v)) = v {
                        let extras = &mut [0u8; 8];
                        BigEndian::write_u64(&mut extras[..], u64::from_str(str::from_utf8(&v.val.bytes[..]).unwrap()).unwrap());

                        rh.cas = v.val.casid;
                        rh.construct(&extras[..], &[], &[]).transmit(c)
                    } else {
                        rh.construct(&[], &[], &[]).transmit(c)
                    }
                }
                memcache::Command::GET | memcache::Command::GETK => {
                    let v = m.get(key);
                    rh.status = v.0 as u16;

                    if let (memcache::Status::SUCCESS, Ok(v)) = v {
                        rh.cas = v.val.casid;
                        let extras = &mut [0u8; 4];
                        BigEndian::write_u32(&mut extras[..], v.val.flags);
                        rh.construct(
                            &extras[..],
                            if op.noq() == memcache::Command::GETK {key} else {&[]},
                            &v.val.bytes[..]
                        ).transmit(c)
                    } else {
                        rh.construct(&[], &[], &[]).transmit(c)
                    }
                }
                memcache::Command::APPEND | memcache::Command::PREPEND => {
                    // XXX: unfortunate copy...
                    let b = body.to_vec();

                    let res = match op.noq() {
                        memcache::Command::PREPEND => m.prepend(key, b, req.cas),
                        memcache::Command::APPEND | _ => m.append(key, b, req.cas),
                    };

                    rh.status = res.0 as u16;
                    if let (memcache::Status::SUCCESS, Ok(v)) = res {
                        rh.cas = v.val.casid;
                    }

                    rh.construct(&[], &[], &[]).transmit(c)
                }
                memcache::Command::VERSION => {
                    rh.status = memcache::Status::SUCCESS as u16;
                    rh.construct(&[], &[], "0.1.0".as_bytes()).transmit(c)
                }
                memcache::Command::NOOP => {
                    rh.status = memcache::Status::SUCCESS as u16;
                    rh.construct(&[], &[], &[]).transmit(c)
                }
                _ => {
                    warn!("client sent unhandled command: {:?}", op);
                    rh.status = memcache::Status::EINVAL as u16;
                    rh.construct(&[], &[], &[]).transmit(c)
                }
            }
        }
        None => {
            rh.status = memcache::Status::UNKNOWN_COMMAND as u16;
            rh.construct(&[], &[], &[]).transmit(c)
        }
    }
}

fn handle_client(m : &cucache::Map, mut c : net::TcpStream) {
    let mut magic = [0_u8; 1];
    let mut body = Vec::with_capacity(100);
    'outer: loop {
        if let Err(_) = read_full(&mut c, &mut magic) {
            break 'outer;
        }

        unsafe { body.set_len(100); }
        body.shrink_to_fit();
        body.clear();
        unsafe { body.set_len(24); }
        match magic[0] {
            0x80 => {
                // binary protocol
                // memcache request
                body[0] = magic[0];
                if let Err(_) = read_full(&mut c, &mut body[1..24]) {
                    break 'outer;
                }

                let blen = BigEndian::read_u32(&body[8..12]) as usize;
                body.reserve_exact(blen);
                unsafe { body.set_len(24 + blen); }

                if let Err(_) = read_full(&mut c, &mut body[24..(24+blen)]) {
                    break 'outer;
                }

                let req = memcache::Request::parse(&mut body[..]);
                if !execute(m, req, &mut c) {
                    break 'outer;
                }
            }
            0x81 => {
                warn!("expected request from client, got response magic");
                break 'outer;
            }
            _ => {
                // text protocol
            }
        }
    };

    info!("client {:?} hung up", c);
}

fn handle_clients(m : &cucache::Map, rxmx : &sync::Mutex<mpsc::Receiver<net::TcpStream>>) {
    loop {
        if let Ok(rx) = rxmx.lock() {
            if let Ok(c) = rx.recv() {
                drop(rx);
                handle_client(&m, c);
            } else {
                // channel receive error
                return;
            }
        } else {
            // lock poisoned by panic
            return;
        }
    }
}

struct SimpleLogger;
impl log::Log for SimpleLogger {
    fn enabled(&self, metadata: &LogMetadata) -> bool {
        metadata.level() <= LogLevel::Warn
    }

    #[allow(unused_must_use)]
    fn log(&self, record: &LogRecord) {
        if self.enabled(record.metadata()) {
            writeln!(&mut std::io::stderr(), "{} - {}", record.level(), record.args());
        }
    }
}

<|endoftext|>


<commit_before>#![allow(dead_code)]

use std::io::prelude::*;
use std::env;
use std::fs::File;
use std::error::Error;
use std::default::Default;

mod tokenizer;      // Tokenizes files into words, sentences, and paragraphs.
mod calculator;     // Does all the calculations for stats

#[derive(Default, Copy, Clone)]
struct Metadata {
    word_total: f32,    // Intermediary for calculator.
    wordlen: f32,
    sentlen: f32,
    paralen: i32,
    comma: i32,
    semicolon: i32,
    quote: i32,
    bangs: i32,
    dashes: i32,
    ands: i32,
    buts: i32,
    however: i32,
    condition: i32,		// Prevent confusion with if.
    thats: i32,
    more: i32,
    musts: i32,
    mights: i32,
    thises: i32,
    very: i32,
}

impl Metadata {
    fn words(&mut self, s: Vec<&str>) {
        self.word_total = calculator::num_words(&s);
        self.wordlen = calculator::word_mean(&s);

        // Let me throw all elegance out the window.
        let (ands, buts, however, condition, thats, more,
            musts, mights, thises, very) = calculator::word_freq(&s);

        self.ands = ands;
        self.buts = buts;
        self.however = however;
        self.condition = condition;
        self.thats = thats;
        self.more = more;
        self.musts = musts;
        self.mights = mights;
        self.thises = thises;
        self.very = very;
    }

    fn sents(&mut self, s: Vec<&str>) {
        self.sentlen = calculator::sent_mean(&self.word_total, &s);
    }
}

fn main() {

    let mut off = Metadata {..Default::default()};
    let mut pseu = Metadata {..Default::default()};

    let args: Vec<String> = env::args().collect();

    let mut file1 = match File::open(&args[1]) {
        Err(why) => panic!("Couldn't open {}: {}", &args[1], Error::description(&why)),
        Ok(file1) => file1,
    };

    let mut file2 = match File::open(&args[2]) {
        Err(why) => panic!("Couldn't open {}: {}", &args[2], Error::description(&why)),
        Ok(file2) => file2,
    };

    let (mut s1, mut s2) = (String::new(), String::new());

    match file1.read_to_string(&mut s1) {
        Err(why) => panic!("Couldn't read file1: {}", Error::description(&why)),
        Ok(_) => print!(""),
    };

    match file2.read_to_string(&mut s2) {
        Err(why) => panic!("Couldn't read file2: {}", Error::description(&why)),
        Ok(_) => print!(""),
    };

    let s1_token: Vec<&str> = tokenizer::word_token(&s1);
    let s2_token: Vec<&str> = tokenizer::word_token(&s2);
    let s1_sent: Vec<&str> = tokenizer::sent_token(&s1);
    let s2_sent: Vec<&str> = tokenizer::sent_token(&s2);

    off.words(s1_token);
    pseu.words(s2_token);
    off.sents(s1_sent);
    pseu.sents(s2_sent);
    print_data(off, pseu);

}

fn print_data (st1: Metadata, st2: Metadata) {
    println!("\t\t\t\tOfficial\t\tPseudonym");
    println!("\t\t\t\t-----------------------------------------");
    println!("Word total\t\t\t{}\t\t\t{}", st1.word_total, st2.word_total);
    println!("Mean word length\t\t{}\t\t{}", st1.wordlen, st2.wordlen);
    println!("Mean sentence length\t\t{}\t\t{}", st1.sentlen, st2.sentlen);
    println!("Frequency of 'and'\t\t{}\t\t\t{}", st1.ands, st2.ands);
    println!("Frequency of 'but'\t\t{}\t\t\t{}", st1.buts, st2.buts);
    println!("Frequency of 'however'\t\t{}\t\t\t{}", st1.however, st2.however);
    println!("Frequency of 'if'\t\t{}\t\t\t{}", st1.condition, st2.condition);
    println!("Frequency of 'that'\t\t{}\t\t\t{}", st1.thats, st2.thats);
    println!("Frequency of 'more'\t\t{}\t\t\t{}", st1.more, st2.more);
    println!("Frequency of 'must'\t\t{}\t\t\t{}", st1.musts, st2.musts);
    println!("Frequency of 'might'\t\t{}\t\t\t{}", st1.mights, st2.mights);
    println!("Frequency of 'this'\t\t{}\t\t\t{}", st1.thises, st2.thises);
    println!("Frequency of 'very'\t\t{}\t\t\t{}", st1.very, st2.very);
}<commit_msg>Much saner way of formatting data<commit_after>#![allow(dead_code)]

use std::io::prelude::*;
use std::env;
use std::fs::File;
use std::error::Error;
use std::default::Default;

mod tokenizer;      // Tokenizes files into words, sentences, and paragraphs.
mod calculator;     // Does all the calculations for stats

#[derive(Default, Copy, Clone)]
struct Metadata {
    word_total: f32,    // Intermediary for calculator.
    wordlen: f32,
    sentlen: f32,
    paralen: i32,
    comma: i32,
    semicolon: i32,
    quote: i32,
    bangs: i32,
    dashes: i32,
    ands: i32,
    buts: i32,
    however: i32,
    condition: i32,		// Prevent confusion with if.
    thats: i32,
    more: i32,
    musts: i32,
    mights: i32,
    thises: i32,
    very: i32,
}

impl Metadata {
    fn words(&mut self, s: Vec<&str>) {
        self.word_total = calculator::num_words(&s);
        self.wordlen = calculator::word_mean(&s);

        // Let me throw all elegance out the window.
        let (ands, buts, however, condition, thats, more,
            musts, mights, thises, very) = calculator::word_freq(&s);

        self.ands = ands;
        self.buts = buts;
        self.however = however;
        self.condition = condition;
        self.thats = thats;
        self.more = more;
        self.musts = musts;
        self.mights = mights;
        self.thises = thises;
        self.very = very;
    }

    fn sents(&mut self, s: Vec<&str>) {
        self.sentlen = calculator::sent_mean(&self.word_total, &s);
    }
}

fn main() {

    let mut off = Metadata {..Default::default()};
    let mut pseu = Metadata {..Default::default()};

    let args: Vec<String> = env::args().collect();

    let mut file1 = match File::open(&args[1]) {
        Err(why) => panic!("Couldn't open {}: {}", &args[1], Error::description(&why)),
        Ok(file1) => file1,
    };

    let mut file2 = match File::open(&args[2]) {
        Err(why) => panic!("Couldn't open {}: {}", &args[2], Error::description(&why)),
        Ok(file2) => file2,
    };

    let (mut s1, mut s2) = (String::new(), String::new());

    match file1.read_to_string(&mut s1) {
        Err(why) => panic!("Couldn't read file1: {}", Error::description(&why)),
        Ok(_) => print!(""),
    };

    match file2.read_to_string(&mut s2) {
        Err(why) => panic!("Couldn't read file2: {}", Error::description(&why)),
        Ok(_) => print!(""),
    };

    let s1_token: Vec<&str> = tokenizer::word_token(&s1);
    let s2_token: Vec<&str> = tokenizer::word_token(&s2);
    let s1_sent: Vec<&str> = tokenizer::sent_token(&s1);
    let s2_sent: Vec<&str> = tokenizer::sent_token(&s2);

    off.words(s1_token);
    pseu.words(s2_token);
    off.sents(s1_sent);
    pseu.sents(s2_sent);
    print_data(off, pseu);

}

fn print_data (st1: Metadata, st2: Metadata) {
    println!("{0: <30} {1: <20} {2: <10}", "", "Official", "Pseudonym");
    println!("{0: <30} {1: <40}", "", "---------------------------------");
    println!("{0: <30} {1: <20} {2:<10}", "Word total", st1.word_total, st2.word_total);
    println!("{0: <30} {1: <20} {2:<10}", "Mean word length", st1.wordlen, st2.wordlen);
    println!("{0: <30} {1: <20} {2:<10}", "Mean sentence length", st1.sentlen, st2.sentlen);
    //println!("{0: <30} {1: <40}", "", "---------------------------------");
    println!("{0: <30} {1: <20} {2:<10}", "Frequency of 'and'", st1.ands, st2.ands);
    println!("{0: <30} {1: <20} {2:<10}", "Frequency of 'but'", st1.buts, st2.buts);
    println!("{0: <30} {1: <20} {2:<10}", "Frequency of 'however'", st1.however, st2.however);
    println!("{0: <30} {1: <20} {2:<10}", "Frequency of 'if'", st1.condition, st2.condition);
    println!("{0: <30} {1: <20} {2:<10}", "Frequency of 'that'", st1.thats, st2.thats);
    println!("{0: <30} {1: <20} {2:<10}", "Frequency of 'more'", st1.more, st2.more);
    println!("{0: <30} {1: <20} {2:<10}", "Frequency of 'must'", st1.musts, st2.musts);
    println!("{0: <30} {1: <20} {2:<10}", "Frequency of 'might'", st1.mights, st2.mights);
    println!("{0: <30} {1: <20} {2:<10}", "Frequency of 'this'", st1.thises, st2.thises);
    println!("{0: <30} {1: <20} {2:<10}", "Frequency of 'very'", st1.very, st2.very);
}<|endoftext|>


<commit_before>#![allow(dead_code)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_imports)]

extern crate toml;
extern crate glob;

use std::io::fs::PathExtensions;
use std::io::fs;
use std::io;
use std::os;

use app_result::{AppResult, app_err};

use dependencies::{
   TagsRoot,
   SourceKind,
   read_dependencies
};

use tags::{
   Tags,
   update_git_tags,
   update_crates_io_tags,
   create_tags,
   merge_tags
};

use dirs::tags_dir;

mod app_result;
mod dependencies;
mod dirs;
mod tags;

fn main()
{
   update_tags().unwrap_or_else(|err| {
      let stderr = &mut io::stderr();
      let _ = writeln!(stderr, "rusty-tags: {}", err);
      os::set_exit_status(1);
   });
}

fn update_tags() -> AppResult<()>
{
   let cwd = try!(os::getcwd());
   let cargo_dir = try!(find_cargo_toml_dir(&cwd));
   let tags_roots = try!(read_dependencies(&cargo_dir));

   for tags_root in tags_roots.iter() {
      let mut tag_files: Vec<Path> = Vec::new();
      let mut tag_dir: Option<Path> = None;

      match *tags_root {
         TagsRoot::Src { ref src_dir, ref dependencies } => {
            let mut src_tags = src_dir.clone();
            src_tags.push("rusty.tags");
            try!(create_tags(src_dir, &src_tags));
            tag_files.push(src_tags);

            for dep in dependencies.iter() {
               tag_files.push(try!(update_tags_of(dep)).tags_file);
            }

            tag_dir = Some(src_dir.clone());
         },

         TagsRoot::Lib { ref src_kind, ref dependencies } => {
            let lib_tags = try!(update_tags_of(src_kind));
            if lib_tags.cached {
               continue;
            }

            tag_files.push(lib_tags.tags_file);
            for dep in dependencies.iter() {
               tag_files.push(try!(update_tags_of(dep)).tags_file);
            }

            tag_dir = Some(lib_tags.src_dir.clone());
         }
      }

      if tag_files.is_empty() || tag_dir.is_none() {
         continue;
      }

      let mut rust_tags = try!(tags_dir());
      rust_tags.push("rust");
      if rust_tags.is_file() {
         tag_files.push(rust_tags);
      }

      let mut tags_file = tag_dir.unwrap();
      tags_file.push("rusty.tags");

      try!(merge_tags(&tag_files, &tags_file));
   }

   Ok(())
}

fn update_tags_of(src_kind: &SourceKind) -> AppResult<Tags>
{
   match *src_kind {
      SourceKind::Git { ref lib_name, ref commit_hash } => {
         update_git_tags(lib_name, commit_hash)
      },

      SourceKind::CratesIo { ref lib_name, ref version } => {
         update_crates_io_tags(lib_name, version)
      }
   }
}

/// Searches for a directory containing a `Cargo.toml` file starting at
/// `start_dir` and continuing the search upwards the directory tree
/// until a directory is found.
fn find_cargo_toml_dir(start_dir: &Path) -> AppResult<Path>
{
   let mut dir = start_dir.clone();
   loop {
      if let Ok(files) = fs::readdir(&dir) {
         for file in files.iter() {
            if file.is_file() {
               if let Some("Cargo.toml") = file.filename_str() {
                  return Ok(dir);
               }
            }
         }
      }

      if ! dir.pop() {
         return Err(app_err(format!("Couldn't find 'Cargo.toml' starting at directory '{}'!", start_dir.display())));
      }
   }
}
<commit_msg>Only skip the tags creation if there's already a merged tags file<commit_after>#![allow(dead_code)]
#![allow(unused_variables)]
#![allow(unused_mut)]
#![allow(unused_imports)]

extern crate toml;
extern crate glob;

use std::io::fs::PathExtensions;
use std::io::fs;
use std::io;
use std::os;

use app_result::{AppResult, app_err};

use dependencies::{
   TagsRoot,
   SourceKind,
   read_dependencies
};

use tags::{
   Tags,
   update_git_tags,
   update_crates_io_tags,
   create_tags,
   merge_tags
};

use dirs::tags_dir;

mod app_result;
mod dependencies;
mod dirs;
mod tags;

fn main()
{
   update_tags().unwrap_or_else(|err| {
      let stderr = &mut io::stderr();
      let _ = writeln!(stderr, "rusty-tags: {}", err);
      os::set_exit_status(1);
   });
}

fn update_tags() -> AppResult<()>
{
   let cwd = try!(os::getcwd());
   let cargo_dir = try!(find_cargo_toml_dir(&cwd));
   let tags_roots = try!(read_dependencies(&cargo_dir));

   for tags_root in tags_roots.iter() {
      let mut tag_files: Vec<Path> = Vec::new();
      let mut tag_dir: Option<Path> = None;

      match *tags_root {
         TagsRoot::Src { ref src_dir, ref dependencies } => {
            let mut src_tags = src_dir.clone();
            src_tags.push("rusty.tags");
            try!(create_tags(src_dir, &src_tags));
            tag_files.push(src_tags);

            for dep in dependencies.iter() {
               tag_files.push(try!(update_tags_of(dep)).tags_file);
            }

            tag_dir = Some(src_dir.clone());
         },

         TagsRoot::Lib { ref src_kind, ref dependencies } => {
            let lib_tags = try!(update_tags_of(src_kind));
            if lib_tags.cached {
               let mut src_tags = lib_tags.src_dir.clone();
               src_tags.push("rusty.tags");
               if src_tags.is_file() {
                  continue;
               }
            }

            tag_files.push(lib_tags.tags_file);

            for dep in dependencies.iter() {
               tag_files.push(try!(update_tags_of(dep)).tags_file);
            }

            tag_dir = Some(lib_tags.src_dir.clone());
         }
      }

      if tag_files.is_empty() || tag_dir.is_none() {
         continue;
      }

      let mut rust_tags = try!(tags_dir());
      rust_tags.push("rust");
      if rust_tags.is_file() {
         tag_files.push(rust_tags);
      }

      let mut tags_file = tag_dir.unwrap();
      tags_file.push("rusty.tags");

      try!(merge_tags(&tag_files, &tags_file));
   }

   Ok(())
}

fn update_tags_of(src_kind: &SourceKind) -> AppResult<Tags>
{
   match *src_kind {
      SourceKind::Git { ref lib_name, ref commit_hash } => {
         update_git_tags(lib_name, commit_hash)
      },

      SourceKind::CratesIo { ref lib_name, ref version } => {
         update_crates_io_tags(lib_name, version)
      }
   }
}

/// Searches for a directory containing a `Cargo.toml` file starting at
/// `start_dir` and continuing the search upwards the directory tree
/// until a directory is found.
fn find_cargo_toml_dir(start_dir: &Path) -> AppResult<Path>
{
   let mut dir = start_dir.clone();
   loop {
      if let Ok(files) = fs::readdir(&dir) {
         for file in files.iter() {
            if file.is_file() {
               if let Some("Cargo.toml") = file.filename_str() {
                  return Ok(dir);
               }
            }
         }
      }

      if ! dir.pop() {
         return Err(app_err(format!("Couldn't find 'Cargo.toml' starting at directory '{}'!", start_dir.display())));
      }
   }
}
<|endoftext|>


<commit_before>extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate docopt;

use docopt::Docopt;

use std::thread;

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_shared::*;
use timely::example_shared::operators::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

mod typedrw;
mod graphmap;
use graphmap::GraphMMap;

mod sorting;
use sorting::{SegmentList, radix_sort_32};

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      file containing list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_spawn(communicators, source);
    }
    else if workers > 1 { pagerank_spawn(ProcessCommunicator::new_vector(workers), source); }
    else { pagerank_spawn(vec![ThreadCommunicator], source); };
}

fn pagerank_spawn<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank_thread(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

// returns [src/peers] degrees, (dst, deg) pairs, and a list of [src/peers] endpoints
fn transpose(mut edges: Vec<Vec<(u32, u32)>>, peers: usize, nodes: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let mut deg = vec![0; (nodes as usize / peers) + 1];
    for list in &edges {
        for &(s, _) in list {
            deg[s as usize / peers] += 1;
        }
    }

    // let start = time::precise_time_s();
    radix_sort_32(&mut edges, &mut Vec::new(), &|&(_,d)| d);
    // println!("sorted in {}s", time::precise_time_s() - start);

    let mut rev = Vec::<(u32,u32)>::with_capacity(deg.len());
    let mut trn = Vec::with_capacity(edges.len() * 1024);
    for list in edges {
        for (s,d) in list {
            let len = rev.len();
            if (len == 0) || (rev[len-1].0 < d) {
                rev.push((d, 0u32));
            }

            let len = rev.len();
            rev[len-1].1 += 1;
            trn.push(s / peers as u32);
        }
    }

    return (deg, rev, trn);
}

// pagerank dataflow graph has a set of edges as input, and a binary vertex that for each epoch of
// received edges initiates an iterative subcomputation to compute the pagerank.

fn pagerank_thread<C>(communicator: C, filename: String, _workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    let start = time::precise_time_s();
    let mut going = start;

    let graph = GraphMMap::new(&filename);
    let nodes = graph.nodes();

    let mut segments = SegmentList::new(1024); // list of edge segments

    let mut src = vec![];   // holds ranks
    let mut deg = vec![];   // holds source degrees
    let mut rev = vec![];   // holds (dst, deg) pairs
    let mut trn = vec![];   // holds transposed sources

    let mut input = root.subcomputation(|builder| {

        let (input, edges) = builder.new_input::<(u32, u32)>();
        let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        edges.binary_notify(&ranks,
                            Exchange::new(|x: &(u32,u32)| x.0 as u64),
                            Exchange::new(|x: &(u32,f32)| x.0 as u64),
                            format!("pagerank"),
                            vec![RootTimestamp::new(0)],
                            move |input1, input2, output, notificator| {

            // receive incoming edges
            while let Some((_index, data)) = input1.pull() {
                segments.push(data.drain_temp());
            }

            while let Some((iter, _)) = notificator.next() {
                // if the very first iteration, prepare some stuff
                if iter.inner == 0 {
                    // println!("received edges in {}s", time::precise_time_s() - start);
                    // (deg, rev, trn) = transpose(srcs, dsts, peers);
                    let (a, b, c) = transpose(segments.finalize(), peers, nodes);
                    deg = a;
                    rev = b;
                    trn = c;
                    src = vec![0.0f32; deg.len()];
                    if index == 0 { println!("transformed; {} x {} shape", deg.len(), rev.len()); }
                }

                if iter.inner == 10 { going = time::precise_time_s(); }
                if iter.inner == 20 && index == 0 { println!("avg: {}", (time::precise_time_s() - going) / 10.0); }

                for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }

                let mut trn_slice = &trn[..];
                let mut rev_slice = &rev[..];
                while rev_slice.len() > 0 {
                    // TODO: session should just flush...
                    let mut session = output.session(&iter);
                    let next = std::cmp::min(100_000, rev_slice.len());
                    for &(dst, deg) in &rev_slice[..next] {
                        let mut accum = 0.0;
                        for &s in &trn_slice[..deg as usize] {
                            accum += src[s as usize];
                        }
                        trn_slice = &trn_slice[deg as usize..];
                        session.give((dst, accum));
                    }
                    rev_slice = &rev_slice[next..];
                }

                for s in &mut src { *s = 0.0; }
            }

            while let Some((iter, data)) = input2.pull() {
                notificator.notify_at(&iter);
                for (node, rank) in data.drain_temp() {
                    src[node as usize / peers] += rank;
                }
            }
        })
        .connect_loop(cycle);

        input
    });

    let mut edges = Vec::new();
    for node in 0..graph.nodes() {
        if node % peers == index {
            for dst in graph.edges(node) {
                edges.push((node as u32, *dst as u32));
            }
            if edges.len() > 1024 {
                input.send_at(0, edges.drain_temp());
                root.step();
            }
        }
    }

    input.send_at(0, edges.drain_temp());
    input.close();
    while root.step() { }

    if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
}

    // .unary_notify(
    //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
    //     format!("Aggregation"),
    //     vec![],
    //     move |input, output, iterator| {
    //         while let Some((iter, data)) = input.pull() {
    //             iterator.notify_at(&iter);
    //             for (node, rank) in data.drain_temp() {
    //                 acc[node as usize / workers] += rank;
    //             }
    //         }
    //
    //         while let Some((item, _)) = iterator.next() {
    //
    //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
    //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
    //
    //             for _ in 0..(1 + (nodes/workers)) { acc.push(0.0); }
    //             assert!(acc.len() == (1 + (nodes/workers)));
    //         }
    //     }
    // )
<commit_msg>prints iteration information<commit_after>extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate docopt;

use docopt::Docopt;

use std::thread;

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_shared::*;
use timely::example_shared::operators::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

mod typedrw;
mod graphmap;
use graphmap::GraphMMap;

mod sorting;
use sorting::{SegmentList, radix_sort_32};

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      file containing list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_spawn(communicators, source);
    }
    else if workers > 1 { pagerank_spawn(ProcessCommunicator::new_vector(workers), source); }
    else { pagerank_spawn(vec![ThreadCommunicator], source); };
}

fn pagerank_spawn<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank_thread(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

// returns [src/peers] degrees, (dst, deg) pairs, and a list of [src/peers] endpoints
fn transpose(mut edges: Vec<Vec<(u32, u32)>>, peers: usize, nodes: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let mut deg = vec![0; (nodes as usize / peers) + 1];
    for list in &edges {
        for &(s, _) in list {
            deg[s as usize / peers] += 1;
        }
    }

    // let start = time::precise_time_s();
    radix_sort_32(&mut edges, &mut Vec::new(), &|&(_,d)| d);
    // println!("sorted in {}s", time::precise_time_s() - start);

    let mut rev = Vec::<(u32,u32)>::with_capacity(deg.len());
    let mut trn = Vec::with_capacity(edges.len() * 1024);
    for list in edges {
        for (s,d) in list {
            let len = rev.len();
            if (len == 0) || (rev[len-1].0 < d) {
                rev.push((d, 0u32));
            }

            let len = rev.len();
            rev[len-1].1 += 1;
            trn.push(s / peers as u32);
        }
    }

    return (deg, rev, trn);
}

// pagerank dataflow graph has a set of edges as input, and a binary vertex that for each epoch of
// received edges initiates an iterative subcomputation to compute the pagerank.

fn pagerank_thread<C>(communicator: C, filename: String, _workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    let start = time::precise_time_s();
    let mut going = start;

    let graph = GraphMMap::new(&filename);
    let nodes = graph.nodes();

    let mut segments = SegmentList::new(1024); // list of edge segments

    let mut src = vec![];   // holds ranks
    let mut deg = vec![];   // holds source degrees
    let mut rev = vec![];   // holds (dst, deg) pairs
    let mut trn = vec![];   // holds transposed sources

    let mut input = root.subcomputation(|builder| {

        let (input, edges) = builder.new_input::<(u32, u32)>();
        let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        edges.binary_notify(&ranks,
                            Exchange::new(|x: &(u32,u32)| x.0 as u64),
                            Exchange::new(|x: &(u32,f32)| x.0 as u64),
                            format!("pagerank"),
                            vec![RootTimestamp::new(0)],
                            move |input1, input2, output, notificator| {

            // receive incoming edges
            while let Some((_index, data)) = input1.pull() {
                segments.push(data.drain_temp());
            }

            while let Some((iter, _)) = notificator.next() {
                // if the very first iteration, prepare some stuff
                if iter.inner == 0 {
                    // println!("received edges in {}s", time::precise_time_s() - start);
                    // (deg, rev, trn) = transpose(srcs, dsts, peers);
                    let (a, b, c) = transpose(segments.finalize(), peers, nodes);
                    deg = a;
                    rev = b;
                    trn = c;
                    src = vec![0.0f32; deg.len()];
                    if index == 0 { println!("transformed; {} x {} shape", deg.len(), rev.len()); }
                }

                if iter.inner == 10 { going = time::precise_time_s(); }
                if iter.inner == 20 && index == 0 { println!("avg: {}", (time::precise_time_s() - going) / 10.0); }

                let iter_start = time::precise_time_s();

                for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }

                let mut trn_slice = &trn[..];
                let mut rev_slice = &rev[..];
                while rev_slice.len() > 0 {
                    // TODO: session should just flush...
                    let mut session = output.session(&iter);
                    let next = std::cmp::min(100_000, rev_slice.len());
                    for &(dst, deg) in &rev_slice[..next] {
                        let mut accum = 0.0;
                        for &s in &trn_slice[..deg as usize] {
                            accum += src[s as usize];
                        }
                        trn_slice = &trn_slice[deg as usize..];
                        session.give((dst, accum));
                    }
                    rev_slice = &rev_slice[next..];
                }

                for s in &mut src { *s = 0.0; }

                if index == 0 { println!("iter: {} took {}s", iter.inner, time::precise_time_s() - iter_start); }
            }

            while let Some((iter, data)) = input2.pull() {
                notificator.notify_at(&iter);
                for (node, rank) in data.drain_temp() {
                    src[node as usize / peers] += rank;
                }
            }
        })
        .connect_loop(cycle);

        input
    });

    let mut edges = Vec::new();
    for node in 0..graph.nodes() {
        if node % peers == index {
            for dst in graph.edges(node) {
                edges.push((node as u32, *dst as u32));
            }
            if edges.len() > 1024 {
                input.send_at(0, edges.drain_temp());
                root.step();
            }
        }
    }

    input.send_at(0, edges.drain_temp());
    input.close();
    while root.step() { }

    if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
}

    // .unary_notify(
    //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
    //     format!("Aggregation"),
    //     vec![],
    //     move |input, output, iterator| {
    //         while let Some((iter, data)) = input.pull() {
    //             iterator.notify_at(&iter);
    //             for (node, rank) in data.drain_temp() {
    //                 acc[node as usize / workers] += rank;
    //             }
    //         }
    //
    //         while let Some((item, _)) = iterator.next() {
    //
    //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
    //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
    //
    //             for _ in 0..(1 + (nodes/workers)) { acc.push(0.0); }
    //             assert!(acc.len() == (1 + (nodes/workers)));
    //         }
    //     }
    // )
<|endoftext|>


<commit_before>use std::io;
use std::io::prelude::*;
use std::net::TcpStream;
use std::fs::File;
use std::env;

extern crate chrono;
use chrono::*;

extern crate aggregated_stats;

#[macro_use]
extern crate clap;

extern crate env_logger;
#[macro_use]
extern crate log;

extern crate stats;

extern crate prometheus;
extern crate hyper;

mod timing_analyzer;
mod error_analyzer;
mod args;
mod filter;
mod log_parser;
use log_parser::log_events::*;
mod render;
mod request_response_matcher;
use request_response_matcher::*;
mod http_handler;
mod result;

fn main() {
    env_logger::init().expect("Failed to initialize logging.");

    let args = args::parse_args(env::args()).expect("Failed to parse arguments.");

    let result = run(&args);

    let mut stream;
    let mut renderer: Box<render::Renderer>;

    renderer = match args.graphite_server {
        Some(ref graphite_server) => {
            stream = TcpStream::connect((graphite_server.as_ref(), args.graphite_port.unwrap()))
                .expect("Could not connect to the Graphite server");

            Box::new(render::graphite::GraphiteRenderer::new(UTC::now(),
                                                             args.graphite_prefix.clone(),
                                                             &mut stream))
        }
        None => Box::new(render::TerminalRenderer::new()),
    };

    renderer.render(result);

    if args.prometheus_listen.is_some() {
        let binding_address = args.prometheus_listen.clone().unwrap();
        http_handler::listen_http(args, &binding_address);
    }

}

fn run(args: &args::RequestLogAnalyzerArgs) -> result::RequestLogAnalyzerResult {
    let input: Box<io::Read> = match args.filename.as_ref() {
        "-" => Box::new(io::stdin()),
        _ => Box::new(File::open(&args.filename).expect("Failed to open file.")),
    };

    let reader = io::BufReader::new(input);

    let mut events_iterator = reader.lines()
        .map(parse_event)
        .filter(|event| event.is_ok())
        .map(|event| event.unwrap());

    let pairs_iterator =
        request_response_matcher::RequestResponsePairIterator::new(&mut events_iterator)
            .filter(|pair| filter::matches_filter(pair, &args.conditions));

    let timing_result = timing_analyzer::analyze_iterator(pairs_iterator);
    // let error_result = error_analyzer::analyze(&pairs);

    result::RequestLogAnalyzerResult {
        count: timing_result.as_ref().unwrap().count,
        timing: timing_result,
        error: None,
    }
}

fn parse_event(line: Result<String, std::io::Error>) -> Result<LogEvent, &'static str> {
    match line {
        Ok(line) => log_parser::parse_line(&line),
        Err(_) => Err("Failed to read line."),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_run() {
        let args = args::RequestLogAnalyzerArgs {
            filename: String::from("src/test/simple-1.log"),
            conditions: filter::FilterConditions {
                include_terms: None,
                exclude_terms: None,
                latest_time: None,
            },
            graphite_server: None,
            graphite_port: Some(2003),
            graphite_prefix: None,
            prometheus_listen: None,
        };

        let result = run(&args);
        assert_eq!(result.count, 2);

        let timing = result.timing.unwrap();
        assert_eq!(timing.min, 7);
        assert_eq!(timing.max, 10);
    }

    #[test]
    fn test_run_ignore_broken_lines() {
        let args = args::RequestLogAnalyzerArgs {
            filename: String::from("src/test/broken.log"),
            conditions: filter::FilterConditions {
                include_terms: None,
                exclude_terms: None,
                latest_time: None,
            },
            graphite_server: None,
            graphite_port: Some(2003),
            graphite_prefix: None,
            prometheus_listen: None,
        };

        let result = run(&args);
        assert_eq!(result.count, 1);
    }

    #[test]
    fn test_parse_event() {
        let lines = vec![Ok(String::from("08/Apr/2016:09:57:47 +0200 [001] -> GET \
                                          /content/some/page.html HTTP/1.1")),
                         Ok(String::from("08/Apr/2016:09:57:47 +0200 [001] <- 200 text/html 7ms"))];

        let result: Vec<Result<LogEvent, &str>> = lines.into_iter().map(parse_event).collect();
        assert_eq!(result.len(), 2);
    }
}
<commit_msg>Unused import<commit_after>use std::io;
use std::io::prelude::*;
use std::net::TcpStream;
use std::fs::File;
use std::env;

extern crate chrono;
use chrono::*;

extern crate aggregated_stats;

#[macro_use]
extern crate clap;

extern crate env_logger;
#[macro_use]
extern crate log;

extern crate stats;

extern crate prometheus;
extern crate hyper;

mod timing_analyzer;
mod error_analyzer;
mod args;
mod filter;
mod log_parser;
use log_parser::log_events::*;
mod render;
mod request_response_matcher;
mod http_handler;
mod result;

fn main() {
    env_logger::init().expect("Failed to initialize logging.");

    let args = args::parse_args(env::args()).expect("Failed to parse arguments.");

    let result = run(&args);

    let mut stream;
    let mut renderer: Box<render::Renderer>;

    renderer = match args.graphite_server {
        Some(ref graphite_server) => {
            stream = TcpStream::connect((graphite_server.as_ref(), args.graphite_port.unwrap()))
                .expect("Could not connect to the Graphite server");

            Box::new(render::graphite::GraphiteRenderer::new(UTC::now(),
                                                             args.graphite_prefix.clone(),
                                                             &mut stream))
        }
        None => Box::new(render::TerminalRenderer::new()),
    };

    renderer.render(result);

    if args.prometheus_listen.is_some() {
        let binding_address = args.prometheus_listen.clone().unwrap();
        http_handler::listen_http(args, &binding_address);
    }

}

fn run(args: &args::RequestLogAnalyzerArgs) -> result::RequestLogAnalyzerResult {
    let input: Box<io::Read> = match args.filename.as_ref() {
        "-" => Box::new(io::stdin()),
        _ => Box::new(File::open(&args.filename).expect("Failed to open file.")),
    };

    let reader = io::BufReader::new(input);

    let mut events_iterator = reader.lines()
        .map(parse_event)
        .filter(|event| event.is_ok())
        .map(|event| event.unwrap());

    let pairs_iterator =
        request_response_matcher::RequestResponsePairIterator::new(&mut events_iterator)
            .filter(|pair| filter::matches_filter(pair, &args.conditions));

    let timing_result = timing_analyzer::analyze_iterator(pairs_iterator);
    // let error_result = error_analyzer::analyze(&pairs);

    result::RequestLogAnalyzerResult {
        count: timing_result.as_ref().unwrap().count,
        timing: timing_result,
        error: None,
    }
}

fn parse_event(line: Result<String, std::io::Error>) -> Result<LogEvent, &'static str> {
    match line {
        Ok(line) => log_parser::parse_line(&line),
        Err(_) => Err("Failed to read line."),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_run() {
        let args = args::RequestLogAnalyzerArgs {
            filename: String::from("src/test/simple-1.log"),
            conditions: filter::FilterConditions {
                include_terms: None,
                exclude_terms: None,
                latest_time: None,
            },
            graphite_server: None,
            graphite_port: Some(2003),
            graphite_prefix: None,
            prometheus_listen: None,
        };

        let result = run(&args);
        assert_eq!(result.count, 2);

        let timing = result.timing.unwrap();
        assert_eq!(timing.min, 7);
        assert_eq!(timing.max, 10);
    }

    #[test]
    fn test_run_ignore_broken_lines() {
        let args = args::RequestLogAnalyzerArgs {
            filename: String::from("src/test/broken.log"),
            conditions: filter::FilterConditions {
                include_terms: None,
                exclude_terms: None,
                latest_time: None,
            },
            graphite_server: None,
            graphite_port: Some(2003),
            graphite_prefix: None,
            prometheus_listen: None,
        };

        let result = run(&args);
        assert_eq!(result.count, 1);
    }

    #[test]
    fn test_parse_event() {
        let lines = vec![Ok(String::from("08/Apr/2016:09:57:47 +0200 [001] -> GET \
                                          /content/some/page.html HTTP/1.1")),
                         Ok(String::from("08/Apr/2016:09:57:47 +0200 [001] <- 200 text/html 7ms"))];

        let result: Vec<Result<LogEvent, &str>> = lines.into_iter().map(parse_event).collect();
        assert_eq!(result.len(), 2);
    }
}
<|endoftext|>


<commit_before>//! Main module of way-cooler

#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate bitflags;

#[cfg(not(test))]
extern crate rustwlc;

extern crate getopts;

#[cfg(test)]
extern crate dummy_rustwlc as rustwlc;

#[macro_use]
extern crate log;
extern crate env_logger;

#[macro_use]
extern crate hlua;
extern crate rustc_serialize;
#[macro_use]
extern crate json_macro;

extern crate nix;

extern crate petgraph;

extern crate uuid;

#[macro_use]
extern crate wayland_client;

extern crate tempfile;

extern crate byteorder;

use std::env;
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::Path;
use getopts::Options;
use log::LogLevel;

use nix::sys::signal::{SigHandler, SigSet, SigAction, SaFlags};
use nix::sys::signal;

use rustwlc::types::LogType;

#[macro_use] // As it happens, it's important to declare the macros first.
mod macros;
mod convert;

mod callbacks;
mod keys;

mod lua;
mod registry;
mod commands;
mod ipc;

mod layout;
mod background;

const VERSION: &'static str = env!("CARGO_PKG_VERSION");
const MODULE_PATH: &'static str = "/proc/modules";

/// Callback to route wlc logs into env_logger
fn log_handler(level: LogType, message: &str) {
    match level {
        LogType::Info => info!("wlc: {}", message),
        LogType::Warn => warn!("wlc: {}", message),
        LogType::Error => error!("wlc: {}", message),
        LogType::Wayland => info!("wayland: {}", message)
    }
}

/// Formats the log strings properly
fn log_format(record: &log::LogRecord) -> String {
    let color = match record.level() {
        LogLevel::Info => "",
        LogLevel::Trace => "\x1B[37m",
        LogLevel::Debug => "\x1B[37m",
        LogLevel::Warn =>  "\x1B[33m",
        LogLevel::Error => "\x1B[31m",
    };
    let location = record.location();
    let file = location.file();
    let line = location.line();
    let mut module_path = location.module_path();
    if let Some(index) = module_path.find("way_cooler::") {
        let index = index + "way_cooler::".len();
        module_path = &module_path[index..];
    }
    format!("{} {} [{}] \x1B[37m{}:{}\x1B[0m{0} {} \x1B[0m",
            color, record.level(), module_path, file, line, record.args())
}

/// Checks the loaded modules, and reports any problematic proprietary ones
fn detect_proprietary() {
    if env::var("DISPLAY").is_ok() {
        // If DISPLAY is present, we are running embedded
        return
    }
    match File::open(Path::new(MODULE_PATH)) {
        Ok(file) => {
            let reader = BufReader::new(&file);
            for line in reader.lines() {
                if let Ok(line) = line {
                    if line.contains("nvidia") {
                        error!("Warning: Proprietary nvidia drivers do NOT support Wayland. Use nouveau.");
                        break;
                    } else if line.contains("fglrx") {
                        error!("Warning: Proprietary AMD drivers do NOT support Wayland. Use radeon.");
                        break;
                    }
                }
            }
        },
        Err(err) => {
            warn!("Could not read proprietary modules at \"{}\", because: {:#?}",
                  MODULE_PATH, err);
        }
    }
}

/// Initializes the logging system.
/// Can be called from within test methods.
pub fn init_logs() {
    // Prepare log builder
    let mut builder = env_logger::LogBuilder::new();
    builder.format(log_format);
    builder.filter(None, log::LogLevelFilter::Trace);
    if env::var("WAY_COOLER_LOG").is_ok() {
        builder.parse(&env::var("WAY_COOLER_LOG").expect("Asserted unwrap!"));
    }
    builder.init().expect("Unable to initialize logging!");
    info!("Logger initialized, setting wlc handlers.");
}

/// Handler for signals, should close the ipc
extern "C" fn sig_handle(_: nix::libc::c_int) {
    rustwlc::terminate();
}

fn main() {
    let args: Vec<String> = env::args().collect();

    let mut opts = Options::new();
    opts.optflag("", "version", "show version information");
    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => {
            println!("{}", f.to_string());
            std::process::exit(1);
        }
    };
    if matches.opt_present("version") {
        println!("Way Cooler {}", VERSION);
        println!("Way Cooler IPC version {}", ipc::VERSION);
        return
    }
    println!("Launching way-cooler...");

    let sig_action = SigAction::new(SigHandler::Handler(sig_handle), SaFlags::empty(), SigSet::empty());
    unsafe {signal::sigaction(signal::SIGINT, &sig_action).unwrap() };

    // Start logging first
    init_logs();
    // Detect proprietary modules loaded, and warn about them.
    detect_proprietary();

    // Initialize callbacks
    callbacks::init();

    // Handle wlc logs
    rustwlc::log_set_rust_handler(log_handler);

    // Prepare to launch wlc
    let run_wlc = rustwlc::init2().expect("Unable to initialize wlc!");

    // (Future config initialization goes here)
    // Initialize commands
    commands::init();
    // Add API to registry
    registry::init();
    // Register Alt+Esc keybinding
    keys::init();
    // Start listening for clients
    let _ipc = ipc::init();

    // Hand control over to wlc's event loop
    info!("Running wlc...");
    run_wlc();
}
<commit_msg>Changed warning text to be more friendly<commit_after>//! Main module of way-cooler

#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate bitflags;

#[cfg(not(test))]
extern crate rustwlc;

extern crate getopts;

#[cfg(test)]
extern crate dummy_rustwlc as rustwlc;

#[macro_use]
extern crate log;
extern crate env_logger;

#[macro_use]
extern crate hlua;
extern crate rustc_serialize;
#[macro_use]
extern crate json_macro;

extern crate nix;

extern crate petgraph;

extern crate uuid;

#[macro_use]
extern crate wayland_client;

extern crate tempfile;

extern crate byteorder;

use std::env;
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::Path;
use getopts::Options;
use log::LogLevel;

use nix::sys::signal::{SigHandler, SigSet, SigAction, SaFlags};
use nix::sys::signal;

use rustwlc::types::LogType;

#[macro_use] // As it happens, it's important to declare the macros first.
mod macros;
mod convert;

mod callbacks;
mod keys;

mod lua;
mod registry;
mod commands;
mod ipc;

mod layout;
mod background;

const VERSION: &'static str = env!("CARGO_PKG_VERSION");
const MODULE_PATH: &'static str = "/proc/modules";

/// Callback to route wlc logs into env_logger
fn log_handler(level: LogType, message: &str) {
    match level {
        LogType::Info => info!("wlc: {}", message),
        LogType::Warn => warn!("wlc: {}", message),
        LogType::Error => error!("wlc: {}", message),
        LogType::Wayland => info!("wayland: {}", message)
    }
}

/// Formats the log strings properly
fn log_format(record: &log::LogRecord) -> String {
    let color = match record.level() {
        LogLevel::Info => "",
        LogLevel::Trace => "\x1B[37m",
        LogLevel::Debug => "\x1B[37m",
        LogLevel::Warn =>  "\x1B[33m",
        LogLevel::Error => "\x1B[31m",
    };
    let location = record.location();
    let file = location.file();
    let line = location.line();
    let mut module_path = location.module_path();
    if let Some(index) = module_path.find("way_cooler::") {
        let index = index + "way_cooler::".len();
        module_path = &module_path[index..];
    }
    format!("{} {} [{}] \x1B[37m{}:{}\x1B[0m{0} {} \x1B[0m",
            color, record.level(), module_path, file, line, record.args())
}

/// Checks the loaded modules, and reports any problematic proprietary ones
fn detect_proprietary() {
    if env::var("DISPLAY").is_ok() {
        // If DISPLAY is present, we are running embedded
        return
    }
    match File::open(Path::new(MODULE_PATH)) {
        Ok(file) => {
            let reader = BufReader::new(&file);
            for line in reader.lines() {
                if let Ok(line) = line {
                    if line.contains("nvidia") {
                        error!("Warning: Proprietary nvidia graphics drivers are installed, \
                               but they are not compatible with Wayland. Consider using nouveau drivers for Wayland.");
                        break;
                    } else if line.contains("fglrx") {
                        error!("Warning: Proprietary AMD graphics drivers are installed, but they are not compatible with Wayland. \
                               Consider using radeon drivers for Waylad.");
                        break;
                    }
                }
            }
        },
        Err(err) => {
            warn!("Could not read proprietary modules at \"{}\", because: {:#?}",
                  MODULE_PATH, err);
        }
    }
}

/// Initializes the logging system.
/// Can be called from within test methods.
pub fn init_logs() {
    // Prepare log builder
    let mut builder = env_logger::LogBuilder::new();
    builder.format(log_format);
    builder.filter(None, log::LogLevelFilter::Trace);
    if env::var("WAY_COOLER_LOG").is_ok() {
        builder.parse(&env::var("WAY_COOLER_LOG").expect("Asserted unwrap!"));
    }
    builder.init().expect("Unable to initialize logging!");
    info!("Logger initialized, setting wlc handlers.");
}

/// Handler for signals, should close the ipc
extern "C" fn sig_handle(_: nix::libc::c_int) {
    rustwlc::terminate();
}

fn main() {
    let args: Vec<String> = env::args().collect();

    let mut opts = Options::new();
    opts.optflag("", "version", "show version information");
    let matches = match opts.parse(&args[1..]) {
        Ok(m) => m,
        Err(f) => {
            println!("{}", f.to_string());
            std::process::exit(1);
        }
    };
    if matches.opt_present("version") {
        println!("Way Cooler {}", VERSION);
        println!("Way Cooler IPC version {}", ipc::VERSION);
        return
    }
    println!("Launching way-cooler...");

    let sig_action = SigAction::new(SigHandler::Handler(sig_handle), SaFlags::empty(), SigSet::empty());
    unsafe {signal::sigaction(signal::SIGINT, &sig_action).unwrap() };

    // Start logging first
    init_logs();
    // Detect proprietary modules loaded, and warn about them.
    detect_proprietary();

    // Initialize callbacks
    callbacks::init();

    // Handle wlc logs
    rustwlc::log_set_rust_handler(log_handler);

    // Prepare to launch wlc
    let run_wlc = rustwlc::init2().expect("Unable to initialize wlc!");

    // (Future config initialization goes here)
    // Initialize commands
    commands::init();
    // Add API to registry
    registry::init();
    // Register Alt+Esc keybinding
    keys::init();
    // Start listening for clients
    let _ipc = ipc::init();

    // Hand control over to wlc's event loop
    info!("Running wlc...");
    run_wlc();
}
<|endoftext|>


<commit_before>extern crate serialize;
extern crate rustbox;
extern crate docopt;
extern crate rdit;

use docopt::Docopt;

#[cfg(not(test))] use rdit::Editor;

static USAGE: &'static str = "
Usage: rdit [<filename>]
       rdit --help

Options:
    -h, --help     Show this message.
";

#[deriving(Decodable, Show)]
struct Args {
    arg_filename: Option<String>,
    flag_help: bool,
}

#[cfg(not(test))]
fn main() {
    let args: Args = Docopt::new(USAGE)
                            .and_then(|d| d.decode())
                            .unwrap_or_else(|e| e.exit());

    let mut editor = Editor::new(args.arg_filename);
    rustbox::init();
    editor.start();
    rustbox::shutdown();
}
<commit_msg>Fix editor not starting on osx<commit_after>extern crate serialize;
extern crate rustbox;
extern crate docopt;
extern crate rdit;

use docopt::Docopt;

#[cfg(not(test))] use rdit::Editor;

static USAGE: &'static str = "
Usage: rdit [<filename>]
       rdit --help

Options:
    -h, --help     Show this message.
";

#[deriving(Decodable, Show)]
struct Args {
    arg_filename: Option<String>,
    flag_help: bool,
}

#[cfg(not(test))]
fn main() {
    let args: Args = Docopt::new(USAGE)
                            .and_then(|d| d.decode())
                            .unwrap_or_else(|e| e.exit());

    rustbox::init();
    let mut editor = Editor::new(args.arg_filename);
    editor.start();
    rustbox::shutdown();
}
<|endoftext|>


<commit_before>#![feature(exit_status)]

extern crate regex;
extern crate getopts;
extern crate lines;

use std::env;
use std::fs::File;
use std::io::{BufReader, Write, stderr, stdout};
use std::io::Result as IoResult;

use getopts::Options;

use regex::Regex;
use lines::linemapper;

fn main() {
    let (cfg, files) = match Config::from_cmdline(env::args().collect()) {
        Ok(cfg) => cfg,
        Err(e) => {
            let _ = stderr().write_all(e.as_ref());
            env::set_exit_status(1);
            return;
        }
    };
    for file in files {
        if let Err(e) = process_arg(&cfg, &file) {
            let _ = write!(&mut stderr(), "{}", e);
            env::set_exit_status(1);
            return;
        }
    }
}

// --------------------------------------------------------------------

struct Config {
    print_line_numbers: bool,
    print_filenames: bool,
    line_regex: Regex,
}

impl Config {
    fn from_cmdline(args: Vec<String>) -> Result<(Config, Vec<String>), String> {
        let mut opts = Options::new();
        opts.optflag("h", "help", "print this help screen");
        opts.optflag("n", "line-numbers", "print line numbers");
        opts.optflag("f", "filename", "print file name");
        opts.optopt("e", "entry", "identify regular lines", "regex");

        let matches = match opts.parse(&args[1..]) {
            Ok(m) => m,
            Err(f) => { return Err(format!("{}", f)); }
        };
        if matches.opt_present("h") || matches.free.len() < 1 {
            return Err(opts.usage("usage: [OPTIONS] <logfile> [<logfile> ...]"));
        }
        let re = {
            let s = matches.opt_str("e").unwrap_or_else(|| {
                r"^\d{4}-\d{2}-\d{2}\s+".to_owned()
            });
            match Regex::new(&s) {
                Ok(r) => r,
                Err(e) => { return Err(format!("{}", e)); }
            }
        };
        Ok((Config {
            print_line_numbers: matches.opt_present("n"),
            print_filenames: matches.opt_present("f"),
            line_regex: re
           },
           matches.free))
    }
}

// --------------------------------------------------------------------

const NEWLINE: &'static [u8] = &[b'\n'];
const SPACE: &'static [u8] = &[b' '];

// ~ XXX use linereader api and return the io error instead of
// handling it here in this function
fn process_arg(cfg: &Config, filename: &str) -> IoResult<()> {
    let f = try!(File::open(filename));
    let r = BufReader::new(f);

    let mut block = Vec::with_capacity(1024);
    let mut stdout = stdout();

    let mut line_no = 0usize;
    let mut collecting_block = false;
    let mut output_produced = false;
    linemapper::map_lines(r, |line| {
        line_no += 1;
        if cfg.line_regex.is_match(&String::from_utf8_lossy(line)) {
            if collecting_block {
                // ~ abort as soon as possible if the write didn't
                // succeed e.g. broken pipe
                if let Err(e) = stdout.write_all(&block) {
                    let _ = write!(&mut stderr(), "{}", e);
                    env::set_exit_status(1);
                    return false;
                }
                output_produced = true;
                collecting_block = false;
            }
            block.clear();

            if output_produced {
                let _ = block.write_all(NEWLINE);
            }
        } else {
            collecting_block = true;
        }

        // XXX prevent the buffer from becoming too large by flushing
        // it out as of a certain size

        if cfg.print_filenames {
            let _ = block.write_all(filename.as_ref());
            let _ = block.write_all(SPACE);
        }
        if cfg.print_line_numbers {
            let _ = write!(&mut block, "{:7} ", line_no);
        }
        let _ = block.write(line);
        true
    })
}
<commit_msg>Return early when reading file fails<commit_after>#![feature(exit_status)]

extern crate regex;
extern crate getopts;

#[macro_use(try_read_lines)]
extern crate lines;

use std::env;
use std::fs::File;
use std::io::{Write, stderr, stdout};
use std::io::Result as IoResult;

use getopts::Options;

use regex::Regex;
use lines::linereader::LineReader;

fn main() {
    let (cfg, files) = match Config::from_cmdline(env::args().collect()) {
        Ok(cfg) => cfg,
        Err(e) => {
            let _ = stderr().write_all(e.as_ref());
            env::set_exit_status(1);
            return;
        }
    };
    for file in files {
        if let Err(e) = process_arg(&cfg, &file) {
            let _ = write!(&mut stderr(), "{}", e);
            env::set_exit_status(1);
            return;
        }
    }
}

// --------------------------------------------------------------------

struct Config {
    print_line_numbers: bool,
    print_filenames: bool,
    line_regex: Regex,
}

impl Config {
    fn from_cmdline(args: Vec<String>) -> Result<(Config, Vec<String>), String> {
        let mut opts = Options::new();
        opts.optflag("h", "help", "print this help screen");
        opts.optflag("n", "line-numbers", "print line numbers");
        opts.optflag("f", "filename", "print file name");
        opts.optopt("e", "entry", "identify regular lines", "regex");

        let matches = match opts.parse(&args[1..]) {
            Ok(m) => m,
            Err(f) => { return Err(format!("{}", f)); }
        };
        if matches.opt_present("h") || matches.free.len() < 1 {
            return Err(opts.usage("usage: [OPTIONS] <logfile> [<logfile> ...]"));
        }
        let re = {
            let s = matches.opt_str("e").unwrap_or_else(|| {
                r"^\d{4}-\d{2}-\d{2}\s+".to_owned()
            });
            match Regex::new(&s) {
                Ok(r) => r,
                Err(e) => { return Err(format!("{}", e)); }
            }
        };
        Ok((Config {
            print_line_numbers: matches.opt_present("n"),
            print_filenames: matches.opt_present("f"),
            line_regex: re
           },
           matches.free))
    }
}

// --------------------------------------------------------------------

const NEWLINE: &'static [u8] = &[b'\n'];
const SPACE: &'static [u8] = &[b' '];

fn process_arg(cfg: &Config, filename: &str) -> IoResult<()> {
    let f = try!(File::open(filename));

    let mut block = Vec::with_capacity(1024);
    let stdout = stdout();
    let mut stdout = stdout.lock();

    let mut line_no = 0usize;
    let mut collecting_block = false;
    let mut output_produced = false;

    // ~ Collect blocks of lines beloning together and spill them out.
    // ~ A block is defined by two or multiple successive lines of
    // which on the first matches the configured regular expression.
    try_read_lines!(line in LineReader::new(f), {
        line_no += 1;
        if cfg.line_regex.is_match(&String::from_utf8_lossy(line)) {
            if collecting_block {
                // ~ abort as soon as possible if the write didn't
                // succeed e.g. broken pipe
                try!(stdout.write_all(&block));
                output_produced = true;
                collecting_block = false;
            }

            // block.clear();
            unsafe { block.set_len(0); }

            if output_produced {
                let _ = block.write_all(NEWLINE);
            }
        } else {
            collecting_block = true;
        }

        // XXX prevent the buffer from becoming too large by flushing
        // it out as of a certain size

        if cfg.print_filenames {
            let _ = block.write_all(filename.as_ref());
            let _ = block.write_all(SPACE);
        }
        if cfg.print_line_numbers {
            let _ = write!(&mut block, "{:7} ", line_no);
        }
        let _ = block.write_all(line);
    });
    Ok(())
}
<|endoftext|>


<commit_before>extern crate regex;
extern crate irc;
extern crate xdg_basedir;
extern crate hyper;
extern crate url;

mod github;

use regex::Regex;
use irc::client::prelude::*;
use irc::client::server::NetIrcServer;
use xdg_basedir::*;
use std::io::Read;
use url::Url;

fn parse_post(post: &str) -> Option<String> {
    let client = hyper::client::Client::new();
    let urlregex = Regex::new(r"https?://\S+\.[:alpha:]{2,}\S+").unwrap();

    if let Some(x) = urlregex.captures(post) {
        if let Ok(parsedurl) = Url::parse(x.at(0).unwrap()) {
            if parsedurl.domain().unwrap().ends_with("github.com") {
                let urlpath = parsedurl.path().unwrap();
                if urlpath.len() == 4 &&
                    (urlpath[2] == "issues" || urlpath[2] == "pull") {
                        return github::get_display_text(&urlpath[0],
                                                        &urlpath[1],
                                                        &urlpath[3],
                                                        false).ok();
                    }
            }
        }
        if let Ok(mut resp) = client.get(x.at(0).unwrap()).send() {
            let mut body = String::new();
            resp.read_to_string(&mut body).unwrap();
            let titleregex = Regex::new(r"<title>(.+)</title>").unwrap();
            if let Some(cap) = titleregex.captures(&body) {
                return Some(format!("Title: {}", cap.at(1).unwrap()));
            }
        }
    }

    let issueregex = Regex::new(r"([^ ()]+)/([^ ()]+)#(\d+)").unwrap();

    if let Some(cap) = issueregex.captures(post) {
        let user = cap.at(1).unwrap();
        let repo = cap.at(2).unwrap();
        let number = cap.at(3).unwrap();
        return github::get_display_text(user, repo, number, true).ok();
    }
    None
}

fn handle_message(server: &NetIrcServer, from: &str, to: &str, message: &str) {
    let nickname = server.config().nickname.as_ref().unwrap();

    if to == nickname && from == "ids1024" {
        let mut words = message.split_whitespace();
        let command = words.next().unwrap_or("");
        let parameter = words.next().unwrap_or("");
        match command {
            "join" => server.send_join(parameter).unwrap(),
            "part" => server.send(
                Command::PART(parameter.to_string(), None)
                ).unwrap(),
            "quit" => server.send_quit("").unwrap(),
            _ => {},
        }
    }

    if let Some(x) = parse_post(&message) {
        for line in x.lines() {
            server.send_privmsg(&to, &line).unwrap();
        }
    }
}

fn main() {
    let mut configpath = get_config_home().unwrap();
    configpath.push("idsbot/config.json");

    let config = Config::load(configpath).unwrap();
    let server = IrcServer::from_config(config).unwrap();
    server.identify().unwrap();

    for message in server.iter() {
        let message = message.unwrap();
        print!("{}", message.into_string());
        if message.command == "PRIVMSG" {
            let from = message.get_source_nickname().unwrap().to_owned();
            let to = &message.args[0];
            let content = message.suffix.unwrap();
            handle_message(&server, &from, to, &content);
        }
    }
}
<commit_msg>Send notice instead of privmsg<commit_after>extern crate regex;
extern crate irc;
extern crate xdg_basedir;
extern crate hyper;
extern crate url;

mod github;

use regex::Regex;
use irc::client::prelude::*;
use irc::client::server::NetIrcServer;
use xdg_basedir::*;
use std::io::Read;
use url::Url;

fn parse_post(post: &str) -> Option<String> {
    let client = hyper::client::Client::new();
    let urlregex = Regex::new(r"https?://\S+\.[:alpha:]{2,}\S+").unwrap();

    if let Some(x) = urlregex.captures(post) {
        if let Ok(parsedurl) = Url::parse(x.at(0).unwrap()) {
            if parsedurl.domain().unwrap().ends_with("github.com") {
                let urlpath = parsedurl.path().unwrap();
                if urlpath.len() == 4 &&
                    (urlpath[2] == "issues" || urlpath[2] == "pull") {
                        return github::get_display_text(&urlpath[0],
                                                        &urlpath[1],
                                                        &urlpath[3],
                                                        false).ok();
                    }
            }
        }
        if let Ok(mut resp) = client.get(x.at(0).unwrap()).send() {
            let mut body = String::new();
            resp.read_to_string(&mut body).unwrap();
            let titleregex = Regex::new(r"<title>(.+)</title>").unwrap();
            if let Some(cap) = titleregex.captures(&body) {
                return Some(format!("Title: {}", cap.at(1).unwrap()));
            }
        }
    }

    let issueregex = Regex::new(r"([^ ()]+)/([^ ()]+)#(\d+)").unwrap();

    if let Some(cap) = issueregex.captures(post) {
        let user = cap.at(1).unwrap();
        let repo = cap.at(2).unwrap();
        let number = cap.at(3).unwrap();
        return github::get_display_text(user, repo, number, true).ok();
    }
    None
}

fn handle_message(server: &NetIrcServer, from: &str, to: &str, message: &str) {
    let nickname = server.config().nickname.as_ref().unwrap();

    if to == nickname && from == "ids1024" {
        let mut words = message.split_whitespace();
        let command = words.next().unwrap_or("");
        let parameter = words.next().unwrap_or("");
        match command {
            "join" => server.send_join(parameter).unwrap(),
            "part" => server.send(
                Command::PART(parameter.to_string(), None)
                ).unwrap(),
            "quit" => server.send_quit("").unwrap(),
            _ => {},
        }
    }

    if let Some(x) = parse_post(&message) {
        for line in x.lines() {
            server.send_notice(&to, &line).unwrap();
        }
    }
}

fn main() {
    let mut configpath = get_config_home().unwrap();
    configpath.push("idsbot/config.json");

    let config = Config::load(configpath).unwrap();
    let server = IrcServer::from_config(config).unwrap();
    server.identify().unwrap();

    for message in server.iter() {
        let message = message.unwrap();
        print!("{}", message.into_string());
        if message.command == "PRIVMSG" {
            let from = message.get_source_nickname().unwrap().to_owned();
            let to = &message.args[0];
            let content = message.suffix.unwrap();
            handle_message(&server, &from, to, &content);
        }
    }
}
<|endoftext|>