<commit_before>extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

extern crate graph_map;
extern crate graph_layout;
extern crate radix_sort;

use timely::progress::timestamp::RootTimestamp;
use timely::progress::nested::Summary::Local;
use timely::dataflow::*;
use timely::dataflow::operators::*;
use timely::dataflow::channels::pact::{Pipeline, Exchange};
use timely::drain::DrainExt;

use graph_layout::layout::*;
use graph_layout::compression::*;

use graph_map::graph_map::GraphMMap;

use radix_sort::*;

fn main () {

    /*
        The plan here is to partition the rows of the matrix into #processes parts, and the columns
        into #threads parts. This could be generally #row_parts and #col_parts, but that requires
        more thinking about things.
    */

    let threads: u32 = ::std::env::args().nth(1).unwrap().parse().unwrap();
    let filename = ::std::env::args().nth(2).unwrap();
    let nodes: u32 = ::std::env::args().nth(3).unwrap().parse().unwrap();

    timely::execute_from_args(::std::env::args().skip(4), move |root| {

        let index = root.index() as u32;
        let peers = root.peers() as u32;
        let processes = (peers / threads) as u32;
        let dst_index = index % threads;

        let start = time::precise_time_s();
        let mut going = start;

        let mut counter = 0;
        let mut sorter = RadixSorter::new();
        let hilbert = Hilbert::new();
        let mut decoder = BytewiseCached::new();
        let mut compressed = vec![];

        // let nodes: u32 = 3_563_602_788 + 1;

        let local_srcs = (nodes / processes) + 1;
        let local_dsts = (nodes / threads) + 1;

        // degree is ignored at the moment.
        // mostly just a pain to put together.
        let deg = vec![1.0f32; local_srcs as usize];
        let mut src = vec![1.0f32; local_srcs as usize];
        let mut dst = vec![0.0f32; local_dsts as usize];

        let mut input = root.scoped(|builder| {

            let (input, edges) = builder.new_input::<(u32, u32)>();
            let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

            let ranks = edges.binary_notify(&ranks,
                                Exchange::new(move |x: &(u32,u32)| ((x.0 % processes) * threads + (x.1 % threads)) as u64),
                                Pipeline,   // data are partitioned as part of the broadcast step.
                                "hilbrank",
                                vec![RootTimestamp::new(0)],
                                move |input1, input2, output, notificator| {

                // receive incoming edges (should only be iter 0)
                while let Some((_index, data)) = input1.next() {
                    counter += data.len();
                    for &(src,dst) in data.iter() {
                        let shrunk = (src / processes, dst / threads);
                        sorter.push(hilbert.entangle(shrunk), &|&x| x);
                    }

                    // should be 1GB per worker
                    if counter > (1 << 27) {
                        let mut results = sorter.finish(&|&x| x);
                        let mut compressor = Compressor::with_capacity(counter);
                        for vec in results.iter_mut() {
                            for elem in vec.drain_temp() {
                                compressor.push(elem);
                            }
                        }
                        sorter.recycle(results);
                        compressed.push(compressor.done());
                        counter = 0;
                    }
                }

                if counter > 0 {
                    // round out any remaining edges
                    let mut results = sorter.finish(&|&x| x);
                    let mut compressor = Compressor::with_capacity(counter);
                    for vec in results.iter_mut() {
                        for elem in vec.drain_temp() {
                            compressor.push(elem);
                        }
                    }
                    compressed.push(compressor.done());
                    counter = 0;
                }

                // all inputs received for iter, commence multiplication
                while let Some((iter, _)) = notificator.next() {

                    let now = time::now();

                    if index == 0 { println!("{}:{}:{}.{} starting iteration {}", now.tm_hour, now.tm_min, now.tm_sec, now.tm_nsec, iter.inner); }

                    // if the very first iteration, prepare some stuff.
                    if iter.inner == 0 {
                        // now we would merge the compressed representations, if we want to.
                        // let's measure both doing this and not doing it. :)

                    }

                    // record some timings in order to estimate per-iteration times
                    // if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                    if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                    if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                    // prepare src for transmitting to destinations
                    for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }
                    for d in 0..dst.len() { dst[d] = 0.0; }

                    let test = time::precise_time_s();

                    // do the multiplication for each compressed sequence
                    for sequence in &compressed {
                        for element in sequence.decompress() {
                            let (s,d) = decoder.detangle(element);
                            dst[d as usize] += src[s as usize];
                        }
                    }

                    if index == 0 { println!("mult: {}", time::precise_time_s() - test);

                    for s in 0..src.len() { src[s] = 0.0; }

                    // now we need to send some stuff to other threads.
                    // TODO : This could be done as blocks, as they mostly go to the same place.
                    let mut session = output.session(&iter);
                    for (index, &value) in dst.iter().enumerate() {
                        if value != 0.0 {
                            session.give((index as u32 * threads + dst_index, value));
                        }
                    }
                }

                // receive data from threads, accumulate in src
                while let Some((iter, data)) = input2.next() {
                    notificator.notify_at(&iter);
                    for &(node, rank) in data.iter() {
                        src[(node / processes) as usize] += rank;
                    }
                }
            });

            let mut aggregates = vec![0.0f32; (nodes / peers) as usize];
            // want to do aggregation so that aggregates are local to the folks who will need them.
            // that is, (node, rank) goes to some worker on machine#: name % processes
            let aggregated = ranks.unary_notify(
                Exchange::new(move |&(n,_): &(u32,f32)| (((n % processes) * threads) + ((n / processes) % threads)) as u64),
                "aggregation",
                vec![],
                move |input, output, notificator | {

                while let Some((time, data)) = input.next() {
                    notificator.notify_at(time);
                    for &(node, rank) in data.iter() {
                        aggregates[(node / peers) as usize] += rank;
                    }
                }

                while let Some((time, _)) = notificator.next() {
                    let mut session = output.session(&time);
                    for (node, &rank) in aggregates.iter().enumerate() {
                        if rank != 0.0 {
                            session.give((node as u32 * peers, rank));
                        }
                    }
                }
            });

            // data are now aggregated, and only need to be broadcast out to the intended recipients
            // ideally, none of this actually ends up hitting the network. also, it seems like it
            // shouldn't even require looking at `node`, on account of already putting all the data
            // on the right processes. We should be able to use (index / threads) * threads, I think.
            let mut exchanged = aggregated.exchange(move |&(node, _)| ((node % processes) * threads) as u64);
            for i in 1..threads {
                exchanged = aggregated.exchange(move |&(node, _)| (((node % processes) * threads) + i) as u64)
                                      .concat(&exchanged);
            }

            exchanged.connect_loop(cycle);

            input
        });

        // introduce edges into the computation;
        // allow mmaped file to drop
        {
            let graph = GraphMMap::new(&filename);
            for node in 0..graph.nodes() {
                if node as u32 % peers == index {
                    for dst in graph.edges(node) {
                        input.send((node as u32, *dst as u32));
                    }
                }
            }
        }
        input.close();
        while root.step() { }

        if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
    });

}
<commit_msg>lolwoops<commit_after>extern crate mmap;
extern crate time;
extern crate timely;
extern crate getopts;

extern crate graph_map;
extern crate graph_layout;
extern crate radix_sort;

use timely::progress::timestamp::RootTimestamp;
use timely::progress::nested::Summary::Local;
use timely::dataflow::*;
use timely::dataflow::operators::*;
use timely::dataflow::channels::pact::{Pipeline, Exchange};
use timely::drain::DrainExt;

use graph_layout::layout::*;
use graph_layout::compression::*;

use graph_map::graph_map::GraphMMap;

use radix_sort::*;

fn main () {

    /*
        The plan here is to partition the rows of the matrix into #processes parts, and the columns
        into #threads parts. This could be generally #row_parts and #col_parts, but that requires
        more thinking about things.
    */

    let threads: u32 = ::std::env::args().nth(1).unwrap().parse().unwrap();
    let filename = ::std::env::args().nth(2).unwrap();
    let nodes: u32 = ::std::env::args().nth(3).unwrap().parse().unwrap();

    timely::execute_from_args(::std::env::args().skip(4), move |root| {

        let index = root.index() as u32;
        let peers = root.peers() as u32;
        let processes = (peers / threads) as u32;
        let dst_index = index % threads;

        let start = time::precise_time_s();
        let mut going = start;

        let mut counter = 0;
        let mut sorter = RadixSorter::new();
        let hilbert = Hilbert::new();
        let mut decoder = BytewiseCached::new();
        let mut compressed = vec![];

        // let nodes: u32 = 3_563_602_788 + 1;

        let local_srcs = (nodes / processes) + 1;
        let local_dsts = (nodes / threads) + 1;

        // degree is ignored at the moment.
        // mostly just a pain to put together.
        let deg = vec![1.0f32; local_srcs as usize];
        let mut src = vec![1.0f32; local_srcs as usize];
        let mut dst = vec![0.0f32; local_dsts as usize];

        let mut input = root.scoped(|builder| {

            let (input, edges) = builder.new_input::<(u32, u32)>();
            let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

            let ranks = edges.binary_notify(&ranks,
                                Exchange::new(move |x: &(u32,u32)| ((x.0 % processes) * threads + (x.1 % threads)) as u64),
                                Pipeline,   // data are partitioned as part of the broadcast step.
                                "hilbrank",
                                vec![RootTimestamp::new(0)],
                                move |input1, input2, output, notificator| {

                // receive incoming edges (should only be iter 0)
                while let Some((_index, data)) = input1.next() {
                    counter += data.len();
                    for &(src,dst) in data.iter() {
                        let shrunk = (src / processes, dst / threads);
                        sorter.push(hilbert.entangle(shrunk), &|&x| x);
                    }

                    // should be 1GB per worker
                    if counter > (1 << 27) {
                        let mut results = sorter.finish(&|&x| x);
                        let mut compressor = Compressor::with_capacity(counter);
                        for vec in results.iter_mut() {
                            for elem in vec.drain_temp() {
                                compressor.push(elem);
                            }
                        }
                        sorter.recycle(results);
                        compressed.push(compressor.done());
                        counter = 0;
                    }
                }

                if counter > 0 {
                    // round out any remaining edges
                    let mut results = sorter.finish(&|&x| x);
                    let mut compressor = Compressor::with_capacity(counter);
                    for vec in results.iter_mut() {
                        for elem in vec.drain_temp() {
                            compressor.push(elem);
                        }
                    }
                    compressed.push(compressor.done());
                    counter = 0;
                }

                // all inputs received for iter, commence multiplication
                while let Some((iter, _)) = notificator.next() {

                    let now = time::now();

                    if index == 0 { println!("{}:{}:{}.{} starting iteration {}", now.tm_hour, now.tm_min, now.tm_sec, now.tm_nsec, iter.inner); }

                    // if the very first iteration, prepare some stuff.
                    if iter.inner == 0 {
                        // now we would merge the compressed representations, if we want to.
                        // let's measure both doing this and not doing it. :)

                    }

                    // record some timings in order to estimate per-iteration times
                    // if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                    if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                    if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                    // prepare src for transmitting to destinations
                    for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }
                    for d in 0..dst.len() { dst[d] = 0.0; }

                    let test = time::precise_time_s();

                    // do the multiplication for each compressed sequence
                    for sequence in &compressed {
                        for element in sequence.decompress() {
                            let (s,d) = decoder.detangle(element);
                            dst[d as usize] += src[s as usize];
                        }
                    }

                    if index == 0 { println!("mult: {}", time::precise_time_s() - test); }

                    for s in 0..src.len() { src[s] = 0.0; }

                    // now we need to send some stuff to other threads.
                    // TODO : This could be done as blocks, as they mostly go to the same place.
                    let mut session = output.session(&iter);
                    for (index, &value) in dst.iter().enumerate() {
                        if value != 0.0 {
                            session.give((index as u32 * threads + dst_index, value));
                        }
                    }
                }

                // receive data from threads, accumulate in src
                while let Some((iter, data)) = input2.next() {
                    notificator.notify_at(&iter);
                    for &(node, rank) in data.iter() {
                        src[(node / processes) as usize] += rank;
                    }
                }
            });

            let mut aggregates = vec![0.0f32; (nodes / peers) as usize];
            // want to do aggregation so that aggregates are local to the folks who will need them.
            // that is, (node, rank) goes to some worker on machine#: name % processes
            let aggregated = ranks.unary_notify(
                Exchange::new(move |&(n,_): &(u32,f32)| (((n % processes) * threads) + ((n / processes) % threads)) as u64),
                "aggregation",
                vec![],
                move |input, output, notificator | {

                while let Some((time, data)) = input.next() {
                    notificator.notify_at(time);
                    for &(node, rank) in data.iter() {
                        aggregates[(node / peers) as usize] += rank;
                    }
                }

                while let Some((time, _)) = notificator.next() {
                    let mut session = output.session(&time);
                    for (node, &rank) in aggregates.iter().enumerate() {
                        if rank != 0.0 {
                            session.give((node as u32 * peers, rank));
                        }
                    }
                }
            });

            // data are now aggregated, and only need to be broadcast out to the intended recipients
            // ideally, none of this actually ends up hitting the network. also, it seems like it
            // shouldn't even require looking at `node`, on account of already putting all the data
            // on the right processes. We should be able to use (index / threads) * threads, I think.
            let mut exchanged = aggregated.exchange(move |&(node, _)| ((node % processes) * threads) as u64);
            for i in 1..threads {
                exchanged = aggregated.exchange(move |&(node, _)| (((node % processes) * threads) + i) as u64)
                                      .concat(&exchanged);
            }

            exchanged.connect_loop(cycle);

            input
        });

        // introduce edges into the computation;
        // allow mmaped file to drop
        {
            let graph = GraphMMap::new(&filename);
            for node in 0..graph.nodes() {
                if node as u32 % peers == index {
                    for dst in graph.edges(node) {
                        input.send((node as u32, *dst as u32));
                    }
                }
            }
        }
        input.close();
        while root.step() { }

        if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
    });

}
<|endoftext|>


<commit_before>#[macro_use]
extern crate imgui;
extern crate mush;
use mush::interface::Interface;
use imgui::Ui;

const CLEAR_COLOR: (f32, f32, f32, f32) = (0.006, 0.006, 0.006, 1.0);

fn main() {
    let mut ifc = Interface::init();
    let mut state = State::default();

    loop {
        let r = ifc.render(CLEAR_COLOR, |ui| { menu(ui, &mut state) } );
        if !r | state.exit{ break }
    }
}

struct State {
    exit: bool,
}
impl Default for State {
    fn default() -> Self {
        State {
            exit: false,
        }
    }
}

fn menu (ui: &Ui, state: &mut State) {
    let mut opening = false;

    ui.main_menu_bar(|| {
        ui.menu(im_str!("File"))
            .build(|| {
                ui.menu_item(im_str!("Open"))
                    .selected(&mut opening)
                    .build();

                ui.menu_item(im_str!("Exit"))
                    .selected(&mut state.exit)
                    .build();
            });
    });

    if opening { ui.open_popup(im_str!("select-file")); }
    ui.popup(im_str!("select-file"), || {
        ui.text(im_str!("Select File"));
        ui.separator();

    });
}
<commit_msg>wip: file selector<commit_after>#[macro_use]
extern crate imgui;
extern crate mush;
use mush::interface::Interface;
use imgui::{Ui,ImStr};

use std::fs;

const CLEAR_COLOR: (f32, f32, f32, f32) = (0.006, 0.006, 0.006, 1.0);

fn main() {
    let mut ifc = Interface::init();
    let mut state = State::default();

    loop {
        let r = ifc.render(CLEAR_COLOR, |ui| {
            menu(ui, &mut state);
            open_file(ui, &mut state);
        });
        if !r | state.exit{ break }
    }
}

struct State {
    exit: bool,
    open_file: bool,
    current_dir: String,
    current_dir_list: i32,
}
impl Default for State {
    fn default() -> Self {
        State {
            exit: false,
            open_file: true,
            current_dir: "./".to_owned(),
            current_dir_list: 0,
        }
    }
}

fn menu (ui: &Ui, state: &mut State) {
    ui.main_menu_bar(|| {
        ui.menu(im_str!("File"))
            .build(|| {
                ui.menu_item(im_str!("Open"))
                    .selected(&mut state.open_file)
                    .build();

                ui.menu_item(im_str!("Exit"))
                    .selected(&mut state.exit)
                    .build();
            });
    });
}

fn open_file (ui: &Ui, state: &mut State) {
    if !state.open_file { return }

    let mut cd = fs::read_dir(&state.current_dir);

    ui.window(im_str!("Select source"))
        .always_auto_resize(true)
        .movable(true)
        .show_borders(true)
        //.opened(&mut state.open_file) // FIXME:borrow check issues with closures
        .build(||{
            ui.text(im_str!("Select a source file to be parsed and evaluated.\nSource files must be parsable through the lichen crate."));
            ui.separator();
            ui.input_text(im_str!("Current Directory"), &mut state.current_dir)
                .build();
            ui.separator();

            if let Ok(ref mut paths) = cd {
                let paths: Vec<ImStr> = paths
                    .filter(|p| p.is_ok())
                    .map(|p| {
                        im_str!("{:}", p.unwrap().path().display())
                    })
                    .collect();

                ui.list_box(im_str!("Files"),
                            &mut state.current_dir_list,
                            &paths[..],
                            paths.len() as i32);

            }
        })
}
<|endoftext|>


<commit_before>#![feature(conservative_impl_trait)]

extern crate curl;
extern crate env_logger;
extern crate futures;
extern crate tokio_core;
extern crate tokio_curl;

#[macro_use]
extern crate clap;
extern crate chrono;
extern crate histogram;

#[macro_use]
#[cfg(test)]
extern crate quickcheck;

mod args;
mod misc;

use args::{Config, parse_args};

use curl::easy::Easy;
use futures::{Future, stream};
use futures::future::{ok, loop_fn, Loop};
use futures::Stream;
use tokio_core::reactor::Core;
use tokio_curl::{Session, PerformError};
use chrono::*;
use std::sync::mpsc::channel;
use histogram::*;
use std::thread;

struct RequestResult {
    latency: Duration,
}

impl RequestResult {
    fn new(latency: Duration) -> Self {
        RequestResult { latency: latency }
    }
}

impl Default for RequestResult {
    fn default() -> Self {
        RequestResult { latency: Duration::zero() }
    }
}

fn send_request(easy_request: Easy,
                session: &Session)
                -> impl Future<Item = (RequestResult, Easy), Error = PerformError> {
    session.perform(easy_request)
        .and_then(|mut easy| {
            let latency = easy.total_time()
                .ok()
                .and_then(|x| Duration::from_std(x).ok())
                .unwrap();
            let request_result = RequestResult::new(latency);
            Ok((request_result, easy))
        })
}

struct RunInfo {
    /// Number of requests successfully completed
    pub requests_completed: usize,
    pub duration: Duration,
    histogram: Histogram,
    /// Number of failed requests
    num_failed_requests: usize,
}

impl RunInfo {
    fn new(duration: Duration) -> Self {
        RunInfo {
            requests_completed: 0,
            num_failed_requests: 0,
            duration: duration,
            histogram: Histogram::new(),
        }
    }
    pub fn requests_per_second(&self) -> f32 {
        (self.requests_completed as f32) / (self.duration.num_seconds() as f32)
    }

    fn merge(&mut self, other: &Self) {
        self.requests_completed += other.requests_completed;
        self.num_failed_requests += other.num_failed_requests;
        self.histogram.merge(&other.histogram);
    }
}

/// Delegates and manages all the work.
struct Boss {
    /// Number of requests successfully completed
    pub requests_completed: usize,
    /// Number of active connections
    pub connections: usize,
    num_threads: usize,
}

fn make_easy(url: &str, timeout: Duration) -> Easy {
    let mut easy_request = Easy::new();
    easy_request.get(true).unwrap();
    easy_request.url(url).unwrap();
    easy_request.timeout(timeout.to_std().unwrap()).unwrap();
    easy_request
}

impl Boss {
    pub fn new(num_threads: usize) -> Self {
        Boss {
            requests_completed: 0,
            connections: 0,
            num_threads: num_threads,
        }
    }

    pub fn start_workforce(&self, config: &Config) -> RunInfo {
        let (tx, rx) = channel();
        let desired_connections_per_worker_iter = misc::split_number(config.num_connections,
                                                                     self.num_threads);
        // start num_threads workers
        for desired_connections_per_worker in desired_connections_per_worker_iter {
            let tx = tx.clone();
            let url = config.url.clone();
            let duration = config.duration;
            let timeout = config.timeout;
            thread::spawn(move || {

                let mut lp = Core::new().unwrap();
                let start_time = Local::now();
                let wanted_end_time = start_time + duration;
                let session = Session::new(lp.handle());

                let iterator = (0..desired_connections_per_worker).map(|_| {

                    let runinfo = RunInfo::new(duration);
                    let easy_request = make_easy(&url, timeout);
                    loop_fn((runinfo, easy_request), |(mut runinfo, easy_request)| {
                        send_request(easy_request, &session)
                            .then(|res| -> Result<_, ()> { match res {
                                // on request success
                                Ok((request_res, easy_request)) => {
                                    // update histogram
                                    request_res.latency.num_nanoseconds()
                                        .map(|x| runinfo.histogram.increment(x as u64).ok());
                                    runinfo.requests_completed += 1;

                                    let state = (runinfo, easy_request);

                                    let now_time = Local::now();
                                    if now_time < wanted_end_time {
                                        Ok(Loop::Continue(state))
                                    } else {
                                        Ok(Loop::Break(state))
                                    }
                                }
                                // on request failure
                                Err(mut err) => {
                                    runinfo.num_failed_requests += 1;
                                    // attempt to recover the easy handle from the error,
                                    // else make a new handle
                                    let easy_request = err.take_easy()
                                        .unwrap_or_else(|| make_easy(&url, timeout));
                                    let state = (runinfo, easy_request);
                                    let now_time = Local::now();
                                    if now_time < wanted_end_time {
                                        Ok(Loop::Continue(state))
                                    } else {
                                        Ok(Loop::Break(state))
                                    }
                                }
                            }})
                            .map_err(|_| ())
                    })
                        // Extract latency histogram
                        .map(|(histogram, _)| histogram)
                });
                let future = stream::futures_unordered(iterator)
                    .fold(RunInfo::new(duration), |mut runinfo_acc, runinfo| {
                        runinfo_acc.merge(&runinfo);
                        ok(runinfo_acc)
                    });
                let res = lp.run(future).unwrap();
                tx.send(res).unwrap();
            });
        }
        // collect information from all workers
        rx.iter()
            .take(self.num_threads)
            .fold(RunInfo::new(config.duration), |mut runinfo_acc, runinfo| {
                runinfo_acc.merge(&runinfo);
                runinfo_acc
            })
    }
}

fn start(config: Config) -> RunInfo {
    let boss = Boss::new(config.num_threads);
    boss.start_workforce(&config)
}

fn nanoseconds_to_milliseconds(nanoseconds: u64) -> f64 {
    nanoseconds as f64 / 1_000_000_f64
}

/// Presents the results to the user
fn present(run_info: RunInfo) {
    let requests_completed = run_info.requests_completed;
    println!("{} requests in {}s",
             requests_completed,
             run_info.duration.num_seconds());
    println!("{} failed requests", run_info.num_failed_requests);
    println!("Requests per second: {}", run_info.requests_per_second());

    // we can only print the latency distribution if there is data in the histogram
    if run_info.histogram.entries() > 0 {
        println!("Latency distribution: \n50%: {} ms\n75%: {} ms\n90%: {} ms\n95%: {} ms\n99%: {} ms",
            run_info.histogram.percentile(50.0).map(nanoseconds_to_milliseconds).unwrap(),
            run_info.histogram.percentile(75.0).map(nanoseconds_to_milliseconds).unwrap(),
            run_info.histogram.percentile(90.0).map(nanoseconds_to_milliseconds).unwrap(),
            run_info.histogram.percentile(95.0).map(nanoseconds_to_milliseconds).unwrap(),
            run_info.histogram.percentile(99.0).map(nanoseconds_to_milliseconds).unwrap(),
        );
    }

}

fn main() {
    env_logger::init().unwrap();
    let config = parse_args();
    let run_info = start(config);
    present(run_info)
}
<commit_msg>print average latency<commit_after>#![feature(conservative_impl_trait)]

extern crate curl;
extern crate env_logger;
extern crate futures;
extern crate tokio_core;
extern crate tokio_curl;

#[macro_use]
extern crate clap;
extern crate chrono;
extern crate histogram;

#[macro_use]
#[cfg(test)]
extern crate quickcheck;

mod args;
mod misc;

use args::{Config, parse_args};

use curl::easy::Easy;
use futures::{Future, stream};
use futures::future::{ok, loop_fn, Loop};
use futures::Stream;
use tokio_core::reactor::Core;
use tokio_curl::{Session, PerformError};
use chrono::*;
use std::sync::mpsc::channel;
use histogram::*;
use std::thread;

struct RequestResult {
    latency: Duration,
}

impl RequestResult {
    fn new(latency: Duration) -> Self {
        RequestResult { latency: latency }
    }
}

impl Default for RequestResult {
    fn default() -> Self {
        RequestResult { latency: Duration::zero() }
    }
}

fn send_request(easy_request: Easy,
                session: &Session)
                -> impl Future<Item = (RequestResult, Easy), Error = PerformError> {
    session.perform(easy_request)
        .and_then(|mut easy| {
            let latency = easy.total_time()
                .ok()
                .and_then(|x| Duration::from_std(x).ok())
                .unwrap();
            let request_result = RequestResult::new(latency);
            Ok((request_result, easy))
        })
}

struct RunInfo {
    /// Number of requests successfully completed
    pub requests_completed: usize,
    pub duration: Duration,
    histogram: Histogram,
    /// Number of failed requests
    num_failed_requests: usize,
}

impl RunInfo {
    fn new(duration: Duration) -> Self {
        RunInfo {
            requests_completed: 0,
            num_failed_requests: 0,
            duration: duration,
            histogram: Histogram::new(),
        }
    }
    pub fn requests_per_second(&self) -> f32 {
        (self.requests_completed as f32) / (self.duration.num_seconds() as f32)
    }

    fn merge(&mut self, other: &Self) {
        self.requests_completed += other.requests_completed;
        self.num_failed_requests += other.num_failed_requests;
        self.histogram.merge(&other.histogram);
    }
}

/// Delegates and manages all the work.
struct Boss {
    /// Number of requests successfully completed
    pub requests_completed: usize,
    /// Number of active connections
    pub connections: usize,
    num_threads: usize,
}

fn make_easy(url: &str, timeout: Duration) -> Easy {
    let mut easy_request = Easy::new();
    easy_request.get(true).unwrap();
    easy_request.url(url).unwrap();
    easy_request.timeout(timeout.to_std().unwrap()).unwrap();
    easy_request
}

impl Boss {
    pub fn new(num_threads: usize) -> Self {
        Boss {
            requests_completed: 0,
            connections: 0,
            num_threads: num_threads,
        }
    }

    pub fn start_workforce(&self, config: &Config) -> RunInfo {
        let (tx, rx) = channel();
        let desired_connections_per_worker_iter = misc::split_number(config.num_connections,
                                                                     self.num_threads);
        // start num_threads workers
        for desired_connections_per_worker in desired_connections_per_worker_iter {
            let tx = tx.clone();
            let url = config.url.clone();
            let duration = config.duration;
            let timeout = config.timeout;
            thread::spawn(move || {

                let mut lp = Core::new().unwrap();
                let start_time = Local::now();
                let wanted_end_time = start_time + duration;
                let session = Session::new(lp.handle());

                let iterator = (0..desired_connections_per_worker).map(|_| {

                    let runinfo = RunInfo::new(duration);
                    let easy_request = make_easy(&url, timeout);
                    loop_fn((runinfo, easy_request), |(mut runinfo, easy_request)| {
                        send_request(easy_request, &session)
                            .then(|res| -> Result<_, ()> { match res {
                                // on request success
                                Ok((request_res, easy_request)) => {
                                    // update histogram
                                    request_res.latency.num_nanoseconds()
                                        .map(|x| runinfo.histogram.increment(x as u64).ok());
                                    runinfo.requests_completed += 1;

                                    let state = (runinfo, easy_request);

                                    let now_time = Local::now();
                                    if now_time < wanted_end_time {
                                        Ok(Loop::Continue(state))
                                    } else {
                                        Ok(Loop::Break(state))
                                    }
                                }
                                // on request failure
                                Err(mut err) => {
                                    runinfo.num_failed_requests += 1;
                                    // attempt to recover the easy handle from the error,
                                    // else make a new handle
                                    let easy_request = err.take_easy()
                                        .unwrap_or_else(|| make_easy(&url, timeout));
                                    let state = (runinfo, easy_request);
                                    let now_time = Local::now();
                                    if now_time < wanted_end_time {
                                        Ok(Loop::Continue(state))
                                    } else {
                                        Ok(Loop::Break(state))
                                    }
                                }
                            }})
                    })
                        // Extract latency histogram
                        .map(|(histogram, _)| histogram)
                });
                let future = stream::futures_unordered(iterator)
                    .fold(RunInfo::new(duration), |mut runinfo_acc, runinfo| {
                        runinfo_acc.merge(&runinfo);
                        ok(runinfo_acc)
                    });
                let res = lp.run(future).unwrap();
                tx.send(res).unwrap();
            });
        }
        // collect information from all workers
        rx.iter()
            .take(self.num_threads)
            .fold(RunInfo::new(config.duration), |mut runinfo_acc, runinfo| {
                runinfo_acc.merge(&runinfo);
                runinfo_acc
            })
    }
}

fn start(config: Config) -> RunInfo {
    let boss = Boss::new(config.num_threads);
    boss.start_workforce(&config)
}

fn nanoseconds_to_milliseconds(nanoseconds: u64) -> f64 {
    nanoseconds as f64 / 1_000_000_f64
}

/// Presents the results to the user
fn present(run_info: RunInfo) {
    let requests_completed = run_info.requests_completed;
    println!("{} requests in {}s",
             requests_completed,
             run_info.duration.num_seconds());
    println!("{} failed requests", run_info.num_failed_requests);
    println!("Requests per second: {}", run_info.requests_per_second());

    // we can only print the latency distribution if there is data in the histogram
    if run_info.histogram.entries() > 0 {
        println!("Average latency: {}ms", run_info.histogram.mean().map(nanoseconds_to_milliseconds).unwrap());
        println!("Latency distribution: \n50%: {} ms\n75%: {} ms\n90%: {} ms\n95%: {} ms\n99%: {} ms",
            run_info.histogram.percentile(50.0).map(nanoseconds_to_milliseconds).unwrap(),
            run_info.histogram.percentile(75.0).map(nanoseconds_to_milliseconds).unwrap(),
            run_info.histogram.percentile(90.0).map(nanoseconds_to_milliseconds).unwrap(),
            run_info.histogram.percentile(95.0).map(nanoseconds_to_milliseconds).unwrap(),
            run_info.histogram.percentile(99.0).map(nanoseconds_to_milliseconds).unwrap(),
        );

    }

}

fn main() {
    env_logger::init().unwrap();
    let config = parse_args();
    let run_info = start(config);
    present(run_info)
}
<|endoftext|>


<commit_before>#![feature(box_patterns)]

use std::error::Error;

extern crate csv;
use csv::{Reader, StringRecords};

/// The data types supported by this database. Currently just u64 and string but others
/// will be added later, including complex types
#[derive(Debug,Clone)]
enum DataType {
    UnsignedLong,
    String
}

/// Definition of a column in a relation (data set).
#[derive(Debug,Clone)]
struct ColumnMeta {
    name: String,
    data_type: DataType,
    nullable: bool
}

/// Definition of a relation (data set) consisting of one or more columns.
#[derive(Debug,Clone)]
struct TupleType {
    columns: Vec<ColumnMeta>
}

/// Value holder for all supported data types
#[derive(Debug,Clone,PartialEq,PartialOrd)]
enum Value {
    UnsignedLong(u64),
    String(String),
    Boolean(bool)
}

/// A tuple represents one row within a relation and is implemented as a trait to allow for
/// specific implementations for different data sources
trait Tuple {
    fn get_value(&self, index: usize) -> Result<Value, Box<std::error::Error>>;
}

/// A simple tuple implementation for testing and initial prototyping
#[derive(Debug)]
struct SimpleTuple {
    values: Vec<Value>
}

impl Tuple for SimpleTuple {

    fn get_value(&self, index: usize) -> Result<Value, Box<std::error::Error>> {
        Ok(self.values[index].clone())
    }

}

/// dumb
//trait DataSource {
//    fn scan(&self) -> Result<Box<Iterator<Item=Tuple>>, Box<Error>>;
//}
//
//struct CsvDataSource {
//}
//
//impl DataSource for CsvDataSource {
//    fn scan(&self) -> Result<Box<Iterator<Item=Tuple>>, Box<Error>> {
//        Err(From::from("not implemented yet"))
//    }
//}

#[derive(Debug)]
enum Operator {
    Eq,
    NotEq,
    Lt,
    LtEq,
    Gt,
    GtEq,
}

#[derive(Debug)]
enum Expr {
    /// index into a value within the tuple
    TupleValue(usize),
    /// literal value
    Literal(Value),
    /// binary expression e.g. "age > 21"
    BinaryExpr { left: Box<Expr>, op: Operator, right: Box<Expr> },
}

/// Query plan
#[derive(Debug)]
enum PlanNode {
    TableScan,
    Filter(Expr),
    Project(Vec<Expr>),
}

fn evaluate(tuple: &Tuple, tt: &TupleType, expr: &Expr) -> Result<Value, Box<std::error::Error>> {

    match expr {
        &Expr::BinaryExpr { box ref left, ref op, box ref right } => {
            //TODO: remove use of unwrap() here
            let left_value = evaluate(tuple, tt, left).unwrap();
            let right_value = evaluate(tuple, tt, right).unwrap();
            match op {
                &Operator::Eq => Ok(Value::Boolean(left_value == right_value)),
                &Operator::NotEq => Ok(Value::Boolean(left_value != right_value)),
                &Operator::Lt => Ok(Value::Boolean(left_value < right_value)),
                &Operator::LtEq => Ok(Value::Boolean(left_value <= right_value)),
                &Operator::Gt => Ok(Value::Boolean(left_value > right_value)),
                &Operator::GtEq => Ok(Value::Boolean(left_value >= right_value)),
            }
        },
        &Expr::TupleValue(index) => tuple.get_value(index),
        &Expr::Literal(ref value) => Ok(value.clone()),
    }

}

fn main() {

    // define schema for data source (csv file)
    let tt = TupleType {
        columns: vec![
            ColumnMeta { name: String::from("id"), data_type: DataType::UnsignedLong, nullable: false },
            ColumnMeta { name: String::from("name"), data_type: DataType::String, nullable: false }
        ]
    };

    // create simple filter expression for "id = 2"
    let filter_expr = Expr::BinaryExpr {
        left: Box::new(Expr::TupleValue(0)),
        op: Operator::Eq,
        right: Box::new(Expr::Literal(Value::UnsignedLong(2)))
    };

    // execute scan with filter expr against csv file
    let mut rdr = csv::Reader::from_file("people.csv").unwrap();
    let mut records = rdr.records();

    // iterate over data
    while let Some(row) = records.next() {
        let data : Vec<String> = row.unwrap();
        println!("Row: {:?}", data);

        // for now, do an expensive translation of strings to the specific tuple type for
        // every single column
        let mut converted : Vec<Value> = vec![];
        for i in 0..data.len() {
            converted.push(match tt.columns[i].data_type {
                    DataType::UnsignedLong => Value::UnsignedLong(data[i].parse::<u64>().unwrap()),
                    DataType::String => Value::String(data[i].clone()),
            });
        }
        let tuple = SimpleTuple { values: converted };

        let is_match = evaluate(&tuple, &tt, &filter_expr).unwrap();
        println!("filter expr evaluates to {:?}", is_match);
    }

}
<commit_msg>working on refactor to create CsvRelation struct<commit_after>#![feature(box_patterns)]

use std::error::Error;
use std::fs::File;

extern crate csv;
use csv::{Reader, StringRecords};

/// The data types supported by this database. Currently just u64 and string but others
/// will be added later, including complex types
#[derive(Debug,Clone)]
enum DataType {
    UnsignedLong,
    String
}

/// Definition of a column in a relation (data set).
#[derive(Debug,Clone)]
struct ColumnMeta {
    name: String,
    data_type: DataType,
    nullable: bool
}

/// Definition of a relation (data set) consisting of one or more columns.
#[derive(Debug,Clone)]
struct TupleType {
    columns: Vec<ColumnMeta>
}

/// Value holder for all supported data types
#[derive(Debug,Clone,PartialEq,PartialOrd)]
enum Value {
    UnsignedLong(u64),
    String(String),
    Boolean(bool)
}

/// A tuple represents one row within a relation and is implemented as a trait to allow for
/// specific implementations for different data sources
trait Tuple {
    fn get_value(&self, index: usize) -> Result<Value, Box<std::error::Error>>;
}

/// A simple tuple implementation for testing and initial prototyping
#[derive(Debug)]
struct SimpleTuple {
    values: Vec<Value>
}

impl Tuple for SimpleTuple {

    fn get_value(&self, index: usize) -> Result<Value, Box<std::error::Error>> {
        Ok(self.values[index].clone())
    }

}

#[derive(Debug)]
enum Operator {
    Eq,
    NotEq,
    Lt,
    LtEq,
    Gt,
    GtEq,
}

#[derive(Debug)]
enum Expr {
    /// index into a value within the tuple
    TupleValue(usize),
    /// literal value
    Literal(Value),
    /// binary expression e.g. "age > 21"
    BinaryExpr { left: Box<Expr>, op: Operator, right: Box<Expr> },
}

/// Query plan
#[derive(Debug)]
enum PlanNode {
    TableScan { schema: String, table: String },
    Filter { expr: Expr, input: Box<PlanNode> },
    Project { expr: Vec<Expr>, input: Box<PlanNode> }
}

fn evaluate(tuple: &Tuple, tt: &TupleType, expr: &Expr) -> Result<Value, Box<std::error::Error>> {

    match expr {
        &Expr::BinaryExpr { box ref left, ref op, box ref right } => {
            //TODO: remove use of unwrap() here
            let left_value = evaluate(tuple, tt, left).unwrap();
            let right_value = evaluate(tuple, tt, right).unwrap();
            match op {
                &Operator::Eq => Ok(Value::Boolean(left_value == right_value)),
                &Operator::NotEq => Ok(Value::Boolean(left_value != right_value)),
                &Operator::Lt => Ok(Value::Boolean(left_value < right_value)),
                &Operator::LtEq => Ok(Value::Boolean(left_value <= right_value)),
                &Operator::Gt => Ok(Value::Boolean(left_value > right_value)),
                &Operator::GtEq => Ok(Value::Boolean(left_value >= right_value)),
            }
        },
        &Expr::TupleValue(index) => tuple.get_value(index),
        &Expr::Literal(ref value) => Ok(value.clone()),
    }

}

trait Relation {
    fn next(&self) -> Result<Option<Box<Tuple>>, Box<Error>>;
}

struct CsvRelation {
    filename: String,
    tuple_type: TupleType,
    reader: csv::Reader<File>,
//    records: &'a csv::StringRecords
}

impl CsvRelation {

    fn open(filename: String, tuple_type: TupleType) -> Self {
        let mut rdr = csv::Reader::from_file(&filename).unwrap();
        CsvRelation {
            filename: filename,
            tuple_type: tuple_type,
            reader: rdr,
//            records: &rdr.records()
        }
    }

}

//impl Relation for CsvRelation {

//    fn next(&self) -> Result<Option<Box<Tuple>>, Box<Error>> {
//        match self.records.next() {
//            Ok(Some(row)) => {
//                let data: Vec<String> = row.unwrap();
//                println!("Row: {:?}", data);
//
//                // for now, do an expensive translation of strings to the specific tuple type for
//                // every single column
//                let mut converted: Vec<Value> = vec![];
//                for i in 0..data.len() {
//                    converted.push(match self.tuple_type.columns[i].data_type {
//                        DataType::UnsignedLong => Value::UnsignedLong(data[i].parse::<u64>().unwrap()),
//                        DataType::String => Value::String(data[i].clone()),
//                    });
//                }
//                Ok(Some(Box::new(SimpleTuple { values: converted })));
//            },
//            Ok(None) => Ok(None),
//            Err(e) => Err(e)
//        }
//        Ok(None)
//    }
//}

//fn execute(node: &PlanNode) {
//    match node {
//        &PlanNode::TableScan { schema, table } => {
//
//        },
//        &PlanNode::Filter { expr, input } => {
//
//        },
//        &PlanNode::Project { expr, input } => {
//
//        }
//    }
//}

fn main() {

    // define schema for data source (csv file)
    let tt = TupleType {
        columns: vec![
            ColumnMeta { name: String::from("id"), data_type: DataType::UnsignedLong, nullable: false },
            ColumnMeta { name: String::from("name"), data_type: DataType::String, nullable: false }
        ]
    };

    // create simple filter expression for "id = 2"
    let filter_expr = Expr::BinaryExpr {
        left: Box::new(Expr::TupleValue(0)),
        op: Operator::Eq,
        right: Box::new(Expr::Literal(Value::UnsignedLong(2)))
    };

    // execute scan with filter expr against csv file
    let mut rdr = csv::Reader::from_file("people.csv").unwrap();
    let mut records = rdr.records();

    // iterate over data
    while let Some(row) = records.next() {
        let data : Vec<String> = row.unwrap();
        println!("Row: {:?}", data);

        // for now, do an expensive translation of strings to the specific tuple type for
        // every single column
        let mut converted : Vec<Value> = vec![];
        for i in 0..data.len() {
            converted.push(match tt.columns[i].data_type {
                    DataType::UnsignedLong => Value::UnsignedLong(data[i].parse::<u64>().unwrap()),
                    DataType::String => Value::String(data[i].clone()),
            });
        }
        let tuple = SimpleTuple { values: converted };

        let is_match = evaluate(&tuple, &tt, &filter_expr).unwrap();
        println!("filter expr evaluates to {:?}", is_match);
    }

}
<|endoftext|>


<commit_before>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

extern crate ansi_term;
extern crate atty;
#[macro_use]
extern crate clap;
extern crate ignore;
#[macro_use]
extern crate lazy_static;
#[cfg(all(unix, not(target_os = "redox")))]
extern crate libc;
extern crate num_cpus;
extern crate regex;
extern crate regex_syntax;

pub mod fshelper;
pub mod lscolors;
mod app;
mod exec;
mod internal;
mod output;
mod walk;

#[cfg(windows)]
mod windows;

use std::env;
use std::error::Error;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time;

use atty::Stream;
use regex::{RegexBuilder, RegexSetBuilder};

use exec::CommandTemplate;
use internal::{error, pattern_has_uppercase_char, transform_args_with_exec, FdOptions};
use lscolors::LsColors;
use walk::FileType;

fn main() {
    let checked_args = transform_args_with_exec(env::args_os());
    let matches = app::build_app().get_matches_from(checked_args);

    // Get the search pattern
    let pattern = matches.value_of("pattern").unwrap_or("");

    // Get the current working directory
    let current_dir = Path::new(".");
    if !fshelper::is_dir(current_dir) {
        error("Error: could not get current directory.");
    }

    //Get one or more root directories to search.
    let mut dir_vec: Vec<_> = match matches.values_of("path") {
        Some(paths) => paths
            .map(|path| {
                let path_buffer = PathBuf::from(path);
                if !fshelper::is_dir(&path_buffer) {
                    error(&format!(
                        "Error: '{}' is not a directory.",
                        path_buffer.to_string_lossy()
                    ));
                }
                path_buffer
            })
            .collect::<Vec<_>>(),
        None => vec![current_dir.to_path_buf()],
    };

    if matches.is_present("absolute-path") {
        dir_vec = dir_vec
            .iter()
            .map(|path_buffer| fshelper::absolute_path(path_buffer).unwrap())
            .collect();
    }

    // Treat pattern as literal string if '--fixed-strings' is used
    let pattern_regex = if matches.is_present("fixed-strings") {
        regex::escape(pattern)
    } else {
        String::from(pattern)
    };

    // The search will be case-sensitive if the command line flag is set or
    // if the pattern has an uppercase character (smart case).
    let case_sensitive = !matches.is_present("ignore-case")
        && (matches.is_present("case-sensitive") || pattern_has_uppercase_char(&pattern_regex));

    let colored_output = match matches.value_of("color") {
        Some("always") => true,
        Some("never") => false,
        _ => atty::is(Stream::Stdout),
    };

    #[cfg(windows)]
    let colored_output = colored_output && windows::enable_colored_output();

    let ls_colors = if colored_output {
        Some(
            env::var("LS_COLORS")
                .ok()
                .map(|val| LsColors::from_string(&val))
                .unwrap_or_default(),
        )
    } else {
        None
    };

    let command = matches.values_of("exec").map(CommandTemplate::new);

    let config = FdOptions {
        case_sensitive,
        search_full_path: matches.is_present("full-path"),
        ignore_hidden: !(matches.is_present("hidden")
            || matches.occurrences_of("rg-alias-hidden-ignore") >= 2),
        read_ignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")),
        read_gitignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")
            || matches.is_present("no-ignore-vcs")),
        follow_links: matches.is_present("follow"),
        null_separator: matches.is_present("null_separator"),
        max_depth: matches
            .value_of("depth")
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        threads: std::cmp::max(
            matches
                .value_of("threads")
                .and_then(|n| usize::from_str_radix(n, 10).ok())
                .unwrap_or_else(num_cpus::get),
            1,
        ),
        max_buffer_time: matches
            .value_of("max-buffer-time")
            .and_then(|n| u64::from_str_radix(n, 10).ok())
            .map(time::Duration::from_millis),
        ls_colors,
        file_types: match matches.values_of("file-type") {
            None => vec![
                FileType::RegularFile,
                FileType::Directory,
                FileType::SymLink,
            ].into_iter()
                .collect(),
            Some(values) => values
                .map(|value| match value {
                    "f" | "file" => FileType::RegularFile,
                    "d" | "directory" => FileType::Directory,
                    "l" | "symlink" => FileType::SymLink,
                    _ => FileType::RegularFile,
                })
                .collect(),
        },
        extensions: matches.values_of("extension").map(|exts| {
            let patterns = exts.map(|e| e.trim_left_matches('.'))
                .map(|e| format!(r".\.{}$", regex::escape(e)));
            match RegexSetBuilder::new(patterns)
                .case_insensitive(true)
                .build()
            {
                Ok(re) => re,
                Err(err) => error(err.description()),
            }
        }),
        command,
        exclude_patterns: matches
            .values_of("exclude")
            .map(|v| v.map(|p| String::from("!") + p).collect())
            .unwrap_or_else(|| vec![]),
    };

    match RegexBuilder::new(&pattern_regex)
        .case_insensitive(!config.case_sensitive)
        .dot_matches_new_line(true)
        .build()
    {
        Ok(re) => walk::scan(&dir_vec, Arc::new(re), Arc::new(config)),
        Err(err) => error(
            format!(
                "{}\nHint: You can use the '--fixed-strings' option to search for a \
                            literal string instead of a regular expression",
                err.description()
            ).as_str(),
        ),
    }
}
<commit_msg>Changes as per the instruction of `cargo fmt -- --write-mode=diff`<commit_after>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

extern crate ansi_term;
extern crate atty;
#[macro_use]
extern crate clap;
extern crate ignore;
#[macro_use]
extern crate lazy_static;
#[cfg(all(unix, not(target_os = "redox")))]
extern crate libc;
extern crate num_cpus;
extern crate regex;
extern crate regex_syntax;

pub mod fshelper;
pub mod lscolors;
mod app;
mod exec;
mod internal;
mod output;
mod walk;

#[cfg(windows)]
mod windows;

use std::env;
use std::error::Error;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time;

use atty::Stream;
use regex::{RegexBuilder, RegexSetBuilder};

use exec::CommandTemplate;
use internal::{error, pattern_has_uppercase_char, transform_args_with_exec, FdOptions};
use lscolors::LsColors;
use walk::FileType;

fn main() {
    let checked_args = transform_args_with_exec(env::args_os());
    let matches = app::build_app().get_matches_from(checked_args);

    // Get the search pattern
    let pattern = matches.value_of("pattern").unwrap_or("");

    // Get the current working directory
    let current_dir = Path::new(".");
    if !fshelper::is_dir(current_dir) {
        error("Error: could not get current directory.");
    }

    //Get one or more root directories to search.
    let mut dir_vec: Vec<_> = match matches.values_of("path") {
        Some(paths) => paths
            .map(|path| {
                let path_buffer = PathBuf::from(path);
                if !fshelper::is_dir(&path_buffer) {
                    error(&format!(
                        "Error: '{}' is not a directory.",
                        path_buffer.to_string_lossy()
                    ));
                }
                path_buffer
            })
            .collect::<Vec<_>>(),
        None => vec![current_dir.to_path_buf()],
    };

    if matches.is_present("absolute-path") {
        dir_vec = dir_vec
            .iter()
            .map(|path_buffer| fshelper::absolute_path(path_buffer).unwrap())
            .collect();
    }

    // Treat pattern as literal string if '--fixed-strings' is used
    let pattern_regex = if matches.is_present("fixed-strings") {
        regex::escape(pattern)
    } else {
        String::from(pattern)
    };

    // The search will be case-sensitive if the command line flag is set or
    // if the pattern has an uppercase character (smart case).
    let case_sensitive = !matches.is_present("ignore-case")
        && (matches.is_present("case-sensitive") || pattern_has_uppercase_char(&pattern_regex));

    let colored_output = match matches.value_of("color") {
        Some("always") => true,
        Some("never") => false,
        _ => atty::is(Stream::Stdout),
    };

    #[cfg(windows)]
    let colored_output = colored_output && windows::enable_colored_output();

    let ls_colors = if colored_output {
        Some(
            env::var("LS_COLORS")
                .ok()
                .map(|val| LsColors::from_string(&val))
                .unwrap_or_default(),
        )
    } else {
        None
    };

    let command = matches.values_of("exec").map(CommandTemplate::new);

    let config = FdOptions {
        case_sensitive,
        search_full_path: matches.is_present("full-path"),
        ignore_hidden: !(matches.is_present("hidden")
            || matches.occurrences_of("rg-alias-hidden-ignore") >= 2),
        read_ignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")),
        read_gitignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")
            || matches.is_present("no-ignore-vcs")),
        follow_links: matches.is_present("follow"),
        null_separator: matches.is_present("null_separator"),
        max_depth: matches
            .value_of("depth")
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        threads: std::cmp::max(
            matches
                .value_of("threads")
                .and_then(|n| usize::from_str_radix(n, 10).ok())
                .unwrap_or_else(num_cpus::get),
            1,
        ),
        max_buffer_time: matches
            .value_of("max-buffer-time")
            .and_then(|n| u64::from_str_radix(n, 10).ok())
            .map(time::Duration::from_millis),
        ls_colors,
        file_types: match matches.values_of("file-type") {
            None => vec![
                FileType::RegularFile,
                FileType::Directory,
                FileType::SymLink,
            ].into_iter()
                .collect(),
            Some(values) => values
                .map(|value| match value {
                    "f" | "file" => FileType::RegularFile,
                    "d" | "directory" => FileType::Directory,
                    "l" | "symlink" => FileType::SymLink,
                    _ => FileType::RegularFile,
                })
                .collect(),
        },
        extensions: matches.values_of("extension").map(|exts| {
            let patterns = exts.map(|e| e.trim_left_matches('.'))
                .map(|e| format!(r".\.{}$", regex::escape(e)));
            match RegexSetBuilder::new(patterns)
                .case_insensitive(true)
                .build()
            {
                Ok(re) => re,
                Err(err) => error(err.description()),
            }
        }),
        command,
        exclude_patterns: matches
            .values_of("exclude")
            .map(|v| v.map(|p| String::from("!") + p).collect())
            .unwrap_or_else(|| vec![]),
    };

    match RegexBuilder::new(&pattern_regex)
        .case_insensitive(!config.case_sensitive)
        .dot_matches_new_line(true)
        .build()
    {
        Ok(re) => walk::scan(&dir_vec, Arc::new(re), Arc::new(config)),
        Err(err) => error(
            format!(
                "{}\nHint: You can use the '--fixed-strings' option to search for a \
                 literal string instead of a regular expression",
                err.description()
            ).as_str(),
        ),
    }
}
<|endoftext|>


<commit_before>// TODO
// impl ToGeojson for ....
// generic number instead of f64 for position?
// rename Position to Pos or Positions

extern crate serialize;

use std::collections::TreeMap;
use serialize::json::ToJson;
use serialize::json;

/// Position
///
/// [GeoJSON Format Specification § 2.1.1](http://geojson.org/geojson-spec.html#positions)
pub struct Position(pub Vec<f64>);

impl ToJson for Position {
    fn to_json(&self) -> json::Json {
        let &Position(ref nums) = self;
        nums.to_json()
    }
}

impl Clone for Position {
    fn clone(&self) -> Position {
        let &Position(ref nums) = self;
        Position(nums.clone())
    }
}

/// Point
///
/// [GeoJSON Format Specification § 2.1.2](http://geojson.org/geojson-spec.html#point)
pub struct Point {
    pub coordinates: Position,
}

impl ToJson for Point {
    fn to_json(&self) -> json::Json {
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "Point".to_string().to_json());
        d.insert("coordinates".to_string(), self.coordinates.to_json());
        d.to_json()
    }
}

/// MultiPoint
///
/// [GeoJSON Format Specification § 2.1.3](http://geojson.org/geojson-spec.html#multipoint)
pub struct MultiPoint {
    pub points: Vec<Point>,
}

impl ToJson for MultiPoint {
    fn to_json(&self) -> json::Json {
        let coordinates: Vec<Position> =
            self.points.iter().map(|p| p.coordinates.clone()).collect();
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "MultiPoint".to_string().to_json());
        d.insert("coordinates".to_string(), coordinates.to_json());
        d.to_json()
    }
}

/// LineString
///
/// [GeoJSON Format Specification § 2.1.4](http://geojson.org/geojson-spec.html#linestring)
pub struct LineString {
    pub coordinates: Vec<Position>,
}

impl ToJson for LineString {
    fn to_json(&self) -> json::Json {
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "LineString".to_string().to_json());
        d.insert("coordinates".to_string(), self.coordinates.to_json());
        d.to_json()
    }
}

/// MultiLineString
///
/// [GeoJSON Format Specification § 2.1.5](http://geojson.org/geojson-spec.html#multilinestring)
pub struct MultiLineString {
    pub line_strings: Vec<LineString>,
}

impl ToJson for MultiLineString {
    fn to_json(&self) -> json::Json {
        let coordinates: Vec<Vec<Position>> =
            self.line_strings.iter().map(|l| l.coordinates.clone()).collect();
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "MultiLineString".to_string().to_json());
        d.insert("coordinates".to_string(), coordinates.to_json());
        d.to_json()
    }
}

/// Polygon
///
/// [GeoJSON Format Specification § 2.1.6](http://geojson.org/geojson-spec.html#polygon)
pub struct Polygon {
    pub exterior: Vec<Position>,
    pub holes: Vec<Vec<Position>>,
}

impl Polygon {
    fn coordinates(&self) -> Vec<Vec<Position>> {
        let mut coordinates = self.holes.clone();
        coordinates.insert(0, self.exterior.clone());
        coordinates
    }
}

impl ToJson for Polygon {
    fn to_json(&self) -> json::Json {
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "Polygon".to_string().to_json());
        d.insert("coordinates".to_string(), self.coordinates().to_json());
        d.to_json()
    }
}

/// MultiPolygon
///
/// [GeoJSON Format Specification § 2.1.7](http://geojson.org/geojson-spec.html#multipolygon)
pub struct MultiPolygon {
    pub polygons: Vec<Polygon>,
}

impl ToJson for MultiPolygon {
    fn to_json(&self) -> json::Json {
        let coordinates: Vec<Vec<Vec<Position>>> =
            self.polygons.iter().map(|p| p.coordinates()).collect();
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "MultiPolygon".to_string().to_json());
        d.insert("coordinates".to_string(), coordinates.to_json());
        d.to_json()
    }
}

/// Geometry
pub enum Geometry {
    Point(Point),
    MultiPoint(MultiPoint),
    LineString(LineString),
    MultiLineString(MultiLineString),
    Polygon(Polygon),
    MultiPolygon(MultiPolygon),
}

impl ToJson for Geometry {
    fn to_json(&self) -> json::Json {
        match *self {
            // TODO: is there a better way of doing this?
            Point(ref geom) => geom.to_json(),
            MultiPoint(ref geom) => geom.to_json(),
            LineString(ref geom) => geom.to_json(),
            MultiLineString(ref geom) => geom.to_json(),
            Polygon(ref geom) => geom.to_json(),
            MultiPolygon(ref geom) => geom.to_json(),
            // TODO: GeometryCollection(ref geom) => geom.to_json(),
        }
    }
}

/// GeometryCollection
///
/// [GeoJSON Format Specification § 2.1.8](http://geojson.org/geojson-spec.html#geometry-collection)
pub struct GeometryCollection {
    geometries: Vec<Geometry>,
}

impl ToJson for GeometryCollection {
    fn to_json(&self) -> json::Json {
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "GeometryCollection".to_string().to_json());
        d.insert("coordinates".to_string(), self.geometries.to_json());
        d.to_json()
    }
}

fn main() {
    let point = Point {
        coordinates: Position(vec![1., 2., 3.]),
    };

    let j: json::Json = point.to_json();
    let s: String = j.to_pretty_str();

    println!("{}", s);
}
<commit_msg>Fix inaccurate GeometryCollection json key<commit_after>// TODO
// impl ToGeojson for ....
// generic number instead of f64 for position?
// rename Position to Pos or Positions

extern crate serialize;

use std::collections::TreeMap;
use serialize::json::ToJson;
use serialize::json;

/// Position
///
/// [GeoJSON Format Specification § 2.1.1](http://geojson.org/geojson-spec.html#positions)
pub struct Position(pub Vec<f64>);

impl ToJson for Position {
    fn to_json(&self) -> json::Json {
        let &Position(ref nums) = self;
        nums.to_json()
    }
}

impl Clone for Position {
    fn clone(&self) -> Position {
        let &Position(ref nums) = self;
        Position(nums.clone())
    }
}

/// Point
///
/// [GeoJSON Format Specification § 2.1.2](http://geojson.org/geojson-spec.html#point)
pub struct Point {
    pub coordinates: Position,
}

impl ToJson for Point {
    fn to_json(&self) -> json::Json {
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "Point".to_string().to_json());
        d.insert("coordinates".to_string(), self.coordinates.to_json());
        d.to_json()
    }
}

/// MultiPoint
///
/// [GeoJSON Format Specification § 2.1.3](http://geojson.org/geojson-spec.html#multipoint)
pub struct MultiPoint {
    pub points: Vec<Point>,
}

impl ToJson for MultiPoint {
    fn to_json(&self) -> json::Json {
        let coordinates: Vec<Position> =
            self.points.iter().map(|p| p.coordinates.clone()).collect();
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "MultiPoint".to_string().to_json());
        d.insert("coordinates".to_string(), coordinates.to_json());
        d.to_json()
    }
}

/// LineString
///
/// [GeoJSON Format Specification § 2.1.4](http://geojson.org/geojson-spec.html#linestring)
pub struct LineString {
    pub coordinates: Vec<Position>,
}

impl ToJson for LineString {
    fn to_json(&self) -> json::Json {
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "LineString".to_string().to_json());
        d.insert("coordinates".to_string(), self.coordinates.to_json());
        d.to_json()
    }
}

/// MultiLineString
///
/// [GeoJSON Format Specification § 2.1.5](http://geojson.org/geojson-spec.html#multilinestring)
pub struct MultiLineString {
    pub line_strings: Vec<LineString>,
}

impl ToJson for MultiLineString {
    fn to_json(&self) -> json::Json {
        let coordinates: Vec<Vec<Position>> =
            self.line_strings.iter().map(|l| l.coordinates.clone()).collect();
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "MultiLineString".to_string().to_json());
        d.insert("coordinates".to_string(), coordinates.to_json());
        d.to_json()
    }
}

/// Polygon
///
/// [GeoJSON Format Specification § 2.1.6](http://geojson.org/geojson-spec.html#polygon)
pub struct Polygon {
    pub exterior: Vec<Position>,
    pub holes: Vec<Vec<Position>>,
}

impl Polygon {
    fn coordinates(&self) -> Vec<Vec<Position>> {
        let mut coordinates = self.holes.clone();
        coordinates.insert(0, self.exterior.clone());
        coordinates
    }
}

impl ToJson for Polygon {
    fn to_json(&self) -> json::Json {
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "Polygon".to_string().to_json());
        d.insert("coordinates".to_string(), self.coordinates().to_json());
        d.to_json()
    }
}

/// MultiPolygon
///
/// [GeoJSON Format Specification § 2.1.7](http://geojson.org/geojson-spec.html#multipolygon)
pub struct MultiPolygon {
    pub polygons: Vec<Polygon>,
}

impl ToJson for MultiPolygon {
    fn to_json(&self) -> json::Json {
        let coordinates: Vec<Vec<Vec<Position>>> =
            self.polygons.iter().map(|p| p.coordinates()).collect();
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "MultiPolygon".to_string().to_json());
        d.insert("coordinates".to_string(), coordinates.to_json());
        d.to_json()
    }
}

/// Geometry
pub enum Geometry {
    Point(Point),
    MultiPoint(MultiPoint),
    LineString(LineString),
    MultiLineString(MultiLineString),
    Polygon(Polygon),
    MultiPolygon(MultiPolygon),
}

impl ToJson for Geometry {
    fn to_json(&self) -> json::Json {
        match *self {
            // TODO: is there a better way of doing this?
            Point(ref geom) => geom.to_json(),
            MultiPoint(ref geom) => geom.to_json(),
            LineString(ref geom) => geom.to_json(),
            MultiLineString(ref geom) => geom.to_json(),
            Polygon(ref geom) => geom.to_json(),
            MultiPolygon(ref geom) => geom.to_json(),
            // TODO: GeometryCollection(ref geom) => geom.to_json(),
        }
    }
}

/// GeometryCollection
///
/// [GeoJSON Format Specification § 2.1.8](http://geojson.org/geojson-spec.html#geometry-collection)
pub struct GeometryCollection {
    geometries: Vec<Geometry>,
}

impl ToJson for GeometryCollection {
    fn to_json(&self) -> json::Json {
        let mut d = TreeMap::new();
        d.insert("type".to_string(), "GeometryCollection".to_string().to_json());
        d.insert("geometries".to_string(), self.geometries.to_json());
        d.to_json()
    }
}

fn main() {
    let point = Point {
        coordinates: Position(vec![1., 2., 3.]),
    };

    let j: json::Json = point.to_json();
    let s: String = j.to_pretty_str();

    println!("{}", s);
}
<|endoftext|>


<commit_before>

fn main() {
    let first_agent = Agent {
        x: 0,
        y: 0,
        reserves: 100,
        chromosomes: vec!["ABCD1234".to_string(), "11100101".to_string()],
    };

    let some_resource = Resource {
        x: 10,
        y: 10,
        value: 100,
    };

    println!("\n")
    println!("I am first_agent my (x, y) is ({}, {})", first_agent.x, first_agent.y);
    println!("I have {} resources.", first_agent.reserves);
    println!("My chromosomes are {}.", first_agent.chromosomes);
    println!("\n")
    println!("I am some_resource my (x, y) is ({}, {})", some_resource.x, some_resource.y);
    println!("I have {} resources.", some_resource.value);
}

struct Agent  {
    x: int,
    y: int,
    reserves: int,
    chromosomes: Vec<String>,
}

struct Resource {
    x: int,
    y: int,
    value: int,
}

<commit_msg>Agent and Point structs and fn travel() are compiling<commit_after>

fn main() {
    let mut first_agent = Agent {
        location: Point {
            x: 0,
            y: 0,
        },
        reserves: 100,
        chromosomes: vec!["ABCD1234".to_string(), "11100101".to_string()],
    };

    let mut some_resource = Resource {
        location: Point {
            x: 10,
            y: 10,
        },
        value: 100,
    };

    println!("\n")
    println!("I am first_agent my (x, y) is ({}, {})", first_agent.location.x, first_agent.location.y);
    println!("I have {} resources.", first_agent.reserves);
    println!("My chromosomes are {}.", first_agent.chromosomes);
    println!("\n")
    println!("I am some_resource my (x, y) is ({}, {})", some_resource.location.x, some_resource.location.y);
    println!("I have {} resources.", some_resource.value);
}

pub struct Point {
    pub y: int,
    pub x: int,
}

pub struct Agent  {
    pub location: Point,
    reserves: int,
    chromosomes: Vec<String>,
}

pub struct Resource {
    pub location: Point,
    pub value: int,
}

trait Mobile {
    fn travel(&mut self, delta_x: int, delta_y: int);
}

impl Mobile for Agent {
    fn travel(&mut self, delta_x: int, delta_y: int) {
        self.location.x = *&self.location.x + delta_x;
        self.location.y = *&self.location.y + delta_y;
    }
}
<|endoftext|>


<commit_before><commit_msg>Increase maximum frame rate to 200<commit_after><|endoftext|>


<commit_before>extern crate serde_json;
#[macro_use]
extern crate quick_error;
#[macro_use]
extern crate clap;
extern crate colored;

extern crate rustfix;

use std::fs::File;
use std::io::{Read, Write};
use std::error::Error;
use std::process::Command;

use colored::Colorize;
use clap::{Arg, App};

use rustfix::Suggestion;
use rustfix::diagnostics::Diagnostic;

const USER_OPTIONS: &'static str = "What do you want to do? \
    [r]eplace | [s]kip | save and [q]uit | [a]bort (without saving)";

fn main() {
    let program = try_main();
    match program {
        Ok(_) => std::process::exit(0),
        Err(ProgramError::UserAbort) => {
            writeln!(std::io::stdout(), "{}", ProgramError::UserAbort).unwrap();
            std::process::exit(0);
        }
        Err(error) => {
            writeln!(std::io::stderr(), "An error occured: {}", error).unwrap();
            writeln!(std::io::stderr(), "{:?}", error).unwrap();
            if let Some(cause) = error.cause() {
                writeln!(std::io::stderr(), "Cause: {:?}", cause).unwrap();
            }
            std::process::exit(1);
        }
    }
}

macro_rules! flush {
    () => (try!(std::io::stdout().flush());)
}

fn try_main() -> Result<(), ProgramError> {
    let matches = App::new("rustfix")
        .about("Automatically apply suggestions made by rustc")
        .version(crate_version!())
        .arg(Arg::with_name("clippy")
            .long("clippy")
            .help("Use `cargo clippy` for suggestions"))
        .arg(Arg::with_name("from_file")
            .long("from-file")
            .value_name("FILE")
            .takes_value(true)
            .help("Read suggestions from file (each line is a JSON object)"))
        .get_matches();

    // Get JSON output from rustc...
    let json = if let Some(file_name) = matches.value_of("from_file") {
        // either by reading a file the user saved (probably only for debugging rustfix itself)...
        try!(json_from_file(&file_name))
    } else {
        // or by spawning a subcommand that runs rustc.
        let subcommand = if matches.is_present("clippy") {
            "clippy"
        } else {
            "rustc"
        };
        try!(json_from_subcommand(subcommand))
    };

    let suggestions: Vec<Suggestion> = json.lines()
        .filter(not_empty)
        // Convert JSON string (and eat parsing errors)
        .flat_map(|line| serde_json::from_str::<Diagnostic>(line))
        // One diagnostic line might have multiple suggestions
        .flat_map(|diagnostic| rustfix::collect_suggestions(&diagnostic, None))
        .collect();

    try!(handle_suggestions(&suggestions));

    Ok(())
}

fn json_from_file(file_name: &str) -> Result<String, ProgramError> {
    read_file_to_string(file_name).map_err(From::from)
}

fn json_from_subcommand(subcommand: &str) -> Result<String, ProgramError> {
    let output = try!(Command::new("cargo")
        .args(&[subcommand,
                "--quiet",
                "--color", "never",
                "--",
                "-Z", "unstable-options",
                "--error-format", "json"])
        .output());

    let content = try!(String::from_utf8(output.stderr));

    if !output.status.success() {
        return Err(ProgramError::SubcommandError(format!("cargo {}", subcommand), content));
    }

    Ok(content)
}

fn handle_suggestions(suggestions: &[Suggestion]) -> Result<(), ProgramError> {
    let mut accepted_suggestions: Vec<&Suggestion> = vec![];

    if suggestions.is_empty() {
        println!("I don't have any suggestions for you right now. Check back later!");
        return Ok(());
    }

    'suggestions: for suggestion in suggestions {
        println!("\n\n{info}: {message}\n\
            {arrow} {file}:{range}\n\
            {suggestion}\n\n\
            {text}\n\n\
            {with}\n\n\
            {replacement}\n",
            info = "Info".green().bold(),
            message = split_at_lint_name(&suggestion.message),
            arrow = "  -->".blue().bold(),
            suggestion = "Suggestion - Replace:".yellow().bold(),
            file = suggestion.file_name,
            range = suggestion.line_range,
            text = indent(4, &reset_indent(&suggestion.text)),
            with = "with:".yellow().bold(),
            replacement = indent(4, &suggestion.replacement));

        'userinput: loop {
            print!("{arrow} {user_options}\n\
                {prompt} ",
                arrow = "==>".green().bold(),
                prompt = "  >".green().bold(),
                user_options = USER_OPTIONS.green());

            flush!();
            let mut input = String::new();
            try!(std::io::stdin().read_line(&mut input));

            match input.trim() {
                "s" => {
                    println!("Skipped.");
                    continue 'suggestions;
                }
                "r" => {
                    accepted_suggestions.push(suggestion);
                    println!("Suggestion accepted. I'll remember that and apply it later.");
                    continue 'suggestions;
                }
                "q" => {
                    println!("Thanks for playing!");
                    break 'suggestions;
                }
                "a" => {
                    return Err(ProgramError::UserAbort);
                }
                _ => {
                    println!("{error}: I didn't quite get that. {user_options}",
                                error = "Error".red().bold(),
                                user_options = USER_OPTIONS);
                    continue 'userinput;
                }
            }
        }
    }

    if !accepted_suggestions.is_empty() {
        println!("Good work. Let me just apply these {} changes!",
                 accepted_suggestions.len());

        for suggestion in accepted_suggestions.iter().rev() {
            try!(apply_suggestion(suggestion));

            print!(".");
            flush!();
        }

        println!("\nDone.");
    }

    Ok(())
}

quick_error! {
    /// All possible errors in programm lifecycle
    #[derive(Debug)]
    pub enum ProgramError {
        UserAbort {
            display("Let's get outta here!")
        }
        /// Missing File
        NoFile {
            display("No input file given")
        }
        SubcommandError(subcommand: String, output: String) {
            display("Error executing subcommand `{}`", subcommand)
            description(output)
        }
        /// Error while dealing with file or stdin/stdout
        Io(err: std::io::Error) {
            from()
            cause(err)
            display("I/O error")
            description(err.description())
        }
        Utf8Error(err: std::string::FromUtf8Error) {
            from()
            display("Error reading input as UTF-8")
        }
        /// Error with deserialization
        Serde(err: serde_json::Error) {
            from()
            cause(err)
            display("Serde JSON error")
            description(err.description())
        }
    }
}

// Helpers
// -------

fn read_file_to_string(file_name: &str) -> Result<String, std::io::Error> {
    let mut file = try!(File::open(file_name));
    let mut buffer = String::new();
    try!(file.read_to_string(&mut buffer));
    Ok(buffer)
}

fn not_empty(s: &&str) -> bool {
    !s.trim().is_empty()
}

fn split_at_lint_name(s: &str) -> String {
    s.split(", #[")
        .collect::<Vec<_>>()
        .join("\n      #[") // Length of whitespace == length of "Info: "
}

fn reset_indent(s: &str) -> String {
    let leading_whitespace = s.lines()
        .nth(0)
        .unwrap_or("")
        .chars()
        .take_while(|&c| char::is_whitespace(c))
        .count();

    s.lines()
        .map(|line| String::from(&line[leading_whitespace..]))
        .collect::<Vec<_>>()
        .join("\n")
}

fn indent(size: u32, s: &str) -> String {
    let whitespace: String = std::iter::repeat(' ').take(size as usize).collect();

    s.lines()
        .map(|l| format!("{}{}", whitespace, l))
        .collect::<Vec<_>>()
        .join("\n")
}

/// Apply suggestion to a file
///
/// Please beware of ugly hacks below! Originally, I wanted to replace byte ranges, but sadly the
/// ranges rustc's JSON output gives me do not correspond to the parts of the file they are meant
/// to correspond to. So, for now, let's just replace lines!
///
/// This function is as stupid as possible. Make sure you call for the replacemnts in one file in
/// reverse order to not mess up the lines for replacements further down the road.
fn apply_suggestion(suggestion: &Suggestion) -> Result<(), ProgramError> {
    use std::cmp::max;

    let file_content = try!(read_file_to_string(&suggestion.file_name));
    let mut new_content = String::new();

    // Add the lines before the section we want to replace
    new_content.push_str(&file_content.lines()
        .take(max(suggestion.line_range.start.line - 1, 0) as usize)
        .collect::<Vec<_>>()
        .join("\n"));
    new_content.push_str("\n");

    // Parts of line before replacement
    new_content.push_str(&file_content.lines()
        .nth(suggestion.line_range.start.line - 1)
        .unwrap_or("")
        .chars()
        .take(suggestion.line_range.start.column - 1)
        .collect::<String>());

    // Insert new content! Finally!
    new_content.push_str(&suggestion.replacement);

    // TODO(killercup): better handling of trailing semicolons
    if suggestion.text.trim().ends_with(';') && !suggestion.replacement.trim().ends_with(';') {
        new_content.push_str(";");
    }

    // Parts of line after replacement
    new_content.push_str(&file_content.lines()
        .nth(suggestion.line_range.end.line - 1)
        .unwrap_or("")
        .chars()
        .skip(suggestion.line_range.end.column - 1)
        .collect::<String>());

    // Add the lines after the section we want to replace
    new_content.push_str("\n");
    new_content.push_str(&file_content.lines()
        .skip(suggestion.line_range.end.line as usize)
        .collect::<Vec<_>>()
        .join("\n"));
    new_content.push_str("\n");

    let mut file = try!(File::create(&suggestion.file_name));
    let new_content = new_content.as_bytes();

    try!(file.set_len(new_content.len() as u64));
    try!(file.write_all(&new_content));

    Ok(())
}
<commit_msg>Let Clippy Worry about Semicolons<commit_after>extern crate serde_json;
#[macro_use]
extern crate quick_error;
#[macro_use]
extern crate clap;
extern crate colored;

extern crate rustfix;

use std::fs::File;
use std::io::{Read, Write};
use std::error::Error;
use std::process::Command;

use colored::Colorize;
use clap::{Arg, App};

use rustfix::Suggestion;
use rustfix::diagnostics::Diagnostic;

const USER_OPTIONS: &'static str = "What do you want to do? \
    [r]eplace | [s]kip | save and [q]uit | [a]bort (without saving)";

fn main() {
    let program = try_main();
    match program {
        Ok(_) => std::process::exit(0),
        Err(ProgramError::UserAbort) => {
            writeln!(std::io::stdout(), "{}", ProgramError::UserAbort).unwrap();
            std::process::exit(0);
        }
        Err(error) => {
            writeln!(std::io::stderr(), "An error occured: {}", error).unwrap();
            writeln!(std::io::stderr(), "{:?}", error).unwrap();
            if let Some(cause) = error.cause() {
                writeln!(std::io::stderr(), "Cause: {:?}", cause).unwrap();
            }
            std::process::exit(1);
        }
    }
}

macro_rules! flush {
    () => (try!(std::io::stdout().flush());)
}

fn try_main() -> Result<(), ProgramError> {
    let matches = App::new("rustfix")
        .about("Automatically apply suggestions made by rustc")
        .version(crate_version!())
        .arg(Arg::with_name("clippy")
            .long("clippy")
            .help("Use `cargo clippy` for suggestions"))
        .arg(Arg::with_name("from_file")
            .long("from-file")
            .value_name("FILE")
            .takes_value(true)
            .help("Read suggestions from file (each line is a JSON object)"))
        .get_matches();

    // Get JSON output from rustc...
    let json = if let Some(file_name) = matches.value_of("from_file") {
        // either by reading a file the user saved (probably only for debugging rustfix itself)...
        try!(json_from_file(&file_name))
    } else {
        // or by spawning a subcommand that runs rustc.
        let subcommand = if matches.is_present("clippy") {
            "clippy"
        } else {
            "rustc"
        };
        try!(json_from_subcommand(subcommand))
    };

    let suggestions: Vec<Suggestion> = json.lines()
        .filter(not_empty)
        // Convert JSON string (and eat parsing errors)
        .flat_map(|line| serde_json::from_str::<Diagnostic>(line))
        // One diagnostic line might have multiple suggestions
        .flat_map(|diagnostic| rustfix::collect_suggestions(&diagnostic, None))
        .collect();

    try!(handle_suggestions(&suggestions));

    Ok(())
}

fn json_from_file(file_name: &str) -> Result<String, ProgramError> {
    read_file_to_string(file_name).map_err(From::from)
}

fn json_from_subcommand(subcommand: &str) -> Result<String, ProgramError> {
    let output = try!(Command::new("cargo")
        .args(&[subcommand,
                "--quiet",
                "--color", "never",
                "--",
                "-Z", "unstable-options",
                "--error-format", "json"])
        .output());

    let content = try!(String::from_utf8(output.stderr));

    if !output.status.success() {
        return Err(ProgramError::SubcommandError(format!("cargo {}", subcommand), content));
    }

    Ok(content)
}

fn handle_suggestions(suggestions: &[Suggestion]) -> Result<(), ProgramError> {
    let mut accepted_suggestions: Vec<&Suggestion> = vec![];

    if suggestions.is_empty() {
        println!("I don't have any suggestions for you right now. Check back later!");
        return Ok(());
    }

    'suggestions: for suggestion in suggestions {
        println!("\n\n{info}: {message}\n\
            {arrow} {file}:{range}\n\
            {suggestion}\n\n\
            {text}\n\n\
            {with}\n\n\
            {replacement}\n",
            info = "Info".green().bold(),
            message = split_at_lint_name(&suggestion.message),
            arrow = "  -->".blue().bold(),
            suggestion = "Suggestion - Replace:".yellow().bold(),
            file = suggestion.file_name,
            range = suggestion.line_range,
            text = indent(4, &reset_indent(&suggestion.text)),
            with = "with:".yellow().bold(),
            replacement = indent(4, &suggestion.replacement));

        'userinput: loop {
            print!("{arrow} {user_options}\n\
                {prompt} ",
                arrow = "==>".green().bold(),
                prompt = "  >".green().bold(),
                user_options = USER_OPTIONS.green());

            flush!();
            let mut input = String::new();
            try!(std::io::stdin().read_line(&mut input));

            match input.trim() {
                "s" => {
                    println!("Skipped.");
                    continue 'suggestions;
                }
                "r" => {
                    accepted_suggestions.push(suggestion);
                    println!("Suggestion accepted. I'll remember that and apply it later.");
                    continue 'suggestions;
                }
                "q" => {
                    println!("Thanks for playing!");
                    break 'suggestions;
                }
                "a" => {
                    return Err(ProgramError::UserAbort);
                }
                _ => {
                    println!("{error}: I didn't quite get that. {user_options}",
                                error = "Error".red().bold(),
                                user_options = USER_OPTIONS);
                    continue 'userinput;
                }
            }
        }
    }

    if !accepted_suggestions.is_empty() {
        println!("Good work. Let me just apply these {} changes!",
                 accepted_suggestions.len());

        for suggestion in accepted_suggestions.iter().rev() {
            try!(apply_suggestion(suggestion));

            print!(".");
            flush!();
        }

        println!("\nDone.");
    }

    Ok(())
}

quick_error! {
    /// All possible errors in programm lifecycle
    #[derive(Debug)]
    pub enum ProgramError {
        UserAbort {
            display("Let's get outta here!")
        }
        /// Missing File
        NoFile {
            display("No input file given")
        }
        SubcommandError(subcommand: String, output: String) {
            display("Error executing subcommand `{}`", subcommand)
            description(output)
        }
        /// Error while dealing with file or stdin/stdout
        Io(err: std::io::Error) {
            from()
            cause(err)
            display("I/O error")
            description(err.description())
        }
        Utf8Error(err: std::string::FromUtf8Error) {
            from()
            display("Error reading input as UTF-8")
        }
        /// Error with deserialization
        Serde(err: serde_json::Error) {
            from()
            cause(err)
            display("Serde JSON error")
            description(err.description())
        }
    }
}

// Helpers
// -------

fn read_file_to_string(file_name: &str) -> Result<String, std::io::Error> {
    let mut file = try!(File::open(file_name));
    let mut buffer = String::new();
    try!(file.read_to_string(&mut buffer));
    Ok(buffer)
}

fn not_empty(s: &&str) -> bool {
    !s.trim().is_empty()
}

fn split_at_lint_name(s: &str) -> String {
    s.split(", #[")
        .collect::<Vec<_>>()
        .join("\n      #[") // Length of whitespace == length of "Info: "
}

fn reset_indent(s: &str) -> String {
    let leading_whitespace = s.lines()
        .nth(0)
        .unwrap_or("")
        .chars()
        .take_while(|&c| char::is_whitespace(c))
        .count();

    s.lines()
        .map(|line| String::from(&line[leading_whitespace..]))
        .collect::<Vec<_>>()
        .join("\n")
}

fn indent(size: u32, s: &str) -> String {
    let whitespace: String = std::iter::repeat(' ').take(size as usize).collect();

    s.lines()
        .map(|l| format!("{}{}", whitespace, l))
        .collect::<Vec<_>>()
        .join("\n")
}

/// Apply suggestion to a file
///
/// Please beware of ugly hacks below! Originally, I wanted to replace byte ranges, but sadly the
/// ranges rustc's JSON output gives me do not correspond to the parts of the file they are meant
/// to correspond to. So, for now, let's just replace lines!
///
/// This function is as stupid as possible. Make sure you call for the replacemnts in one file in
/// reverse order to not mess up the lines for replacements further down the road.
fn apply_suggestion(suggestion: &Suggestion) -> Result<(), ProgramError> {
    use std::cmp::max;

    let file_content = try!(read_file_to_string(&suggestion.file_name));
    let mut new_content = String::new();

    // Add the lines before the section we want to replace
    new_content.push_str(&file_content.lines()
        .take(max(suggestion.line_range.start.line - 1, 0) as usize)
        .collect::<Vec<_>>()
        .join("\n"));
    new_content.push_str("\n");

    // Parts of line before replacement
    new_content.push_str(&file_content.lines()
        .nth(suggestion.line_range.start.line - 1)
        .unwrap_or("")
        .chars()
        .take(suggestion.line_range.start.column - 1)
        .collect::<String>());

    // Insert new content! Finally!
    new_content.push_str(&suggestion.replacement);

    // Parts of line after replacement
    new_content.push_str(&file_content.lines()
        .nth(suggestion.line_range.end.line - 1)
        .unwrap_or("")
        .chars()
        .skip(suggestion.line_range.end.column - 1)
        .collect::<String>());

    // Add the lines after the section we want to replace
    new_content.push_str("\n");
    new_content.push_str(&file_content.lines()
        .skip(suggestion.line_range.end.line as usize)
        .collect::<Vec<_>>()
        .join("\n"));
    new_content.push_str("\n");

    let mut file = try!(File::create(&suggestion.file_name));
    let new_content = new_content.as_bytes();

    try!(file.set_len(new_content.len() as u64));
    try!(file.write_all(&new_content));

    Ok(())
}
<|endoftext|>