<commit_before>extern crate nalgebra as na;
extern crate kiss3d;
extern crate rand;
extern crate num;
extern crate clap;

use kiss3d::window::Window;
use na::{Pnt2, Pnt3, Vec2, Vec3, Norm, FloatPnt, FloatVec};
use rand::{Rng, Closed01};
use num::Zero;
use clap::{Arg, App};
use std::str::FromStr;

struct SpaceColonization<T, F>
    where T: FloatPnt<f32, F>,
          F: FloatVec<f32> + Zero + Copy
{
    node_parents: Vec<usize>,
    node_positions: Vec<T>,
    leaf_node_pointers: Vec<usize>, // XXX: how can a branch be terminated?
    leaf_node_grow_directions: Vec<F>,
}

impl<T, F> SpaceColonization<T, F>
    where T: FloatPnt<f32, F>,
          F: FloatVec<f32> + Zero + Copy
{
    fn new() -> SpaceColonization<T, F> {
        SpaceColonization {
            node_parents: Vec::new(),
            node_positions: Vec::new(),
            leaf_node_pointers: Vec::new(),
            leaf_node_grow_directions: Vec::new(),
        }
    }

    // adds initial root node(s)
    fn add_root_node(&mut self, pos: T) {
        debug_assert!(self.node_positions.len() == self.node_parents.len()); // invariant
        debug_assert!(self.leaf_node_pointers.len() == self.leaf_node_grow_directions.len()); // invariant

        let idx = self.node_positions.len();
        self.node_positions.push(pos);

        // a root node has it's own index as parent
        self.node_parents.push(idx);

        // a root node creates a new "leaf" node (better: branch!)
        self.leaf_node_pointers.push(idx);
        self.leaf_node_grow_directions.push(Zero::zero());

        debug_assert!(self.node_positions.len() == self.node_parents.len()); // invariant
        debug_assert!(self.leaf_node_pointers.len() == self.leaf_node_grow_directions.len()); // invariant
    }

    fn add_moved_leaf_node(&mut self, leaf_node_slot: usize, v: F) {
        let parent = self.leaf_node_pointers[leaf_node_slot];
        let parent_pos = self.node_positions[parent];

        self.update_leaf_node(leaf_node_slot, parent_pos + v)
    }

    fn update_leaf_node(&mut self, leaf_node_slot: usize, new_pos: T) {
        let parent = self.leaf_node_pointers[leaf_node_slot];

        // create new node
        let idx = self.node_positions.len();
        self.node_positions.push(new_pos);
        self.node_parents.push(parent);

        // update the leaf node pointer
        self.leaf_node_pointers[leaf_node_slot] = idx;
    }

    fn find_nearest_leaf_for_attraction_point(&self,
                                              attraction_pt: &T,
                                              radius_influence_sq: f32)
                                              -> Option<(usize, f32)> {
        let mut nearest_leaf: Option<usize> = None;
        let mut nearest_distance_sq: f32 = radius_influence_sq;

        for (i, &leaf_node_ptr) in self.leaf_node_pointers.iter().enumerate() {
            let leaf_pos = &self.node_positions[leaf_node_ptr];
            let dist_sq = leaf_pos.sqdist(attraction_pt);

            // is the attraction point within the radius of influence
            // and closed than the currently best
            if dist_sq < nearest_distance_sq {
                nearest_distance_sq = dist_sq;
                nearest_leaf = Some(i);
            }
        }

        return nearest_leaf.map(|idx| (idx, nearest_distance_sq));
    }

    fn iter_segments<C>(&self, callback: &mut C)
        where C: FnMut(&T, &T)
    {
        for (i, &parent) in self.node_parents.iter().enumerate() {
            if i != parent {
                callback(&self.node_positions[i], &self.node_positions[parent]);
            }
        }
    }

    fn iterate(&mut self,
               attraction_points: &mut [(T, bool)],
               radius_influce_sq: f32,
               move_distance: f32,
               kill_distance_sq: f32) {
        // reset all attraction forces
        for force in &mut self.leaf_node_grow_directions {
            *force = Zero::zero();
        }

        // for each attraction_point, find the nearest (leaf) node that it
        // influences.
        for ap in attraction_points.iter_mut() {
            let active = ap.1;
            if !active {
                continue;
            }
            let nearest_leaf_opt = self.find_nearest_leaf_for_attraction_point(&ap.0,
                                                                               radius_influce_sq);
            if let Some((nearest_leaf_slot, d_sq)) = nearest_leaf_opt {
                let nearest_leaf_node_idx = self.leaf_node_pointers[nearest_leaf_slot];
                // update the force with the normalized vector towards the attraction point
                let v = (ap.0 - self.node_positions[nearest_leaf_node_idx]).normalize();
                self.leaf_node_grow_directions[nearest_leaf_slot] =
                    self.leaf_node_grow_directions[nearest_leaf_slot] + v;

                if d_sq < kill_distance_sq {
                    // set inactive
                    ap.1 = false;
                }
            }
        }

        // now create new leaf nodes
        for i in 0..self.leaf_node_grow_directions.len() {
            let force = self.leaf_node_grow_directions[i];

            // update leaf node only if there is a force!
            if !force.is_zero() {
                let n = force.normalize();
                self.add_moved_leaf_node(i, n * move_distance);
            }
        }

    }
}

fn random_closed01<R: Rng>(rng: &mut R) -> f32 {
    rng.gen::<Closed01<f32>>().0
}

fn random_coord<R: Rng>(rng: &mut R) -> f32 {
    2.0 * random_closed01(rng) - 1.0
}

trait MyPoint {
    fn into_pnt3(self) -> Pnt3<f32>;
    fn random<R: Rng>(rng: &mut R) -> Self;
}

impl MyPoint for Pnt3<f32> {
    fn into_pnt3(self) -> Pnt3<f32> {
        self
    }

    fn random<R: Rng>(rng: &mut R) -> Pnt3<f32> {
        Pnt3::new(random_coord(rng), random_coord(rng), random_coord(rng))
    }
}

impl MyPoint for Pnt2<f32> {
    fn into_pnt3(self) -> Pnt3<f32> {
        Pnt3::new(self.x, self.y, 3.0)
    }

    fn random<R: Rng>(rng: &mut R) -> Pnt2<f32> {
        Pnt2::new(random_coord(rng), random_coord(rng))
    }
}

#[derive(Debug)]
struct Config {
    n_attraction_points: usize,
    n_roots: usize,
    influence_radius: f32,
    move_distance: f32,
    kill_distance: f32,
    use_3d: bool,
    max_iter: Option<usize>,
}

impl Config {
    fn config1() -> Config {
        Config {
            n_attraction_points: 10_000,
            n_roots: 5,
            influence_radius: 0.44,
            kill_distance: 0.22,
            move_distance: 0.02,
            use_3d: true,
            max_iter: None,
        }
    }

    fn config2() -> Config {
        Config {
            n_attraction_points: 10_000,
            n_roots: 5,
            influence_radius: 0.1,
            kill_distance: 0.07,
            move_distance: 0.01,
            use_3d: false,
            max_iter: None,
        }
    }

    fn from_cmd() -> Config {
        let matches = App::new("space-colonization")
                          .arg(Arg::with_name("NUM_POINTS")
                                   .long("num-points")
                                   .help("Number of attraction points (default: 1000)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("NUM_ROOTS")
                                   .long("num-roots")
                                   .help("Number of root nodes (default: 1)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("IR")
                                   .long("radius")
                                   .help("Influence radius (default: 0.25)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("KD")
                                   .long("kill-distance")
                                   .help("Kill distance (default: 0.1)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("MD")
                                   .long("move-distance")
                                   .help("Kill distance (default: 0.05)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("MAX_ITER")
                                   .long("max-iter")
                                   .help("Maximum iterations (default: infinite)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("USE_3D")
                                   .long("use-3d")
                                   .help("Use 3d mode"))
                          .get_matches();

        Config {
            n_attraction_points: usize::from_str(matches.value_of("NUM_POINTS").unwrap_or("1000"))
                                     .unwrap(),
            n_roots: usize::from_str(matches.value_of("NUM_ROOTS").unwrap_or("1")).unwrap(),
            influence_radius: f32::from_str(matches.value_of("IR").unwrap_or("0.25")).unwrap(),
            kill_distance: f32::from_str(matches.value_of("KD").unwrap_or("0.1")).unwrap(),
            move_distance: f32::from_str(matches.value_of("MD").unwrap_or("0.05")).unwrap(),
            use_3d: matches.is_present("USE_3D"),
            max_iter: usize::from_str(matches.value_of("MAX_ITER").unwrap_or("INVALID")).ok(),
        }
    }
}

fn run<T, F>(config: &Config)
    where T: MyPoint + FloatPnt<f32, F>,
          F: FloatVec<f32> + Zero + Copy
{
    let mut rng = rand::thread_rng();

    // generate initial attraction point configuration
    // stores status of attraction point in the second tuple element (true=active, false=used)
    let mut attraction_points: Vec<(T, bool)> = (0..config.n_attraction_points)
                                                    .into_iter()
                                                    .map(|_| {
                                                        (<T as MyPoint>::random(&mut rng), true)
                                                    })
                                                    .collect();

    let mut sc: SpaceColonization<T, F> = SpaceColonization::new();
    for _ in 0..config.n_roots {
        sc.add_root_node(<T as MyPoint>::random(&mut rng));
    }

    let mut window = Window::new("Space Colonization");
    let white = Pnt3::new(1.0, 1.0, 1.0);

    let mut i = 0;

    while window.render() {
        i += 1;

        if let Some(m) = config.max_iter {
            if i > m {
                break;
            }
        }

        println!("Iteration: {}", i);
        for &(pt, status) in &attraction_points {
            if status {
                window.draw_point(&pt.into_pnt3(), &white);
            }
        }

        sc.iter_segments(&mut |&a, &b| {
            window.draw_line(&a.into_pnt3(), &b.into_pnt3(), &white);
        });

        sc.iterate(&mut attraction_points,
                   config.influence_radius * config.influence_radius,
                   config.move_distance,
                   config.kill_distance * config.kill_distance);

    }
}

fn main() {
    let config = Config::from_cmd();

    println!("{:?}", config);

    if config.use_3d {
        run::<Pnt3<f32>, Vec3<f32>>(&config);
    } else {
        run::<Pnt2<f32>, Vec2<f32>>(&config);
    }
}
<commit_msg>Use red for lines<commit_after>extern crate nalgebra as na;
extern crate kiss3d;
extern crate rand;
extern crate num;
extern crate clap;

use kiss3d::window::Window;
use na::{Pnt2, Pnt3, Vec2, Vec3, Norm, FloatPnt, FloatVec};
use rand::{Rng, Closed01};
use num::Zero;
use clap::{Arg, App};
use std::str::FromStr;

struct SpaceColonization<T, F>
    where T: FloatPnt<f32, F>,
          F: FloatVec<f32> + Zero + Copy
{
    node_parents: Vec<usize>,
    node_positions: Vec<T>,
    leaf_node_pointers: Vec<usize>, // XXX: how can a branch be terminated?
    leaf_node_grow_directions: Vec<F>,
}

impl<T, F> SpaceColonization<T, F>
    where T: FloatPnt<f32, F>,
          F: FloatVec<f32> + Zero + Copy
{
    fn new() -> SpaceColonization<T, F> {
        SpaceColonization {
            node_parents: Vec::new(),
            node_positions: Vec::new(),
            leaf_node_pointers: Vec::new(),
            leaf_node_grow_directions: Vec::new(),
        }
    }

    // adds initial root node(s)
    fn add_root_node(&mut self, pos: T) {
        debug_assert!(self.node_positions.len() == self.node_parents.len()); // invariant
        debug_assert!(self.leaf_node_pointers.len() == self.leaf_node_grow_directions.len()); // invariant

        let idx = self.node_positions.len();
        self.node_positions.push(pos);

        // a root node has it's own index as parent
        self.node_parents.push(idx);

        // a root node creates a new "leaf" node (better: branch!)
        self.leaf_node_pointers.push(idx);
        self.leaf_node_grow_directions.push(Zero::zero());

        debug_assert!(self.node_positions.len() == self.node_parents.len()); // invariant
        debug_assert!(self.leaf_node_pointers.len() == self.leaf_node_grow_directions.len()); // invariant
    }

    fn add_moved_leaf_node(&mut self, leaf_node_slot: usize, v: F) {
        let parent = self.leaf_node_pointers[leaf_node_slot];
        let parent_pos = self.node_positions[parent];

        self.update_leaf_node(leaf_node_slot, parent_pos + v)
    }

    fn update_leaf_node(&mut self, leaf_node_slot: usize, new_pos: T) {
        let parent = self.leaf_node_pointers[leaf_node_slot];

        // create new node
        let idx = self.node_positions.len();
        self.node_positions.push(new_pos);
        self.node_parents.push(parent);

        // update the leaf node pointer
        self.leaf_node_pointers[leaf_node_slot] = idx;
    }

    fn find_nearest_leaf_for_attraction_point(&self,
                                              attraction_pt: &T,
                                              radius_influence_sq: f32)
                                              -> Option<(usize, f32)> {
        let mut nearest_leaf: Option<usize> = None;
        let mut nearest_distance_sq: f32 = radius_influence_sq;

        for (i, &leaf_node_ptr) in self.leaf_node_pointers.iter().enumerate() {
            let leaf_pos = &self.node_positions[leaf_node_ptr];
            let dist_sq = leaf_pos.sqdist(attraction_pt);

            // is the attraction point within the radius of influence
            // and closed than the currently best
            if dist_sq < nearest_distance_sq {
                nearest_distance_sq = dist_sq;
                nearest_leaf = Some(i);
            }
        }

        return nearest_leaf.map(|idx| (idx, nearest_distance_sq));
    }

    fn iter_segments<C>(&self, callback: &mut C)
        where C: FnMut(&T, &T)
    {
        for (i, &parent) in self.node_parents.iter().enumerate() {
            if i != parent {
                callback(&self.node_positions[i], &self.node_positions[parent]);
            }
        }
    }

    fn iterate(&mut self,
               attraction_points: &mut [(T, bool)],
               radius_influce_sq: f32,
               move_distance: f32,
               kill_distance_sq: f32) {
        // reset all attraction forces
        for force in &mut self.leaf_node_grow_directions {
            *force = Zero::zero();
        }

        // for each attraction_point, find the nearest (leaf) node that it
        // influences.
        for ap in attraction_points.iter_mut() {
            let active = ap.1;
            if !active {
                continue;
            }
            let nearest_leaf_opt = self.find_nearest_leaf_for_attraction_point(&ap.0,
                                                                               radius_influce_sq);
            if let Some((nearest_leaf_slot, d_sq)) = nearest_leaf_opt {
                let nearest_leaf_node_idx = self.leaf_node_pointers[nearest_leaf_slot];
                // update the force with the normalized vector towards the attraction point
                let v = (ap.0 - self.node_positions[nearest_leaf_node_idx]).normalize();
                self.leaf_node_grow_directions[nearest_leaf_slot] =
                    self.leaf_node_grow_directions[nearest_leaf_slot] + v;

                if d_sq < kill_distance_sq {
                    // set inactive
                    ap.1 = false;
                }
            }
        }

        // now create new leaf nodes
        for i in 0..self.leaf_node_grow_directions.len() {
            let force = self.leaf_node_grow_directions[i];

            // update leaf node only if there is a force!
            if !force.is_zero() {
                let n = force.normalize();
                self.add_moved_leaf_node(i, n * move_distance);
            }
        }

    }
}

fn random_closed01<R: Rng>(rng: &mut R) -> f32 {
    rng.gen::<Closed01<f32>>().0
}

fn random_coord<R: Rng>(rng: &mut R) -> f32 {
    2.0 * random_closed01(rng) - 1.0
}

trait MyPoint {
    fn into_pnt3(self) -> Pnt3<f32>;
    fn random<R: Rng>(rng: &mut R) -> Self;
}

impl MyPoint for Pnt3<f32> {
    fn into_pnt3(self) -> Pnt3<f32> {
        self
    }

    fn random<R: Rng>(rng: &mut R) -> Pnt3<f32> {
        Pnt3::new(random_coord(rng), random_coord(rng), random_coord(rng))
    }
}

impl MyPoint for Pnt2<f32> {
    fn into_pnt3(self) -> Pnt3<f32> {
        Pnt3::new(self.x, self.y, 3.0)
    }

    fn random<R: Rng>(rng: &mut R) -> Pnt2<f32> {
        Pnt2::new(random_coord(rng), random_coord(rng))
    }
}

#[derive(Debug)]
struct Config {
    n_attraction_points: usize,
    n_roots: usize,
    influence_radius: f32,
    move_distance: f32,
    kill_distance: f32,
    use_3d: bool,
    max_iter: Option<usize>,
}

impl Config {
    fn config1() -> Config {
        Config {
            n_attraction_points: 10_000,
            n_roots: 5,
            influence_radius: 0.44,
            kill_distance: 0.22,
            move_distance: 0.02,
            use_3d: true,
            max_iter: None,
        }
    }

    fn config2() -> Config {
        Config {
            n_attraction_points: 10_000,
            n_roots: 5,
            influence_radius: 0.1,
            kill_distance: 0.07,
            move_distance: 0.01,
            use_3d: false,
            max_iter: None,
        }
    }

    fn from_cmd() -> Config {
        let matches = App::new("space-colonization")
                          .arg(Arg::with_name("NUM_POINTS")
                                   .long("num-points")
                                   .help("Number of attraction points (default: 1000)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("NUM_ROOTS")
                                   .long("num-roots")
                                   .help("Number of root nodes (default: 1)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("IR")
                                   .long("radius")
                                   .help("Influence radius (default: 0.25)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("KD")
                                   .long("kill-distance")
                                   .help("Kill distance (default: 0.1)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("MD")
                                   .long("move-distance")
                                   .help("Kill distance (default: 0.05)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("MAX_ITER")
                                   .long("max-iter")
                                   .help("Maximum iterations (default: infinite)")
                                   .takes_value(true)
                                   .required(false))
                          .arg(Arg::with_name("USE_3D")
                                   .long("use-3d")
                                   .help("Use 3d mode"))
                          .get_matches();

        Config {
            n_attraction_points: usize::from_str(matches.value_of("NUM_POINTS").unwrap_or("1000"))
                                     .unwrap(),
            n_roots: usize::from_str(matches.value_of("NUM_ROOTS").unwrap_or("1")).unwrap(),
            influence_radius: f32::from_str(matches.value_of("IR").unwrap_or("0.25")).unwrap(),
            kill_distance: f32::from_str(matches.value_of("KD").unwrap_or("0.1")).unwrap(),
            move_distance: f32::from_str(matches.value_of("MD").unwrap_or("0.05")).unwrap(),
            use_3d: matches.is_present("USE_3D"),
            max_iter: usize::from_str(matches.value_of("MAX_ITER").unwrap_or("INVALID")).ok(),
        }
    }
}

fn run<T, F>(config: &Config)
    where T: MyPoint + FloatPnt<f32, F>,
          F: FloatVec<f32> + Zero + Copy
{
    let mut rng = rand::thread_rng();

    // generate initial attraction point configuration
    // stores status of attraction point in the second tuple element (true=active, false=used)
    let mut attraction_points: Vec<(T, bool)> = (0..config.n_attraction_points)
                                                    .into_iter()
                                                    .map(|_| {
                                                        (<T as MyPoint>::random(&mut rng), true)
                                                    })
                                                    .collect();

    let mut sc: SpaceColonization<T, F> = SpaceColonization::new();
    for _ in 0..config.n_roots {
        sc.add_root_node(<T as MyPoint>::random(&mut rng));
    }

    let mut window = Window::new("Space Colonization");
    let white = Pnt3::new(1.0, 1.0, 1.0);
    let red = Pnt3::new(1.0, 0.0, 0.0);

    let mut i = 0;

    while window.render() {
        i += 1;

        if let Some(m) = config.max_iter {
            if i > m {
                break;
            }
        }

        println!("Iteration: {}", i);
        for &(pt, status) in &attraction_points {
            if status {
                window.draw_point(&pt.into_pnt3(), &white);
            }
        }

        sc.iter_segments(&mut |&a, &b| {
            window.draw_line(&a.into_pnt3(), &b.into_pnt3(), &red);
        });

        sc.iterate(&mut attraction_points,
                   config.influence_radius * config.influence_radius,
                   config.move_distance,
                   config.kill_distance * config.kill_distance);

    }
}

fn main() {
    let config = Config::from_cmd();

    println!("{:?}", config);

    if config.use_3d {
        run::<Pnt3<f32>, Vec3<f32>>(&config);
    } else {
        run::<Pnt2<f32>, Vec2<f32>>(&config);
    }
}
<|endoftext|>


<commit_before>// Copyright 2015 Till Höppner
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![feature(libc, plugin)]
#![plugin(regex_macros)]

extern crate ilc;
extern crate chrono;
extern crate docopt;
extern crate rustc_serialize;
extern crate libc;
extern crate regex;
#[macro_use]
extern crate log;
extern crate env_logger;

use std::process;
use std::io::{ self, BufReader };
use std::fs::File;
use std::error::Error;
use std::str::FromStr;
use std::collections::HashMap;

use docopt::Docopt;

use chrono::offset::fixed::FixedOffset;
use chrono::naive::date::NaiveDate;

use ilc::context::Context;
use ilc::format::{ self, Encode, Decode, DecodeBox };
use ilc::event::{ Event, Type };

static USAGE: &'static str = r#"
d8b   888
Y8P   888
      888
888   888    .d8888b
888   888   d88P"
888   888   888
888   888   Y88b.
888   888    "Y8888P

A converter and statistics utility for IRC log files.

Usage:
  ilc parse <file>...
  ilc convert <informat> <outformat> [--date DATE] [--tz SECS] [--channel CH]
  ilc freq
  ilc (-h | --help | -v | --version)

Options:
  -h --help         Show this screen.
  -v --version      Show the version (duh).
  --date DATE       Override the date for this log. ISO 8601, YYYY-MM-DD.
  --tz SECONDS      UTC offset in the direction of the western hemisphere. [default: 0]
  --channel CH      Set a channel for the given log.
"#;

#[derive(RustcDecodable, Debug)]
struct Args {
    cmd_parse: bool,
    cmd_convert: bool,
    cmd_freq: bool,
    arg_file: Vec<String>,
    arg_informat: Option<String>,
    arg_outformat: Option<String>,
    flag_help: bool,
    flag_version: bool,
    flag_date: Option<String>,
    flag_tz: i32,
    flag_channel: Option<String>
}

fn error(e: Box<Error>) -> ! {
    println!("{}", e.description());
    let mut e = e.cause();
    while let Some(err) = e {
        println!("\t{}", err.description());
        e = err.cause();
    }
    process::exit(1)
}

fn main() {
    env_logger::init().unwrap();
    let args: Args = Docopt::new(USAGE)
               .and_then(|d| d.decode())
               .unwrap_or_else(|e| e.exit());
    if args.flag_help {
        println!("{}", USAGE);
        process::exit(1)
    }

    let context = Context {
        timezone: FixedOffset::west(args.flag_tz),
        override_date: args.flag_date.and_then(|d| NaiveDate::from_str(&d).ok()),
        channel: args.flag_channel.clone()
    };

    if args.cmd_parse {
        let mut parser = format::energymech::Energymech;
        let formatter = format::binary::Binary;
        for file in args.arg_file {
            let f: BufReader<File> = BufReader::new(File::open(file).unwrap());
            let iter = parser.decode(&context, f);
            for e in iter {
                info!("Parsed: {:?}", e);
                drop(formatter.encode(&context, io::stdout(), &e.unwrap()));
            }
        }
    }

    if args.cmd_convert {
        let stdin = io::stdin();

        let informat = args.arg_informat.expect("Must provide informat");
        let outformat = args.arg_outformat.expect("Must provide outformat");
        let mut decoder = format::decoder(&informat).expect("Decoder not available");
        let encoder = format::encoder(&outformat).expect("Encoder not available");

        let mut lock = stdin.lock();
        for e in decoder.decode_box(&context, &mut lock) {
            match e {
                Ok(e) => {
                    let _ = encoder.encode(&context, &mut io::stdout(), &e);
                },
                Err(e) => error(Box::new(e))
            }
        }
    }

    if args.cmd_freq {
        struct Person {
            lines: u32,
            words: u32
        }

        fn words(s: &str) -> u32 {
            s.split_whitespace().filter(|s| !s.is_empty()).count() as u32
        }

        fn strip_nick_prefix(s: &str) -> &str {
            if s.is_empty() { return s }
            match s.as_bytes()[0] {
                b'~' | b'&' | b'@' | b'%' | b'+' => &s[1..],
                _ => s
            }
        }

        let stdin = io::stdin();

        let mut stats: HashMap<String, Person> = HashMap::new();
        let context = Context {
            timezone: FixedOffset::west(0),
            override_date: Some(NaiveDate::from_ymd(2015, 6, 10)),
            channel: Some("#code".to_owned())
        };

        let mut parser = format::weechat3::Weechat3;
        for e in parser.decode(&context, stdin.lock()) {
            let m = match e {
                Ok(m) => m,
                Err(err) => panic!(err)
            };

            match m {
                Event { ty: Type::Msg { ref from, ref content, .. }, .. } => {
                    let nick = strip_nick_prefix(from);
                    if stats.contains_key(nick) {
                        let p: &mut Person = stats.get_mut(nick).unwrap();
                        p.lines += 1;
                        p.words += words(content);
                    } else {
                        stats.insert(nick.to_owned(), Person {
                            lines: 1,
                            words: words(content)
                        });
                    }
                },
                _ => ()
            }
        }

        let mut stats: Vec<(String, Person)> = stats.into_iter().collect();
        stats.sort_by(|&(_, ref a), &(_, ref b)| b.words.cmp(&a.words));

        for &(ref name, ref stat) in stats.iter().take(10) {
            println!("{}:\n\tLines: {}\n\tWords: {}", name, stat.lines, stat.words)
        }
    }
}
<commit_msg>Use existing context, instead of mock context for freq<commit_after>// Copyright 2015 Till Höppner
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#![feature(libc, plugin)]
#![plugin(regex_macros)]

extern crate ilc;
extern crate chrono;
extern crate docopt;
extern crate rustc_serialize;
extern crate libc;
extern crate regex;
#[macro_use]
extern crate log;
extern crate env_logger;

use std::process;
use std::io::{ self, BufReader };
use std::fs::File;
use std::error::Error;
use std::str::FromStr;
use std::collections::HashMap;

use docopt::Docopt;

use chrono::offset::fixed::FixedOffset;
use chrono::naive::date::NaiveDate;

use ilc::context::Context;
use ilc::format::{ self, Encode, Decode, DecodeBox };
use ilc::event::{ Event, Type };

static USAGE: &'static str = r#"
d8b   888
Y8P   888
      888
888   888    .d8888b
888   888   d88P"
888   888   888
888   888   Y88b.
888   888    "Y8888P

A converter and statistics utility for IRC log files.

Usage:
  ilc parse <file>...
  ilc convert <informat> <outformat> [--date DATE] [--tz SECS] [--channel CH]
  ilc freq
  ilc (-h | --help | -v | --version)

Options:
  -h --help         Show this screen.
  -v --version      Show the version (duh).
  --date DATE       Override the date for this log. ISO 8601, YYYY-MM-DD.
  --tz SECONDS      UTC offset in the direction of the western hemisphere. [default: 0]
  --channel CH      Set a channel for the given log.
"#;

#[derive(RustcDecodable, Debug)]
struct Args {
    cmd_parse: bool,
    cmd_convert: bool,
    cmd_freq: bool,
    arg_file: Vec<String>,
    arg_informat: Option<String>,
    arg_outformat: Option<String>,
    flag_help: bool,
    flag_version: bool,
    flag_date: Option<String>,
    flag_tz: i32,
    flag_channel: Option<String>
}

fn error(e: Box<Error>) -> ! {
    println!("{}", e.description());
    let mut e = e.cause();
    while let Some(err) = e {
        println!("\t{}", err.description());
        e = err.cause();
    }
    process::exit(1)
}

fn main() {
    env_logger::init().unwrap();
    let args: Args = Docopt::new(USAGE)
               .and_then(|d| d.decode())
               .unwrap_or_else(|e| e.exit());
    if args.flag_help {
        println!("{}", USAGE);
        process::exit(1)
    }

    let context = Context {
        timezone: FixedOffset::west(args.flag_tz),
        override_date: args.flag_date.and_then(|d| NaiveDate::from_str(&d).ok()),
        channel: args.flag_channel.clone()
    };

    if args.cmd_parse {
        let mut parser = format::energymech::Energymech;
        let formatter = format::binary::Binary;
        for file in args.arg_file {
            let f: BufReader<File> = BufReader::new(File::open(file).unwrap());
            let iter = parser.decode(&context, f);
            for e in iter {
                info!("Parsed: {:?}", e);
                drop(formatter.encode(&context, io::stdout(), &e.unwrap()));
            }
        }
    }

    if args.cmd_convert {
        let stdin = io::stdin();

        let informat = args.arg_informat.expect("Must provide informat");
        let outformat = args.arg_outformat.expect("Must provide outformat");
        let mut decoder = format::decoder(&informat).expect("Decoder not available");
        let encoder = format::encoder(&outformat).expect("Encoder not available");

        let mut lock = stdin.lock();
        for e in decoder.decode_box(&context, &mut lock) {
            match e {
                Ok(e) => {
                    let _ = encoder.encode(&context, &mut io::stdout(), &e);
                },
                Err(e) => error(Box::new(e))
            }
        }
    }

    if args.cmd_freq {
        struct Person {
            lines: u32,
            words: u32
        }

        fn words(s: &str) -> u32 {
            s.split_whitespace().filter(|s| !s.is_empty()).count() as u32
        }

        fn strip_nick_prefix(s: &str) -> &str {
            if s.is_empty() { return s }
            match s.as_bytes()[0] {
                b'~' | b'&' | b'@' | b'%' | b'+' => &s[1..],
                _ => s
            }
        }

        let stdin = io::stdin();

        let mut stats: HashMap<String, Person> = HashMap::new();

        let mut parser = format::weechat3::Weechat3;
        for e in parser.decode(&context, stdin.lock()) {
            let m = match e {
                Ok(m) => m,
                Err(err) => panic!(err)
            };

            match m {
                Event { ty: Type::Msg { ref from, ref content, .. }, .. } => {
                    let nick = strip_nick_prefix(from);
                    if stats.contains_key(nick) {
                        let p: &mut Person = stats.get_mut(nick).unwrap();
                        p.lines += 1;
                        p.words += words(content);
                    } else {
                        stats.insert(nick.to_owned(), Person {
                            lines: 1,
                            words: words(content)
                        });
                    }
                },
                _ => ()
            }
        }

        let mut stats: Vec<(String, Person)> = stats.into_iter().collect();
        stats.sort_by(|&(_, ref a), &(_, ref b)| b.words.cmp(&a.words));

        for &(ref name, ref stat) in stats.iter().take(10) {
            println!("{}:\n\tLines: {}\n\tWords: {}", name, stat.lines, stat.words)
        }
    }
}
<|endoftext|>


<commit_before>extern crate hyper;
extern crate url;

use std::fs::OpenOptions;
use std::io::Read;
use std::sync::mpsc::channel;
use std::sync::mpsc::Sender;
use std::io::SeekFrom;
use std::io::Seek;
use std::thread;
use std::fs::{self, File};
use std::io::Write;
use std::path::PathBuf;
use std::process::Command;
use hyper::client::RequestBuilder;
use hyper::Url as DownloadUrl;
use std::collections::HashMap;
use hyper::header::{Connection, AcceptRanges};
use hyper::header::{ByteRangeSpec, Range, ContentLength};
use hyper::Client;
use url::Url;

#[derive(Debug)]
pub enum State {
    Initial,
    Ready,
    Downloading,
    Completed(u64),
    Paused,
    Stopped,
}

#[derive(Debug)]
struct Downloader {
    id: usize,
    url: String,
    start: u64,
    end: u64,
    file_name: String,
    cursor: u64,
}

impl Downloader {
    fn new(id: usize, url: &str, start: u64, end: u64, file_name: &str, cursor: u64) -> Self {
        Downloader {
            id: id,
            url: url.to_owned(),
            start: start,
            end: end,
            file_name: file_name.to_owned(),
            cursor: cursor,
        }
    }
    fn download(&self, sender:Sender<String>) {
        let client = Client::new();
        println!("Worker {} Downloading", self.id);
        let mut res = client.get(&self.url)
                            .header(if self.end == 0 {
                                Range::Bytes(vec![ByteRangeSpec::AllFrom(self.start)])
                            } else {
                                Range::bytes(self.start, self.end)
                            })
                            .header(Connection::keep_alive())
                            .send()
                            .unwrap();
        let mut body: Vec<u8> = Vec::new();
        res.read_to_end(&mut body).unwrap();
        // Id incremented file parts
        let file_name = format!("{}{}", self.file_name, self.id);
        let mut file = DownloadManager::request_file(&file_name[..]);
        file.write_all(body.as_slice());
        sender.send(file_name).unwrap();
    }
}

#[derive(Debug)]
pub struct DownloadManager {
    task_queue: Vec<Downloader>,
    url: Option<Url>,
    max_connection: usize,
    file_path: Option<PathBuf>,
    state: State,
    block_size: usize,
    resume: bool,
    complete_queue: Vec<String>
}

impl DownloadManager {
    pub fn new() -> Self {
        DownloadManager {
            task_queue: vec![],
            url: None,
            max_connection: 0,
            file_path: None,
            state: State::Initial,
            block_size: 1024,
            resume: false,
            complete_queue: vec![]
        }
    }
    pub fn add_url(&mut self, url: Url) -> &mut DownloadManager {
        self.url = Some(url);
        self
    }
    pub fn max_connection(&mut self, max_con: usize) -> &mut DownloadManager {
        self.max_connection = max_con;
        self
    }
    pub fn file(&mut self, file_path: &str) -> &mut DownloadManager {
        self.file_path = Some(PathBuf::from(file_path));
        self
    }

    pub fn finish(&mut self) {
        self.state = State::Ready;
    }
    pub fn start(&mut self) -> State {

        let mut content_length: u64 = 0;
        let (tx, rx) = channel();

        match self.check_resume() {
            (true, len) => {
                let ContentLength(length) = len;
                content_length = length;
            }
            (false, _) => println!("Download does not support resume"),
        }

        let mut start_range: u64 = 0;
        let mut end_range: u64 = (content_length / self.max_connection as u64) - 1;
        let mut parts_suffix = 0;
        let file_path = "./".to_owned() + self.file_path.clone().unwrap().to_str().unwrap();
        let url_str = self.url.clone().unwrap().to_string();
        while !(end_range > content_length) {
            let worker = Downloader::new(parts_suffix,&url_str,start_range,end_range,&file_path,start_range);
            self.task_queue.push(worker);
            start_range = end_range + 1;
            end_range = ((start_range - 1) * 2)+1;
            parts_suffix+=1;
            }

            let mut remaining_bytes = content_length - start_range;
            if remaining_bytes != 0 {
                let last_range = (start_range+remaining_bytes)-1;
                let last_worker = Downloader::new(parts_suffix, &url_str, start_range, last_range, &file_path, start_range);
                self.task_queue.push(last_worker);
            }

            for i in 0..self.task_queue.len() {
                self.task_queue[i].download(tx.clone());
                self.complete_queue.push(rx.recv().unwrap());
            }
        State::Completed(content_length)
    }

    fn join(&self,file_path:&String) {

        let final_download = OpenOptions::new()
            .read(true)
            .write(true)
            .create(true)
            .append(true)
            .open(&file_path).unwrap();

        let joiner = Command::new("python")
                                 .arg("join.py")
                                 .arg(&self.complete_queue.len().to_string())
                                 .output()
                                 .unwrap_or_else(|e| { panic!("failed to execute process: {}", e) });
        for i in self.complete_queue.iter().map(|ref i|{
                println!("{:?}",i );
                fs::remove_file(i);
            }){};

    }

    fn check_resume(&self) -> (bool, ContentLength) {
        let client = Client::new();
        let url_str = self.url.clone().unwrap().to_string();
        let head_req = client.head(DownloadUrl::parse(&url_str).unwrap());
        match head_req.send() {
            Ok(res) => {
                (res.headers.has::<AcceptRanges>(),
                 *res.headers.get::<ContentLength>().unwrap())
            }
            Err(_) => (false, ContentLength(0)),
        }
    }

    fn request_file(path: &str) -> File {
        let file = OpenOptions::new()
                       .read(true)
                       .write(true)
                       .create(true)
                       .open(path);
        if let Ok(file) = file {
            file
        } else {
            panic!("File open error");
        }
    }
}

fn main() {

}

#[test]
fn test_download_sublime_deb_package() {
    let mut manager = DownloadManager::new();
    let download_url = Url::parse("https://download.sublimetext.com/sublime-text_build-3103_amd64.deb").unwrap();
    manager.add_url(download_url.clone())
           .max_connection(4)
           .file(
            match download_url.path() {
                Some(path_vec) => &path_vec[path_vec.len()-1],
                None => "file.txt"
                }
            )
           .finish();

    let download_thread = thread::spawn(move || {
        match manager.start() {
            State::Completed(bytes) => { println!("Download complete of {} bytes",bytes ); },
            _ => {}
        }
        let joiner = Command::new("python")
                                 .arg("join.py")
                                 .arg("6")
                                 .output()
                                 .unwrap_or_else(|e| { panic!("failed to execute process: {}", e) });
    }).join();
}<commit_msg>run through rustfmt<commit_after>extern crate hyper;
extern crate url;

use std::fs::OpenOptions;
use std::io::Read;
use std::sync::mpsc::channel;
use std::sync::mpsc::Sender;
use std::io::SeekFrom;
use std::io::Seek;
use std::thread;
use std::fs::{self, File};
use std::io::Write;
use std::path::PathBuf;
use std::process::Command;
use hyper::client::RequestBuilder;
use hyper::Url as DownloadUrl;
use std::collections::HashMap;
use hyper::header::{Connection, AcceptRanges};
use hyper::header::{ByteRangeSpec, Range, ContentLength};
use hyper::Client;
use url::Url;

#[derive(Debug)]
pub enum State {
    Initial,
    Ready,
    Downloading,
    Completed(u64),
    Paused,
    Stopped,
}

#[derive(Debug)]
struct Downloader {
    id: usize,
    url: String,
    start: u64,
    end: u64,
    file_name: String,
    cursor: u64,
}

impl Downloader {
    fn new(id: usize, url: &str, start: u64, end: u64, file_name: &str, cursor: u64) -> Self {
        Downloader {
            id: id,
            url: url.to_owned(),
            start: start,
            end: end,
            file_name: file_name.to_owned(),
            cursor: cursor,
        }
    }
    fn download(&self, sender: Sender<String>) {
        let client = Client::new();
        println!("Worker {} Downloading", self.id);
        let mut res = client.get(&self.url)
                            .header(if self.end == 0 {
                                Range::Bytes(vec![ByteRangeSpec::AllFrom(self.start)])
                            } else {
                                Range::bytes(self.start, self.end)
                            })
                            .header(Connection::keep_alive())
                            .send()
                            .unwrap();
        let mut body: Vec<u8> = Vec::new();
        res.read_to_end(&mut body).unwrap();
        // Id incremented file parts
        let file_name = format!("{}{}", self.file_name, self.id);
        let mut file = DownloadManager::request_file(&file_name[..]);
        file.write_all(body.as_slice());
        sender.send(file_name).unwrap();
    }
}

#[derive(Debug)]
pub struct DownloadManager {
    task_queue: Vec<Downloader>,
    url: Option<Url>,
    max_connection: usize,
    file_path: Option<PathBuf>,
    state: State,
    block_size: usize,
    resume: bool,
    complete_queue: Vec<String>,
}

impl DownloadManager {
    pub fn new() -> Self {
        DownloadManager {
            task_queue: vec![],
            url: None,
            max_connection: 0,
            file_path: None,
            state: State::Initial,
            block_size: 1024,
            resume: false,
            complete_queue: vec![],
        }
    }
    pub fn add_url(&mut self, url: Url) -> &mut DownloadManager {
        self.url = Some(url);
        self
    }
    pub fn max_connection(&mut self, max_con: usize) -> &mut DownloadManager {
        self.max_connection = max_con;
        self
    }
    pub fn file(&mut self, file_path: &str) -> &mut DownloadManager {
        self.file_path = Some(PathBuf::from(file_path));
        self
    }

    pub fn finish(&mut self) {
        self.state = State::Ready;
    }
    pub fn start(&mut self) -> State {

        let mut content_length: u64 = 0;
        let (tx, rx) = channel();

        match self.check_resume() {
            (true, len) => {
                let ContentLength(length) = len;
                content_length = length;
            }
            (false, _) => println!("Download does not support resume"),
        }

        let mut start_range: u64 = 0;
        let mut end_range: u64 = (content_length / self.max_connection as u64) - 1;
        let mut parts_suffix = 0;
        let file_path = "./".to_owned() + self.file_path.clone().unwrap().to_str().unwrap();
        let url_str = self.url.clone().unwrap().to_string();
        while !(end_range > content_length) {
            let worker = Downloader::new(parts_suffix,
                                         &url_str,
                                         start_range,
                                         end_range,
                                         &file_path,
                                         start_range);
            self.task_queue.push(worker);
            start_range = end_range + 1;
            end_range = ((start_range - 1) * 2) + 1;
            parts_suffix += 1;
        }

        let mut remaining_bytes = content_length - start_range;
        if remaining_bytes != 0 {
            let last_range = (start_range + remaining_bytes) - 1;
            let last_worker = Downloader::new(parts_suffix,
                                              &url_str,
                                              start_range,
                                              last_range,
                                              &file_path,
                                              start_range);
            self.task_queue.push(last_worker);
        }

        for i in 0..self.task_queue.len() {
            self.task_queue[i].download(tx.clone());
            self.complete_queue.push(rx.recv().unwrap());
        }
        State::Completed(content_length)
    }

    fn join(&self, file_path: &String) {

        let final_download = OpenOptions::new()
                                 .read(true)
                                 .write(true)
                                 .create(true)
                                 .append(true)
                                 .open(&file_path)
                                 .unwrap();

        let joiner = Command::new("python")
                         .arg("join.py")
                         .arg(&self.complete_queue.len().to_string())
                         .output()
                         .unwrap_or_else(|e| panic!("failed to execute process: {}", e));
        for i in self.complete_queue.iter().map(|ref i| {
            println!("{:?}", i);
            fs::remove_file(i);
        }) {}

    }

    fn check_resume(&self) -> (bool, ContentLength) {
        let client = Client::new();
        let url_str = self.url.clone().unwrap().to_string();
        let head_req = client.head(DownloadUrl::parse(&url_str).unwrap());
        match head_req.send() {
            Ok(res) => {
                (res.headers.has::<AcceptRanges>(),
                 *res.headers.get::<ContentLength>().unwrap())
            }
            Err(_) => (false, ContentLength(0)),
        }
    }

    fn request_file(path: &str) -> File {
        let file = OpenOptions::new()
                       .read(true)
                       .write(true)
                       .create(true)
                       .open(path);
        if let Ok(file) = file {
            file
        } else {
            panic!("File open error");
        }
    }
}

fn main() {}

#[test]
fn test_download_sublime_deb_package() {
    let mut manager = DownloadManager::new();
    let download_url = Url::parse("https://download.sublimetext.\
                                   com/sublime-text_build-3103_amd64.deb")
                           .unwrap();
    manager.add_url(download_url.clone())
           .max_connection(4)
           .file(match download_url.path() {
               Some(path_vec) => &path_vec[path_vec.len() - 1],
               None => "file.txt",
           })
           .finish();

    let download_thread = thread::spawn(move || {
                              match manager.start() {
                                  State::Completed(bytes) => {
                                      println!("Download complete of {} bytes", bytes);
                                  }
                                  _ => {}
                              }
                              let joiner = Command::new("python")
                                               .arg("join.py")
                                               .arg("6")
                                               .output()
                                               .unwrap_or_else(|e| {
                                                   panic!("failed to execute process: {}", e)
                                               });
                          })
                              .join();
}
<|endoftext|>


<commit_before>extern crate clap;

use clap::{Arg, App};
use std::cmp::Ordering::Equal;
use std::error::Error;
use std::f32;
use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
use std::path::Path;
use std::str::FromStr;

fn print_results(mut timings: Vec<f32>) {
    timings.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Equal));

    let start: usize = 0;
    let end: usize = timings.len() - 1;
    if end == 0 {
        return;
    };
    let median: usize = end / 2;
    let p70: usize = end.checked_mul(70).unwrap().checked_div(100).unwrap();
    let p80: usize = end.checked_mul(80).unwrap().checked_div(100).unwrap();
    let p95: usize = end.checked_mul(95).unwrap().checked_div(100).unwrap();
    let p99: usize = end.checked_mul(99).unwrap().checked_div(100).unwrap();

    println!("");
    println!("Results:");
    println!("    Total count:   {}", timings.len());
    println!("    Min:           {}", timings[start]);
    println!("    Median:        {} ({})", timings[median], median + 1);
    println!("    70 percentile: {} ({})", timings[p70], p70 + 1);
    println!("    80 percentile: {} ({})", timings[p80], p80 + 1);
    println!("    95 percentile: {} ({})", timings[p95], p95 + 1);
    println!("    99 percentile: {} ({})", timings[p99], p99 + 1);
    println!("    Max:           {} ({})", timings[end], end + 1);
}

// This is the main function
fn main() {

    let matches = App::new("Calculate response time percentiles on logs")
        .version("0.0.1")
        .author("Vladimir Prokhoda <vladimirbright@gmail.com>")
        .arg(Arg::with_name("separator")
            .short("s")
            .long("separator")
            .default_value(" ")
            .help("Column separator")
            .takes_value(true))
        .arg(Arg::with_name("column")
            .short("c")
            .long("column")
            .default_value("8")
            .use_delimiter(false)
            .help("Column number to use")
            .takes_value(true))
        .arg(Arg::with_name("input")
            .help("Sets the input file to use")
            .required(true)
            .index(1))
            .arg(Arg::with_name("print")
            .short("p")
            .long("print")
            .help("Print matched rows"))
        .get_matches();

    let sep = match matches.value_of("separator") {
        Some(f) => f,
        None => panic!("provide --separator option"),
    };
    let separator = match sep {
        "\\t" => "\t",
        _ => sep,
    };
    let column = match matches.value_of("column") {
        Some(f) => f,
        None => panic!("provide --column option"),
    };
    let timing_index: usize = match usize::from_str(column) {
        Ok(f) => f,
        Err(_) => panic!("--column must be integer"),
    };

	let file_path = matches.value_of("input").unwrap();
    let path = Path::new(&file_path);
    let display = path.display();

    let f = match File::open(&path) {
        Err(why) => panic!("couldn't open {}: {}", display, why.description()),
        Ok(f) => f,
    };
	let file = BufReader::with_capacity(128 * 1024, &f);

    let mut timings: Vec<f32> = vec![];

	for line in file.lines().filter_map(|result| result.ok()) {
		let v: Vec<&str> = line.split(separator).collect();
        let t = v[timing_index];

        if t == "-" {
            continue;
        }
		if matches.is_present("print") {
			println!("{}", line);
		}
        match f32::from_str(&t) {
            Err(_) => continue,
            Ok(f) => timings.push(f),
        };
    }

    print_results(timings);
}
<commit_msg>Explicit error when the column has no numbers<commit_after>extern crate clap;

use clap::{Arg, App};
use std::cmp::Ordering::Equal;
use std::error::Error;
use std::f32;
use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
use std::path::Path;
use std::str::FromStr;

fn print_results(mut timings: Vec<f32>) {
    timings.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Equal));

    let start: usize = 0;
    let end: usize = timings.len() - 1;
    if end == 0 {
        return;
    };
    let median: usize = end / 2;
    let p70: usize = end.checked_mul(70).unwrap().checked_div(100).unwrap();
    let p80: usize = end.checked_mul(80).unwrap().checked_div(100).unwrap();
    let p95: usize = end.checked_mul(95).unwrap().checked_div(100).unwrap();
    let p99: usize = end.checked_mul(99).unwrap().checked_div(100).unwrap();

    println!("");
    println!("Results:");
    println!("    Total count:   {}", timings.len());
    println!("    Min:           {}", timings[start]);
    println!("    Median:        {} ({})", timings[median], median + 1);
    println!("    70 percentile: {} ({})", timings[p70], p70 + 1);
    println!("    80 percentile: {} ({})", timings[p80], p80 + 1);
    println!("    95 percentile: {} ({})", timings[p95], p95 + 1);
    println!("    99 percentile: {} ({})", timings[p99], p99 + 1);
    println!("    Max:           {} ({})", timings[end], end + 1);
}

// This is the main function
fn main() {

    let matches = App::new("Calculate response time percentiles on logs")
        .version("0.0.1")
        .author("Vladimir Prokhoda <vladimirbright@gmail.com>")
        .arg(Arg::with_name("separator")
            .short("s")
            .long("separator")
            .default_value(" ")
            .help("Column separator")
            .takes_value(true))
        .arg(Arg::with_name("column")
            .short("c")
            .long("column")
            .default_value("8")
            .use_delimiter(false)
            .help("Column number to use")
            .takes_value(true))
        .arg(Arg::with_name("input")
            .help("Sets the input file to use")
            .required(true)
            .index(1))
            .arg(Arg::with_name("print")
            .short("p")
            .long("print")
            .help("Print matched rows"))
        .get_matches();

    let sep = match matches.value_of("separator") {
        Some(f) => f,
        None => panic!("provide --separator option"),
    };
    let separator = match sep {
        "\\t" => "\t",
        _ => sep,
    };
    let column = match matches.value_of("column") {
        Some(f) => f,
        None => panic!("provide --column option"),
    };
    let timing_index: usize = match usize::from_str(column) {
        Ok(f) => f,
        Err(_) => panic!("--column must be integer"),
    };

	let file_path = matches.value_of("input").unwrap();
    let path = Path::new(&file_path);
    let display = path.display();

    let f = match File::open(&path) {
        Err(why) => panic!("couldn't open {}: {}", display, why.description()),
        Ok(f) => f,
    };
	let file = BufReader::with_capacity(128 * 1024, &f);

    let mut timings: Vec<f32> = vec![];
    let mut has_errors: bool = false;

	for line in file.lines().filter_map(|result| result.ok()) {
		let v: Vec<&str> = line.split(separator).collect();
        let t = v[timing_index];

        if t == "-" {
            continue;
        }
		if matches.is_present("print") {
			println!("{}", line);
		}
        match f32::from_str(&t) {
            Err(_) => {
                has_errors = true;
                continue;
            },
            Ok(f) => timings.push(f),
        };
    }

    if timings.len() == 0 && has_errors {
        panic!("column {} does not contain valid numbers", timing_index)
    }

    print_results(timings);
}
<|endoftext|>


<commit_before>
/// Returns version number 0.2
pub fn get_version() -> String {
    "version 0.2".to_string()
}

fn main() {
    println!("{}", get_version());
}
<commit_msg>Update main.rs<commit_after>
/// Returns version number 0.2.1
pub fn get_version() -> String {
    "version 0.2.1".to_string()
}

fn main() {
    println!("{}", get_version());
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate clap;

mod args;
mod macros;

use crate::args::AppArgs;
use monolith::html::{html_to_dom, stringify_document, walk_and_embed_assets};
use monolith::http::retrieve_asset;
use monolith::utils::is_valid_url;
use reqwest::blocking::Client;
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
use std::collections::HashMap;
use std::fs::{remove_file, File};
use std::io::{Error, Write};
use std::time::Duration;

fn create_file(file_path: &String, content: String) -> Result<(), Error> {
    let file = File::create(file_path.as_str());

    let mut file = match file {
        Ok(file) => file,
        Err(error) => return Err(error),
    };

    if content != str!() {
        file.write_all(content.as_bytes())?;
        file.write_all("\n".as_bytes())?;
        file.sync_all()?;
    } else {
        // Remove the file right away if it had no content
        remove_file(file_path.as_str())?;
    }

    Ok(())
}

fn main() {
    let app_args = AppArgs::get();
    let cache = &mut HashMap::new();

    // Attempt to create output file
    if app_args.output != str!() {
        create_file(&app_args.output, str!()).unwrap();
    }

    if is_valid_url(app_args.url_target.as_str()) {
        // Initialize client
        let mut header_map = HeaderMap::new();
        match HeaderValue::from_str(&app_args.user_agent) {
            Ok(header) => header_map.insert(USER_AGENT, header),
            Err(err) => {
                eprintln!("Invalid user agent! {}", err);
                return;
            }
        };
        let client = Client::builder()
            .timeout(Duration::from_secs(10))
            .danger_accept_invalid_certs(app_args.insecure)
            .default_headers(header_map)
            .build()
            .expect("Failed to initialize HTTP client");

        // Retrieve root document
        let (data, final_url) = retrieve_asset(
            cache,
            &client,
            app_args.url_target.as_str(),
            false,
            "",
            app_args.silent,
        )
        .unwrap();
        let downloaded_time = time::now();
        let dom = html_to_dom(&data);

        walk_and_embed_assets(
            cache,
            &client,
            &final_url,
            &dom.document,
            app_args.no_css,
            app_args.no_js,
            app_args.no_images,
            app_args.silent,
            app_args.no_frames,
        );

        let mut html: String = stringify_document(
            &dom.document,
            app_args.no_css,
            app_args.no_frames,
            app_args.no_js,
            app_args.no_images,
            app_args.isolate,
        );

        if !app_args.no_context {
            html.insert_str(
                0,
                &format!(
                    "<!--- Downloaded from {} on {}using {} v{} -->\n",
                    &final_url,
                    downloaded_time.rfc822(),
                    env!("CARGO_PKG_NAME"),
                    env!("CARGO_PKG_VERSION"),
                ),
            );
        }

        if app_args.output == str!() {
            println!("{}", html);
        } else {
            create_file(&app_args.output, html).unwrap();
        }
    }
}
<commit_msg>Clean user, pass, and fragment from URL before writing<commit_after>#[macro_use]
extern crate clap;

mod args;
mod macros;

use crate::args::AppArgs;
use monolith::html::{html_to_dom, stringify_document, walk_and_embed_assets};
use monolith::http::retrieve_asset;
use monolith::utils::is_valid_url;
use reqwest::blocking::Client;
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
use reqwest::Url;
use std::collections::HashMap;
use std::fs::{remove_file, File};
use std::io::{Error, Write};
use std::time::Duration;

fn create_file(file_path: &String, content: String) -> Result<(), Error> {
    let file = File::create(file_path.as_str());

    let mut file = match file {
        Ok(file) => file,
        Err(error) => return Err(error),
    };

    if content != str!() {
        file.write_all(content.as_bytes())?;
        file.write_all("\n".as_bytes())?;
        file.sync_all()?;
    } else {
        // Remove the file right away if it had no content
        remove_file(file_path.as_str())?;
    }

    Ok(())
}

fn main() {
    let app_args = AppArgs::get();
    let cache = &mut HashMap::new();

    // Attempt to create output file
    if app_args.output != str!() {
        create_file(&app_args.output, str!()).unwrap();
    }

    if is_valid_url(app_args.url_target.as_str()) {
        // Initialize client
        let mut header_map = HeaderMap::new();
        match HeaderValue::from_str(&app_args.user_agent) {
            Ok(header) => header_map.insert(USER_AGENT, header),
            Err(err) => {
                eprintln!("Invalid user agent! {}", err);
                return;
            }
        };
        let client = Client::builder()
            .timeout(Duration::from_secs(10))
            .danger_accept_invalid_certs(app_args.insecure)
            .default_headers(header_map)
            .build()
            .expect("Failed to initialize HTTP client");

        // Retrieve root document
        let (data, final_url) = retrieve_asset(
            cache,
            &client,
            app_args.url_target.as_str(),
            false,
            "",
            app_args.silent,
        )
        .unwrap();
        let downloaded_time = time::now();
        let dom = html_to_dom(&data);

        walk_and_embed_assets(
            cache,
            &client,
            &final_url,
            &dom.document,
            app_args.no_css,
            app_args.no_js,
            app_args.no_images,
            app_args.silent,
            app_args.no_frames,
        );

        let mut html: String = stringify_document(
            &dom.document,
            app_args.no_css,
            app_args.no_frames,
            app_args.no_js,
            app_args.no_images,
            app_args.isolate,
        );

        if !app_args.no_context {
            // Safe to unwrap: We just put this through an HTTP request
            let mut clean_url = Url::parse(&final_url).unwrap();
            clean_url.set_fragment(None);
            // Safe to unwrap: must have a protocol and thus base 'cause we just used it.
            clean_url.set_password(None).unwrap();
            clean_url.set_username("").unwrap();
            html.insert_str(
                0,
                &format!(
                    "<!--- Downloaded from {} on {}using {} v{} -->\n",
                    &clean_url,
                    downloaded_time.rfc822(),
                    env!("CARGO_PKG_NAME"),
                    env!("CARGO_PKG_VERSION"),
                ),
            );
        }

        if app_args.output == str!() {
            println!("{}", html);
        } else {
            create_file(&app_args.output, html).unwrap();
        }
    }
}
<|endoftext|>


<commit_before>#![feature(core)]
#![feature(path)]

#![feature(plugin)]
#![plugin(regex_macros)]
extern crate regex;
extern crate git2;
extern crate "rustc-serialize" as rustc_serialize;
extern crate docopt;

use git2::{Repository, Branch, BranchType, DiffLine,
    Commit, Diff, DiffFormat, DiffDelta, DiffHunk, Error, Oid};
use docopt::Docopt;
use std::old_io as io;
use std::collections::HashMap;
use std::old_io::BufferedReader;
use std::old_io::File;
use std::str;
use regex::Regex;

// Write the Docopt usage string.
static USAGE: &'static str = "
Usage: transit <repo> [<old> <new>]

If no commits are given, transit will revwalk from latest to oldest.
";

#[derive(RustcDecodable, Debug)]
struct Args {
    arg_repo: String,
    arg_old: Option<String>,
    arg_new: Option<String>,
}

fn main() {
    // Parse the args above or die.
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());
    // Pull up the revwalk.
    let path = Path::new(args.arg_repo);
    let repo = Repository::discover(&path)
        .ok().expect("Unable to find repo.");
    // If we can destructures the two optional args into real things.
    if let (Some(old_string), Some(new_string)) = (args.arg_old, args.arg_new) {
        // Compare a specific commit pair.
        // Fist, get the commits. (Error checked)
        let old = Oid::from_str(&old_string[]).and_then(|oid| repo.find_commit(oid));
        let new = Oid::from_str(&new_string[]).and_then(|oid| repo.find_commit(oid));
        if old.is_ok() && new.is_ok() {
            let output = find_moves(&repo, &old.unwrap(), &new.unwrap()).unwrap();
            make_output(output);
        } else {
            panic!("Commit ids were not valid.");
        }
    } else {
        // Revwalk.
        let mut revwalk = repo.revwalk()
            .ok().expect("Unable to get revwalk.");
        // Setup some options.
        revwalk.simplify_first_parent(); // TODO: Maybe remove?
        let mut flags = git2::Sort::empty();
        flags.insert(git2::SORT_TIME);
        flags.insert(git2::SORT_TOPOLOGICAL);
        revwalk.set_sorting(flags);
        // Push HEAD to the revwalk.
        revwalk.push_head()
            .ok().expect("Unable to push HEAD.");
        // We sadly must collect here to use `.windows()`
        let history = revwalk.filter_map(|id| repo.find_commit(id).ok())
            .collect::<Vec<Commit>>();
        // Walk through each pair of commits.
        for pair in history.windows(2) {let (old, new) = (&pair[1], &pair[0]);
			let output = find_moves(&repo, old.clone(), new.clone()).unwrap();
			// mangle the vectors here to make pretty output
			make_output(output);
		}
	}
}

fn make_output(output: Vec<Output>) {
	println!("make_output:");
    println!("\toutput.len()={}", output.len());
    for i in range(0, output.len()) {
            println!("\told_commit={}",         output[i].old_commit);
            println!("\tnew_commit={}",         output[i].new_commit);
            println!("\torigin_line={}",        output[i].origin_line);
            println!("\tdestintation_line={}",  output[i].destination_line);
            println!("\tnum_lines={}",          output[i].num_lines);
            println!("\tnew_filename={}",       output[i].new_filename);
            println!("\told_filename={}",       output[i].old_filename);
    }
}

#[derive(Debug, PartialEq, Eq)]
enum FoundState {
    Added, Deleted
}

fn dump_diffline(line: &DiffLine) {
    // 'origin' is wrapped in pipes to ease displaying space characters.
    print!("line: old={:?} new={:?} offset={} |origin|=|{}|\n      content={}",
             line.old_lineno(), line.new_lineno(), line.content_offset(),
             line.origin(), str::from_utf8(line.content()).unwrap());
}

fn dump_diffdelta(delta: &DiffDelta) {
    println!("delta: nfiles={} status={:?} old_file=(id={} path_bytes={:?} path={:?} tsize={}) new_file=(id={} path_bytes={:?} path={:?} tsize={})",
            delta.nfiles(), delta.status(),
            delta.old_file().id(), delta.old_file().path_bytes(), delta.old_file().path(), delta.old_file().size(),
            delta.new_file().id(), delta.new_file().path_bytes(), delta.new_file().path(), delta.new_file().size());
}

fn dump_diffhunk(hunk: &DiffHunk) {
    println!("hunk: old_start={} old_lines={} new_start={} new_lines={} header={}",
            hunk.old_start(), hunk.old_lines(),
            hunk.new_start(), hunk.new_lines(),
            str::from_utf8(hunk.header()).unwrap());
}

fn format_key(key: String) -> String {
    let remove_whitespace = regex!(r"\s{2,}"); // 2 or more whitespaces    // TODO Removes whitespace from a string.
    let trim = regex!(r"^[\s]+|[\s]+$");
    let result = remove_whitespace.replace_all(key.as_slice(), "");
    trim.replace_all(result.as_slice(), "")
}

#[derive(Debug)]
struct Found {
    filename: Path,
    key: String,
    state: FoundState,
    start_position: u32,
    line_count: u32,
}

fn find_keys(diff: Diff) -> Vec<Found> {

    #[derive(Debug)]
    enum State {
        Other, Addition, Deletion
    }

    let mut founds: Vec<Found> = Vec::new();

    let mut state = State::Other;
    let mut added = String::new();
    let mut deleted = String::new();
    let mut old_path = Path::new("");
    let mut new_path = Path::new("");

    let mut line_count: u32 = 0;
    let mut start_position: u32 = 0;

	// Read about this function in http://alexcrichton.com/git2-rs/git2/struct.Diff.html#method.print
	// It's a bit weird, but I think it will provide the necessary information.
	diff.print(DiffFormat::Patch, |delta, maybe_hunk, line| -> bool {

        assert!(delta.nfiles() == 2, "This only works on diffs between exactly 2 files. Found {} files.", delta.nfiles());

		// Thinking:
		//  * If is not a hunk, keep going.
		//  * If it's a hunk, do regex magic.
		//  * Stick regex output into a hashmap as a hash.
		//  * Later, we will iterate through and output pased on the values.
		// Filter out all the boring context lines.
		// If we're not interested in this line just return since it will iterate to the next.
		if maybe_hunk.is_none() { return true }; // Return early.

        //dump_diffline(&line);
        //dump_diffdelta(&delta);
        //dump_diffhunk(&maybe_hunk.unwrap());

        old_path = delta.old_file().path().unwrap().clone();
        new_path = delta.new_file().path().unwrap().clone();

		match line.origin() {
			// Additions
			'+' | '>' => {
                added.push_str(str::from_utf8(line.content()).unwrap());

                match state {
                    State::Deletion => {
                        founds.push(Found {
                            filename: old_path.clone(),
                            key: format_key(deleted.clone()),
                            state: FoundState::Deleted,
                            start_position: start_position,
                            line_count: line_count,
                        });
                        deleted = String::new();
                        line_count = 0;
                        start_position = line.new_lineno().unwrap();
                    },
                    State::Addition => {
                        line_count += 1;
                    },
                    State::Other => {
                        line_count = 1;
                        start_position = line.new_lineno().unwrap();
                    },
                }

                state = State::Addition;
				true
			},
			// Deletions
			'-' | '<' => {
                deleted.push_str(str::from_utf8(line.content()).unwrap());

                match state {
                    State::Addition => {
                        founds.push(Found {
                            filename: new_path.clone(),
                            key: format_key(added.clone()),
                            state: FoundState::Added,
                            start_position: start_position,
                            line_count: line_count,
                        });
                        added = String::new();
                        line_count = 0;
                        start_position = line.old_lineno().unwrap();
                    },
                    State::Deletion => {
                        line_count += 1;
                    },
                    State::Other => {
                        line_count = 1;
                        start_position = line.old_lineno().unwrap();
                    },
                }

                state = State::Deletion;
				true
			},
			// Other
			_         => {
                match state {
                    State::Addition => {
                        founds.push(Found {
                            filename: new_path.clone(),
                            key: format_key(added.clone()),
                            state: FoundState::Added,
                            start_position: start_position,
                            line_count: line_count,
                        });
                        added = String::new();
                    },
                    State::Deletion => {
                        founds.push(Found {
                            filename: old_path.clone(),
                            key: format_key(deleted.clone()),
                            state: FoundState::Deleted,
                            start_position: start_position,
                            line_count: line_count,
                        });
                        deleted = String::new();
                    },
                    _ => (),
                }

                state = State::Other;
                true
            }
		}
	});

    // Grab last one.
    match state {
        State::Addition => {
            if added.len() > 0 {
                founds.push(Found {
                    filename: new_path,
                    key: format_key(added.clone()),
                    state: FoundState::Added,
                    start_position: start_position,
                    line_count: line_count,
                });
            }
        },
        State::Deletion => {
            if deleted.len() > 0 {
                founds.push(Found {
                    filename: old_path,
                    key: format_key(deleted.clone()),
                    state: FoundState::Deleted,
                    start_position: start_position,
                    line_count: line_count,
                });
            }
        },
        _ => (),
    }

    return founds;
}

fn path_to_string(path: Path) -> String {
    String::from_utf8(path.into_vec()).unwrap()
}

fn find_moves(repo: &Repository, old: &Commit, new: &Commit) -> Result<Vec<Output>, Error> {
	let old_tree = try!(old.tree());
	let new_tree = try!(new.tree());
	// Build up a diff of the two trees.
	let diff = try!(Diff::tree_to_tree(repo, Some(&old_tree), Some(&new_tree), None));

    let founds: Vec<Found> = find_keys(diff);

    let mut moves: Vec<Output> = Vec::new();
    let mut map: HashMap<String, Found> = HashMap::new();

    for f in founds {
        if f.key.len() == 0 { continue; }

        if map.contains_key(&f.key) {
            let q = map.get(&f.key).unwrap();
            if q.line_count == f.line_count {

                assert!(f.state != q.state, "One state must be an addition and the other state must be a deletion.");

                let output: Output;

                match f.state {
                    FoundState::Added => {
                        output = Output {
                            old_commit: old.id(),
                            new_commit: new.id(),
                            old_filename: path_to_string(q.filename.clone()),
                            new_filename: path_to_string(f.filename.clone()),
                            origin_line: q.start_position,
                            destination_line: f.start_position,
                            num_lines: f.line_count,
                        };
                    },
                    FoundState::Deleted => {
                        output = Output {
                            old_commit: old.id(),
                            new_commit: new.id(),
                            old_filename: path_to_string(f.filename.clone()),
                            new_filename: path_to_string(q.filename.clone()),
                            origin_line: f.start_position,
                            destination_line: q.start_position,
                            num_lines: f.line_count,
                        };
                    },
                }

                moves.push(output);
            }
        } else {
            map.insert(f.key.clone(), f);
        }
    }

    return Ok(moves);
}

#[derive(Debug)]
struct Output {
    old_commit: Oid,
    new_commit: Oid,
    old_filename: String,
    new_filename: String,
    origin_line: u32,
    destination_line: u32,
    num_lines: u32,
}
<commit_msg>Print output lenght on same line as function name.<commit_after>#![feature(core)]
#![feature(path)]

#![feature(plugin)]
#![plugin(regex_macros)]
extern crate regex;
extern crate git2;
extern crate "rustc-serialize" as rustc_serialize;
extern crate docopt;

use git2::{Repository, Branch, BranchType, DiffLine,
    Commit, Diff, DiffFormat, DiffDelta, DiffHunk, Error, Oid};
use docopt::Docopt;
use std::old_io as io;
use std::collections::HashMap;
use std::old_io::BufferedReader;
use std::old_io::File;
use std::str;
use regex::Regex;

// Write the Docopt usage string.
static USAGE: &'static str = "
Usage: transit <repo> [<old> <new>]

If no commits are given, transit will revwalk from latest to oldest.
";

#[derive(RustcDecodable, Debug)]
struct Args {
    arg_repo: String,
    arg_old: Option<String>,
    arg_new: Option<String>,
}

fn main() {
    // Parse the args above or die.
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());
    // Pull up the revwalk.
    let path = Path::new(args.arg_repo);
    let repo = Repository::discover(&path)
        .ok().expect("Unable to find repo.");
    // If we can destructures the two optional args into real things.
    if let (Some(old_string), Some(new_string)) = (args.arg_old, args.arg_new) {
        // Compare a specific commit pair.
        // Fist, get the commits. (Error checked)
        let old = Oid::from_str(&old_string[]).and_then(|oid| repo.find_commit(oid));
        let new = Oid::from_str(&new_string[]).and_then(|oid| repo.find_commit(oid));
        if old.is_ok() && new.is_ok() {
            let output = find_moves(&repo, &old.unwrap(), &new.unwrap()).unwrap();
            make_output(output);
        } else {
            panic!("Commit ids were not valid.");
        }
    } else {
        // Revwalk.
        let mut revwalk = repo.revwalk()
            .ok().expect("Unable to get revwalk.");
        // Setup some options.
        revwalk.simplify_first_parent(); // TODO: Maybe remove?
        let mut flags = git2::Sort::empty();
        flags.insert(git2::SORT_TIME);
        flags.insert(git2::SORT_TOPOLOGICAL);
        revwalk.set_sorting(flags);
        // Push HEAD to the revwalk.
        revwalk.push_head()
            .ok().expect("Unable to push HEAD.");
        // We sadly must collect here to use `.windows()`
        let history = revwalk.filter_map(|id| repo.find_commit(id).ok())
            .collect::<Vec<Commit>>();
        // Walk through each pair of commits.
        for pair in history.windows(2) {let (old, new) = (&pair[1], &pair[0]);
			let output = find_moves(&repo, old.clone(), new.clone()).unwrap();
			// mangle the vectors here to make pretty output
			make_output(output);
		}
	}
}

fn make_output(output: Vec<Output>) {
	println!("make_output: output.len()={}", output.len());
    for i in range(0, output.len()) {
            println!("\told_commit={}",         output[i].old_commit);
            println!("\tnew_commit={}",         output[i].new_commit);
            println!("\torigin_line={}",        output[i].origin_line);
            println!("\tdestintation_line={}",  output[i].destination_line);
            println!("\tnum_lines={}",          output[i].num_lines);
            println!("\tnew_filename={}",       output[i].new_filename);
            println!("\told_filename={}",       output[i].old_filename);
    }
}

#[derive(Debug, PartialEq, Eq)]
enum FoundState {
    Added, Deleted
}

fn dump_diffline(line: &DiffLine) {
    // 'origin' is wrapped in pipes to ease displaying space characters.
    print!("line: old={:?} new={:?} offset={} |origin|=|{}|\n      content={}",
             line.old_lineno(), line.new_lineno(), line.content_offset(),
             line.origin(), str::from_utf8(line.content()).unwrap());
}

fn dump_diffdelta(delta: &DiffDelta) {
    println!("delta: nfiles={} status={:?} old_file=(id={} path_bytes={:?} path={:?} tsize={}) new_file=(id={} path_bytes={:?} path={:?} tsize={})",
            delta.nfiles(), delta.status(),
            delta.old_file().id(), delta.old_file().path_bytes(), delta.old_file().path(), delta.old_file().size(),
            delta.new_file().id(), delta.new_file().path_bytes(), delta.new_file().path(), delta.new_file().size());
}

fn dump_diffhunk(hunk: &DiffHunk) {
    println!("hunk: old_start={} old_lines={} new_start={} new_lines={} header={}",
            hunk.old_start(), hunk.old_lines(),
            hunk.new_start(), hunk.new_lines(),
            str::from_utf8(hunk.header()).unwrap());
}

fn format_key(key: String) -> String {
    let remove_whitespace = regex!(r"\s{2,}"); // 2 or more whitespaces    // TODO Removes whitespace from a string.
    let trim = regex!(r"^[\s]+|[\s]+$");
    let result = remove_whitespace.replace_all(key.as_slice(), "");
    trim.replace_all(result.as_slice(), "")
}

#[derive(Debug)]
struct Found {
    filename: Path,
    key: String,
    state: FoundState,
    start_position: u32,
    line_count: u32,
}

fn find_keys(diff: Diff) -> Vec<Found> {

    #[derive(Debug)]
    enum State {
        Other, Addition, Deletion
    }

    let mut founds: Vec<Found> = Vec::new();

    let mut state = State::Other;
    let mut added = String::new();
    let mut deleted = String::new();
    let mut old_path = Path::new("");
    let mut new_path = Path::new("");

    let mut line_count: u32 = 0;
    let mut start_position: u32 = 0;

	// Read about this function in http://alexcrichton.com/git2-rs/git2/struct.Diff.html#method.print
	// It's a bit weird, but I think it will provide the necessary information.
	diff.print(DiffFormat::Patch, |delta, maybe_hunk, line| -> bool {

        assert!(delta.nfiles() == 2, "This only works on diffs between exactly 2 files. Found {} files.", delta.nfiles());

		// Thinking:
		//  * If is not a hunk, keep going.
		//  * If it's a hunk, do regex magic.
		//  * Stick regex output into a hashmap as a hash.
		//  * Later, we will iterate through and output pased on the values.
		// Filter out all the boring context lines.
		// If we're not interested in this line just return since it will iterate to the next.
		if maybe_hunk.is_none() { return true }; // Return early.

        //dump_diffline(&line);
        //dump_diffdelta(&delta);
        //dump_diffhunk(&maybe_hunk.unwrap());

        old_path = delta.old_file().path().unwrap().clone();
        new_path = delta.new_file().path().unwrap().clone();

		match line.origin() {
			// Additions
			'+' | '>' => {
                added.push_str(str::from_utf8(line.content()).unwrap());

                match state {
                    State::Deletion => {
                        founds.push(Found {
                            filename: old_path.clone(),
                            key: format_key(deleted.clone()),
                            state: FoundState::Deleted,
                            start_position: start_position,
                            line_count: line_count,
                        });
                        deleted = String::new();
                        line_count = 0;
                        start_position = line.new_lineno().unwrap();
                    },
                    State::Addition => {
                        line_count += 1;
                    },
                    State::Other => {
                        line_count = 1;
                        start_position = line.new_lineno().unwrap();
                    },
                }

                state = State::Addition;
				true
			},
			// Deletions
			'-' | '<' => {
                deleted.push_str(str::from_utf8(line.content()).unwrap());

                match state {
                    State::Addition => {
                        founds.push(Found {
                            filename: new_path.clone(),
                            key: format_key(added.clone()),
                            state: FoundState::Added,
                            start_position: start_position,
                            line_count: line_count,
                        });
                        added = String::new();
                        line_count = 0;
                        start_position = line.old_lineno().unwrap();
                    },
                    State::Deletion => {
                        line_count += 1;
                    },
                    State::Other => {
                        line_count = 1;
                        start_position = line.old_lineno().unwrap();
                    },
                }

                state = State::Deletion;
				true
			},
			// Other
			_         => {
                match state {
                    State::Addition => {
                        founds.push(Found {
                            filename: new_path.clone(),
                            key: format_key(added.clone()),
                            state: FoundState::Added,
                            start_position: start_position,
                            line_count: line_count,
                        });
                        added = String::new();
                    },
                    State::Deletion => {
                        founds.push(Found {
                            filename: old_path.clone(),
                            key: format_key(deleted.clone()),
                            state: FoundState::Deleted,
                            start_position: start_position,
                            line_count: line_count,
                        });
                        deleted = String::new();
                    },
                    _ => (),
                }

                state = State::Other;
                true
            }
		}
	});

    // Grab last one.
    match state {
        State::Addition => {
            if added.len() > 0 {
                founds.push(Found {
                    filename: new_path,
                    key: format_key(added.clone()),
                    state: FoundState::Added,
                    start_position: start_position,
                    line_count: line_count,
                });
            }
        },
        State::Deletion => {
            if deleted.len() > 0 {
                founds.push(Found {
                    filename: old_path,
                    key: format_key(deleted.clone()),
                    state: FoundState::Deleted,
                    start_position: start_position,
                    line_count: line_count,
                });
            }
        },
        _ => (),
    }

    return founds;
}

fn path_to_string(path: Path) -> String {
    String::from_utf8(path.into_vec()).unwrap()
}

fn find_moves(repo: &Repository, old: &Commit, new: &Commit) -> Result<Vec<Output>, Error> {
	let old_tree = try!(old.tree());
	let new_tree = try!(new.tree());
	// Build up a diff of the two trees.
	let diff = try!(Diff::tree_to_tree(repo, Some(&old_tree), Some(&new_tree), None));

    let founds: Vec<Found> = find_keys(diff);

    let mut moves: Vec<Output> = Vec::new();
    let mut map: HashMap<String, Found> = HashMap::new();

    for f in founds {
        if f.key.len() == 0 { continue; }

        if map.contains_key(&f.key) {
            let q = map.get(&f.key).unwrap();
            if q.line_count == f.line_count {

                assert!(f.state != q.state, "One state must be an addition and the other state must be a deletion.");

                let output: Output;

                match f.state {
                    FoundState::Added => {
                        output = Output {
                            old_commit: old.id(),
                            new_commit: new.id(),
                            old_filename: path_to_string(q.filename.clone()),
                            new_filename: path_to_string(f.filename.clone()),
                            origin_line: q.start_position,
                            destination_line: f.start_position,
                            num_lines: f.line_count,
                        };
                    },
                    FoundState::Deleted => {
                        output = Output {
                            old_commit: old.id(),
                            new_commit: new.id(),
                            old_filename: path_to_string(f.filename.clone()),
                            new_filename: path_to_string(q.filename.clone()),
                            origin_line: f.start_position,
                            destination_line: q.start_position,
                            num_lines: f.line_count,
                        };
                    },
                }

                moves.push(output);
            }
        } else {
            map.insert(f.key.clone(), f);
        }
    }

    return Ok(moves);
}

#[derive(Debug)]
struct Output {
    old_commit: Oid,
    new_commit: Oid,
    old_filename: String,
    new_filename: String,
    origin_line: u32,
    destination_line: u32,
    num_lines: u32,
}
<|endoftext|>


<commit_before>extern crate clap;
extern crate rusqlite;
#[macro_use]

extern crate rustc_serialize;

use clap::{App, Arg};

mod loader;
mod db;
mod dbFsDirType;
mod dbFsDir;
mod dbFsFile;
mod dbProvider;
mod dbJob;
mod dbJobProvide;

fn main() {

    // Once all App settings (including all arguments) have been set, you call get_matches() which
    // parses the string provided by the user, and returns all the valid matches to the ones you
    // specified.
    //
    // You can then query the matches struct to get information about how the user ran the program
    // at startup.
    //
    // For this example, let's assume you created an App which accepts three arguments (plus two
    // generated by clap), a flag to display debugging information triggered with "-d" or
    // "--debug" as well as an option argument which specifies a custom configuration file to use
    // triggered with "-c file" or "--config file" or "--config=file" and finally a positional
    // argument which is the input file we want to work with, this will be the only required
    // argument.
    let matches = App::new("whenenv")
                        .about("Parses an input file to do awesome things")
                        .version("1.0")
                        .author("Kevin K. <kbknapp@gmail.com>")
                        .arg(Arg::with_name("verbose")
                                    .help("Increase log output.")
                                    .short("v")
                                    .multiple(true)
                                    .long("verbose"))
                        .arg(Arg::with_name("quiet")
                                    .help("Decrease log output.")
                                    .short("q")
                                    .multiple(true)
                                    .long("quiet"))
                        .arg(Arg::with_name("env")
                                    .short("e")
                                    .long("env")
                                    .value_name("ENVIROMENT_VARIABLE")
                                    .help("Which enviroment variables to process")
                                    .multiple(true)
                                    .takes_value(true))
                        .arg(Arg::with_name("dir-jobs")
                                    .long("dir-jobs")
                                    .value_name("DIR_JOB")
                                    .help("directory storing json jobs.")
                                    .multiple(true)
                                    .takes_value(true))
                        .arg(Arg::with_name("dir-sh")
                                    .long("dir-sh")
                                    .value_name("DIR_SHELL")
                                    .help("directory storing jobs shell scripts.")
                                    .multiple(true)
                                    .takes_value(true))
                        .arg(Arg::with_name("config")
                                    .short("c")
                                    .long("config")
                                    .value_name("FILE")
                                    .help("Sets a custom config file")
                                    .takes_value(true))
                        .get_matches();

    // We can find out whether or not debugging was turned on
    if matches.is_present("debug") {
        println!("Debugging is turned on");
    }

    // If we wanted to some custom initialization based off some configuration file provided
    // by the user, we could get the file (A string of the file)
    if let Some(ref file) = matches.value_of("config") {
        println!("Using config file: {}", file);
    }
    if let Some(ref env) = matches.value_of("env") {
        println!("Using config file: {}", env);
    }
    // Continued program logic goes here...
    loader::deligate(matches)
}
<commit_msg>Add dbJobDepend<commit_after>extern crate clap;
extern crate rusqlite;
#[macro_use]

extern crate rustc_serialize;

use clap::{App, Arg};

mod loader;
mod db;
mod dbFsDirType;
mod dbFsDir;
mod dbFsFile;
mod dbProvider;
mod dbJob;
mod dbJobProvide;
mod dbJobDepend;

fn main() {

    // Once all App settings (including all arguments) have been set, you call get_matches() which
    // parses the string provided by the user, and returns all the valid matches to the ones you
    // specified.
    //
    // You can then query the matches struct to get information about how the user ran the program
    // at startup.
    //
    // For this example, let's assume you created an App which accepts three arguments (plus two
    // generated by clap), a flag to display debugging information triggered with "-d" or
    // "--debug" as well as an option argument which specifies a custom configuration file to use
    // triggered with "-c file" or "--config file" or "--config=file" and finally a positional
    // argument which is the input file we want to work with, this will be the only required
    // argument.
    let matches = App::new("whenenv")
                        .about("Parses an input file to do awesome things")
                        .version("1.0")
                        .author("Kevin K. <kbknapp@gmail.com>")
                        .arg(Arg::with_name("verbose")
                                    .help("Increase log output.")
                                    .short("v")
                                    .multiple(true)
                                    .long("verbose"))
                        .arg(Arg::with_name("quiet")
                                    .help("Decrease log output.")
                                    .short("q")
                                    .multiple(true)
                                    .long("quiet"))
                        .arg(Arg::with_name("env")
                                    .short("e")
                                    .long("env")
                                    .value_name("ENVIROMENT_VARIABLE")
                                    .help("Which enviroment variables to process")
                                    .multiple(true)
                                    .takes_value(true))
                        .arg(Arg::with_name("dir-jobs")
                                    .long("dir-jobs")
                                    .value_name("DIR_JOB")
                                    .help("directory storing json jobs.")
                                    .multiple(true)
                                    .takes_value(true))
                        .arg(Arg::with_name("dir-sh")
                                    .long("dir-sh")
                                    .value_name("DIR_SHELL")
                                    .help("directory storing jobs shell scripts.")
                                    .multiple(true)
                                    .takes_value(true))
                        .arg(Arg::with_name("config")
                                    .short("c")
                                    .long("config")
                                    .value_name("FILE")
                                    .help("Sets a custom config file")
                                    .takes_value(true))
                        .get_matches();

    // We can find out whether or not debugging was turned on
    if matches.is_present("debug") {
        println!("Debugging is turned on");
    }

    // If we wanted to some custom initialization based off some configuration file provided
    // by the user, we could get the file (A string of the file)
    if let Some(ref file) = matches.value_of("config") {
        println!("Using config file: {}", file);
    }
    if let Some(ref env) = matches.value_of("env") {
        println!("Using config file: {}", env);
    }
    // Continued program logic goes here...
    loader::deligate(matches)
}
<|endoftext|>