<commit_before>// This file is part of sokoban-rs
// Copyright 2015 Sébastien Watteau
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! This is an implementation of Sokoban in Rust.

#[macro_use]
extern crate bitflags;
extern crate clap;
extern crate sdl2;
extern crate sdl2_image;
extern crate sdl2_ttf;
extern crate xml;

use std::path::Path;
use std::fs::File;
use std::io::BufReader;
use std::str::FromStr;
use clap::{App, Arg};
use sdl2::event::Event;
use sdl2::keyboard::Keycode;
use sdl2_image::INIT_PNG;
use xml::reader::EventReader;
use xml::reader::events::*;

pub mod error;
pub mod game;
pub mod render;

use game::Level;
use render::Drawer;

pub fn main() {
    let matches = App::new("sokoban-rs")
        .version("1.0.1")
        .author("Sébastien Watteau")
        .about("An implementation of Sokoban in the Rust programming language.")
        .arg(Arg::with_name("slc_file")
             .help("a Sokoban level collection (SLC) file")
             .required(true)
        )
        .arg(Arg::with_name("fullscreen")
             .help("Launches the game in fullscreen mode")
             .short("f")
             .long("fullscreen")
        )
        .arg(Arg::with_name("width")
             .help("The width of the window in pixels")
             .short("w")
             .long("width")
             .takes_value(true)
             .requires("height")
        )
        .arg(Arg::with_name("height")
             .help("The height of the window in pixels")
             .short("h")
             .long("height")
             .takes_value(true)
             .requires("width")
        )
        .get_matches();

    let slc_file = matches.value_of("slc_file").unwrap();

    let width: u32 = matches.value_of("width")
        .unwrap_or("1024")
        .parse()
        .unwrap_or_else(|err| {
            panic!("Couldn't parse width as an integer: {}", err);
        });
    let height: u32 = matches.value_of("height")
        .unwrap_or("768")
        .parse()
        .unwrap_or_else(|err| {
            panic!("Couldn't parse height as an integer: {}", err);
        });

    let sdl_context = sdl2::init().unwrap_or_else(|err| {
        println!("Failed to initialize an SDL context: {}", err);
        std::process::exit(1);
    });

    let video_subsystem = sdl_context.video().unwrap_or_else(|err| {
        println!("Failed to initialize the video subsystem: {}", err);
        std::process::exit(1);
    });

    sdl2_image::init(INIT_PNG);
    sdl2_ttf::init();

    let mut window_builder = video_subsystem.window("sokoban-rs", width, height);
    if matches.is_present("fullscreen") {
        window_builder.fullscreen();
    } else {
        window_builder.position_centered();
    }

    let window = window_builder.opengl()
        .build()
        .unwrap_or_else(|err| {
            println!("Failed to create the window: {}", err);
            std::process::exit(1);
        });

    let renderer = window.renderer()
        .build()
        .unwrap_or_else(|err| {
            println!("Failed to get an SDL renderer for the main window: {}", err);
            std::process::exit(1);
        });

    let mut drawer = Drawer::new(renderer);

    let mut collection = load_slc_file(Path::new(&slc_file))
        .unwrap_or_else(|err| {
            panic!("{}", err);
        })
        .into_iter();
    let mut reference_level = collection.next().unwrap();
    let mut level = reference_level.clone();

    let mut running = true;
    let mut event_pump = sdl_context.event_pump().unwrap();
    let mut skip = false;
    while running {
        if level.is_completed() || skip {
            match collection.next() {
                Some(l) => {
                    level = l;
                    reference_level = level.clone();
                    skip = false;
                },
                None => {
                    break;
                }
            }
        }
        drawer.draw(&level);

        match event_pump.wait_event() {
            Event::Quit {..} | Event::KeyDown { keycode: Some(Keycode::Escape), .. } => {
                running = false
            },
            Event::KeyDown { keycode: Some(Keycode::Left), .. } => {
                level.step(game::Direction::Left);
            }
            Event::KeyDown { keycode: Some(Keycode::Right), .. } => {
                level.step(game::Direction::Right);
            }
            Event::KeyDown { keycode: Some(Keycode::Up), .. } => {
                level.step(game::Direction::Up);
            }
            Event::KeyDown { keycode: Some(Keycode::Down), .. } => {
                level.step(game::Direction::Down);
            }
            Event::KeyDown { keycode: Some(Keycode::R), .. } => {
                level = reference_level.clone();
            }
            Event::KeyDown { keycode: Some(Keycode::N), .. } => {
                skip = true;
            }
            _ => {}
        }
    }
    sdl2_ttf::quit();
    sdl2_image::quit();
}

/// Builds levels from a level collection file in the SLC format.
fn load_slc_file(path: &Path) -> Result<Vec<Level>, error::SokobanError> {
    let mut collection = Vec::new();
    let file = try!(File::open(&path));
    let reader = BufReader::new(file);
    let mut parser = EventReader::new(reader);

    let mut level_title = String::new();
    let mut level_str = String::new();
    let mut reading_level = false;
    for e in parser.events() {
        match e {
            XmlEvent::StartElement { ref name, ref attributes, .. } => {
                if name.local_name == "L" {
                    reading_level = true;
                } else if name.local_name == "Level" {
                    if let Some(id) = attributes.iter().find(|&attr| attr.name.local_name == "Id") {
                        level_title = id.value.clone();
                    }
                }
            },
            XmlEvent::EndElement { name } => {
                if name.local_name == "Level" {
                    let mut level = try!(Level::from_str(&level_str));
                    level.set_title(level_title.clone());
                    collection.push(level);
                    level_str.clear();
                }
            },
            XmlEvent::Characters(ref data) => {
                if reading_level {
                    level_str.push_str(data);
                    level_str.push('\n');
                }
            }
            _ => {}
        }
    }

    Ok(collection)
}
<commit_msg>Use clap macros to parse width and height options<commit_after>// This file is part of sokoban-rs
// Copyright 2015 Sébastien Watteau
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! This is an implementation of Sokoban in Rust.

#[macro_use]
extern crate bitflags;
#[macro_use]
extern crate clap;
extern crate sdl2;
extern crate sdl2_image;
extern crate sdl2_ttf;
extern crate xml;

use std::path::Path;
use std::fs::File;
use std::io::BufReader;
use std::str::FromStr;
use clap::{App, Arg};
use sdl2::event::Event;
use sdl2::keyboard::Keycode;
use sdl2_image::INIT_PNG;
use xml::reader::EventReader;
use xml::reader::events::*;

pub mod error;
pub mod game;
pub mod render;

use game::Level;
use render::Drawer;

pub fn main() {
    let matches = App::new("sokoban-rs")
        .version("1.0.1")
        .author("Sébastien Watteau")
        .about("An implementation of Sokoban in the Rust programming language.")
        .arg(Arg::with_name("slc_file")
             .help("a Sokoban level collection (SLC) file")
             .required(true)
        )
        .arg(Arg::with_name("fullscreen")
             .help("Launches the game in fullscreen mode")
             .short("f")
             .long("fullscreen")
        )
        .arg(Arg::with_name("width")
             .help("The width of the window in pixels")
             .short("w")
             .long("width")
             .takes_value(true)
             .requires("height")
        )
        .arg(Arg::with_name("height")
             .help("The height of the window in pixels")
             .short("h")
             .long("height")
             .takes_value(true)
             .requires("width")
        )
        .get_matches();

    let slc_file = matches.value_of("slc_file").unwrap();

    let width = value_t!(matches.value_of("width"), u32).unwrap_or(1024);
    let height = value_t!(matches.value_of("height"), u32).unwrap_or(768);

    let sdl_context = sdl2::init().unwrap_or_else(|err| {
        println!("Failed to initialize an SDL context: {}", err);
        std::process::exit(1);
    });

    let video_subsystem = sdl_context.video().unwrap_or_else(|err| {
        println!("Failed to initialize the video subsystem: {}", err);
        std::process::exit(1);
    });

    sdl2_image::init(INIT_PNG);
    sdl2_ttf::init();

    let mut window_builder = video_subsystem.window("sokoban-rs", width, height);
    if matches.is_present("fullscreen") {
        window_builder.fullscreen();
    } else {
        window_builder.position_centered();
    }

    let window = window_builder.opengl()
        .build()
        .unwrap_or_else(|err| {
            println!("Failed to create the window: {}", err);
            std::process::exit(1);
        });

    let renderer = window.renderer()
        .build()
        .unwrap_or_else(|err| {
            println!("Failed to get an SDL renderer for the main window: {}", err);
            std::process::exit(1);
        });

    let mut drawer = Drawer::new(renderer);

    let mut collection = load_slc_file(Path::new(&slc_file))
        .unwrap_or_else(|err| {
            panic!("{}", err);
        })
        .into_iter();
    let mut reference_level = collection.next().unwrap();
    let mut level = reference_level.clone();

    let mut running = true;
    let mut event_pump = sdl_context.event_pump().unwrap();
    let mut skip = false;
    while running {
        if level.is_completed() || skip {
            match collection.next() {
                Some(l) => {
                    level = l;
                    reference_level = level.clone();
                    skip = false;
                },
                None => {
                    break;
                }
            }
        }
        drawer.draw(&level);

        match event_pump.wait_event() {
            Event::Quit {..} | Event::KeyDown { keycode: Some(Keycode::Escape), .. } => {
                running = false
            },
            Event::KeyDown { keycode: Some(Keycode::Left), .. } => {
                level.step(game::Direction::Left);
            }
            Event::KeyDown { keycode: Some(Keycode::Right), .. } => {
                level.step(game::Direction::Right);
            }
            Event::KeyDown { keycode: Some(Keycode::Up), .. } => {
                level.step(game::Direction::Up);
            }
            Event::KeyDown { keycode: Some(Keycode::Down), .. } => {
                level.step(game::Direction::Down);
            }
            Event::KeyDown { keycode: Some(Keycode::R), .. } => {
                level = reference_level.clone();
            }
            Event::KeyDown { keycode: Some(Keycode::N), .. } => {
                skip = true;
            }
            _ => {}
        }
    }
    sdl2_ttf::quit();
    sdl2_image::quit();
}

/// Builds levels from a level collection file in the SLC format.
fn load_slc_file(path: &Path) -> Result<Vec<Level>, error::SokobanError> {
    let mut collection = Vec::new();
    let file = try!(File::open(&path));
    let reader = BufReader::new(file);
    let mut parser = EventReader::new(reader);

    let mut level_title = String::new();
    let mut level_str = String::new();
    let mut reading_level = false;
    for e in parser.events() {
        match e {
            XmlEvent::StartElement { ref name, ref attributes, .. } => {
                if name.local_name == "L" {
                    reading_level = true;
                } else if name.local_name == "Level" {
                    if let Some(id) = attributes.iter().find(|&attr| attr.name.local_name == "Id") {
                        level_title = id.value.clone();
                    }
                }
            },
            XmlEvent::EndElement { name } => {
                if name.local_name == "Level" {
                    let mut level = try!(Level::from_str(&level_str));
                    level.set_title(level_title.clone());
                    collection.push(level);
                    level_str.clear();
                }
            },
            XmlEvent::Characters(ref data) => {
                if reading_level {
                    level_str.push_str(data);
                    level_str.push('\n');
                }
            }
            _ => {}
        }
    }

    Ok(collection)
}
<|endoftext|>


<commit_before>extern crate raytracer;
use raytracer::vec3::Vec3;
use raytracer::ray::Ray;

use std::io::{self, Write};

const NX: i32 = 200;
const NY: i32 = 100;

fn color(r: &Ray) -> Vec3 {
    let unit_dir = r.dir.normalized();
    let t = 0.5 * (unit_dir.y + 1.0);
    Vec3::new(1.0,1.0,1.0).mul_scalar(1.0-t) + Vec3::new(0.5, 0.7, 1.0).mul_scalar(t)
}

fn main() {
    io::stdout().write_fmt(format_args!("P3\n{} {}\n{}\n", NX, NY, 255)).unwrap();

    let lower_left_corner = Vec3::new(-2.0, -1.0, -1.0);
    let horizontal = Vec3::new(4.0, 0.0, 0.0);
    let vertical = Vec3::new(0.0, 2.0, 0.0);
    let origin = Vec3::new(0.0, 0.0, 0.0);

    for j in (0..NY).rev() {
        for i in 0..NX {
            let u = (i as f64) / (NX as f64);
            let v = (j as f64) / (NY as f64);

            let r = Ray {
                origin: origin,
                dir: lower_left_corner + horizontal.mul_scalar(u) + vertical.mul_scalar(v),
            };
            let c = color(&r);
            let c = c.mul_scalar(255.99);
            println!("{} {} {}", c.x as i32, c.y as i32, c.z as i32);
        }
    }
}
<commit_msg>Chapter 4, add sphere intersection<commit_after>extern crate raytracer;
use raytracer::vec3::Vec3;
use raytracer::ray::Ray;

use std::io::{self, Write};

const NX: i32 = 200;
const NY: i32 = 100;

fn hit_sphere(center: Vec3, radius: f64, ray: &Ray) -> bool {
    let oc = ray.origin - center;
    let a = ray.dir.dot(ray.dir);
    let b = 2.0 * oc.dot(ray.dir);
    let c = oc.dot(oc) - radius * radius;
    let discriminant = b*b - 4.0*a*c;
    discriminant > 0.0
}

fn color(r: &Ray) -> Vec3 {
    if (hit_sphere(Vec3::new(0.0,0.0,-1.0), 0.5, r)) {
        // Sphere color
        Vec3::new(1.0,0.0,0.0)
    } else {
        // Background
        let unit_dir = r.dir.normalized();
        let t = 0.5 * (unit_dir.y + 1.0);
        Vec3::new(1.0,1.0,1.0).mul_scalar(1.0-t) + Vec3::new(0.5, 0.7, 1.0).mul_scalar(t)
    }

}

fn main() {
    io::stdout().write_fmt(format_args!("P3\n{} {}\n{}\n", NX, NY, 255)).unwrap();

    let lower_left_corner = Vec3::new(-2.0, -1.0, -1.0);
    let horizontal = Vec3::new(4.0, 0.0, 0.0);
    let vertical = Vec3::new(0.0, 2.0, 0.0);
    let origin = Vec3::new(0.0, 0.0, 0.0);

    for j in (0..NY).rev() {
        for i in 0..NX {
            let u = (i as f64) / (NX as f64);
            let v = (j as f64) / (NY as f64);

            let r = Ray {
                origin: origin,
                dir: lower_left_corner + horizontal.mul_scalar(u) + vertical.mul_scalar(v),
            };
            let c = color(&r);
            let c = c.mul_scalar(255.99);
            println!("{} {} {}", c.x as i32, c.y as i32, c.z as i32);
        }
    }
}
<|endoftext|>


<commit_before>//    Copyright 2017 Vector Creations Limited
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

// extern crate base64;
// extern crate blake2_rfc;
#[macro_use]
extern crate clap;
extern crate humansize;
extern crate linear_map;
extern crate rand;
extern crate rusqlite;
extern crate twox_hash;
extern crate walkdir;
extern crate indicatif;

// use blake2_rfc::blake2b::Blake2b;
use humansize::{FileSize, file_size_opts as options};
use clap::{App, Arg};
use linear_map::LinearMap;
use rand::Rng;
use std::io;
use std::io::Read;
use std::fs::File;
use std::hash::Hasher;
use std::collections::BTreeMap;
use std::path::PathBuf;
use walkdir::WalkDir;

fn copy<R: io::Read, W: Hasher>(reader: &mut R, writer: &mut W) -> io::Result<u64> {
    let mut buf = [0; 64 * 1024];
    let mut written = 0;
    loop {
        let len = match reader.read(&mut buf) {
            Ok(0) => return Ok(written),
            Ok(len) => len,
            Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
            Err(e) => return Err(e),
        };
        writer.write(&buf[..len]);
        written += len as u64;
    }
}

fn to_hash(path: &PathBuf) -> u64 {
    let mut file = File::open(path).unwrap();
    let mut hasher = twox_hash::XxHash::default();
    copy(&mut file, &mut hasher).unwrap();
    hasher.finish()
}

fn read_file(path: &PathBuf) -> Vec<u8> {
    let mut file = File::open(path).unwrap();
    let mut vec = Vec::new();
    file.read_to_end(&mut vec).unwrap();
    vec
}

fn partition_by<I, F, R>(paths: I, f: F) -> LinearMap<R, Vec<PathBuf>>
    where I: Iterator<Item=PathBuf>, F: Fn(&PathBuf) -> R, R: Eq
{
    let mut map = LinearMap::with_capacity(paths.size_hint().0);
    for path in paths {
        let key = f(&path);
        map.entry(key).or_insert_with(Vec::new).push(path);
    }
    map
}

const DB_TABLE_SCHEMA: &'static str = r#"
CREATE TABLE files (
    hash BIGINT NOT NULL,
    path TEXT NOT NULL,
    size BIGINT NOT NULL
);
"#;

fn main() {
    let matches = App::new(crate_name!())
        .version(crate_version!())
        .author(crate_authors!("\n"))
        .arg(Arg::with_name("media_directory")
            .help("The location of the media store")
            .index(1)
            .multiple(true)
            .required(true))
        .get_matches();

    let paths_to_search = matches.values_of("media_directory").unwrap();

    let db = rusqlite::Connection::open_in_memory().expect("failed to open sqlite db");
    db.execute_batch(DB_TABLE_SCHEMA).expect("failed to create db schema");

    let mut paths_by_size = BTreeMap::new();
    let mut total_files = 0;
    let mut total_size = 0;

    let pb = indicatif::ProgressBar::new_spinner();
    pb.set_style(
        indicatif::ProgressStyle::default_spinner()
        .template("{spinner} Collected metadata for {pos} files...")
    );

    for path in paths_to_search {
        for entry in WalkDir::new(path) {
            let entry = entry.unwrap();
            if !entry.file_type().is_file() {
                continue
            }

            let file_size = entry.metadata().unwrap().len() as usize;
            paths_by_size.entry(file_size).or_insert_with(Vec::new).push(entry.path().to_owned());

            total_files += 1;
            total_size += file_size;

            pb.inc(1);
        }
    }

    pb.finish_and_clear();

    println!("  Collected metadata for {} files", total_files);

    let pb = indicatif::ProgressBar::new(total_files);
    pb.set_style(
        indicatif::ProgressStyle::default_bar()
        .template("  Searching for possible duplicates  {bar:40} {pos:>8}/{len}")
    );

    let mut possible_total_size = 0;
    let mut possible_duplicates = Vec::new();
    for (file_size, paths) in paths_by_size.into_iter() {
        if paths.len() > 1 {
            possible_total_size += file_size * paths.len();
            possible_duplicates.push((file_size, paths))
        }
        pb.inc(1);
    }

    pb.finish();

    // We shuffle the vector as otherwise we do the small files first, and
    // that can skew the progress estimates massively.
    let mut rng = rand::thread_rng();
    rng.shuffle(possible_duplicates.as_mut_slice());

    let pb = indicatif::ProgressBar::new(possible_total_size as u64);
    pb.set_style(
        indicatif::ProgressStyle::default_bar()
        .template("  Comparing hashes                   {bar:40} {bytes:>8}/{total_bytes}")
    );

    let mut total_wasted_size = 0;

    for (file_size, paths) in possible_duplicates {
        if paths.len() == 1 {
            continue
        }

        let by_hash = partition_by(paths.into_iter(), to_hash);

        for (hash, paths) in by_hash {
            if paths.len() == 1 {
                pb.inc(file_size as u64);
                continue
            }

            let by_contents = partition_by(paths.into_iter(), read_file);

            for (_, paths) in by_contents {
                if paths.len() == 1 {
                    pb.inc(file_size as u64);
                    continue
                }

                for path in &paths {
                    db.execute("INSERT INTO files (hash, path, size) VALUES (?, ?, ?)", &[&(hash as i64), &path.to_str().unwrap(), &(file_size as i64)]).expect("failed to write to db");
                }

                let wasted = file_size * (paths.len() - 1);

                total_wasted_size += wasted;

                pb.inc((file_size * paths.len()) as u64);
            }
        }
    }

    pb.finish();

    println!();
    println!(
        "Total wasted size: {} out of {}. Percentage: {:.2}%",
        total_wasted_size.file_size(options::CONVENTIONAL).unwrap(),
        total_size.file_size(options::CONVENTIONAL).unwrap(),
        (total_wasted_size * 100) as f64 / total_size as f64,
    );

    let mut disk_db = rusqlite::Connection::open("media_store_sizes.db").expect("failed to open sqlite db");
    let backup = rusqlite::backup::Backup::new(&db, &mut disk_db).expect("failed to create backup");
    backup.run_to_completion(5, std::time::Duration::from_millis(0), None).expect("failed to write to disk");
}
<commit_msg>Add time elapsed<commit_after>//    Copyright 2017 Vector Creations Limited
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

// extern crate base64;
// extern crate blake2_rfc;
#[macro_use]
extern crate clap;
extern crate humansize;
extern crate linear_map;
extern crate rand;
extern crate rusqlite;
extern crate twox_hash;
extern crate walkdir;
extern crate indicatif;

// use blake2_rfc::blake2b::Blake2b;
use humansize::{FileSize, file_size_opts as options};
use clap::{App, Arg};
use linear_map::LinearMap;
use rand::Rng;
use std::io;
use std::io::Read;
use std::fs::File;
use std::hash::Hasher;
use std::collections::BTreeMap;
use std::path::PathBuf;
use walkdir::WalkDir;

fn copy<R: io::Read, W: Hasher>(reader: &mut R, writer: &mut W) -> io::Result<u64> {
    let mut buf = [0; 64 * 1024];
    let mut written = 0;
    loop {
        let len = match reader.read(&mut buf) {
            Ok(0) => return Ok(written),
            Ok(len) => len,
            Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
            Err(e) => return Err(e),
        };
        writer.write(&buf[..len]);
        written += len as u64;
    }
}

fn to_hash(path: &PathBuf) -> u64 {
    let mut file = File::open(path).unwrap();
    let mut hasher = twox_hash::XxHash::default();
    copy(&mut file, &mut hasher).unwrap();
    hasher.finish()
}

fn read_file(path: &PathBuf) -> Vec<u8> {
    let mut file = File::open(path).unwrap();
    let mut vec = Vec::new();
    file.read_to_end(&mut vec).unwrap();
    vec
}

fn partition_by<I, F, R>(paths: I, f: F) -> LinearMap<R, Vec<PathBuf>>
    where I: Iterator<Item=PathBuf>, F: Fn(&PathBuf) -> R, R: Eq
{
    let mut map = LinearMap::with_capacity(paths.size_hint().0);
    for path in paths {
        let key = f(&path);
        map.entry(key).or_insert_with(Vec::new).push(path);
    }
    map
}

const DB_TABLE_SCHEMA: &'static str = r#"
CREATE TABLE files (
    hash BIGINT NOT NULL,
    path TEXT NOT NULL,
    size BIGINT NOT NULL
);
"#;

fn main() {
    let matches = App::new(crate_name!())
        .version(crate_version!())
        .author(crate_authors!("\n"))
        .arg(Arg::with_name("media_directory")
            .help("The location of the media store")
            .index(1)
            .multiple(true)
            .required(true))
        .get_matches();

    let paths_to_search = matches.values_of("media_directory").unwrap();

    let db = rusqlite::Connection::open_in_memory().expect("failed to open sqlite db");
    db.execute_batch(DB_TABLE_SCHEMA).expect("failed to create db schema");

    let mut paths_by_size = BTreeMap::new();
    let mut total_files = 0;
    let mut total_size = 0;

    let pb = indicatif::ProgressBar::new_spinner();
    pb.set_style(
        indicatif::ProgressStyle::default_spinner()
        .template("{spinner} Collected metadata for {pos} files [{elapsed}]")
    );

    for path in paths_to_search {
        for entry in WalkDir::new(path) {
            let entry = entry.unwrap();
            if !entry.file_type().is_file() {
                continue
            }

            let file_size = entry.metadata().unwrap().len() as usize;
            paths_by_size.entry(file_size).or_insert_with(Vec::new).push(entry.path().to_owned());

            total_files += 1;
            total_size += file_size;

            pb.inc(1);
        }
    }

    pb.finish();
    pb.set_position(total_files);

    // println!("  Collected metadata for {} files", total_files);

    let pb = indicatif::ProgressBar::new(total_files);
    pb.set_style(
        indicatif::ProgressStyle::default_bar()
        .template("  Searching for possible duplicates  {bar:40} {pos:>9}/{len:9} [{elapsed}]")
    );

    let mut possible_total_size = 0;
    let mut possible_duplicates = Vec::new();
    for (file_size, paths) in paths_by_size.into_iter() {
        if paths.len() > 1 {
            possible_total_size += file_size * paths.len();
            possible_duplicates.push((file_size, paths))
        }
        pb.inc(1);
    }

    pb.finish();

    // We shuffle the vector as otherwise we do the small files first, and
    // that can skew the progress estimates massively.
    let mut rng = rand::thread_rng();
    rng.shuffle(possible_duplicates.as_mut_slice());

    let pb = indicatif::ProgressBar::new(possible_total_size as u64);
    pb.set_style(
        indicatif::ProgressStyle::default_bar()
        .template("  Comparing hashes                   {bar:40} {bytes:>9}/{total_bytes:9} [{elapsed}]")
    );

    let mut total_wasted_size = 0;

    for (file_size, paths) in possible_duplicates {
        if paths.len() == 1 {
            continue
        }

        let by_hash = partition_by(paths.into_iter(), to_hash);

        for (hash, paths) in by_hash {
            if paths.len() == 1 {
                pb.inc(file_size as u64);
                continue
            }

            let by_contents = partition_by(paths.into_iter(), read_file);

            for (_, paths) in by_contents {
                if paths.len() == 1 {
                    pb.inc(file_size as u64);
                    continue
                }

                for path in &paths {
                    db.execute("INSERT INTO files (hash, path, size) VALUES (?, ?, ?)", &[&(hash as i64), &path.to_str().unwrap(), &(file_size as i64)]).expect("failed to write to db");
                }

                let wasted = file_size * (paths.len() - 1);

                total_wasted_size += wasted;

                pb.inc((file_size * paths.len()) as u64);
            }
        }
    }

    pb.finish();

    println!();
    println!(
        "Total wasted size: {} out of {}. Percentage: {:.2}%",
        total_wasted_size.file_size(options::CONVENTIONAL).unwrap(),
        total_size.file_size(options::CONVENTIONAL).unwrap(),
        (total_wasted_size * 100) as f64 / total_size as f64,
    );

    let mut disk_db = rusqlite::Connection::open("media_store_sizes.db").expect("failed to open sqlite db");
    let backup = rusqlite::backup::Backup::new(&db, &mut disk_db).expect("failed to create backup");
    backup.run_to_completion(5, std::time::Duration::from_millis(0), None).expect("failed to write to disk");
}
<|endoftext|>


<commit_before>/*
* rust-clean - Copyright (c) 2014 Gomez Guillaume.
*
* This software is provided 'as-is', without any express or implied warranty.
* In no event will the authors be held liable for any damages arising from
* the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must not claim
*    that you wrote the original software. If you use this software in a product,
*    an acknowledgment in the product documentation would be appreciated but is
*    not required.
*
* 2. Altered source versions must be plainly marked as such, and must not be
*    misrepresented as being the original software.
*
* 3. This notice may not be removed or altered from any source distribution.
*/

#![feature(core, path, os, collections, fs, old_io, env)]

use std::fs::{self, PathExt};
use std::path::Path;

struct CleanOptions {
    recursive: bool,
    verbose: bool,
    confirmation: bool
}

fn ask_confirmation(file: &Path) -> bool {
    loop {
        print!("clean: remove \x1b[37;1m'{}'\x1b[0m (y/n) ? ", file.to_str().unwrap());
        match std::old_io::stdio::stdin().read_line() {
            Ok(s) => {
                let tmp_s = s.replace("\r\n", "").replace("\n", "");

                if tmp_s.as_slice() == "y" || tmp_s.as_slice() == "yes" {
                    return true;
                } else if tmp_s.as_slice() == "n" || tmp_s.as_slice() == "no" {
                    return false;
                }
            }
            Err(_) => {}
        }
    }
}

fn start_clean(options: &CleanOptions, entry: &Path) {
    if entry.exists() {
        if entry.is_dir() {
            if options.recursive || entry.to_str().unwrap() == "." {
                match fs::read_dir(entry) {
                    Ok(res) => {
                        if options.verbose {
                            println!("\x1b[36;1m-> Entering {}\x1b[0m", entry.to_str().unwrap());
                        }
                        for tmp in res {
                            match tmp {
                                Ok(current) => start_clean(options, &current.path()),
                                Err(e) => println!("Error: {:?}", e)
                            };
                        }
                        if options.verbose {
                            println!("\x1b[34;1m<- Leaving {}\x1b[0m", entry.to_str().unwrap());
                        }
                    }
                    Err(e) => {
                        println!("\x1b[31;1mProblem with this directory: {} -> {}\x1b[0m", entry.to_str().unwrap(), e);
                    }
                }
            }
        } else {
            match entry.file_name() {
                Some(s) => {
                    match s.to_str() {
                        Some(ss) => if ss.ends_with("~") {
                            if !options.confirmation || ask_confirmation(&Path::new(s)) {
                                match fs::remove_file(entry) {
                                    Ok(_) => {
                                        if options.verbose {
                                            println!("\x1b[32;1m{} deleted\x1b[0m", entry.to_str().unwrap());
                                        }
                                    }
                                    Err(e) => {
                                        println!("\x1b[31;1mProblem with this file: {} -> {}\x1b[0m", entry.to_str().unwrap(), e);
                                    }
                                }
                            }
                        },
                        _ => {}
                    }
                }
                None => {
                    println!("\x1b[31;1mProblem with this file: {}\x1b[0m", entry.to_str().unwrap());
                }
            }
        }
    } else {
        println!("\x1b[31;1mProblem with this entry: {}\x1b[0m", entry.to_str().unwrap());
    }
}

fn print_help() {
    println!("./clean [options] [files | dirs]");
    println!("    -r : recursive mode");
    println!("    -v : verbose mode");
    println!("    -i : prompt before every removal");
    println!("--help : print this help");
}

fn main() {
    let mut args = Vec::new();

    for argument in std::env::args() {
        args.push(argument);
    }
    let mut options = CleanOptions{recursive: false, verbose: false, confirmation: false};
    let mut files = Vec::new();

    args.remove(0);
    for tmp in args.iter() {
        if tmp.clone().into_bytes()[0] == '-' as u8 {
            let mut tmp_arg = String::from_str(tmp.as_slice());

            tmp_arg.remove(0);
            if tmp_arg.len() > 0 {
                for character in tmp_arg.into_bytes().iter() {
                    match *character as char {
                        '-' => {
                            if tmp.as_slice() == "--help" {
                                print_help();
                                return;
                            }
                        }
                        'r' => {
                            options.recursive = true;
                        }
                        'v' => {
                            options.verbose = true;
                        }
                        'i' => {
                            options.confirmation = true;
                        }
                        _ => {
                            panic!("Unknown option: '{}', to have the options list, please launch with '-h' option\n", character);
                        }
                    }
                }
            } else {
                files.push(Path::new(tmp.as_slice()));
            }
        }
    }
    if files.len() == 0 {
        files.push(Path::new("."));
    }
    if options.verbose {
        println!("\x1b[33;1m=== VERBOSE MODE ===\x1b[0m");
    }
    for tmp in files.iter() {
        start_clean(&options, tmp);
    }
    if options.verbose {
        println!("\x1b[33;1mEnd of execution\x1b[0m");
    }
}<commit_msg>Update to last rustc version<commit_after>/*
* rust-clean - Copyright (c) 2014 Gomez Guillaume.
*
* This software is provided 'as-is', without any express or implied warranty.
* In no event will the authors be held liable for any damages arising from
* the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must not claim
*    that you wrote the original software. If you use this software in a product,
*    an acknowledgment in the product documentation would be appreciated but is
*    not required.
*
* 2. Altered source versions must be plainly marked as such, and must not be
*    misrepresented as being the original software.
*
* 3. This notice may not be removed or altered from any source distribution.
*/

#![feature(core, path, os, collections, fs, old_io)]

use std::fs::{self, PathExt};
use std::path::Path;

struct CleanOptions {
    recursive: bool,
    verbose: bool,
    confirmation: bool
}

fn ask_confirmation(file: &Path) -> bool {
    loop {
        print!("clean: remove \x1b[37;1m'{}'\x1b[0m (y/n) ? ", file.to_str().unwrap());
        match std::old_io::stdio::stdin().read_line() {
            Ok(s) => {
                let tmp_s = s.replace("\r\n", "").replace("\n", "");

                if tmp_s.as_slice() == "y" || tmp_s.as_slice() == "yes" {
                    return true;
                } else if tmp_s.as_slice() == "n" || tmp_s.as_slice() == "no" {
                    return false;
                }
            }
            Err(_) => {}
        }
    }
}

fn start_clean(options: &CleanOptions, entry: &Path) {
    if entry.exists() {
        if entry.is_dir() {
            if options.recursive || entry.to_str().unwrap() == "." {
                match fs::read_dir(entry) {
                    Ok(res) => {
                        if options.verbose {
                            println!("\x1b[36;1m-> Entering {}\x1b[0m", entry.to_str().unwrap());
                        }
                        for tmp in res {
                            match tmp {
                                Ok(current) => start_clean(options, &current.path()),
                                Err(e) => println!("Error: {:?}", e)
                            };
                        }
                        if options.verbose {
                            println!("\x1b[34;1m<- Leaving {}\x1b[0m", entry.to_str().unwrap());
                        }
                    }
                    Err(e) => {
                        println!("\x1b[31;1mProblem with this directory: {} -> {}\x1b[0m", entry.to_str().unwrap(), e);
                    }
                }
            }
        } else {
            match entry.file_name() {
                Some(s) => {
                    match s.to_str() {
                        Some(ss) => if ss.ends_with("~") {
                            if !options.confirmation || ask_confirmation(&Path::new(s)) {
                                match fs::remove_file(entry) {
                                    Ok(_) => {
                                        if options.verbose {
                                            println!("\x1b[32;1m{} deleted\x1b[0m", entry.to_str().unwrap());
                                        }
                                    }
                                    Err(e) => {
                                        println!("\x1b[31;1mProblem with this file: {} -> {}\x1b[0m", entry.to_str().unwrap(), e);
                                    }
                                }
                            }
                        },
                        _ => {}
                    }
                }
                None => {
                    println!("\x1b[31;1mProblem with this file: {}\x1b[0m", entry.to_str().unwrap());
                }
            }
        }
    } else {
        println!("\x1b[31;1mProblem with this entry: {}\x1b[0m", entry.to_str().unwrap());
    }
}

fn print_help() {
    println!("./clean [options] [files | dirs]");
    println!("    -r : recursive mode");
    println!("    -v : verbose mode");
    println!("    -i : prompt before every removal");
    println!("--help : print this help");
}

fn main() {
    let mut args = Vec::new();

    for argument in std::env::args() {
        args.push(argument);
    }
    let mut options = CleanOptions{recursive: false, verbose: false, confirmation: false};
    let mut files = Vec::new();

    args.remove(0);
    for tmp in args.iter() {
        if tmp.clone().into_bytes()[0] == '-' as u8 {
            let mut tmp_arg = String::from_str(tmp.as_slice());

            tmp_arg.remove(0);
            if tmp_arg.len() > 0 {
                for character in tmp_arg.into_bytes().iter() {
                    match *character as char {
                        '-' => {
                            if tmp.as_slice() == "--help" {
                                print_help();
                                return;
                            }
                        }
                        'r' => {
                            options.recursive = true;
                        }
                        'v' => {
                            options.verbose = true;
                        }
                        'i' => {
                            options.confirmation = true;
                        }
                        _ => {
                            panic!("Unknown option: '{}', to have the options list, please launch with '-h' option\n", character);
                        }
                    }
                }
            } else {
                files.push(Path::new(tmp.as_slice()));
            }
        }
    }
    if files.len() == 0 {
        files.push(Path::new("."));
    }
    if options.verbose {
        println!("\x1b[33;1m=== VERBOSE MODE ===\x1b[0m");
    }
    for tmp in files.iter() {
        start_clean(&options, tmp);
    }
    if options.verbose {
        println!("\x1b[33;1mEnd of execution\x1b[0m");
    }
}<|endoftext|>


<commit_before>#[macro_use]
extern crate log;
extern crate env_logger;
extern crate irc;
extern crate github;
#[macro_use]
extern crate lazy_static;
extern crate regex;

use irc::client::prelude::*;
use regex::Regex;

fn main() {
    env_logger::init().unwrap();

    // This could be in a JSON config, but then we need to figure out how
    // to find that JSON config
    let irc_config: Config = Config {
        owners: Some(vec![format!("dbaron")]),
        nickname: Some(format!("wgmeeting-github-bot")),
        alt_nicks: Some(vec![format!("wgmeeting-github-bot-"),
                             format!("wgmeeting-github-bot--")]),
        username: Some(format!("dbaron-gh-bot")),
        realname: Some(format!("Bot to add meeting minutes to github issues.")),
        server: Some(format!("irc.w3.org")),
        port: Some(6667),
        use_ssl: Some(false),
        encoding: Some(format!("UTF-8")),
        channels: Some(vec![format!("#cssbottest")]),
        user_info: Some(format!("Bot to add meeting minutes to github issues.")),
        // FIXME: why doesn't this work as documented?
        //source: Some(format!("https://github.com/dbaron/wgmeeting-github-ircbot")),
        ..Default::default()
    };

    // FIXME: Eventually this should support multiple channels, plus
    // options to ask the bot which channels it's in, and which channels
    // it currently has buffers in.  (Then we can do things like ask the
    // bot to reboot itself, but it will only do so if it's not busy.)
    let mut channel_data = ChannelData::new();

    let server = IrcServer::from_config(irc_config).unwrap();
    server.identify().unwrap();
    for message in server.iter() {
        let message = message.unwrap(); // panic if there's an error

        match message.command {
            Command::PRIVMSG(ref target, ref msg) => {
                match message.source_nickname() {
                    None => {
                        // FIXME: trailing \n
                        warn!("PRIVMSG without a source! {}", message);
                    }
                    Some(ref source) => {
                        let mynick = server.current_nickname();
                        if target == mynick {
                            handle_bot_command(&server, msg, source, None)
                        } else if target.starts_with('#') {
                            let source_ = String::from(*source);
                            let line = if msg.starts_with("\x01ACTION ") && msg.ends_with("\x01") {
                                ChannelLine {
                                    source: source_,
                                    is_action: true,
                                    message: String::from(&msg[8..msg.len() - 1]),
                                }
                            } else {
                                ChannelLine {
                                    source: source_,
                                    is_action: false,
                                    message: msg.clone(),
                                }
                            };

                            // FIXME: This needs to handle requests in /me
                            match check_command_in_channel(mynick, msg) {
                                Some(ref command) => {
                                    handle_bot_command(&server, command, target, Some(source))
                                }
                                None => {
                                    match channel_data.add_line(line) {
                                        None => (),
                                        Some(response) => {
                                            server.send_privmsg(target, &*response).unwrap();
                                        }
                                    }
                                }
                            }
                        } else {
                            // FIXME: trailing \n
                            warn!("UNEXPECTED TARGET {} in message {}", target, message);
                        }
                    }
                }
            }
            _ => (),
        }
    }
}

// Take a message in the channel, and see if it was a message sent to
// this bot.
fn check_command_in_channel(mynick: &str, msg: &String) -> Option<String> {
    if !msg.starts_with(mynick) {
        return None;
    }
    let after_nick = &msg[mynick.len()..];
    if !after_nick.starts_with(":") && !after_nick.starts_with(",") {
        return None;
    }
    let after_punct = &after_nick[1..];
    Some(String::from(after_punct.trim_left()))
}

fn handle_bot_command(server: &IrcServer,
                      command: &str,
                      response_target: &str,
                      response_username: Option<&str>) {

    let send_line = |response_username: Option<&str>, line: &str| {
        let adjusted_line = match response_username {
            None => String::from(line),
            Some(username) => String::from(username) + ", " + line,
        };
        server
            .send_privmsg(response_target, &adjusted_line)
            .unwrap();
    };

    if command == "help" {
        send_line(response_username, "The commands I understand are:");
        send_line(None, "  help     Send this message.");
        return;
    }

    send_line(response_username,
              "Sorry, I don't understand that command.  Try 'help'.");
}

struct ChannelLine {
    source: String,
    is_action: bool,
    message: String,
}

struct TopicData {
    topic: String,
    github_url: Option<String>,
    lines: Vec<ChannelLine>,
}

struct ChannelData {
    current_topic: Option<TopicData>,
}

impl TopicData {
    fn new(topic: &str) -> TopicData {
        let topic_ = String::from(topic);
        TopicData {
            topic: topic_,
            github_url: None,
            lines: vec![],
        }
    }
}

fn ci_starts_with(s: &str, prefix: &str) -> bool {
    assert!(prefix.to_lowercase() == prefix);

    s.len() >= prefix.len() && s[0..prefix.len()].to_lowercase() == prefix
}

fn strip_ci_prefix(s: &str, prefix: &str) -> Option<String> {
    if ci_starts_with(s, prefix) {
        Some(String::from(s[prefix.len()..].trim_left()))
    } else {
        None
    }
}

impl ChannelData {
    fn new() -> ChannelData {
        ChannelData { current_topic: None }
    }

    // Returns the response that should be sent to the message over IRC.
    fn add_line(&mut self, line: ChannelLine) -> Option<String> {
        match strip_ci_prefix(&line.message, "topic:") {
            None => (),
            Some(ref topic) => {
                self.start_topic(line.message[6..].trim_left());
            }
        }
        if line.source == "trackbot" && line.is_action == true &&
           line.message == "is ending a teleconference." {
            self.end_topic();
        }
        match self.current_topic {
            None => None,
            Some(ref mut data) => {
                let new_url_option = extract_github_url(&line.message);
                let response = match (new_url_option.as_ref(), data.github_url.as_ref()) {
                    (None, _) => None,
                    // FIXME: Add and implement instructions to cancel!
                    (Some(new_url), None) => {
                        Some(format!("OK, I'll post this discussion to {}", new_url))
                    }
                    (Some(new_url), Some(old_url)) if old_url == new_url => None,
                    (Some(new_url), Some(old_url)) => {
                        Some(format!("OK, I'll post this discussion to {} instead of {} like you said before",
                                     new_url,
                                     old_url))
                    }
                };

                if let Some(new_url) = new_url_option {
                    data.github_url = Some(new_url);
                }

                data.lines.push(line);

                response
            }
        }
    }

    fn start_topic(&mut self, topic: &str) {
        if self.current_topic.is_some() {
            self.end_topic();
        }

        self.current_topic = Some(TopicData::new(topic));
    }

    fn end_topic(&mut self) {
        // TODO: Test the topic boundary code.
        // FIXME: Do something with the data rather than throwing it away!
        self.current_topic = None;
    }
}

fn extract_github_url(message: &str) -> Option<String> {
    lazy_static! {
        static ref GITHUB_URL_RE: Regex =
            Regex::new(r"^https://github.com/(?P<repo>[^/]*/[^/]*)/issues/(?P<number>[0-9]+)$")
            .unwrap();
        static ref ALLOWED_REPOS: [String; 1] = [format!("dbaron/wgmeeting-github-ircbot")];
    }
    for prefix in ["topic:", "github topic:"].into_iter() {
        match strip_ci_prefix(&message, prefix) {
            None => (),
            Some(ref maybe_url) => {
                match GITHUB_URL_RE.captures(maybe_url) {
                    None => (),
                    Some(ref caps) => {
                        for repo in ALLOWED_REPOS.into_iter() {
                            if caps["repo"] == *repo {
                                return Some(maybe_url.clone());
                            }
                        }
                    }
                }
            }
        }
    }
    None
}
<commit_msg>Use "if let" more now that I know about it.<commit_after>#[macro_use]
extern crate log;
extern crate env_logger;
extern crate irc;
extern crate github;
#[macro_use]
extern crate lazy_static;
extern crate regex;

use irc::client::prelude::*;
use regex::Regex;

fn main() {
    env_logger::init().unwrap();

    // This could be in a JSON config, but then we need to figure out how
    // to find that JSON config
    let irc_config: Config = Config {
        owners: Some(vec![format!("dbaron")]),
        nickname: Some(format!("wgmeeting-github-bot")),
        alt_nicks: Some(vec![format!("wgmeeting-github-bot-"),
                             format!("wgmeeting-github-bot--")]),
        username: Some(format!("dbaron-gh-bot")),
        realname: Some(format!("Bot to add meeting minutes to github issues.")),
        server: Some(format!("irc.w3.org")),
        port: Some(6667),
        use_ssl: Some(false),
        encoding: Some(format!("UTF-8")),
        channels: Some(vec![format!("#cssbottest")]),
        user_info: Some(format!("Bot to add meeting minutes to github issues.")),
        // FIXME: why doesn't this work as documented?
        //source: Some(format!("https://github.com/dbaron/wgmeeting-github-ircbot")),
        ..Default::default()
    };

    // FIXME: Eventually this should support multiple channels, plus
    // options to ask the bot which channels it's in, and which channels
    // it currently has buffers in.  (Then we can do things like ask the
    // bot to reboot itself, but it will only do so if it's not busy.)
    let mut channel_data = ChannelData::new();

    let server = IrcServer::from_config(irc_config).unwrap();
    server.identify().unwrap();
    for message in server.iter() {
        let message = message.unwrap(); // panic if there's an error

        match message.command {
            Command::PRIVMSG(ref target, ref msg) => {
                match message.source_nickname() {
                    None => {
                        // FIXME: trailing \n
                        warn!("PRIVMSG without a source! {}", message);
                    }
                    Some(ref source) => {
                        let mynick = server.current_nickname();
                        if target == mynick {
                            handle_bot_command(&server, msg, source, None)
                        } else if target.starts_with('#') {
                            let source_ = String::from(*source);
                            let line = if msg.starts_with("\x01ACTION ") && msg.ends_with("\x01") {
                                ChannelLine {
                                    source: source_,
                                    is_action: true,
                                    message: String::from(&msg[8..msg.len() - 1]),
                                }
                            } else {
                                ChannelLine {
                                    source: source_,
                                    is_action: false,
                                    message: msg.clone(),
                                }
                            };

                            // FIXME: This needs to handle requests in /me
                            match check_command_in_channel(mynick, msg) {
                                Some(ref command) => {
                                    handle_bot_command(&server, command, target, Some(source))
                                }
                                None => {
                                    if let Some(response) = channel_data.add_line(line) {
                                        server.send_privmsg(target, &*response).unwrap();
                                    }
                                }
                            }
                        } else {
                            // FIXME: trailing \n
                            warn!("UNEXPECTED TARGET {} in message {}", target, message);
                        }
                    }
                }
            }
            _ => (),
        }
    }
}

// Take a message in the channel, and see if it was a message sent to
// this bot.
fn check_command_in_channel(mynick: &str, msg: &String) -> Option<String> {
    if !msg.starts_with(mynick) {
        return None;
    }
    let after_nick = &msg[mynick.len()..];
    if !after_nick.starts_with(":") && !after_nick.starts_with(",") {
        return None;
    }
    let after_punct = &after_nick[1..];
    Some(String::from(after_punct.trim_left()))
}

fn handle_bot_command(server: &IrcServer,
                      command: &str,
                      response_target: &str,
                      response_username: Option<&str>) {

    let send_line = |response_username: Option<&str>, line: &str| {
        let adjusted_line = match response_username {
            None => String::from(line),
            Some(username) => String::from(username) + ", " + line,
        };
        server
            .send_privmsg(response_target, &adjusted_line)
            .unwrap();
    };

    if command == "help" {
        send_line(response_username, "The commands I understand are:");
        send_line(None, "  help     Send this message.");
        return;
    }

    send_line(response_username,
              "Sorry, I don't understand that command.  Try 'help'.");
}

struct ChannelLine {
    source: String,
    is_action: bool,
    message: String,
}

struct TopicData {
    topic: String,
    github_url: Option<String>,
    lines: Vec<ChannelLine>,
}

struct ChannelData {
    current_topic: Option<TopicData>,
}

impl TopicData {
    fn new(topic: &str) -> TopicData {
        let topic_ = String::from(topic);
        TopicData {
            topic: topic_,
            github_url: None,
            lines: vec![],
        }
    }
}

fn ci_starts_with(s: &str, prefix: &str) -> bool {
    assert!(prefix.to_lowercase() == prefix);

    s.len() >= prefix.len() && s[0..prefix.len()].to_lowercase() == prefix
}

fn strip_ci_prefix(s: &str, prefix: &str) -> Option<String> {
    if ci_starts_with(s, prefix) {
        Some(String::from(s[prefix.len()..].trim_left()))
    } else {
        None
    }
}

impl ChannelData {
    fn new() -> ChannelData {
        ChannelData { current_topic: None }
    }

    // Returns the response that should be sent to the message over IRC.
    fn add_line(&mut self, line: ChannelLine) -> Option<String> {
        if let Some(ref topic) = strip_ci_prefix(&line.message, "topic:") {
            self.start_topic(line.message[6..].trim_left());
        }
        if line.source == "trackbot" && line.is_action == true &&
           line.message == "is ending a teleconference." {
            self.end_topic();
        }
        match self.current_topic {
            None => None,
            Some(ref mut data) => {
                let new_url_option = extract_github_url(&line.message);
                let response = match (new_url_option.as_ref(), data.github_url.as_ref()) {
                    (None, _) => None,
                    // FIXME: Add and implement instructions to cancel!
                    (Some(new_url), None) => {
                        Some(format!("OK, I'll post this discussion to {}", new_url))
                    }
                    (Some(new_url), Some(old_url)) if old_url == new_url => None,
                    (Some(new_url), Some(old_url)) => {
                        Some(format!("OK, I'll post this discussion to {} instead of {} like you said before",
                                     new_url,
                                     old_url))
                    }
                };

                if let Some(new_url) = new_url_option {
                    data.github_url = Some(new_url);
                }

                data.lines.push(line);

                response
            }
        }
    }

    fn start_topic(&mut self, topic: &str) {
        if self.current_topic.is_some() {
            self.end_topic();
        }

        self.current_topic = Some(TopicData::new(topic));
    }

    fn end_topic(&mut self) {
        // TODO: Test the topic boundary code.
        // FIXME: Do something with the data rather than throwing it away!
        self.current_topic = None;
    }
}

fn extract_github_url(message: &str) -> Option<String> {
    lazy_static! {
        static ref GITHUB_URL_RE: Regex =
            Regex::new(r"^https://github.com/(?P<repo>[^/]*/[^/]*)/issues/(?P<number>[0-9]+)$")
            .unwrap();
        static ref ALLOWED_REPOS: [String; 1] = [format!("dbaron/wgmeeting-github-ircbot")];
    }
    for prefix in ["topic:", "github topic:"].into_iter() {
        if let Some(ref maybe_url) = strip_ci_prefix(&message, prefix) {
            if let Some(ref caps) = GITHUB_URL_RE.captures(maybe_url) {
                for repo in ALLOWED_REPOS.into_iter() {
                    if caps["repo"] == *repo {
                        return Some(maybe_url.clone());
                    }
                }
            }
        }
    }
    None
}
<|endoftext|>


<commit_before>extern crate osmio;
extern crate image;
extern crate gif;
extern crate clap;

use clap::{Arg, App, AppSettings};

use std::fs;
use osmio::OSMReader;
use osmio::pbf::PBFReader;
use std::io::{Read, Write, BufReader, BufRead, BufWriter};
use std::collections::HashMap;

use gif::SetParameter;

// (frame_no, (pixel_index, num_changes))
type Frames = Vec<(u32, Vec<(u32, u16)>)>;

struct ColourRamp {
    empty_colour: (u8, u8, u8),
    steps: Vec<(u32, (u8, u8, u8))>,
}

impl ColourRamp {
    fn new_from_filename(filename: &str) -> Self {
        let mut contents = String::new();
        let mut file = fs::File::open(filename).unwrap();
        file.read_to_string(&mut contents).expect("Couldn't read colour ramp file");
        Self::new_from_text(&contents)

    }
    fn new_from_text(source: &str) -> Self {
        let lines: Vec<_> = source.lines().collect();
        let empty_vec = lines[0].split(",").filter_map(|x| x.parse::<u8>().ok()).take(3).collect::<Vec<_>>();
        let empty = (empty_vec[0], empty_vec[1], empty_vec[2]);

        let mut steps = Vec::new();
        for line in lines.iter().skip(1) {
            let line = line.split(",").filter_map(|x| x.parse::<u32>().ok()).take(4).collect::<Vec<_>>();
            let age = line[0];
            let colour = (line[1] as u8, line[2] as u8, line[3] as u8);
            steps.push((age, colour));
        }

        if steps.len() > 254 {
            panic!("Too many steps");
        }

        ColourRamp{ empty_colour: empty, steps: steps }
    }

    fn palette(&self) -> Vec<u8> {
        let mut results = Vec::with_capacity((self.steps.len()+1)*3);
        results.push(self.empty_colour.0);
        results.push(self.empty_colour.1);
        results.push(self.empty_colour.2);

        for &(_, (r, g, b)) in self.steps.iter() {
            results.push(r);
            results.push(g);
            results.push(b);
        }

        results
    }

    fn index_for_magnitude(&self, magnitude: Option<u32>) -> u8 {
        match magnitude {
            None => 0,
            Some(magnitude) => {
                if magnitude > 255 {
                    1
                } else {
                    (255 - magnitude) as u8
                }
            },
            //Some(age) => match self.steps.iter().position(|&(x, _)| x == age) {
            //    None => 0,
            //    Some(i) => (i+1) as u8,
            //}
        }
    }
}

fn read_pbf(filename: &str, height: u32, sec_per_frame: u32, bbox: &[f32; 4]) -> Frames {
    let file = BufReader::new(fs::File::open(&filename).unwrap());
    let mut node_reader = PBFReader::new(file);
    let node_reader = node_reader.nodes();

    let mut results: HashMap<u32, HashMap<u32, u16>> = HashMap::new();

    let left = bbox[0]; let bottom = bbox[1]; let right = bbox[2]; let top = bbox[3];
    let bbox_width = right - left;
    let bbox_height = top - bottom;
    let width = ((bbox_width / bbox_height) * (height as f32)) as u32;

    // 1st April 2005, midnight GMT. We presume no OSM editing before then.
    let osm_epoch = 1109635200;

    // FIXME use a btreemap?

    let mut first_frame_no = std::u32::MAX;
    let mut last_frame_no = 0;

    let mut num_nodes: u64 = 0;
    for node in node_reader {
        if let (Some(lat), Some(lon)) = (node.lat, node.lon) {
            // FIXME should be able to do non-equals but it fails for point at south pole
            if lat >= top || lat <= bottom || lon >= right || lon <= left {
                continue;
            }

            let timestamp = node.timestamp.to_epoch_number() as u64;
            if timestamp < osm_epoch {
                panic!("timestamp before epoch. Change code. {}", timestamp);
            }
            let offset = (timestamp - osm_epoch) as u32;
            // TODO double check that this rounds down
            let frame_no = offset / sec_per_frame;

            if frame_no < first_frame_no {
                first_frame_no = frame_no;
            }
            if frame_no > last_frame_no {
                last_frame_no = frame_no;
            }

            let pixel_idx = latlon_to_pixel_index(lat, lon, width, height, &bbox);
            let curr_val = results.entry(frame_no).or_insert(HashMap::new()).entry(pixel_idx).or_insert(0);
            if *curr_val < std::u16::MAX {
                *curr_val += 1;
            }

            num_nodes += 1;
            if num_nodes % 50_000_000 == 0 {
                println!("Done {} million points", num_nodes/1_000_000);
            }
        }
    }
    println!("Done {} points", num_nodes);
    let num_frames = last_frame_no - first_frame_no + 1;
    println!("There are {} frames, which is {} sec", num_frames, num_frames as f32/30.);

    let mut sorted_results = Vec::with_capacity((last_frame_no-first_frame_no+1) as usize);

    for frame_no in 0..num_frames {
        match results.remove(&(frame_no+first_frame_no)) {
            None => { sorted_results.push((frame_no, Vec::with_capacity(0))) },
            Some(pixels) => {
                let pixels = pixels.into_iter().collect();
                sorted_results.push((frame_no, pixels));
            }
        }
    }

    sorted_results
}

fn write_frames(frames: Frames, filename: &str) {
    let mut file = BufWriter::new(fs::File::create(&filename).unwrap());
    for (frame_no, pixels) in frames.into_iter() {
        write!(file, "{}", frame_no).unwrap();
        for p in pixels {
            write!(file, ",{},{}", p.0, p.1).unwrap();
        }
        write!(file, "\n").unwrap()
    }
}

fn read_frames(filename: &str) -> Frames {
    let file = BufReader::new(fs::File::open(&filename).unwrap());
    let mut results = Frames::new();

    for line in file.lines() {
        let line = line.unwrap();
        let frame_no = line.split(",").nth(0).unwrap().parse().unwrap();
        let pixels: Vec<(u32, u16)> = line.split(",").skip(1).collect::<Vec<_>>().chunks(2).map(|pair| (pair[0].parse().unwrap(), pair[1].parse().unwrap())).collect();
        results.push((frame_no, pixels))
    }

    results
}

fn latlon_to_pixel_index(lat: f32, lon: f32, width: u32, height: u32, bbox: &[f32; 4]) -> u32 {
    let left = bbox[0]; let bottom = bbox[1]; let right = bbox[2]; let top = bbox[3];
    let bbox_width = right - left;
    let bbox_height = top - bottom;

    let lat0 = top - lat;
    let lon0 = lon - left;

    let x = ((lon0/bbox_width)*(width as f32)) as u32;
    let y = ((lat0/bbox_height)*(height as f32)) as u32;

    let i = y * width + x;

    assert!(i < width*height, "{} L{}, lat = {} lon = {} width = {} height = {} bbox = {:?} x = {} y = {} i = {}", file!(), line!(), lat, lon, width, height, bbox, x, y, i);

    i
}

fn create_equirectangular_map(frames: Frames, output_image_filename: &str, height: u32, bbox: &[f32; 4], colour_ramp: &ColourRamp) {
    let mut output_file = fs::File::create(output_image_filename).expect("Can't create image");

    let left = bbox[0]; let bottom = bbox[1]; let right = bbox[2]; let top = bbox[3];
    let bbox_width = right - left;
    let bbox_height = top - bottom;
    let width = ((bbox_width / bbox_height) * (height as f32)) as u32;

    // FIXME change width/height to u16?
    let mut encoder = gif::Encoder::new(&mut output_file, width as u16, height as u16, &colour_ramp.palette()).expect("Couldn't create encoder");
    encoder.set(gif::Repeat::Infinite).expect("Couldn't get inf repeat");

    // TODO have a global palette

    let mut image = vec![None; (width*height) as usize];

    for (frame_no, pixels) in frames.into_iter() {

        for i in 0..image.len() {
            if image[i].is_some() && image[i].unwrap() > 0 {
                image[i] = image[i].map(|x| x-1);
            }
        }

        for (i, magnitude) in pixels {
            // FIXME sometimes the value is invalid
            //assert!(i < width*height, "{} L{}, width = {} height = {} i = {}", file!(), line!(), width, height, i);
            if i < width*height {
                image[i as usize] = Some(magnitude as u32);
            }
        }

        let mut pixels = Vec::with_capacity(image.len() * 4);
        for p in image.iter().cloned() {
            let index = colour_ramp.index_for_magnitude(p);
            pixels.push(index);
        }

        let mut frame = gif::Frame::from_indexed_pixels(width as u16, height as u16, pixels.as_mut_slice());
        // 30 fps, and delay is in units of 10ms.
        frame.delay = 100 / 30;

        encoder.write_frame(&frame).expect("Couldn't write frame");

        if frame_no % 30 == 0 {
            println!("Wrote frame {}", frame_no);
        }

    }

}

fn main() {
    let matches = App::new("osm-history-animation")
                          .arg(Arg::with_name("input").long("input").short("i")
                               .takes_value(true))
                          .arg(Arg::with_name("output") .long("output").short("o")
                               .takes_value(true))
                          .arg(Arg::with_name("height") .long("height").short("h")
                               .takes_value(true))
                          .arg(Arg::with_name("spf") .long("sec-per-frame").short("s")
                               .takes_value(true))
                          .arg(Arg::with_name("colour_ramp") .long("colour-ramp")
                               .takes_value(true))
                          .arg(Arg::with_name("save-intermediate").long("save-intermediate"))
                          .arg(Arg::with_name("load-intermediate").long("load-intermediate"))
                          .arg(Arg::with_name("bbox").long("bbox").takes_value(true).short("b"))
                          .setting(AppSettings::AllowLeadingHyphen)
                          .get_matches();

    let input_filename = matches.value_of("input").unwrap();
    let output_filename = matches.value_of("output").unwrap();
    let height: u32 = matches.value_of("height").unwrap().parse().unwrap();
    let frames_per_sec: u32 = matches.value_of("spf").unwrap().parse().unwrap();

    let bbox = match matches.value_of("bbox") {
        None => [-180., -90., 180., 90.],
        Some(text) => {
            let coords: Vec<f32> = text.split(",").map(|x| x.parse().unwrap() ).collect();
            [coords[0], coords[1], coords[2], coords[3]]
        }
    };

    let frames = if matches.is_present("load-intermediate") {
        println!("Reading frames from {}", input_filename);
        read_frames(&input_filename)
    } else {
        println!("Reading PBF file {}", input_filename);
        read_pbf(&input_filename, height, frames_per_sec, &bbox)
    };

    if matches.is_present("save-intermediate") {
        println!("Saving frame details to {}", output_filename);
        write_frames(frames, &output_filename);
    } else {
        let colour_ramp = ColourRamp::new_from_filename(matches.value_of("colour_ramp").unwrap());
        println!("Creating image {}", output_filename);
        create_equirectangular_map(frames, &output_filename, height, &bbox, &colour_ramp);
    }
    println!("\nFinished");
}
<commit_msg>correct var name<commit_after>extern crate osmio;
extern crate image;
extern crate gif;
extern crate clap;

use clap::{Arg, App, AppSettings};

use std::fs;
use osmio::OSMReader;
use osmio::pbf::PBFReader;
use std::io::{Read, Write, BufReader, BufRead, BufWriter};
use std::collections::HashMap;

use gif::SetParameter;

// (frame_no, (pixel_index, num_changes))
type Frames = Vec<(u32, Vec<(u32, u16)>)>;

struct ColourRamp {
    empty_colour: (u8, u8, u8),
    steps: Vec<(u32, (u8, u8, u8))>,
}

impl ColourRamp {
    fn new_from_filename(filename: &str) -> Self {
        let mut contents = String::new();
        let mut file = fs::File::open(filename).unwrap();
        file.read_to_string(&mut contents).expect("Couldn't read colour ramp file");
        Self::new_from_text(&contents)

    }
    fn new_from_text(source: &str) -> Self {
        let lines: Vec<_> = source.lines().collect();
        let empty_vec = lines[0].split(",").filter_map(|x| x.parse::<u8>().ok()).take(3).collect::<Vec<_>>();
        let empty = (empty_vec[0], empty_vec[1], empty_vec[2]);

        let mut steps = Vec::new();
        for line in lines.iter().skip(1) {
            let line = line.split(",").filter_map(|x| x.parse::<u32>().ok()).take(4).collect::<Vec<_>>();
            let age = line[0];
            let colour = (line[1] as u8, line[2] as u8, line[3] as u8);
            steps.push((age, colour));
        }

        if steps.len() > 254 {
            panic!("Too many steps");
        }

        ColourRamp{ empty_colour: empty, steps: steps }
    }

    fn palette(&self) -> Vec<u8> {
        let mut results = Vec::with_capacity((self.steps.len()+1)*3);
        results.push(self.empty_colour.0);
        results.push(self.empty_colour.1);
        results.push(self.empty_colour.2);

        for &(_, (r, g, b)) in self.steps.iter() {
            results.push(r);
            results.push(g);
            results.push(b);
        }

        results
    }

    fn index_for_magnitude(&self, magnitude: Option<u32>) -> u8 {
        match magnitude {
            None => 0,
            Some(magnitude) => {
                if magnitude > 255 {
                    1
                } else {
                    (255 - magnitude) as u8
                }
            },
            //Some(age) => match self.steps.iter().position(|&(x, _)| x == age) {
            //    None => 0,
            //    Some(i) => (i+1) as u8,
            //}
        }
    }
}

fn read_pbf(filename: &str, height: u32, sec_per_frame: u32, bbox: &[f32; 4]) -> Frames {
    let file = BufReader::new(fs::File::open(&filename).unwrap());
    let mut node_reader = PBFReader::new(file);
    let node_reader = node_reader.nodes();

    let mut results: HashMap<u32, HashMap<u32, u16>> = HashMap::new();

    let left = bbox[0]; let bottom = bbox[1]; let right = bbox[2]; let top = bbox[3];
    let bbox_width = right - left;
    let bbox_height = top - bottom;
    let width = ((bbox_width / bbox_height) * (height as f32)) as u32;

    // 1st April 2005, midnight GMT. We presume no OSM editing before then.
    let osm_epoch = 1109635200;

    // FIXME use a btreemap?

    let mut first_frame_no = std::u32::MAX;
    let mut last_frame_no = 0;

    let mut num_nodes: u64 = 0;
    for node in node_reader {
        if let (Some(lat), Some(lon)) = (node.lat, node.lon) {
            // FIXME should be able to do non-equals but it fails for point at south pole
            if lat >= top || lat <= bottom || lon >= right || lon <= left {
                continue;
            }

            let timestamp = node.timestamp.to_epoch_number() as u64;
            if timestamp < osm_epoch {
                panic!("timestamp before epoch. Change code. {}", timestamp);
            }
            let offset = (timestamp - osm_epoch) as u32;
            // TODO double check that this rounds down
            let frame_no = offset / sec_per_frame;

            if frame_no < first_frame_no {
                first_frame_no = frame_no;
            }
            if frame_no > last_frame_no {
                last_frame_no = frame_no;
            }

            let pixel_idx = latlon_to_pixel_index(lat, lon, width, height, &bbox);
            let curr_val = results.entry(frame_no).or_insert(HashMap::new()).entry(pixel_idx).or_insert(0);
            if *curr_val < std::u16::MAX {
                *curr_val += 1;
            }

            num_nodes += 1;
            if num_nodes % 50_000_000 == 0 {
                println!("Done {} million points", num_nodes/1_000_000);
            }
        }
    }
    println!("Done {} points", num_nodes);
    let num_frames = last_frame_no - first_frame_no + 1;
    println!("There are {} frames, which is {} sec", num_frames, num_frames as f32/30.);

    let mut sorted_results = Vec::with_capacity((last_frame_no-first_frame_no+1) as usize);

    for frame_no in 0..num_frames {
        match results.remove(&(frame_no+first_frame_no)) {
            None => { sorted_results.push((frame_no, Vec::with_capacity(0))) },
            Some(pixels) => {
                let pixels = pixels.into_iter().collect();
                sorted_results.push((frame_no, pixels));
            }
        }
    }

    sorted_results
}

fn write_frames(frames: Frames, filename: &str) {
    let mut file = BufWriter::new(fs::File::create(&filename).unwrap());
    for (frame_no, pixels) in frames.into_iter() {
        write!(file, "{}", frame_no).unwrap();
        for p in pixels {
            write!(file, ",{},{}", p.0, p.1).unwrap();
        }
        write!(file, "\n").unwrap()
    }
}

fn read_frames(filename: &str) -> Frames {
    let file = BufReader::new(fs::File::open(&filename).unwrap());
    let mut results = Frames::new();

    for line in file.lines() {
        let line = line.unwrap();
        let frame_no = line.split(",").nth(0).unwrap().parse().unwrap();
        let pixels: Vec<(u32, u16)> = line.split(",").skip(1).collect::<Vec<_>>().chunks(2).map(|pair| (pair[0].parse().unwrap(), pair[1].parse().unwrap())).collect();
        results.push((frame_no, pixels))
    }

    results
}

fn latlon_to_pixel_index(lat: f32, lon: f32, width: u32, height: u32, bbox: &[f32; 4]) -> u32 {
    let left = bbox[0]; let bottom = bbox[1]; let right = bbox[2]; let top = bbox[3];
    let bbox_width = right - left;
    let bbox_height = top - bottom;

    let lat0 = top - lat;
    let lon0 = lon - left;

    let x = ((lon0/bbox_width)*(width as f32)) as u32;
    let y = ((lat0/bbox_height)*(height as f32)) as u32;

    let i = y * width + x;

    assert!(i < width*height, "{} L{}, lat = {} lon = {} width = {} height = {} bbox = {:?} x = {} y = {} i = {}", file!(), line!(), lat, lon, width, height, bbox, x, y, i);

    i
}

fn create_equirectangular_map(frames: Frames, output_image_filename: &str, height: u32, bbox: &[f32; 4], colour_ramp: &ColourRamp) {
    let mut output_file = fs::File::create(output_image_filename).expect("Can't create image");

    let left = bbox[0]; let bottom = bbox[1]; let right = bbox[2]; let top = bbox[3];
    let bbox_width = right - left;
    let bbox_height = top - bottom;
    let width = ((bbox_width / bbox_height) * (height as f32)) as u32;

    // FIXME change width/height to u16?
    let mut encoder = gif::Encoder::new(&mut output_file, width as u16, height as u16, &colour_ramp.palette()).expect("Couldn't create encoder");
    encoder.set(gif::Repeat::Infinite).expect("Couldn't get inf repeat");

    // TODO have a global palette

    let mut image = vec![None; (width*height) as usize];

    for (frame_no, pixels) in frames.into_iter() {

        for i in 0..image.len() {
            if image[i].is_some() && image[i].unwrap() > 0 {
                image[i] = image[i].map(|x| x-1);
            }
        }

        for (i, magnitude) in pixels {
            // FIXME sometimes the value is invalid
            //assert!(i < width*height, "{} L{}, width = {} height = {} i = {}", file!(), line!(), width, height, i);
            if i < width*height {
                image[i as usize] = Some(magnitude as u32);
            }
        }

        let mut pixels = Vec::with_capacity(image.len() * 4);
        for p in image.iter().cloned() {
            let index = colour_ramp.index_for_magnitude(p);
            pixels.push(index);
        }

        let mut frame = gif::Frame::from_indexed_pixels(width as u16, height as u16, pixels.as_mut_slice());
        // 30 fps, and delay is in units of 10ms.
        frame.delay = 100 / 30;

        encoder.write_frame(&frame).expect("Couldn't write frame");

        if frame_no % 30 == 0 {
            println!("Wrote frame {}", frame_no);
        }

    }

}

fn main() {
    let matches = App::new("osm-history-animation")
                          .arg(Arg::with_name("input").long("input").short("i")
                               .takes_value(true))
                          .arg(Arg::with_name("output") .long("output").short("o")
                               .takes_value(true))
                          .arg(Arg::with_name("height") .long("height").short("h")
                               .takes_value(true))
                          .arg(Arg::with_name("spf") .long("sec-per-frame").short("s")
                               .takes_value(true))
                          .arg(Arg::with_name("colour_ramp") .long("colour-ramp")
                               .takes_value(true))
                          .arg(Arg::with_name("save-intermediate").long("save-intermediate"))
                          .arg(Arg::with_name("load-intermediate").long("load-intermediate"))
                          .arg(Arg::with_name("bbox").long("bbox").takes_value(true).short("b"))
                          .setting(AppSettings::AllowLeadingHyphen)
                          .get_matches();

    let input_filename = matches.value_of("input").unwrap();
    let output_filename = matches.value_of("output").unwrap();
    let height: u32 = matches.value_of("height").unwrap().parse().unwrap();
    let sec_per_frame: u32 = matches.value_of("spf").unwrap().parse().unwrap();

    let bbox = match matches.value_of("bbox") {
        None => [-180., -90., 180., 90.],
        Some(text) => {
            let coords: Vec<f32> = text.split(",").map(|x| x.parse().unwrap() ).collect();
            [coords[0], coords[1], coords[2], coords[3]]
        }
    };

    let frames = if matches.is_present("load-intermediate") {
        println!("Reading frames from {}", input_filename);
        read_frames(&input_filename)
    } else {
        println!("Reading PBF file {}", input_filename);
        read_pbf(&input_filename, height, sec_per_frame, &bbox)
    };

    if matches.is_present("save-intermediate") {
        println!("Saving frame details to {}", output_filename);
        write_frames(frames, &output_filename);
    } else {
        let colour_ramp = ColourRamp::new_from_filename(matches.value_of("colour_ramp").unwrap());
        println!("Creating image {}", output_filename);
        create_equirectangular_map(frames, &output_filename, height, &bbox, &colour_ramp);
    }
    println!("\nFinished");
}
<|endoftext|>


<commit_before>extern crate base64;
extern crate hyper;
extern crate hyper_native_tls;
extern crate pem;
#[macro_use]
extern crate prettytable;
extern crate ring;
extern crate serde_json;
extern crate url;

extern crate ct_submitter;

use std::{env, process};
use std::fs::File;
use std::io::Read;

use ring::digest;

use ct_submitter::{fetch_trusted_ct_logs, submit_cert_to_logs, AddChainRequest};

fn pems_to_chain(data: &str) -> Vec<Vec<u8>> {
    return pem::parse_many(data)
               .into_iter()
               .filter(|p| p.tag == "CERTIFICATE")
               .map(|p| p.contents)
               .collect();
}

fn build_chain_for_cert(http_client: &hyper::Client, cert: &[u8]) -> Vec<Vec<u8>> {
    let body = url::form_urlencoded::Serializer::new(String::new())
        .append_pair("b64cert", &base64::encode(cert))
        .finish();
    let body_bytes = body.as_bytes();
    let response = http_client
        .post("https://crt.sh/gen-add-chain")
        .header(hyper::header::ContentType::form_url_encoded())
        .body(hyper::client::Body::BufBody(body_bytes, body_bytes.len()))
        .send()
        .unwrap();

    let add_chain_request: AddChainRequest = serde_json::from_reader(response).unwrap();
    return add_chain_request
               .chain
               .iter()
               .map(|c| base64::decode(c).unwrap())
               .collect();
}

fn crtsh_url_for_cert(cert: &[u8]) -> String {
    return format!("https://crt.sh?q={}",
             digest::digest(&digest::SHA256, &cert)
                 .as_ref()
                 .iter()
                 .map(|b| format!("{:02X}", b))
                 .collect::<String>());
}

fn main() {
    if env::args().len() == 1 {
        println!("Usage: {} <cert-or-chain.pem ...>",
                 env::args().next().unwrap());
        process::exit(1);
    }

    let http_client = hyper::Client::with_connector(
        hyper::net::HttpsConnector::new(hyper_native_tls::NativeTlsClient::new().unwrap())
    );
    // TODO: timeout on the http_client, but for submit_cert_to_logs only, not build_chain
    let logs = fetch_trusted_ct_logs(&http_client);

    for path in env::args().skip(1) {
        println!("Submitting {} ...", path);

        let mut contents = String::new();
        File::open(path)
            .unwrap()
            .read_to_string(&mut contents)
            .unwrap();

        let mut chain = pems_to_chain(&contents);
        if chain.len() == 1 {
            // TODO: There's got to be some way to do this ourselves, instead of using crt.sh as a
            // glorified AIA chaser.
            println!("Only one certificate in chain, using crt.sh to build a full chain ...");
            chain = build_chain_for_cert(&http_client, &chain[0]);
        }
        let scts = submit_cert_to_logs(&http_client, &logs, &chain);

        // TODO: is there a better way to do this hex-encoding?
        println!("Find the cert on crt.sh: {}", crtsh_url_for_cert(&chain[0]));

        let mut table = prettytable::Table::new();
        table.add_row(row!["Log", "SCT"]);
        for (log, sct) in scts {
            table.add_row(row![log.description, base64::encode(&sct.to_raw_bytes())]);
        }
        table.printstd();
        println!();
        println!();
    }

}
<commit_msg>cargo fmt<commit_after>extern crate base64;
extern crate hyper;
extern crate hyper_native_tls;
extern crate pem;
#[macro_use]
extern crate prettytable;
extern crate ring;
extern crate serde_json;
extern crate url;

extern crate ct_submitter;

use std::{env, process};
use std::fs::File;
use std::io::Read;

use ring::digest;

use ct_submitter::{fetch_trusted_ct_logs, submit_cert_to_logs, AddChainRequest};

fn pems_to_chain(data: &str) -> Vec<Vec<u8>> {
    return pem::parse_many(data)
               .into_iter()
               .filter(|p| p.tag == "CERTIFICATE")
               .map(|p| p.contents)
               .collect();
}

fn build_chain_for_cert(http_client: &hyper::Client, cert: &[u8]) -> Vec<Vec<u8>> {
    let body = url::form_urlencoded::Serializer::new(String::new())
        .append_pair("b64cert", &base64::encode(cert))
        .finish();
    let body_bytes = body.as_bytes();
    let response = http_client
        .post("https://crt.sh/gen-add-chain")
        .header(hyper::header::ContentType::form_url_encoded())
        .body(hyper::client::Body::BufBody(body_bytes, body_bytes.len()))
        .send()
        .unwrap();

    let add_chain_request: AddChainRequest = serde_json::from_reader(response).unwrap();
    return add_chain_request
               .chain
               .iter()
               .map(|c| base64::decode(c).unwrap())
               .collect();
}

fn crtsh_url_for_cert(cert: &[u8]) -> String {
    return format!("https://crt.sh?q={}",
                   digest::digest(&digest::SHA256, &cert)
                       .as_ref()
                       .iter()
                       .map(|b| format!("{:02X}", b))
                       .collect::<String>());
}

fn main() {
    if env::args().len() == 1 {
        println!("Usage: {} <cert-or-chain.pem ...>",
                 env::args().next().unwrap());
        process::exit(1);
    }

    let http_client = hyper::Client::with_connector(
        hyper::net::HttpsConnector::new(hyper_native_tls::NativeTlsClient::new().unwrap())
    );
    // TODO: timeout on the http_client, but for submit_cert_to_logs only, not build_chain
    let logs = fetch_trusted_ct_logs(&http_client);

    for path in env::args().skip(1) {
        println!("Submitting {} ...", path);

        let mut contents = String::new();
        File::open(path)
            .unwrap()
            .read_to_string(&mut contents)
            .unwrap();

        let mut chain = pems_to_chain(&contents);
        if chain.len() == 1 {
            // TODO: There's got to be some way to do this ourselves, instead of using crt.sh as a
            // glorified AIA chaser.
            println!("Only one certificate in chain, using crt.sh to build a full chain ...");
            chain = build_chain_for_cert(&http_client, &chain[0]);
        }
        let scts = submit_cert_to_logs(&http_client, &logs, &chain);

        // TODO: is there a better way to do this hex-encoding?
        println!("Find the cert on crt.sh: {}", crtsh_url_for_cert(&chain[0]));

        let mut table = prettytable::Table::new();
        table.add_row(row!["Log", "SCT"]);
        for (log, sct) in scts {
            table.add_row(row![log.description, base64::encode(&sct.to_raw_bytes())]);
        }
        table.printstd();
        println!();
        println!();
    }

}
<|endoftext|>


<commit_before>extern crate afterparty;
extern crate env_logger;
extern crate envy;
extern crate hyper;
#[macro_use]
extern crate log;
extern crate jira_transit;
#[macro_use]
extern crate chan;
extern crate chan_signal;

use chan_signal::Signal;
use std::process::exit;
use afterparty::Hub;
use hyper::Server;
use jira_transit::{Config, DefaultGithub, DefaultJira, Transit};

fn run(_sdone: chan::Sender<()>, config: Config) {
    let github_secret = config.github_secret.clone();
    let github = DefaultGithub::new(hyper::Client::new(), config.clone());
    let jira = DefaultJira::new(hyper::Client::new(), config.clone());
    let transit = Transit::new(Box::new(github), Box::new(jira));

    // wire up wehbook registrations
    let mut hub = Hub::new();
    // register interest in _all_ github events
    hub.handle_authenticated("pull_request", github_secret, transit);
    let svc = Server::http("0.0.0.0:4567")
        .unwrap()
        .handle(hub);
    info!("ready to go");
    svc.unwrap();
}

fn main() {
    env_logger::init().unwrap();
    match envy::from_env::<Config>() {
        Ok(config) => {
            let signal = chan_signal::notify(&[Signal::INT, Signal::TERM]);
            let (sdone, rdone) = chan::sync(0);
            // start the application in a thread
            ::std::thread::spawn(move || run(sdone, config));
            // wait for process signal or application to stop
            chan_select! {
                signal.recv() -> signal => {
                    println!("received signal: {:?}", signal)
                },
                rdone.recv() => {
                    println!("Program completed normally.")
                }
            }
        },
        Err(envy::Error::MissingValue(field)) => {
            panic!("missing required env var {}",
                   field.to_owned().to_uppercase())
        }
        Err(envy::Error::Custom(msg)) => panic!(msg),
    }
}
<commit_msg>unused imports<commit_after>extern crate afterparty;
extern crate env_logger;
extern crate envy;
extern crate hyper;
#[macro_use]
extern crate log;
extern crate jira_transit;
#[macro_use]
extern crate chan;
extern crate chan_signal;

use chan_signal::Signal;
use afterparty::Hub;
use hyper::Server;
use jira_transit::{Config, DefaultGithub, DefaultJira, Transit};

fn run(_sdone: chan::Sender<()>, config: Config) {
    let github_secret = config.github_secret.clone();
    let github = DefaultGithub::new(hyper::Client::new(), config.clone());
    let jira = DefaultJira::new(hyper::Client::new(), config.clone());
    let transit = Transit::new(Box::new(github), Box::new(jira));

    // wire up wehbook registrations
    let mut hub = Hub::new();
    // register interest in _all_ github events
    hub.handle_authenticated("pull_request", github_secret, transit);
    let svc = Server::http("0.0.0.0:4567")
        .unwrap()
        .handle(hub);
    info!("ready to go");
    svc.unwrap();
}

fn main() {
    env_logger::init().unwrap();
    match envy::from_env::<Config>() {
        Ok(config) => {
            let signal = chan_signal::notify(&[Signal::INT, Signal::TERM]);
            let (sdone, rdone) = chan::sync(0);
            // start the application in a thread
            ::std::thread::spawn(move || run(sdone, config));
            // wait for process signal or application to stop
            chan_select! {
                signal.recv() -> signal => {
                    println!("received signal: {:?}", signal)
                },
                rdone.recv() => {
                    println!("Program completed normally.")
                }
            }
        },
        Err(envy::Error::MissingValue(field)) => {
            panic!("missing required env var {}",
                   field.to_owned().to_uppercase())
        }
        Err(envy::Error::Custom(msg)) => panic!(msg),
    }
}
<|endoftext|>