<commit_before>#![feature(plugin)]
#![plugin(regex_macros)]

extern crate argparse;
extern crate html5ever;
extern crate html5ever_dom_sink;
extern crate hyper;
extern crate iron;
extern crate regex;
extern crate rustc_serialize;
extern crate tempfile;
extern crate tendril;
extern crate url;

mod json;
mod wiki;

use argparse::ArgumentParser;
use argparse::Store;
use std::fs::OpenOptions;
use std::io::Read;
use std::io::Write;
use std::process::Command;
use std::process::Stdio;
use std::str::FromStr;

use html5ever::Attribute;
use html5ever::tree_builder::interface::TreeSink;
use html5ever_dom_sink::common::NodeEnum;
use html5ever_dom_sink::rcdom::Handle;
use html5ever_dom_sink::rcdom::RcDom;
use hyper::Client;
use hyper::header::Connection;
use iron::Iron;
use iron::IronResult;
use iron::Request;
use iron::Response;
use iron::headers::ContentType;
use iron::middleware::Handler;
use iron::mime::Mime;
use iron::mime::SubLevel;
use iron::mime::TopLevel;
use tempfile::NamedTempFile;

use wiki::Revision;
use wiki::Wiki;

// TODO: consider doing s/en.wikipedia.org/this app's url/ on the HTML before serving it. This
// currently works fine, but might not over HTTPS.
// TODO: there are some places where I've handrolled try!() equivalents. Fix those.
// TODO: make sure I'm returning Results everywhere, and propagating errors correctly. Remove all
// uses of unwrap() that might panic.
// TODO: The page "Battle_of_Palo_Alto" is truncated. Figure out the best way to debug.

// TODO: I'm not so sure these parameter names aren't terrible.
// TODO: would Result make more sense than Option for this return value?
/// Does a 3-way merge, merging `new1` and `new2` under the assumption that both diverged from
/// `old`. Returns None if the strings do not merge together cleanly.
fn merge(old: &str, new1: &str, new2: &str) -> Option<String> {
    fn write_to_temp_file(contents: &str) -> NamedTempFile {
        let tempfile = NamedTempFile::new().unwrap();
        let mut file = OpenOptions::new().write(true).open(tempfile.path()).unwrap();
        file.write_all(contents.as_bytes()).unwrap();
        file.flush().unwrap();
        tempfile
    }

    let old_tempfile = write_to_temp_file(old);
    let new1_tempfile = write_to_temp_file(new1);
    let new2_tempfile = write_to_temp_file(new2);

    let mut process = Command::new("diff3");
    process.arg("-m").args(&[new1_tempfile.path(), old_tempfile.path(), new2_tempfile.path()])
        .stdout(Stdio::piped()).stderr(Stdio::null());
    let output = process.output().unwrap();
    if output.status.success() {
        Some(String::from_utf8(output.stdout).unwrap())
    } else {
        None
    }
}

fn replace_node_with_placeholder(original_html: &str, div_id: &str, placeholder: &str)
    -> Result<String, String> {
    // TODO: check errors
    let html = tendril::StrTendril::from_str(original_html).unwrap();
    let mut dom: RcDom = html5ever::parse(html5ever::one_input(html), Default::default());

    let handle = try!(find_node_by_id(&dom.get_document(), div_id));
    let child_handles =
        (&handle.borrow().children).into_iter().map(|child| child.clone()).collect::<Vec<_>>();
    for child_handle in child_handles {
        dom.remove_from_parent(child_handle);
    }
    dom.append(handle,
               html5ever::tree_builder::interface::NodeOrText::AppendText(
                   tendril::StrTendril::from_str(placeholder).unwrap()));
    let mut serialized: Vec<u8> = vec![];
    html5ever::serialize::serialize(&mut serialized, &dom.document, Default::default());
    // TODO: error handling
    Ok(String::from_utf8(serialized).unwrap())
}

fn find_node_by_id(handle: &Handle, id: &str) -> Result<Handle, String> {
    fn has_matching_id(attributes: &Vec<Attribute>, id: &str) -> bool {
        // TODO: could also do this with filter() and is_empty(). Not sure if that would be better.
        for attribute in attributes {
            // TODO: do I seriously have to construct a StrTendril here?
            // There has to be a better way.
            if attribute.name.local.as_slice() == "id" &&
                attribute.value == tendril::StrTendril::from_str(id).unwrap() {
                    return true;
                }
        }
        return false;
    }

    let node = handle.borrow();
    match node.node {
        NodeEnum::Element(_, ref attributes) if has_matching_id(attributes, id) => Ok(handle.clone()),
        _ => {
            for child in &node.children {
                match find_node_by_id(child, id) {
                    // TODO: this looks weird. Is this an abnormal way to do things?
                    Ok(node) => return Ok(node),
                    _ => continue,
                }
            }
            Err(format!("No node with ID {} found", id))
        },
    }
}

// TODO: rename?
struct WikipediaWithoutWikipedians {
    wiki: Wiki,
}

impl WikipediaWithoutWikipedians {
    fn new(wiki: Wiki) -> WikipediaWithoutWikipedians {
        WikipediaWithoutWikipedians { wiki: wiki }
    }

    /// Returns a vector of Revisions representing all reversions of vandalism for the page `title`.
    fn get_vandalism_reversions(&self, title: &str) -> Result<Vec<Revision>, String> {
        let revisions = try!(self.wiki.get_revisions(title, 60));
        Ok(revisions.into_iter().filter(|revision| revision.comment.contains("vandal")).collect())
    }

    // TODO: this name is terrible.
    fn get_page_with_vandalism_restored(&self, title: &str) -> Result<String, String> {
        let canonical_title = self.wiki.get_canonical_title(title).unwrap();

        let latest_revid = self.wiki.get_latest_revision(&canonical_title).unwrap().revid;
        let mut accumulated_contents =
            self.wiki.get_revision_content(&canonical_title, latest_revid).unwrap();
        let mut revisions = vec![];
        for revision in self.get_vandalism_reversions(&canonical_title).ok().unwrap() {
            let reverted_contents =
                self.wiki.get_revision_content(&canonical_title, revision.revid).unwrap();
            let vandalized_contents =
                self.wiki.get_revision_content(&canonical_title, revision.parentid).unwrap();
            match merge(&reverted_contents, &vandalized_contents, &accumulated_contents) {
                Some(merged_contents) => {
                    // TODO: replace this with a log statement, if there's a good logging framework.
                    println!(
                        "For page \"{}\", restored vandalism https://{}/w/index.php?title={}&diff=prev&oldid={}",
                        &title, self.wiki.hostname, &canonical_title, revision.revid);
                    accumulated_contents = merged_contents;
                    revisions.push(revision.revid);
                }
                None => (),
            }
        }

        // TODO: replace this with a log statement, if there's a good logging framework.
        println!("For page \"{}\", restored vandalisms reverted in: {:?}", &title, revisions);

        let body = self.wiki.parse_wikitext(&canonical_title, &accumulated_contents).unwrap();
        // Note: "title" rather than "canonical_title", so that redirects look right.
        let current_page_contents = self.wiki.get_current_page_content(&title);
        // TODO: randomize the placeholder string per-request
        let page_contents_with_placeholder =
            replace_node_with_placeholder(
                &current_page_contents, "mw-content-text", "WMW_PLACEHOLDER_TEXT").unwrap();
        Ok(page_contents_with_placeholder.replace("WMW_PLACEHOLDER_TEXT", &body))
    }
}

impl Handler for WikipediaWithoutWikipedians {
    fn handle(&self, request: &mut Request) -> IronResult<Response> {
        if request.url.path.len() == 2 && request.url.path[0] == "wiki" {
            let mut response = Response::with(
                (iron::status::Ok,
                 self.get_page_with_vandalism_restored(&request.url.path[1]).unwrap()));
            response.headers.set(ContentType(Mime(TopLevel::Text, SubLevel::Html, vec![])));
            Ok(response)
        } else {
            // TODO: should I use an HTTP redirect here instead? Would that work? Would it be desirable?
            let client = Client::new();
            // TODO: error handling
            let mut wikipedia_response =
                // TODO: not good enough. Needs to include query string. Maybe should be moved to wiki module.
                client.get(
                    &format!("https://{}/{}", self.wiki.hostname, request.url.path.join("/")))
                .header(Connection::close())
                .send().unwrap();
            let mut wikipedia_body: Vec<u8> = Vec::new();
            wikipedia_response.read_to_end(&mut wikipedia_body);

            let mut response = Response::with(wikipedia_body);
            response.status = Some(wikipedia_response.status);
            response.headers = wikipedia_response.headers.clone();
            println!("Forwarded request for {} to {}", request.url.path.join("/"),
                     self.wiki.hostname);
            Ok(response)
        }
    }
}

fn main() {
    let mut port = 3000;
    let mut wiki = "en.wikipedia.org".to_string();
    {
        let mut parser = ArgumentParser::new();
        parser.set_description("TODO: Usage description");
        parser.refer(&mut port).add_option(&["-p", "--port"], Store, "The port to serve HTTP on.");
        parser.refer(&mut wiki).add_option(&["--wiki"], Store, "The wiki to mirror.");
        parser.parse_args_or_exit();
    }
    let wikipedia_without_wikipedians =
        WikipediaWithoutWikipedians::new(Wiki::new(wiki.to_string(), Client::new()));
    Iron::new(wikipedia_without_wikipedians).http(("localhost", port)).unwrap();
}

#[cfg(test)]
mod tests {
    use super::{merge, replace_node_with_placeholder};

    #[test]
    fn test_merge_clean() {
        let old = "First line.\n\nSecond line.\n";
        let new1 = "First line.\n\nSecond line changed.\n";
        let new2 = "First line changed.\n\nSecond line.\n";
        assert_eq!("First line changed.\n\nSecond line changed.\n", merge(old, new1, new2).unwrap());
    }

    #[test]
    fn test_merge_conflicting() {
        let old = "First line.\n\nSecond line.\n";
        let new1 = "First line.\n\nSecond line changed one way.\n";
        let new2 = "First line changed.\n\nSecond line changed a different way.\n";
        assert_eq!(None, merge(old, new1, new2));
    }

    #[test]
    fn test_merge_special_characters() {
        let old = "First line.\n\nSecond line.\n";
        let new1 = "First line.\n\nSecond line 𐅃.\n";
        let new2 = "First line さようなら.\n\nSecond line.\n";
        assert_eq!("First line さようなら.\n\nSecond line 𐅃.\n", merge(old, new1, new2).unwrap());
    }

    #[test]
    fn test_replace_html_content() {
        let original_html = "

original text

Other text

";
        let expected_html = "

replaced text

Other text

";
        let processed_html = replace_node_with_placeholder(original_html, "mw-content-text", "replaced text").unwrap();
        assert_eq!(expected_html, processed_html);
    }
}
<commit_msg>Store a Client as a member of WikipediaWithoutWikipedians rather than constructing a new one for every fetch.<commit_after>#![feature(plugin)]
#![plugin(regex_macros)]

extern crate argparse;
extern crate html5ever;
extern crate html5ever_dom_sink;
extern crate hyper;
extern crate iron;
extern crate regex;
extern crate rustc_serialize;
extern crate tempfile;
extern crate tendril;
extern crate url;

mod json;
mod wiki;

use argparse::ArgumentParser;
use argparse::Store;
use std::fs::OpenOptions;
use std::io::Read;
use std::io::Write;
use std::process::Command;
use std::process::Stdio;
use std::str::FromStr;

use html5ever::Attribute;
use html5ever::tree_builder::interface::TreeSink;
use html5ever_dom_sink::common::NodeEnum;
use html5ever_dom_sink::rcdom::Handle;
use html5ever_dom_sink::rcdom::RcDom;
use hyper::Client;
use hyper::header::Connection;
use iron::Iron;
use iron::IronResult;
use iron::Request;
use iron::Response;
use iron::headers::ContentType;
use iron::middleware::Handler;
use iron::mime::Mime;
use iron::mime::SubLevel;
use iron::mime::TopLevel;
use tempfile::NamedTempFile;

use wiki::Revision;
use wiki::Wiki;

// TODO: consider doing s/en.wikipedia.org/this app's url/ on the HTML before serving it. This
// currently works fine, but might not over HTTPS.
// TODO: there are some places where I've handrolled try!() equivalents. Fix those.
// TODO: make sure I'm returning Results everywhere, and propagating errors correctly. Remove all
// uses of unwrap() that might panic.
// TODO: The page "Battle_of_Palo_Alto" is truncated. Figure out the best way to debug.

// TODO: I'm not so sure these parameter names aren't terrible.
// TODO: would Result make more sense than Option for this return value?
/// Does a 3-way merge, merging `new1` and `new2` under the assumption that both diverged from
/// `old`. Returns None if the strings do not merge together cleanly.
fn merge(old: &str, new1: &str, new2: &str) -> Option<String> {
    fn write_to_temp_file(contents: &str) -> NamedTempFile {
        let tempfile = NamedTempFile::new().unwrap();
        let mut file = OpenOptions::new().write(true).open(tempfile.path()).unwrap();
        file.write_all(contents.as_bytes()).unwrap();
        file.flush().unwrap();
        tempfile
    }

    let old_tempfile = write_to_temp_file(old);
    let new1_tempfile = write_to_temp_file(new1);
    let new2_tempfile = write_to_temp_file(new2);

    let mut process = Command::new("diff3");
    process.arg("-m").args(&[new1_tempfile.path(), old_tempfile.path(), new2_tempfile.path()])
        .stdout(Stdio::piped()).stderr(Stdio::null());
    let output = process.output().unwrap();
    if output.status.success() {
        Some(String::from_utf8(output.stdout).unwrap())
    } else {
        None
    }
}

fn replace_node_with_placeholder(original_html: &str, div_id: &str, placeholder: &str)
    -> Result<String, String> {
    // TODO: check errors
    let html = tendril::StrTendril::from_str(original_html).unwrap();
    let mut dom: RcDom = html5ever::parse(html5ever::one_input(html), Default::default());

    let handle = try!(find_node_by_id(&dom.get_document(), div_id));
    let child_handles =
        (&handle.borrow().children).into_iter().map(|child| child.clone()).collect::<Vec<_>>();
    for child_handle in child_handles {
        dom.remove_from_parent(child_handle);
    }
    dom.append(handle,
               html5ever::tree_builder::interface::NodeOrText::AppendText(
                   tendril::StrTendril::from_str(placeholder).unwrap()));
    let mut serialized: Vec<u8> = vec![];
    html5ever::serialize::serialize(&mut serialized, &dom.document, Default::default());
    // TODO: error handling
    Ok(String::from_utf8(serialized).unwrap())
}

fn find_node_by_id(handle: &Handle, id: &str) -> Result<Handle, String> {
    fn has_matching_id(attributes: &Vec<Attribute>, id: &str) -> bool {
        // TODO: could also do this with filter() and is_empty(). Not sure if that would be better.
        for attribute in attributes {
            // TODO: do I seriously have to construct a StrTendril here?
            // There has to be a better way.
            if attribute.name.local.as_slice() == "id" &&
                attribute.value == tendril::StrTendril::from_str(id).unwrap() {
                    return true;
                }
        }
        return false;
    }

    let node = handle.borrow();
    match node.node {
        NodeEnum::Element(_, ref attributes) if has_matching_id(attributes, id) => Ok(handle.clone()),
        _ => {
            for child in &node.children {
                match find_node_by_id(child, id) {
                    // TODO: this looks weird. Is this an abnormal way to do things?
                    Ok(node) => return Ok(node),
                    _ => continue,
                }
            }
            Err(format!("No node with ID {} found", id))
        },
    }
}

// TODO: rename?
struct WikipediaWithoutWikipedians {
    wiki: Wiki,
    client: Client
}

impl WikipediaWithoutWikipedians {
    fn new(wiki: Wiki, client: Client) -> WikipediaWithoutWikipedians {
        WikipediaWithoutWikipedians { wiki: wiki, client: client }
    }

    /// Returns a vector of Revisions representing all reversions of vandalism for the page `title`.
    fn get_vandalism_reversions(&self, title: &str) -> Result<Vec<Revision>, String> {
        let revisions = try!(self.wiki.get_revisions(title, 60));
        Ok(revisions.into_iter().filter(|revision| revision.comment.contains("vandal")).collect())
    }

    // TODO: this name is terrible.
    fn get_page_with_vandalism_restored(&self, title: &str) -> Result<String, String> {
        let canonical_title = self.wiki.get_canonical_title(title).unwrap();

        let latest_revid = self.wiki.get_latest_revision(&canonical_title).unwrap().revid;
        let mut accumulated_contents =
            self.wiki.get_revision_content(&canonical_title, latest_revid).unwrap();
        let mut revisions = vec![];
        for revision in self.get_vandalism_reversions(&canonical_title).ok().unwrap() {
            let reverted_contents =
                self.wiki.get_revision_content(&canonical_title, revision.revid).unwrap();
            let vandalized_contents =
                self.wiki.get_revision_content(&canonical_title, revision.parentid).unwrap();
            match merge(&reverted_contents, &vandalized_contents, &accumulated_contents) {
                Some(merged_contents) => {
                    // TODO: replace this with a log statement, if there's a good logging framework.
                    println!(
                        "For page \"{}\", restored vandalism https://{}/w/index.php?title={}&diff=prev&oldid={}",
                        &title, self.wiki.hostname, &canonical_title, revision.revid);
                    accumulated_contents = merged_contents;
                    revisions.push(revision.revid);
                }
                None => (),
            }
        }

        // TODO: replace this with a log statement, if there's a good logging framework.
        println!("For page \"{}\", restored vandalisms reverted in: {:?}", &title, revisions);

        let body = self.wiki.parse_wikitext(&canonical_title, &accumulated_contents).unwrap();
        // Note: "title" rather than "canonical_title", so that redirects look right.
        let current_page_contents = self.wiki.get_current_page_content(&title);
        // TODO: randomize the placeholder string per-request
        let page_contents_with_placeholder =
            replace_node_with_placeholder(
                &current_page_contents, "mw-content-text", "WMW_PLACEHOLDER_TEXT").unwrap();
        Ok(page_contents_with_placeholder.replace("WMW_PLACEHOLDER_TEXT", &body))
    }
}

impl Handler for WikipediaWithoutWikipedians {
    fn handle(&self, request: &mut Request) -> IronResult<Response> {
        if request.url.path.len() == 2 && request.url.path[0] == "wiki" {
            let mut response = Response::with(
                (iron::status::Ok,
                 self.get_page_with_vandalism_restored(&request.url.path[1]).unwrap()));
            response.headers.set(ContentType(Mime(TopLevel::Text, SubLevel::Html, vec![])));
            Ok(response)
        } else {
            // TODO: should I use an HTTP redirect here instead? Would that work? Would it be desirable?
            // TODO: error handling
            let mut wikipedia_response =
                // TODO: not good enough. Needs to include query string. Maybe should be moved to wiki module.
                self.client.get(
                    &format!("https://{}/{}", self.wiki.hostname, request.url.path.join("/")))
                .header(Connection::close())
                .send().unwrap();
            let mut wikipedia_body: Vec<u8> = Vec::new();
            wikipedia_response.read_to_end(&mut wikipedia_body);

            let mut response = Response::with(wikipedia_body);
            response.status = Some(wikipedia_response.status);
            response.headers = wikipedia_response.headers.clone();
            println!("Forwarded request for {} to {}", request.url.path.join("/"),
                     self.wiki.hostname);
            Ok(response)
        }
    }
}

fn main() {
    let mut port = 3000;
    let mut wiki = "en.wikipedia.org".to_string();
    {
        let mut parser = ArgumentParser::new();
        parser.set_description("TODO: Usage description");
        parser.refer(&mut port).add_option(&["-p", "--port"], Store, "The port to serve HTTP on.");
        parser.refer(&mut wiki).add_option(&["--wiki"], Store, "The wiki to mirror.");
        parser.parse_args_or_exit();
    }
    let wikipedia_without_wikipedians =
        WikipediaWithoutWikipedians::new(Wiki::new(wiki.to_string(), Client::new()), Client::new());
    Iron::new(wikipedia_without_wikipedians).http(("localhost", port)).unwrap();
}

#[cfg(test)]
mod tests {
    use super::{merge, replace_node_with_placeholder};

    #[test]
    fn test_merge_clean() {
        let old = "First line.\n\nSecond line.\n";
        let new1 = "First line.\n\nSecond line changed.\n";
        let new2 = "First line changed.\n\nSecond line.\n";
        assert_eq!("First line changed.\n\nSecond line changed.\n", merge(old, new1, new2).unwrap());
    }

    #[test]
    fn test_merge_conflicting() {
        let old = "First line.\n\nSecond line.\n";
        let new1 = "First line.\n\nSecond line changed one way.\n";
        let new2 = "First line changed.\n\nSecond line changed a different way.\n";
        assert_eq!(None, merge(old, new1, new2));
    }

    #[test]
    fn test_merge_special_characters() {
        let old = "First line.\n\nSecond line.\n";
        let new1 = "First line.\n\nSecond line 𐅃.\n";
        let new2 = "First line さようなら.\n\nSecond line.\n";
        assert_eq!("First line さようなら.\n\nSecond line 𐅃.\n", merge(old, new1, new2).unwrap());
    }

    #[test]
    fn test_replace_html_content() {
        let original_html = "

original text

Other text

";
        let expected_html = "

replaced text

Other text

";
        let processed_html = replace_node_with_placeholder(original_html, "mw-content-text", "replaced text").unwrap();
        assert_eq!(expected_html, processed_html);
    }
}
<|endoftext|>


<commit_before>#[macro_use] extern crate error_chain;
#[macro_use] extern crate prettytable;
#[macro_use] extern crate stderr;
extern crate chan;
extern crate clap;
extern crate filetime;
extern crate globset;
extern crate pretty_bytes;
extern crate separator;

#[macro_use] mod errors;
mod compression;
mod lists;
mod structs;

use clap::{App, Arg};
use errors::*;
use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
use lists::*;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::mpsc;
use structs::*;

quick_main!(run);

fn run() -> Result<()> {
    let matches = App::new("static-compress")
        .version("0.1")
        .about("Create statically-compresed copies of matching files")
        .author("NeoSmart Technologies")
        .arg(Arg::with_name("compressor")
            .short("c")
            .long("compressor")
            .value_name("[brotli|gzip|zopfli]")
            .help("The compressor to use (default: gzip)")
            .takes_value(true))
        .arg(Arg::with_name("threads")
            .short("j")
            .long("threads")
            .value_name("COUNT")
            .help("The number of simultaneous compressions (default: 1)")
            .takes_value(true))
        .arg(Arg::with_name("filters")
            .value_name("FILTER")
            .multiple(true)
            .required(true))
        .arg(Arg::with_name("ext")
            .short("e")
            .value_name("EXT")
            .long("extension")
            .help("The extension to use for compressed files (default: gz or br)"))
        /*.arg(Arg::with_name("excludes")
            .short("x")
            .value_name("FILTER")
            .long("exclude")
            .multiple(true)
            .help("Exclude files matching this glob expression"))*/
        .get_matches();

    fn get_parameter<'a, T>(matches: &clap::ArgMatches, name: &str, default_value: T) -> Result<T>
        where T: std::str::FromStr
    {
        match matches.value_of(name) {
            Some(v) => {
                Ok(v.parse().map_err(|_| ErrorKind::InvalidParameterValue(name.to_owned()))?)
            }
            None => Ok(default_value),
        }
    }

    let case_sensitive = !matches.is_present("nocase");
    let compressor = get_parameter(&matches, "compressor", CompressionAlgorithm::GZip)?;
    let temp = Parameters {
        extension: matches.value_of("ext")
            .unwrap_or(compressor.extension())
            .trim_matches(|c: char| c.is_whitespace() || c.is_control() || c == '.')
            .to_owned(),
        compressor: compressor,
        threads: get_parameter(&matches, "threads", 1)?,
    };

    /*let exclude_filters = match matches.values_of("exclude") {
        Some(values)=> values.map(|s| s.to_owned()).collect(),
        None => Vec::<String>::new(),
    };*/

    let parameters = Arc::<Parameters>::new(temp);
    let (send_queue, stats_rx, wait_group) = start_workers(&parameters);

    let include_filters: Vec<String> = match matches.values_of("filters") {
        Some(values) => Ok(values.map(|s| s.to_owned()).collect()),
        None => Err(ErrorKind::InvalidUsage),
    }?;

    let mut builder = GlobSetBuilder::new();
    for filter in include_filters.iter() {
        let glob = GlobBuilder::new(filter)
            .case_insensitive(!case_sensitive)
            //.literal_separator(true) //this would prevent repeated entry into subdirs
            .build().map_err(|_| ErrorKind::InvalidIncludeFilter)?;
        builder.add(glob);
    }
    let globset = builder.build().map_err(|_| ErrorKind::InvalidIncludeFilter)?;

    //convert filters to paths and deal out conversion jobs
    dispatch_jobs(send_queue, globset/*, exclude_filters*/)?;

    //wait for all jobs to finish
    wait_group.wait();

    //merge statistics from all threads
    let mut stats = Statistics::new();
    while let Ok(thread_stats) = stats_rx.recv() {
        stats.merge(&thread_stats);
    }

    println!("{}", stats);

    Ok(())
}

type ThreadParam = std::path::PathBuf;

fn start_workers<'a>(params: &Arc<Parameters>) -> (chan::Sender<ThreadParam>, mpsc::Receiver<Statistics>, chan::WaitGroup) {
    let (tx, rx) = chan::sync::<ThreadParam>(params.threads);
    let (stats_tx, stats_rx) = std::sync::mpsc::channel::<Statistics>();
    let wg = chan::WaitGroup::new();

    for _ in 0..params.threads {
        let local_params = params.clone();
        let local_rx = rx.clone();
        let local_stats_tx = stats_tx.clone();
        let local_wg = wg.clone();
        wg.add(1);
        std::thread::spawn(move || {
            worker_thread(local_params, local_stats_tx, local_rx);
            local_wg.done();
        });
    }

    (tx, stats_rx, wg)
}

fn yield_file<F>(path: PathBuf, globset: &GlobSet, callback: &F) -> Result<()>
    where F: Fn(PathBuf) -> Result<()>
{
    if is_hidden(&path)? {
        //we are ignoring .files and .directories
        //we may add a command-line switch to control this behavior in the future
        return Ok(());
    }
    if path.is_dir() {
        for child in path.read_dir()? {
            let child_path = child?.path();
            yield_file(child_path, globset, callback)?;
        }
        Ok(())
    }
    else {
        if globset.is_match(&path) && !is_blacklisted(&path)? {
            callback(path)?;
        }
        Ok(())
    }
}

fn dispatch_jobs(send_queue: chan::Sender<ThreadParam>, globset: GlobSet/*, exclude_filters: Vec<String>*/) -> Result<()> {
    yield_file(Path::new("./").to_path_buf(), &globset, &|path: PathBuf| {
        if is_blacklisted(&path)? {
            //this path has been excluded
            return Ok(());
        }
        //make sure this is a file, not a folder
        match std::fs::metadata(&path) {
            Ok(metadata) => {
                if metadata.is_file() {
                    send_queue.send(path);
                }
            }
            Err(e) => errstln!("{}: {}", path.to_string_lossy(), e),
        };
        Ok(())
    })
}

fn worker_thread(params: Arc<Parameters>, stats_tx: mpsc::Sender<Statistics>, rx: chan::Receiver<ThreadParam>) {
    let mut local_stats = Statistics::new();

    loop {
        let src = match rx.recv() {
            Some(task) => task,
            None => break, //no more tasks
        };

        //in a nested function so we can handle errors centrally
        fn compress_single(src: &ThreadParam, params: &Parameters, mut local_stats: &mut Statistics) -> Result<()> {
            let dst_path = format!("{}.{}",
                                   src.to_str().ok_or(ErrorKind::InvalidCharactersInPath)?,
                                   params.extension);
            let dst = Path::new(&dst_path);

            //again, in a scope for error handling
            |local_stats: &mut Statistics| -> Result<()> {
                    println!("{}", src.to_string_lossy());
                    let src_metadata = std::fs::metadata(src)?;

                    //don't compress files that are already compressed that haven't changed
                    if let Ok(dst_metadata) = std::fs::metadata(dst) {
                        //the destination already exists
                        let src_seconds = src_metadata.modified()?.duration_since(std::time::UNIX_EPOCH)?.as_secs();
                        let dst_seconds = dst_metadata.modified()?.duration_since(std::time::UNIX_EPOCH)?.as_secs();
                        match src_seconds == dst_seconds {
                            true => {
                                local_stats.update(src_metadata.len(), dst_metadata.len(), false);
                                return Ok(());//no need to recompress
                            },
                            false => {
                                std::fs::remove_file(dst)?; //throw if we can't
                            }
                        };
                    }
                    params.compressor.compress(src.as_path(), dst)?;
                    let dst_metadata = std::fs::metadata(dst)?;
                    local_stats.update(src_metadata.len(), dst_metadata.len(), true);
                    let src_modified = filetime::FileTime::from_last_modification_time(&src_metadata);
                    filetime::set_file_times(dst, filetime::FileTime::zero(), src_modified).unwrap_or_default();

                    Ok(())
                }(&mut local_stats)
                .map_err(|e| {
                    //try deleting the invalid destination file, but don't care if we can't
                    std::fs::remove_file(dst).unwrap_or_default();
                    e //return the same error
                })
        }

        if let Err(e) = compress_single(&src, &params, &mut local_stats) {
            errstln!("Error compressing {}: {}", src.to_string_lossy(), e);
        }
    }

    if !stats_tx.send(local_stats).is_ok() {
        errstln!("Error compiling statistics!");
    }
}

fn str_search(sorted: &[&str], search_term: &str, case_sensitive: bool) -> std::result::Result<usize, usize> {
    let term = match case_sensitive {
        true => search_term.to_owned(),
        false => search_term.to_lowercase(),
    };

    sorted.binary_search_by(|probe| probe.cmp(&&*term))
}

fn is_hidden(path: &Path) -> Result<bool> {
    let hidden = match path.file_name() {
        Some(x) => x.to_str().ok_or(ErrorKind::InvalidCharactersInPath)?
            .starts_with("."),
        None => false
    };
    Ok(hidden)
}

fn is_blacklisted(path: &Path) -> Result<bool> {

    let r = match path.extension() {
        Some(x) => {
            let ext = x.to_str().ok_or(ErrorKind::InvalidCharactersInPath)?;
            str_search(COMP_EXTS, &ext, false).is_ok()
        },
        None => false,
    };

    return Ok(r);
}

<commit_msg>Optimizations and improvements<commit_after>#[macro_use] extern crate error_chain;
#[macro_use] extern crate prettytable;
#[macro_use] extern crate stderr;
extern crate chan;
extern crate clap;
extern crate filetime;
extern crate globset;
extern crate pretty_bytes;
extern crate separator;

#[macro_use] mod errors;
mod compression;
mod lists;
mod structs;

use clap::{App, Arg};
use errors::*;
use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
use lists::*;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::mpsc;
use structs::*;

quick_main!(run);

fn run() -> Result<()> {
    let matches = App::new("static-compress")
        .version("0.1")
        .about("Create statically-compresed copies of matching files")
        .author("NeoSmart Technologies")
        .arg(Arg::with_name("compressor")
            .short("c")
            .long("compressor")
            .value_name("[brotli|gzip|zopfli]")
            .help("The compressor to use (default: gzip)")
            .takes_value(true))
        .arg(Arg::with_name("threads")
            .short("j")
            .long("threads")
            .value_name("COUNT")
            .help("The number of simultaneous compressions (default: 1)")
            .takes_value(true))
        .arg(Arg::with_name("filters")
            .value_name("FILTER")
            .multiple(true)
            .required(true))
        .arg(Arg::with_name("ext")
            .short("e")
            .value_name("EXT")
            .long("extension")
            .help("The extension to use for compressed files (default: gz or br)"))
        /*.arg(Arg::with_name("excludes")
            .short("x")
            .value_name("FILTER")
            .long("exclude")
            .multiple(true)
            .help("Exclude files matching this glob expression"))*/
        .get_matches();

    fn get_parameter<'a, T>(matches: &clap::ArgMatches, name: &str, default_value: T) -> Result<T>
        where T: std::str::FromStr
    {
        match matches.value_of(name) {
            Some(v) => {
                Ok(v.parse().map_err(|_| ErrorKind::InvalidParameterValue(name.to_owned()))?)
            }
            None => Ok(default_value),
        }
    }

    let case_sensitive = !matches.is_present("nocase");
    let compressor = get_parameter(&matches, "compressor", CompressionAlgorithm::GZip)?;
    let temp = Parameters {
        extension: matches.value_of("ext")
            .unwrap_or(compressor.extension())
            .trim_matches(|c: char| c.is_whitespace() || c.is_control() || c == '.')
            .to_owned(),
        compressor: compressor,
        threads: get_parameter(&matches, "threads", 1)?,
    };

    /*let exclude_filters = match matches.values_of("exclude") {
        Some(values)=> values.map(|s| s.to_owned()).collect(),
        None => Vec::<String>::new(),
    };*/

    let parameters = Arc::<Parameters>::new(temp);
    let (send_queue, stats_rx, wait_group) = start_workers(&parameters);

    let include_filters: Vec<String> = match matches.values_of("filters") {
        Some(values) => Ok(values.map(|s| s.to_owned()).collect()),
        None => Err(ErrorKind::InvalidUsage),
    }?;

    let mut builder = GlobSetBuilder::new();
    for filter in include_filters.iter() {
        let glob = GlobBuilder::new(filter)
            .case_insensitive(!case_sensitive)
            //.literal_separator(true) //this would prevent repeated entry into subdirs
            .build().map_err(|_| ErrorKind::InvalidIncludeFilter)?;
        builder.add(glob);
    }
    let globset = builder.build().map_err(|_| ErrorKind::InvalidIncludeFilter)?;

    //convert filters to paths and deal out conversion jobs
    dispatch_jobs(send_queue, globset/*, exclude_filters*/)?;

    //wait for all jobs to finish
    wait_group.wait();

    //merge statistics from all threads
    let mut stats = Statistics::new();
    while let Ok(thread_stats) = stats_rx.recv() {
        stats.merge(&thread_stats);
    }

    println!("{}", stats);

    Ok(())
}

type ThreadParam = std::path::PathBuf;

fn start_workers<'a>(params: &Arc<Parameters>) -> (chan::Sender<ThreadParam>, mpsc::Receiver<Statistics>, chan::WaitGroup) {
    let (tx, rx) = chan::sync::<ThreadParam>(params.threads);
    let (stats_tx, stats_rx) = std::sync::mpsc::channel::<Statistics>();
    let wg = chan::WaitGroup::new();

    for _ in 0..params.threads {
        let local_params = params.clone();
        let local_rx = rx.clone();
        let local_stats_tx = stats_tx.clone();
        let local_wg = wg.clone();
        wg.add(1);
        std::thread::spawn(move || {
            worker_thread(local_params, local_stats_tx, local_rx);
            local_wg.done();
        });
    }

    (tx, stats_rx, wg)
}

fn yield_file<F>(path: PathBuf, globset: &GlobSet, callback: &F) -> Result<()>
    where F: Fn(PathBuf) -> Result<()>
{
    if is_hidden(&path)? {
        //we are ignoring .files and .directories
        //we may add a command-line switch to control this behavior in the future
        return Ok(());
    }

    if path.is_dir() {
        for child in path.read_dir()? {
            let child_path = child?.path();
            yield_file(child_path, globset, callback)?;
        }
    }
    else {
        //I'm presuming the binary search in is_blacklisted is faster
        //than globset.is_match, but we should benchmark it at some point
        if !is_blacklisted(&path)? && globset.is_match(&path) {
            callback(path)?;
        }
    }

    Ok(())
}

fn dispatch_jobs(send_queue: chan::Sender<ThreadParam>, globset: GlobSet/*, exclude_filters: Vec<String>*/) -> Result<()> {
    yield_file(Path::new("./").to_path_buf(), &globset, &|path: PathBuf| {
        send_queue.send(path);
        Ok(())
    })
}

fn worker_thread(params: Arc<Parameters>, stats_tx: mpsc::Sender<Statistics>, rx: chan::Receiver<ThreadParam>) {
    let mut local_stats = Statistics::new();

    loop {
        let src = match rx.recv() {
            Some(task) => task,
            None => break, //no more tasks
        };

        //in a nested function so we can handle errors centrally
        fn compress_single(src: &ThreadParam, params: &Parameters, mut local_stats: &mut Statistics) -> Result<()> {
            let dst_path = format!("{}.{}",
                                   src.to_str().ok_or(ErrorKind::InvalidCharactersInPath)?,
                                   params.extension);
            let dst = Path::new(&dst_path);

            //again, in a scope for error handling
            |local_stats: &mut Statistics| -> Result<()> {
                    println!("{}", src.to_string_lossy());
                    let src_metadata = std::fs::metadata(src)?;

                    //don't compress files that are already compressed that haven't changed
                    if let Ok(dst_metadata) = std::fs::metadata(dst) {
                        //the destination already exists
                        let src_seconds = src_metadata.modified()?.duration_since(std::time::UNIX_EPOCH)?.as_secs();
                        let dst_seconds = dst_metadata.modified()?.duration_since(std::time::UNIX_EPOCH)?.as_secs();
                        match src_seconds == dst_seconds {
                            true => {
                                local_stats.update(src_metadata.len(), dst_metadata.len(), false);
                                return Ok(());//no need to recompress
                            },
                            false => {
                                std::fs::remove_file(dst)?; //throw if we can't
                            }
                        };
                    }
                    params.compressor.compress(src.as_path(), dst)?;
                    let dst_metadata = std::fs::metadata(dst)?;
                    local_stats.update(src_metadata.len(), dst_metadata.len(), true);
                    let src_modified = filetime::FileTime::from_last_modification_time(&src_metadata);
                    filetime::set_file_times(dst, filetime::FileTime::zero(), src_modified).unwrap_or_default();

                    Ok(())
                }(&mut local_stats)
                .map_err(|e| {
                    //try deleting the invalid destination file, but don't care if we can't
                    std::fs::remove_file(dst).unwrap_or_default();
                    e //return the same error
                })
        }

        if let Err(e) = compress_single(&src, &params, &mut local_stats) {
            errstln!("Error compressing {}: {}", src.to_string_lossy(), e);
        }
    }

    if !stats_tx.send(local_stats).is_ok() {
        errstln!("Error compiling statistics!");
    }
}

fn str_search(sorted: &[&str], search_term: &str, case_sensitive: bool) -> std::result::Result<usize, usize> {
    let term = match case_sensitive {
        true => search_term.to_owned(),
        false => search_term.to_lowercase(),
    };

    sorted.binary_search_by(|probe| probe.cmp(&&*term))
}

fn is_hidden(path: &Path) -> Result<bool> {
    let hidden = match path.file_name() {
        Some(x) => x.to_str().ok_or(ErrorKind::InvalidCharactersInPath)?
            .starts_with("."),
        None => false
    };
    Ok(hidden)
}

fn is_blacklisted(path: &Path) -> Result<bool> {

    let r = match path.extension() {
        Some(x) => {
            let ext = x.to_str().ok_or(ErrorKind::InvalidCharactersInPath)?;
            str_search(COMP_EXTS, &ext, false).is_ok()
        },
        None => false,
    };

    return Ok(r);
}

<|endoftext|>


<commit_before>extern crate clap;
extern crate walkdir;
extern crate twox_hash;

use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};

use clap::{App, Arg};
use walkdir::WalkDir;
use twox_hash::XxHash;
use std::hash::Hasher;

struct Stats {
    file_count: u64,
    total_size: u64,
    duplicate_count: u64,
}

fn collect_files(path: &str, stats: &mut Stats) -> HashMap<u64, Vec<PathBuf>> {

    let mut files = HashMap::new();

    for entry in WalkDir::new(path) {
        let entry = entry.unwrap();

        if !entry.file_type().is_file() {
            continue;
        }

        let metadata = entry.metadata().unwrap();

        if metadata.len() == 0 {
            continue;
        }

        stats.file_count += 1;
        stats.total_size += metadata.len();

        let path = entry.path().to_path_buf();

        files.entry(metadata.len()).or_insert(Vec::new()).push(path);
    }

    files
}

fn main() {
    let matches = App::new("dupr")
        .version("0.1.0")
        .author("Mike Sampson")
        .about("Duplicate file finder")
        .arg(
            Arg::with_name("DIR")
                .help("Directory to process")
                .required(true),
        )
        .arg(
            Arg::with_name("summary")
                .short("s")
                .long("summary")
                .help("Print out summary information after duplicates")
        )
        .get_matches();

    let mut stats = Stats {
        file_count: 0,
        total_size: 0,
        duplicate_count: 0,
    };

    let files = collect_files(matches.value_of("DIR").unwrap(), &mut stats);

    for f_paths in files.values() {
        if f_paths.len() == 1 {
            continue;
        }

        let mut hashes = HashMap::new();
        for f_path in f_paths {
            hashes
                .entry(hash_file(&f_path))
                .or_insert(Vec::new())
                .push(f_path);
        }

        for (hash, paths) in hashes {
            if paths.len() < 2 {
                continue;
            }
            stats.duplicate_count += paths.len() as u64;
            for p in paths {
                println!("{:016x} - {}", hash, p.display());
            }
        }
    }

    if matches.is_present("summary")  {
        println!(
            "Processed {} files with a total size of {} bytes. {} duplicates found.",
            stats.file_count,
            stats.total_size,
            stats.duplicate_count
        );
    }
}

fn hash_file<P: AsRef<Path>>(path: P) -> u64 {
    let mut file = File::open(path).unwrap();

    let mut hash = XxHash::with_seed(0);

    let mut buffer = [0; 32 * 1024];

    while let Ok(n) = file.read(&mut buffer) {
        if n == 0 {
            break;
        }
        hash.write(&buffer[0..n]);
    }

    hash.finish()
}
<commit_msg>Use Default.<commit_after>extern crate clap;
extern crate walkdir;
extern crate twox_hash;

use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};

use clap::{App, Arg};
use walkdir::WalkDir;
use twox_hash::XxHash;
use std::hash::Hasher;

#[derive(Default)]
struct Stats {
    file_count: u64,
    total_size: u64,
    duplicate_count: u64,
}

fn collect_files(path: &str, stats: &mut Stats) -> HashMap<u64, Vec<PathBuf>> {

    let mut files = HashMap::new();

    for entry in WalkDir::new(path) {
        let entry = entry.unwrap();

        if !entry.file_type().is_file() {
            continue;
        }

        let metadata = entry.metadata().unwrap();

        if metadata.len() == 0 {
            continue;
        }

        stats.file_count += 1;
        stats.total_size += metadata.len();

        let path = entry.path().to_path_buf();

        files.entry(metadata.len()).or_insert(Vec::new()).push(path);
    }

    files
}

fn main() {
    let matches = App::new("dupr")
        .version("0.1.0")
        .author("Mike Sampson")
        .about("Duplicate file finder")
        .arg(
            Arg::with_name("DIR")
                .help("Directory to process")
                .required(true),
        )
        .arg(
            Arg::with_name("summary")
                .short("s")
                .long("summary")
                .help("Print out summary information after duplicates")
        )
        .get_matches();

    let mut stats: Stats = Default::default();

    let files = collect_files(matches.value_of("DIR").unwrap(), &mut stats);

    for f_paths in files.values() {
        if f_paths.len() == 1 {
            continue;
        }

        let mut hashes = HashMap::new();
        for f_path in f_paths {
            hashes
                .entry(hash_file(&f_path))
                .or_insert(Vec::new())
                .push(f_path);
        }

        for (hash, paths) in hashes {
            if paths.len() < 2 {
                continue;
            }
            stats.duplicate_count += paths.len() as u64;
            for p in paths {
                println!("{:016x} - {}", hash, p.display());
            }
        }
    }

    if matches.is_present("summary")  {
        println!(
            "Processed {} files with a total size of {} bytes. {} duplicates found.",
            stats.file_count,
            stats.total_size,
            stats.duplicate_count
        );
    }
}

fn hash_file<P: AsRef<Path>>(path: P) -> u64 {
    let mut file = File::open(path).unwrap();

    let mut hash = XxHash::with_seed(0);

    let mut buffer = [0; 32 * 1024];

    while let Ok(n) = file.read(&mut buffer) {
        if n == 0 {
            break;
        }
        hash.write(&buffer[0..n]);
    }

    hash.finish()
}
<|endoftext|>


<commit_before>use std::fs;
use std::env;
use std::fs::File;
use std::io::Read;
use std::hash::Hasher;
use std::path::PathBuf;
use std::collections::HashMap;
use std::os::unix::fs::MetadataExt;
use std::collections::hash_map::DefaultHasher;

struct FileEntry {
    inode: u64,
    dev:  u64,
    path : String,
}

fn collect_files(dir: &String, h: &mut HashMap<u64, Vec<FileEntry>>) {

    let entries = match fs::read_dir(dir) {
        Ok(x) => x,
        _ => {
            println!("{}: invalid directory", dir);
            return
        },
    };

    for path in  entries.filter( |x| x.is_ok() ).map( |x| x.unwrap().path() )
                .collect::<Vec<PathBuf>>() {
        if let (Some(path_str), Ok(metadata)) =
                (path.to_str(), path.symlink_metadata()) {
            let ft = metadata.file_type();
            if ft.is_symlink() {
                continue;
            } else if ft.is_file() {
                match metadata.len() {
                    0 => continue,
                    i => h.entry(i).or_insert_with(Vec::new)
                          .push(
                           FileEntry{
                               inode : metadata.ino(),
                               dev   : metadata.dev(),
                               path  : String::from(path_str),
                           }),
                };
            } else if ft.is_dir() {
                collect_files(&(String::from(path_str)), h);
            }
        }
    }
}

fn find_duplicates(duplicates: &mut HashMap<u64, Vec<FileEntry>>,
                   vec: &Vec<FileEntry> ) {
    for file_entry in vec {
        if let Ok(mut file) = File::open(&file_entry.path) {
            let mut buf: [u8; 1024] = [0; 1024];
            if let Ok(nbytes) = file.read(&mut buf) {
                let mut hasher = DefaultHasher::new();
                hasher.write(&buf[..nbytes as usize]);
                let k = hasher.finish();
                duplicates.entry(k).or_insert_with(Vec::new)
                    .push(
                        FileEntry{
                            inode : file_entry.inode,
                            dev   : file_entry.dev,
                            path  : file_entry.path.to_string(),
                        });
            }
        }
    }
}

fn main() {
    let args: Vec<String> = env::args().collect();

    // Print out usage if no directories are given.
    if args.len() == 1 {
        println!("Usage: {} <dir1> [dir2 [dir3 ...]]", args[0]);
        ::std::process::exit(0);
    }

    let mut hmap: HashMap<u64, Vec<FileEntry> > = HashMap::new();

    for dir in &args[1..] {
        collect_files(dir, &mut hmap);
    }

    // Get rid of all the single entries.
    hmap.retain(|_, v| v.len() >= 2);

    let mut cluster = 1;

    for (key, val) in hmap.iter() {
        let mut duplicates: HashMap<u64, Vec<FileEntry> > = HashMap::new();
        find_duplicates(&mut duplicates, &val);

        for (hash, vec) in duplicates {
            if vec.len() >= 2 {
                println!("{} files in cluster {} (size: {}, digest: {})",
                         vec.len(), cluster, key, hash);
                // for_each becomes stable v1.22.0 onwards. Should uncomment then.
                //vec.iter().for_each(|f| println!("{}", i.path));
                for file_entry in vec {
                    println!("{}", file_entry.path);
                }
                cluster += 1;
            }
        }
    }
}
<commit_msg>Fix a typo<commit_after>use std::fs;
use std::env;
use std::fs::File;
use std::io::Read;
use std::hash::Hasher;
use std::path::PathBuf;
use std::collections::HashMap;
use std::os::unix::fs::MetadataExt;
use std::collections::hash_map::DefaultHasher;

struct FileEntry {
    inode: u64,
    dev:  u64,
    path : String,
}

fn collect_files(dir: &String, h: &mut HashMap<u64, Vec<FileEntry>>) {

    let entries = match fs::read_dir(dir) {
        Ok(x) => x,
        _ => {
            println!("{}: invalid directory", dir);
            return
        },
    };

    for path in  entries.filter( |x| x.is_ok() ).map( |x| x.unwrap().path() )
                .collect::<Vec<PathBuf>>() {
        if let (Some(path_str), Ok(metadata)) =
                (path.to_str(), path.symlink_metadata()) {
            let ft = metadata.file_type();
            if ft.is_symlink() {
                continue;
            } else if ft.is_file() {
                match metadata.len() {
                    0 => continue,
                    i => h.entry(i).or_insert_with(Vec::new)
                          .push(
                           FileEntry{
                               inode : metadata.ino(),
                               dev   : metadata.dev(),
                               path  : String::from(path_str),
                           }),
                };
            } else if ft.is_dir() {
                collect_files(&(String::from(path_str)), h);
            }
        }
    }
}

fn find_duplicates(duplicates: &mut HashMap<u64, Vec<FileEntry>>,
                   vec: &Vec<FileEntry> ) {
    for file_entry in vec {
        if let Ok(mut file) = File::open(&file_entry.path) {
            let mut buf: [u8; 1024] = [0; 1024];
            if let Ok(nbytes) = file.read(&mut buf) {
                let mut hasher = DefaultHasher::new();
                hasher.write(&buf[..nbytes as usize]);
                let k = hasher.finish();
                duplicates.entry(k).or_insert_with(Vec::new)
                    .push(
                        FileEntry{
                            inode : file_entry.inode,
                            dev   : file_entry.dev,
                            path  : file_entry.path.to_string(),
                        });
            }
        }
    }
}

fn main() {
    let args: Vec<String> = env::args().collect();

    // Print out usage if no directories are given.
    if args.len() == 1 {
        println!("Usage: {} <dir1> [dir2 [dir3 ...]]", args[0]);
        ::std::process::exit(0);
    }

    let mut hmap: HashMap<u64, Vec<FileEntry> > = HashMap::new();

    for dir in &args[1..] {
        collect_files(dir, &mut hmap);
    }

    // Get rid of all the single entries.
    hmap.retain(|_, v| v.len() >= 2);

    let mut cluster = 1;

    for (key, val) in hmap.iter() {
        let mut duplicates: HashMap<u64, Vec<FileEntry> > = HashMap::new();
        find_duplicates(&mut duplicates, &val);

        for (hash, vec) in duplicates {
            if vec.len() >= 2 {
                println!("{} files in cluster {} (size: {}, digest: {})",
                         vec.len(), cluster, key, hash);
                // for_each becomes stable v1.22.0 onwards. Should uncomment then.
                //vec.iter().for_each(|f| println!("{}", f.path));
                for file_entry in vec {
                    println!("{}", file_entry.path);
                }
                cluster += 1;
            }
        }
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate clap;
extern crate time;
extern crate clog;
extern crate semver;
#[cfg(feature = "color")]
extern crate ansi_term;

use clap::{App, Arg, ArgGroup, ArgMatches};

use clog::{LinkStyle, Clog};
use clog::fmt::ChangelogFormat;

#[macro_use]
mod macros;
mod error;
mod fmt;

use error::CliError;

pub type CliResult<T> = Result<T, CliError>;
const CLOG_CONFIG_FILE: &'static str = ".clog.toml";

fn main() {
    let styles = LinkStyle::variants();
    let matches = App::new("clog")
        // Pull version from Cargo.toml
        .version(crate_version!())
        .about("a conventional changelog for the rest of us")
        .args_from_usage("-r, --repository [URL]      'Repository used for generating commit and issue links \
                                                       (without the .git, e.g. https://github.com/thoughtram/clog)'
                          -f, --from [HASH]           'e.g. 12a8546'
                          -T, --format [FORMAT]       'The output format, defaults to markdown \
                                                       (valid values: markdown, json)'
                          -M, --major                 'Increment major version by one (Sets minor and patch to 0)'
                          -g, --git-dir [PATH]        'Local .git directory (defaults to current dir + \'.git\')*'
                          -w, --work-tree [PATH]      'Local working tree of the git project \
                                                       (defaults to current dir)*'
                          -m, --minor                 'Increment minor version by one (Sets patch to 0)'
                          -p, --patch                 'Increment patch version by one'
                          -s, --subtitle [TITLE]      'e.g. \"Crazy Release Title\"'
                          -t, --to [HASH]             'e.g. 8057684 (Defaults to HEAD when omitted)'
                          -o, --outfile [FILE]        'Where to write the changelog (Defaults to stdout when omitted)'
                          -c, --config [FILE]         'The Clog Configuration TOML file to use (Defaults to \
                                                       \'.clog.toml\')**'
                          -i, --infile [FILE]         'A changelog to append to, but *NOT* write to (Useful in \
                                                       conjunction with --outfile)'
                          --setversion [VER]          'e.g. 1.0.1'")
        // Because --from-latest-tag can't be used with --from, we add it seperately so we can
        // specify a .conflicts_with()
        .arg(Arg::from_usage("-F, --from-latest-tag 'use latest tag as start (instead of --from)'")
                .conflicts_with("from"))
        // Because we may want to add more "flavors" at a later date, we can automate the process
        // of enumerating all possible values with clap
        .arg(Arg::from_usage("-l, --link-style [STYLE]     'The style of repository link to generate (Defaults to github)'")
            .possible_values(&styles))
        // Because no one should use --changelog and either an --infile or --outfile, we add those
        // to conflicting lists
        .arg(Arg::from_usage("-C, --changelog [FILE]       'A previous changelog to prepend new changes to (this is like \
                                                           using the same file for both --infile and --outfile and \
                                                           should not be used in conjuction with either)'")
            .conflicts_with("infile")
            .conflicts_with("outfile"))
        // Since --setversion shouldn't be used with any of the --major, --minor, or --match, we
        // set those as exclusions
        .group(ArgGroup::with_name("setver")
                .args(&["major", "minor", "patch", "ver"]))
        .after_help("\
* If your .git directory is a child of your project directory (most common, such as \
/myproject/.git) AND not in the current working directory (i.e you need to use --work-tree or \
--git-dir) you only need to specify either the --work-tree (i.e. /myproject) OR --git-dir (i.e. \
/myproject/.git), you don't need to use both.\n\n\

** If using the --config to specify a clog configuration TOML file NOT in the current working \
directory (meaning you need to use --work-tree or --git-dir) AND the TOML file is inside your \
project directory (i.e. /myproject/.clog.toml) you do not need to use --work-tree or --git-dir.")
        .get_matches();

    let start_nsec = time::get_time().nsec;

    let clog = from_matches(&matches).unwrap_or_else(|e| e.exit());

    if let Some(ref file) = clog.outfile {
        clog.write_changelog_to(file).unwrap_or_else(|e| e.exit());

        let end_nsec = time::get_time().nsec;
        let elapsed_mssec = (end_nsec - start_nsec) / 1000000;
        println!("changelog written. (took {} ms)", elapsed_mssec);
    } else {
        clog.write_changelog().unwrap_or_else(|e| e.exit());
    }
}

/// Creates a `Clog` struct from command line `clap::ArgMatches`
///
/// # Example
///
/// ```ignore
/// # use clog::Clog;
///
/// let matches = // clap settings...
///
/// let clog = Clog::from_matches(matches).unwrap_or_else(|e| {
///     e.exit();
/// });
/// ```
pub fn from_matches(matches: &ArgMatches) -> CliResult<Clog> {
    debugln!("Creating clog from matches");
    let mut clog = if let Some(cfg) = matches.value_of("config") {
        debugln!("User passed in config file: {:?}", cfg);
        if matches.is_present("work-dir") && matches.is_present("gitdir") {
            debugln!("User passed in both\n\tworking dir: {:?}\n\tgit dir: {:?}",
                     matches.value_of("work-dir"),
                     matches.value_of("git-dir"));
           // use --config --work-tree --git-dir
            try!(Clog::with_all(matches.value_of("git-dir").unwrap(),
                                matches.value_of("work-dir").unwrap(),
                                cfg))
        } else if let Some(dir) = matches.value_of("work-dir") {
            debugln!("User passed in working dir: {:?}", dir);
           // use --config --work-tree
            try!(Clog::with_dir_and_file(dir, cfg))
        } else if let Some(dir) = matches.value_of("git-dir") {
            debugln!("User passed in git dir: {:?}", dir);
           // use --config --git-dir
            try!(Clog::with_dir_and_file(dir, cfg))
        } else {
            debugln!("User only passed config");
           // use --config only
            try!(Clog::from_file(cfg))
        }
    } else {
        debugln!("User didn't pass in a config");
        if matches.is_present("git-dir") && matches.is_present("work-dir") {
            let wdir = matches.value_of("work-dir").unwrap();
            let gdir = matches.value_of("git-dir").unwrap();
            debugln!("User passed in both\n\tworking dir: {:?}\n\tgit dir: {:?}",
                     wdir,
                     gdir);
            try!(Clog::with_dirs(gdir, wdir))
        } else if let Some(dir) = matches.value_of("git-dir") {
            debugln!("User passed in git dir: {:?}", dir);
            try!(Clog::with_dir(dir))
        } else if let Some(dir) = matches.value_of("work-dir") {
            debugln!("User passed in working dir: {:?}", dir);
            try!(Clog::with_dir(dir))
        } else {
            debugln!("Trying the default config file");
            try!(Clog::from_file(CLOG_CONFIG_FILE))
        }
    };

    // compute version early, so we can exit on error
    clog.version = {
        // less typing later...
        let (major, minor, patch) = (matches.is_present("major"),
                                     matches.is_present("minor"),
                                     matches.is_present("patch"));
        if matches.is_present("ver") {
            matches.value_of("ver").unwrap().to_owned()
        } else if major || minor || patch {
            let mut had_v = false;
            let v_string = clog.get_latest_tag_ver();
            let first_char = v_string.chars().nth(0).unwrap_or(' ');
            let v_slice = if first_char == 'v' || first_char == 'V' {
                had_v = true;
                v_string.trim_left_matches(|c| c == 'v' || c == 'V')
            } else {
                &v_string[..]
            };
            match semver::Version::parse(v_slice) {
                Ok(ref mut v) => {
                    // if-else may be quicker, but it's longer mentally, and this isn't slow
                    match (major, minor, patch) {
                        (true,_,_) => {
                            v.major += 1;
                            v.minor = 0;
                            v.patch = 0;
                        }
                        (_,true,_) => {
                            v.minor += 1;
                            v.patch = 0;
                        }
                        (_,_,true) => {
                            v.patch += 1;
                            clog.patch_ver = true;
                        }
                        _ => unreachable!(),
                    }
                    format!("{}{}",
                            if had_v {
                                "v"
                            } else {
                                ""
                            },
                            v)
                }
                Err(e) => {
                    return Err(CliError::Semver(Box::new(e),
                                                String::from("Failed to parse version into \
                                                              valid SemVer. Ensure the version \
                                                              is in the X.Y.Z format.")));
                }
            }
        } else {
            clog.version
        }
    };

    if let Some(from) = matches.value_of("from") {
        clog.from = from.to_owned();
    } else if matches.is_present("from-latest-tag") {
        clog.from = clog.get_latest_tag();
    }

    if let Some(to) = matches.value_of("to") {
        clog.to = to.to_owned();
    }

    if let Some(repo) = matches.value_of("repository") {
        clog.repo = repo.to_owned();
    }

    if matches.is_present("link-style") {
        clog.link_style = value_t!(matches.value_of("link-style"), LinkStyle)
                              .unwrap_or(LinkStyle::Github);
    }

    if let Some(subtitle) = matches.value_of("subtitle") {
        clog.subtitle = subtitle.to_owned();
    }

    if let Some(file) = matches.value_of("outfile") {
        clog.outfile = Some(file.to_owned());
    }

    if let Some(file) = matches.value_of("infile") {
        clog.infile = Some(file.to_owned());
    }

    if let Some(file) = matches.value_of("changelog") {
        clog.infile = Some(file.to_owned());
        clog.outfile = Some(file.to_owned());
    }

    if matches.is_present("format") {
        clog.out_format = value_t_or_exit!(matches.value_of("format"), ChangelogFormat);
    }

    debugln!("Returning clog:\n{:?}", clog);

    Ok(clog)
}
<commit_msg>fix: `setversion` option name typo<commit_after>#[macro_use]
extern crate clap;
extern crate time;
extern crate clog;
extern crate semver;
#[cfg(feature = "color")]
extern crate ansi_term;

use clap::{App, Arg, ArgGroup, ArgMatches};

use clog::{LinkStyle, Clog};
use clog::fmt::ChangelogFormat;

#[macro_use]
mod macros;
mod error;
mod fmt;

use error::CliError;

pub type CliResult<T> = Result<T, CliError>;
const CLOG_CONFIG_FILE: &'static str = ".clog.toml";

fn main() {
    let styles = LinkStyle::variants();
    let matches = App::new("clog")
        // Pull version from Cargo.toml
        .version(crate_version!())
        .about("a conventional changelog for the rest of us")
        .args_from_usage("-r, --repository [URL]      'Repository used for generating commit and issue links \
                                                       (without the .git, e.g. https://github.com/thoughtram/clog)'
                          -f, --from [HASH]           'e.g. 12a8546'
                          -T, --format [FORMAT]       'The output format, defaults to markdown \
                                                       (valid values: markdown, json)'
                          -M, --major                 'Increment major version by one (Sets minor and patch to 0)'
                          -g, --git-dir [PATH]        'Local .git directory (defaults to current dir + \'.git\')*'
                          -w, --work-tree [PATH]      'Local working tree of the git project \
                                                       (defaults to current dir)*'
                          -m, --minor                 'Increment minor version by one (Sets patch to 0)'
                          -p, --patch                 'Increment patch version by one'
                          -s, --subtitle [TITLE]      'e.g. \"Crazy Release Title\"'
                          -t, --to [HASH]             'e.g. 8057684 (Defaults to HEAD when omitted)'
                          -o, --outfile [FILE]        'Where to write the changelog (Defaults to stdout when omitted)'
                          -c, --config [FILE]         'The Clog Configuration TOML file to use (Defaults to \
                                                       \'.clog.toml\')**'
                          -i, --infile [FILE]         'A changelog to append to, but *NOT* write to (Useful in \
                                                       conjunction with --outfile)'
                          --setversion [VER]          'e.g. 1.0.1'")
        // Because --from-latest-tag can't be used with --from, we add it seperately so we can
        // specify a .conflicts_with()
        .arg(Arg::from_usage("-F, --from-latest-tag 'use latest tag as start (instead of --from)'")
                .conflicts_with("from"))
        // Because we may want to add more "flavors" at a later date, we can automate the process
        // of enumerating all possible values with clap
        .arg(Arg::from_usage("-l, --link-style [STYLE]     'The style of repository link to generate (Defaults to github)'")
            .possible_values(&styles))
        // Because no one should use --changelog and either an --infile or --outfile, we add those
        // to conflicting lists
        .arg(Arg::from_usage("-C, --changelog [FILE]       'A previous changelog to prepend new changes to (this is like \
                                                           using the same file for both --infile and --outfile and \
                                                           should not be used in conjuction with either)'")
            .conflicts_with("infile")
            .conflicts_with("outfile"))
        // Since --setversion shouldn't be used with any of the --major, --minor, or --match, we
        // set those as exclusions
        .group(ArgGroup::with_name("setver")
                .args(&["major", "minor", "patch", "setversion"]))
        .after_help("\
* If your .git directory is a child of your project directory (most common, such as \
/myproject/.git) AND not in the current working directory (i.e you need to use --work-tree or \
--git-dir) you only need to specify either the --work-tree (i.e. /myproject) OR --git-dir (i.e. \
/myproject/.git), you don't need to use both.\n\n\

** If using the --config to specify a clog configuration TOML file NOT in the current working \
directory (meaning you need to use --work-tree or --git-dir) AND the TOML file is inside your \
project directory (i.e. /myproject/.clog.toml) you do not need to use --work-tree or --git-dir.")
        .get_matches();

    let start_nsec = time::get_time().nsec;

    let clog = from_matches(&matches).unwrap_or_else(|e| e.exit());

    if let Some(ref file) = clog.outfile {
        clog.write_changelog_to(file).unwrap_or_else(|e| e.exit());

        let end_nsec = time::get_time().nsec;
        let elapsed_mssec = (end_nsec - start_nsec) / 1000000;
        println!("changelog written. (took {} ms)", elapsed_mssec);
    } else {
        clog.write_changelog().unwrap_or_else(|e| e.exit());
    }
}

/// Creates a `Clog` struct from command line `clap::ArgMatches`
///
/// # Example
///
/// ```ignore
/// # use clog::Clog;
///
/// let matches = // clap settings...
///
/// let clog = Clog::from_matches(matches).unwrap_or_else(|e| {
///     e.exit();
/// });
/// ```
pub fn from_matches(matches: &ArgMatches) -> CliResult<Clog> {
    debugln!("Creating clog from matches");
    let mut clog = if let Some(cfg) = matches.value_of("config") {
        debugln!("User passed in config file: {:?}", cfg);
        if matches.is_present("work-dir") && matches.is_present("gitdir") {
            debugln!("User passed in both\n\tworking dir: {:?}\n\tgit dir: {:?}",
                     matches.value_of("work-dir"),
                     matches.value_of("git-dir"));
           // use --config --work-tree --git-dir
            try!(Clog::with_all(matches.value_of("git-dir").unwrap(),
                                matches.value_of("work-dir").unwrap(),
                                cfg))
        } else if let Some(dir) = matches.value_of("work-dir") {
            debugln!("User passed in working dir: {:?}", dir);
           // use --config --work-tree
            try!(Clog::with_dir_and_file(dir, cfg))
        } else if let Some(dir) = matches.value_of("git-dir") {
            debugln!("User passed in git dir: {:?}", dir);
           // use --config --git-dir
            try!(Clog::with_dir_and_file(dir, cfg))
        } else {
            debugln!("User only passed config");
           // use --config only
            try!(Clog::from_file(cfg))
        }
    } else {
        debugln!("User didn't pass in a config");
        if matches.is_present("git-dir") && matches.is_present("work-dir") {
            let wdir = matches.value_of("work-dir").unwrap();
            let gdir = matches.value_of("git-dir").unwrap();
            debugln!("User passed in both\n\tworking dir: {:?}\n\tgit dir: {:?}",
                     wdir,
                     gdir);
            try!(Clog::with_dirs(gdir, wdir))
        } else if let Some(dir) = matches.value_of("git-dir") {
            debugln!("User passed in git dir: {:?}", dir);
            try!(Clog::with_dir(dir))
        } else if let Some(dir) = matches.value_of("work-dir") {
            debugln!("User passed in working dir: {:?}", dir);
            try!(Clog::with_dir(dir))
        } else {
            debugln!("Trying the default config file");
            try!(Clog::from_file(CLOG_CONFIG_FILE))
        }
    };

    // compute version early, so we can exit on error
    clog.version = {
        // less typing later...
        let (major, minor, patch) = (matches.is_present("major"),
                                     matches.is_present("minor"),
                                     matches.is_present("patch"));
        if matches.is_present("setversion") {
            matches.value_of("setversion").unwrap().to_owned()
        } else if major || minor || patch {
            let mut had_v = false;
            let v_string = clog.get_latest_tag_ver();
            let first_char = v_string.chars().nth(0).unwrap_or(' ');
            let v_slice = if first_char == 'v' || first_char == 'V' {
                had_v = true;
                v_string.trim_left_matches(|c| c == 'v' || c == 'V')
            } else {
                &v_string[..]
            };
            match semver::Version::parse(v_slice) {
                Ok(ref mut v) => {
                    // if-else may be quicker, but it's longer mentally, and this isn't slow
                    match (major, minor, patch) {
                        (true,_,_) => {
                            v.major += 1;
                            v.minor = 0;
                            v.patch = 0;
                        }
                        (_,true,_) => {
                            v.minor += 1;
                            v.patch = 0;
                        }
                        (_,_,true) => {
                            v.patch += 1;
                            clog.patch_ver = true;
                        }
                        _ => unreachable!(),
                    }
                    format!("{}{}",
                            if had_v {
                                "v"
                            } else {
                                ""
                            },
                            v)
                }
                Err(e) => {
                    return Err(CliError::Semver(Box::new(e),
                                                String::from("Failed to parse version into \
                                                              valid SemVer. Ensure the version \
                                                              is in the X.Y.Z format.")));
                }
            }
        } else {
            clog.version
        }
    };

    if let Some(from) = matches.value_of("from") {
        clog.from = from.to_owned();
    } else if matches.is_present("from-latest-tag") {
        clog.from = clog.get_latest_tag();
    }

    if let Some(to) = matches.value_of("to") {
        clog.to = to.to_owned();
    }

    if let Some(repo) = matches.value_of("repository") {
        clog.repo = repo.to_owned();
    }

    if matches.is_present("link-style") {
        clog.link_style = value_t!(matches.value_of("link-style"), LinkStyle)
                              .unwrap_or(LinkStyle::Github);
    }

    if let Some(subtitle) = matches.value_of("subtitle") {
        clog.subtitle = subtitle.to_owned();
    }

    if let Some(file) = matches.value_of("outfile") {
        clog.outfile = Some(file.to_owned());
    }

    if let Some(file) = matches.value_of("infile") {
        clog.infile = Some(file.to_owned());
    }

    if let Some(file) = matches.value_of("changelog") {
        clog.infile = Some(file.to_owned());
        clog.outfile = Some(file.to_owned());
    }

    if matches.is_present("format") {
        clog.out_format = value_t_or_exit!(matches.value_of("format"), ChangelogFormat);
    }

    debugln!("Returning clog:\n{:?}", clog);

    Ok(clog)
}
<|endoftext|>


<commit_before>extern crate ansi_term;
extern crate clap;
extern crate entities;

mod css;
mod html;
mod js;

trait CharEncoder: 'static {
    fn encode(iter: &mut Iterator<Item = char>) -> Option<String>;
    fn wrap_in_quotes() -> bool;
}

trait Named {
    fn name() -> &'static str;
}

const HELP_TEMPLATE: &'static str = "
{bin}: {about}

Usage:
    {usage}

Options:
{flags}
{after-help}";

const AFTER_HELP_TEMPLATE: &'static str = "
Example:
  $ how-do-i-escape \u{00A7}

    \"\\00A7\"    -- css

    §     -- html

    \"\\u00A7\"   -- javascript
";

fn main() {
    let matches = clap::App::new("how-do-i-escape")
        .about("Prints escape sequences for unicode graphemes")
        .version(env!("CARGO_PKG_VERSION"))
        .arg(clap::Arg::with_name("grapheme").required(true))
        .template(HELP_TEMPLATE)
        .after_help(AFTER_HELP_TEMPLATE)
        .get_matches();

    let grapheme = matches.value_of("grapheme").unwrap();

    println!();
    println!("{}", language_output(&grapheme, css::Css));
    println!();
    println!("{}", language_output(&grapheme, html::Html));
    println!();
    println!("{}", language_output(&grapheme, js::Js));
    println!();
}

fn language_output<T: CharEncoder + Named>(grapheme: &str, t: T) -> String {
    let grey = ansi_term::Colour::Black.bold();
    let escape = escape_grapheme(grapheme, t);
    let lang = grey.paint(format!("-- {}", T::name()));

    format!("  {:<10} {}", escape, lang)
}

fn escape_grapheme<T: CharEncoder>(grapheme: &str, _: T) -> String {
    let mut result = String::new();
    let mut iter = grapheme.chars();

    loop {
        match T::encode(&mut iter) {
            Some(s) => result.push_str(&s),
            None => break,
        }
    }

    if T::wrap_in_quotes() {
        format!(r#""{}""#, result)
    } else {
        result
    }
}

#[cfg(test)]
mod tests {
    use super::{escape_grapheme, language_output, CharEncoder, Named};

    #[test]
    fn test_escape_grapheme() {
        struct AlwaysHello;

        impl CharEncoder for AlwaysHello {
            fn encode(iter: &mut Iterator<Item = char>) -> Option<String> {
                iter.next().map(|_| "hello".to_string())
            }
            fn wrap_in_quotes() -> bool {
                false
            }
        }

        let always_hello = escape_grapheme("a", AlwaysHello);
        assert_eq!(always_hello, "hello");

        struct Simple;

        impl CharEncoder for Simple {
            fn encode(iter: &mut Iterator<Item = char>) -> Option<String> {
                iter.next().map(|i| format!("{}", i as u32))
            }
            fn wrap_in_quotes() -> bool {
                true
            }
        }

        let simple = escape_grapheme("a", Simple);
        assert_eq!(simple, r#""97""#);
    }

    #[test]
    fn test_output() {
        struct AlwaysHello;

        impl CharEncoder for AlwaysHello {
            fn encode(iter: &mut Iterator<Item = char>) -> Option<String> {
                iter.next().map(|_| "hello".to_string())
            }
            fn wrap_in_quotes() -> bool {
                false
            }
        }

        impl Named for AlwaysHello {
            fn name() -> &'static str {
                "AlwaysHello"
            }
        }

        let grey = super::ansi_term::Colour::Black.bold();

        let actual = language_output("a", AlwaysHello);
        let expected = format!("  hello      {}", grey.paint("-- AlwaysHello"));
        assert_eq!(actual, expected);
    }
}
<commit_msg>Colourise help text<commit_after>extern crate ansi_term;
extern crate clap;
extern crate entities;

mod css;
mod html;
mod js;

trait CharEncoder: 'static {
    fn encode(iter: &mut Iterator<Item = char>) -> Option<String>;
    fn wrap_in_quotes() -> bool;
}

trait Named {
    fn name() -> &'static str;
}

const HELP_TEMPLATE: &'static str = "
{bin}: {about}

Usage:
    {usage}

Options:
{flags}
{after-help}";

fn main() {
    let matches = clap::App::new("how-do-i-escape")
        .about("Prints escape sequences for unicode graphemes")
        .version(env!("CARGO_PKG_VERSION"))
        .arg(clap::Arg::with_name("grapheme").required(true))
        .template(HELP_TEMPLATE)
        .after_help(example_text().as_ref())
        .get_matches();

    let grapheme = matches.value_of("grapheme").unwrap();

    println!();
    println!("{}", language_output(&grapheme, css::Css));
    println!();
    println!("{}", language_output(&grapheme, html::Html));
    println!();
    println!("{}", language_output(&grapheme, js::Js));
    println!();
}

fn language_output<T: CharEncoder + Named>(grapheme: &str, t: T) -> String {
    let grey = ansi_term::Colour::Black.bold();
    let escape = escape_grapheme(grapheme, t);
    let lang = grey.paint(format!("-- {}", T::name()));

    format!("  {:<10} {}", escape, lang)
}

fn example_text() -> String {
    let grey = ansi_term::Colour::Black.bold();

    format!(
        "
Example:
  $ how-do-i-escape \u{00A7}

    \"\\00A7\"    {css}

    §     {html}

    \"\\u00A7\"   {js}
",
        css = grey.paint("-- css"),
        html = grey.paint("-- html"),
        js = grey.paint("-- javascript")
    )
}

fn escape_grapheme<T: CharEncoder>(grapheme: &str, _: T) -> String {
    let mut result = String::new();
    let mut iter = grapheme.chars();

    loop {
        match T::encode(&mut iter) {
            Some(s) => result.push_str(&s),
            None => break,
        }
    }

    if T::wrap_in_quotes() {
        format!(r#""{}""#, result)
    } else {
        result
    }
}

#[cfg(test)]
mod tests {
    use super::{escape_grapheme, language_output, CharEncoder, Named};

    #[test]
    fn test_escape_grapheme() {
        struct AlwaysHello;

        impl CharEncoder for AlwaysHello {
            fn encode(iter: &mut Iterator<Item = char>) -> Option<String> {
                iter.next().map(|_| "hello".to_string())
            }
            fn wrap_in_quotes() -> bool {
                false
            }
        }

        let always_hello = escape_grapheme("a", AlwaysHello);
        assert_eq!(always_hello, "hello");

        struct Simple;

        impl CharEncoder for Simple {
            fn encode(iter: &mut Iterator<Item = char>) -> Option<String> {
                iter.next().map(|i| format!("{}", i as u32))
            }
            fn wrap_in_quotes() -> bool {
                true
            }
        }

        let simple = escape_grapheme("a", Simple);
        assert_eq!(simple, r#""97""#);
    }

    #[test]
    fn test_output() {
        struct AlwaysHello;

        impl CharEncoder for AlwaysHello {
            fn encode(iter: &mut Iterator<Item = char>) -> Option<String> {
                iter.next().map(|_| "hello".to_string())
            }
            fn wrap_in_quotes() -> bool {
                false
            }
        }

        impl Named for AlwaysHello {
            fn name() -> &'static str {
                "AlwaysHello"
            }
        }

        let grey = super::ansi_term::Colour::Black.bold();

        let actual = language_output("a", AlwaysHello);
        let expected = format!("  hello      {}", grey.paint("-- AlwaysHello"));
        assert_eq!(actual, expected);
    }
}
<|endoftext|>


<commit_before>extern crate toml;

#[cfg(not(test))]
fn main() {
    let home_dir_bin = get_ship_dir();
    let target_name = build_target_name();

    let args = vec!["test".to_owned()];
    let test_exit_status = run_command("cargo".to_owned(), args);
    if !test_exit_status.success() {
        println!("Stopping ship due to tests.");
        exit_with_code(test_exit_status.code());
    }

    let build_args = vec!["build".to_owned(), "--release".to_owned()];
    let build_exit_status = run_command("cargo".to_owned(), build_args);
    if !build_exit_status.success() {
        println!("Stopping ship due to compilation error.");
        exit_with_code(build_exit_status.code());
    }

    let cp_args = vec![target_name, home_dir_bin];
    println!("copying {} to {}", cp_args[0], cp_args[1]);
    run_command("cp".to_owned(), cp_args);
}

#[cfg(not(test))]
fn build_target_name() -> String {
    let value = read_cargo_config();
    let binary_name  = value.lookup("bin.0.name");
    let release_name = value.lookup("release.name");
    let default_name = value.lookup("package.name");
    "./target/release/".to_owned() + binary_name.or(release_name).or(default_name).expect("unable to determine target name").as_str().unwrap()
}

#[cfg(not(test))]
fn get_ship_dir() -> String {
    use std::env;
    env::home_dir().expect("unable to determine publish directory").to_str().unwrap().to_owned() + "/.bin/"
}

#[cfg(not(test))]
fn read_cargo_config() -> toml::Value {
    use std::io::prelude::*;
    use std::fs::File;

    let mut f = File::open("Cargo.toml").unwrap();
    let mut toml = String::new();
    f.read_to_string(&mut toml).ok();

    //toml::Parser::new(&toml).parse().unwrap()
    toml.parse().unwrap()
}

#[cfg(not(test))]
fn exit_with_code(code: Option<i32>) {
    use std::process::{self};
    process::exit(code.unwrap_or(0));
}

#[cfg(not(test))]
fn run_command(command: String, args: Vec<String>) -> std::process::ExitStatus {
    use std::process::{Command};

    let mut command = Command::new(command);
    command.args(&args);
    let mut child = command.spawn().unwrap_or_else(|e| panic!("{}", e));
    child.wait().unwrap_or_else(|e| panic!("{}", e))
}
<commit_msg>format code with rustfmt<commit_after>extern crate toml;

#[cfg(not(test))]
fn main() {
    let home_dir_bin = get_ship_dir();
    let target_name = build_target_name();

    let args = vec!["test".to_owned()];
    let test_exit_status = run_command("cargo".to_owned(), args);
    if !test_exit_status.success() {
        println!("Stopping ship due to tests.");
        exit_with_code(test_exit_status.code());
    }

    let build_args = vec!["build".to_owned(), "--release".to_owned()];
    let build_exit_status = run_command("cargo".to_owned(), build_args);
    if !build_exit_status.success() {
        println!("Stopping ship due to compilation error.");
        exit_with_code(build_exit_status.code());
    }

    let cp_args = vec![target_name, home_dir_bin];
    println!("copying {} to {}", cp_args[0], cp_args[1]);
    run_command("cp".to_owned(), cp_args);
}

#[cfg(not(test))]
fn build_target_name() -> String {
    let value = read_cargo_config();
    let binary_name = value.lookup("bin.0.name");
    let release_name = value.lookup("release.name");
    let default_name = value.lookup("package.name");
    "./target/release/".to_owned() +
    binary_name.or(release_name)
               .or(default_name)
               .expect("unable to determine target name")
               .as_str()
               .unwrap()
}

#[cfg(not(test))]
fn get_ship_dir() -> String {
    use std::env;
    env::home_dir().expect("unable to determine publish directory").to_str().unwrap().to_owned() +
    "/.bin/"
}

#[cfg(not(test))]
fn read_cargo_config() -> toml::Value {
    use std::io::prelude::*;
    use std::fs::File;

    let mut f = File::open("Cargo.toml").unwrap();
    let mut toml = String::new();
    f.read_to_string(&mut toml).ok();

    //toml::Parser::new(&toml).parse().unwrap()
    toml.parse().unwrap()
}

#[cfg(not(test))]
fn exit_with_code(code: Option<i32>) {
    use std::process;
    process::exit(code.unwrap_or(0));
}

#[cfg(not(test))]
fn run_command(command: String, args: Vec<String>) -> std::process::ExitStatus {
    use std::process::Command;

    let mut command = Command::new(command);
    command.args(&args);
    let mut child = command.spawn().unwrap_or_else(|e| panic!("{}", e));
    child.wait().unwrap_or_else(|e| panic!("{}", e))
}
<|endoftext|>