<commit_before>#![feature(core)]
#![feature(path)]

#![feature(plugin)]
#![plugin(regex_macros)]
extern crate regex;
extern crate git2;
extern crate "rustc-serialize" as rustc_serialize;
extern crate docopt;

use git2::{Repository, Branch, BranchType, DiffLine,
    Commit, Diff, DiffFormat, DiffDelta, DiffHunk, Error, Oid};
use docopt::Docopt;
use std::old_io as io;
use std::old_io::BufferedReader;
use std::old_io::File;
use std::str;
use regex::Regex;

// Write the Docopt usage string.
static USAGE: &'static str = "
Usage: transit <repo> [<old> <new>]

If no commits are given, transit will revwalk from latest to oldest.
";

#[derive(RustcDecodable, Debug)]
struct Args {
    arg_repo: String,
    arg_old: Option<String>,
    arg_new: Option<String>,
}

fn main() {
    // Parse the args above or die.
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());
    // Pull up the revwalk.
    let path = Path::new(args.arg_repo);
    let repo = Repository::discover(&path)
        .ok().expect("Unable to find repo.");
    // If we can destructures the two optional args into real things.
    if let (Some(old_string), Some(new_string)) = (args.arg_old, args.arg_new) {
        // Compare a specific commit pair.
        // Fist, get the commits. (Error checked)
        let old = Oid::from_str(&old_string[]).and_then(|oid| repo.find_commit(oid));
        let new = Oid::from_str(&new_string[]).and_then(|oid| repo.find_commit(oid));
        if old.is_ok() && new.is_ok() {
            let output = find_moves(&repo, &old.unwrap(), &new.unwrap()).unwrap();
            make_output(output);
        } else {
            panic!("Commit ids were not valid.");
        }
    } else {
        // Revwalk.
        let mut revwalk = repo.revwalk()
            .ok().expect("Unable to get revwalk.");
        // Setup some options.
        revwalk.simplify_first_parent(); // TODO: Maybe remove?
        let mut flags = git2::Sort::empty();
        flags.insert(git2::SORT_TIME);
        flags.insert(git2::SORT_TOPOLOGICAL);
        revwalk.set_sorting(flags);
        // Push HEAD to the revwalk.
        revwalk.push_head()
            .ok().expect("Unable to push HEAD.");
        // We sadly must collect here to use `.windows()`
        let history = revwalk.filter_map(|id| repo.find_commit(id).ok())
            .collect::<Vec<Commit>>();
        // Walk through each pair of commits.
        for pair in history.windows(2) {let (old, new) = (&pair[1], &pair[0]);
			let output = find_moves(&repo, old.clone(), new.clone()).unwrap();
			// mangle the vectors here to make pretty output
			make_output(output);
		}
	}
}

fn make_output(output: Vec<Output>) {
	println!("make_output:");
    println!("\toutput.len()={}", output.len());
    for i in range(0, output.len()) {
            println!("\told_commit={}",         output[i].old_commit);
            println!("\tnew_commit={}",         output[i].new_commit);
            println!("\torigin_line={}",        output[i].origin_line);
            println!("\tdestintation_line={}",  output[i].destination_line);
            println!("\tnum_lines={}",          output[i].num_lines);
    }
}

#[derive(Debug)]
enum State {
    Other, Addition, Deletion
}

#[derive(Debug)]
enum FoundState {
    Added, Deleted
}

fn dump_diffline(line: &DiffLine) {
    // 'origin' is wrapped in pipes to ease displaying space characters.
    print!("line: old={:?} new={:?} offset={} |origin|=|{}|\n      content={}",
             line.old_lineno(), line.new_lineno(), line.content_offset(),
             line.origin(), str::from_utf8(line.content()).unwrap());
}

fn dump_diffdelta(delta: &DiffDelta) {
    println!("delta: nfiles={} status={:?} old_file=(id={} path_bytes={:?} path={:?} tsize={}) new_file=(id={} path_bytes={:?} path={:?} tsize={})",
            delta.nfiles(), delta.status(),
            delta.old_file().id(), delta.old_file().path_bytes(), delta.old_file().path(), delta.old_file().size(),
            delta.new_file().id(), delta.new_file().path_bytes(), delta.new_file().path(), delta.new_file().size());
}

fn format_key(key: String) -> String {
    let removeWhiteSpace = regex!(r"\s{2,}"); // 2 or more whitespaces    // TODO Removes whitespace from a string.
    removeWhiteSpace.replace_all(key.as_slice(), "")
}

#[derive(Debug)]
struct Found {
    filename: Path,
    key: String,
    state: FoundState,
}

fn find_moves(repo: &Repository, old: &Commit, new: &Commit) -> Result<Vec<Output>, Error> {

    println!("\nFIND MOVES---------------------");

	let old_tree = try!(old.tree());
	let new_tree = try!(new.tree());
	// Build up a diff of the two trees.
	let diff = try!(Diff::tree_to_tree(repo, Some(&old_tree), Some(&new_tree), None));

    let mut state = State::Other;
    let mut added = String::new();
    let mut deleted = String::new();
    let mut old_path: Path = Path::new("");
    let mut new_path: Path = Path::new("");

    let mut keys: Vec<String> = Vec::new();  // TODO Will become hashmap.

    let mut founds: Vec<Found> = Vec::new();

	// Read about this function in http://alexcrichton.com/git2-rs/git2/struct.Diff.html#method.print
	// It's a bit weird, but I think it will provide the necessary information.
	diff.print(DiffFormat::Patch, |delta, maybe_hunk, line| -> bool {

        assert!(delta.nfiles() == 2, "This only works on diffs between exactly 2 files.");

		// Thinking:
		//  * If is not a hunk, keep going.
		//  * If it's a hunk, do regex magic.
		//  * Stick regex output into a hashmap as a hash.
		//  * Later, we will iterate through and output pased on the values.
		// Filter out all the boring context lines.
		// If we're not interested in this line just return since it will iterate to the next.
		if maybe_hunk.is_none() { return true }; // Return early.

        //println!("top of loop: keys={:?}", keys);
        println!("top of loop: founds={:?}", founds);
        println!("top of loop: added={:?} deleted={:?}", added, deleted);

        dump_diffline(&line);
        //dump_diffdelta(&delta);

        old_path = delta.old_file().path().unwrap().clone();
        new_path = delta.new_file().path().unwrap().clone();

		match line.origin() {
			// Additions
			'+' | '>' => {
                println!("In additions. state={:?}", state);

                added.push_str(str::from_utf8(line.content()).unwrap());

                match state {
                    State::Deletion => {
                        founds.push(Found {
                            filename: old_path.clone(),
                            key: format_key(deleted.clone()),
                            state: FoundState::Deleted,
                        });
                        deleted = String::new();
                    },
                    _ => (),
                }

                state = State::Addition;
				true
			},
			// Deletions
			'-' | '<' => {
                println!("In deletion. state={:?}", state);

                deleted.push_str(str::from_utf8(line.content()).unwrap());

                match state {
                    State::Addition => {
                        founds.push(Found {
                            filename: new_path.clone(),
                            key: format_key(added.clone()),
                            state: FoundState::Added,
                        });
                        added = String::new();
                    },
                    _ => (),
                }

                state = State::Deletion;
				true
			},
			// Other
			_         => {
                println!("in _. state={:?}", state);

                match state {
                    State::Addition => {
                        founds.push(Found {
                            filename: new_path.clone(),
                            key: format_key(added.clone()),
                            state: FoundState::Added,
                        });
                        added = String::new();
                    },
                    State::Deletion => {
                        founds.push(Found {
                            filename: old_path.clone(),
                            key: format_key(deleted.clone()),
                            state: FoundState::Deleted,
                        });
                        deleted = String::new();
                    },
                    _ => (),
                }

                state = State::Other;
                true
            }
		}
	});

    // Grab one.
    match state {
        State::Addition => {
            if added.len() > 0 {
                founds.push(Found {
                    filename: new_path,
                    key: format_key(added.clone()),
                    state: FoundState::Added,
                });
            }
        },
        State::Deletion => {
            if deleted.len() > 0 {
                founds.push(Found {
                    filename: old_path,
                    key: format_key(deleted.clone()),
                    state: FoundState::Deleted,
                });
            }
        },
        _ => (),
    }

    println!("KEYS={:?}", keys);
    println!("FOUNDS={:?}", founds);

    Ok(vec![Output {
	    old_commit: old.id(),
	    new_commit: new.id(),
        old_filename: String::from_str("oldfilename"),
        new_filename: String::from_str("newfilename"),
	    origin_line: 0,
	    destination_line: 0,
	    num_lines: 0
    }])
}

#[derive(Debug)]
struct Output {
    old_commit: Oid,
    new_commit: Oid,
    old_filename: String,
    new_filename: String,
    origin_line: u32,
    destination_line: u32,
    num_lines: u32,
}
<commit_msg>Remove unused variable.<commit_after>#![feature(core)]
#![feature(path)]

#![feature(plugin)]
#![plugin(regex_macros)]
extern crate regex;
extern crate git2;
extern crate "rustc-serialize" as rustc_serialize;
extern crate docopt;

use git2::{Repository, Branch, BranchType, DiffLine,
    Commit, Diff, DiffFormat, DiffDelta, DiffHunk, Error, Oid};
use docopt::Docopt;
use std::old_io as io;
use std::old_io::BufferedReader;
use std::old_io::File;
use std::str;
use regex::Regex;

// Write the Docopt usage string.
static USAGE: &'static str = "
Usage: transit <repo> [<old> <new>]

If no commits are given, transit will revwalk from latest to oldest.
";

#[derive(RustcDecodable, Debug)]
struct Args {
    arg_repo: String,
    arg_old: Option<String>,
    arg_new: Option<String>,
}

fn main() {
    // Parse the args above or die.
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());
    // Pull up the revwalk.
    let path = Path::new(args.arg_repo);
    let repo = Repository::discover(&path)
        .ok().expect("Unable to find repo.");
    // If we can destructures the two optional args into real things.
    if let (Some(old_string), Some(new_string)) = (args.arg_old, args.arg_new) {
        // Compare a specific commit pair.
        // Fist, get the commits. (Error checked)
        let old = Oid::from_str(&old_string[]).and_then(|oid| repo.find_commit(oid));
        let new = Oid::from_str(&new_string[]).and_then(|oid| repo.find_commit(oid));
        if old.is_ok() && new.is_ok() {
            let output = find_moves(&repo, &old.unwrap(), &new.unwrap()).unwrap();
            make_output(output);
        } else {
            panic!("Commit ids were not valid.");
        }
    } else {
        // Revwalk.
        let mut revwalk = repo.revwalk()
            .ok().expect("Unable to get revwalk.");
        // Setup some options.
        revwalk.simplify_first_parent(); // TODO: Maybe remove?
        let mut flags = git2::Sort::empty();
        flags.insert(git2::SORT_TIME);
        flags.insert(git2::SORT_TOPOLOGICAL);
        revwalk.set_sorting(flags);
        // Push HEAD to the revwalk.
        revwalk.push_head()
            .ok().expect("Unable to push HEAD.");
        // We sadly must collect here to use `.windows()`
        let history = revwalk.filter_map(|id| repo.find_commit(id).ok())
            .collect::<Vec<Commit>>();
        // Walk through each pair of commits.
        for pair in history.windows(2) {let (old, new) = (&pair[1], &pair[0]);
			let output = find_moves(&repo, old.clone(), new.clone()).unwrap();
			// mangle the vectors here to make pretty output
			make_output(output);
		}
	}
}

fn make_output(output: Vec<Output>) {
	println!("make_output:");
    println!("\toutput.len()={}", output.len());
    for i in range(0, output.len()) {
            println!("\told_commit={}",         output[i].old_commit);
            println!("\tnew_commit={}",         output[i].new_commit);
            println!("\torigin_line={}",        output[i].origin_line);
            println!("\tdestintation_line={}",  output[i].destination_line);
            println!("\tnum_lines={}",          output[i].num_lines);
    }
}

#[derive(Debug)]
enum State {
    Other, Addition, Deletion
}

#[derive(Debug)]
enum FoundState {
    Added, Deleted
}

fn dump_diffline(line: &DiffLine) {
    // 'origin' is wrapped in pipes to ease displaying space characters.
    print!("line: old={:?} new={:?} offset={} |origin|=|{}|\n      content={}",
             line.old_lineno(), line.new_lineno(), line.content_offset(),
             line.origin(), str::from_utf8(line.content()).unwrap());
}

fn dump_diffdelta(delta: &DiffDelta) {
    println!("delta: nfiles={} status={:?} old_file=(id={} path_bytes={:?} path={:?} tsize={}) new_file=(id={} path_bytes={:?} path={:?} tsize={})",
            delta.nfiles(), delta.status(),
            delta.old_file().id(), delta.old_file().path_bytes(), delta.old_file().path(), delta.old_file().size(),
            delta.new_file().id(), delta.new_file().path_bytes(), delta.new_file().path(), delta.new_file().size());
}

fn format_key(key: String) -> String {
    let removeWhiteSpace = regex!(r"\s{2,}"); // 2 or more whitespaces    // TODO Removes whitespace from a string.
    removeWhiteSpace.replace_all(key.as_slice(), "")
}

#[derive(Debug)]
struct Found {
    filename: Path,
    key: String,
    state: FoundState,
}

fn find_moves(repo: &Repository, old: &Commit, new: &Commit) -> Result<Vec<Output>, Error> {

    println!("\nFIND MOVES---------------------");

	let old_tree = try!(old.tree());
	let new_tree = try!(new.tree());
	// Build up a diff of the two trees.
	let diff = try!(Diff::tree_to_tree(repo, Some(&old_tree), Some(&new_tree), None));

    let mut state = State::Other;
    let mut added = String::new();
    let mut deleted = String::new();
    let mut old_path: Path = Path::new("");
    let mut new_path: Path = Path::new("");

    let mut founds: Vec<Found> = Vec::new();

	// Read about this function in http://alexcrichton.com/git2-rs/git2/struct.Diff.html#method.print
	// It's a bit weird, but I think it will provide the necessary information.
	diff.print(DiffFormat::Patch, |delta, maybe_hunk, line| -> bool {

        assert!(delta.nfiles() == 2, "This only works on diffs between exactly 2 files.");

		// Thinking:
		//  * If is not a hunk, keep going.
		//  * If it's a hunk, do regex magic.
		//  * Stick regex output into a hashmap as a hash.
		//  * Later, we will iterate through and output pased on the values.
		// Filter out all the boring context lines.
		// If we're not interested in this line just return since it will iterate to the next.
		if maybe_hunk.is_none() { return true }; // Return early.

        println!("top of loop: founds={:?}", founds);
        println!("top of loop: added={:?} deleted={:?}", added, deleted);

        dump_diffline(&line);
        //dump_diffdelta(&delta);

        old_path = delta.old_file().path().unwrap().clone();
        new_path = delta.new_file().path().unwrap().clone();

		match line.origin() {
			// Additions
			'+' | '>' => {
                println!("In additions. state={:?}", state);

                added.push_str(str::from_utf8(line.content()).unwrap());

                match state {
                    State::Deletion => {
                        founds.push(Found {
                            filename: old_path.clone(),
                            key: format_key(deleted.clone()),
                            state: FoundState::Deleted,
                        });
                        deleted = String::new();
                    },
                    _ => (),
                }

                state = State::Addition;
				true
			},
			// Deletions
			'-' | '<' => {
                println!("In deletion. state={:?}", state);

                deleted.push_str(str::from_utf8(line.content()).unwrap());

                match state {
                    State::Addition => {
                        founds.push(Found {
                            filename: new_path.clone(),
                            key: format_key(added.clone()),
                            state: FoundState::Added,
                        });
                        added = String::new();
                    },
                    _ => (),
                }

                state = State::Deletion;
				true
			},
			// Other
			_         => {
                println!("in _. state={:?}", state);

                match state {
                    State::Addition => {
                        founds.push(Found {
                            filename: new_path.clone(),
                            key: format_key(added.clone()),
                            state: FoundState::Added,
                        });
                        added = String::new();
                    },
                    State::Deletion => {
                        founds.push(Found {
                            filename: old_path.clone(),
                            key: format_key(deleted.clone()),
                            state: FoundState::Deleted,
                        });
                        deleted = String::new();
                    },
                    _ => (),
                }

                state = State::Other;
                true
            }
		}
	});

    // Grab one.
    match state {
        State::Addition => {
            if added.len() > 0 {
                founds.push(Found {
                    filename: new_path,
                    key: format_key(added.clone()),
                    state: FoundState::Added,
                });
            }
        },
        State::Deletion => {
            if deleted.len() > 0 {
                founds.push(Found {
                    filename: old_path,
                    key: format_key(deleted.clone()),
                    state: FoundState::Deleted,
                });
            }
        },
        _ => (),
    }

    println!("FOUNDS={:?}", founds);

    Ok(vec![Output {
	    old_commit: old.id(),
	    new_commit: new.id(),
        old_filename: String::from_str("oldfilename"),
        new_filename: String::from_str("newfilename"),
	    origin_line: 0,
	    destination_line: 0,
	    num_lines: 0
    }])
}

#[derive(Debug)]
struct Output {
    old_commit: Oid,
    new_commit: Oid,
    old_filename: String,
    new_filename: String,
    origin_line: u32,
    destination_line: u32,
    num_lines: u32,
}
<|endoftext|>


<commit_before>// Copyright 2015 click2stream, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Arrow Client definitions.

extern crate libc;
extern crate native_tls;
extern crate openssl;
extern crate time;
extern crate uuid;

#[macro_use]
extern crate json;

extern crate bytes;

#[macro_use]
extern crate futures;

extern crate tokio_core;
extern crate tokio_io;
extern crate tokio_timer;
extern crate tokio_tls;

pub mod futures_ex;

#[macro_use]
pub mod utils;

pub mod net;

pub mod config;
pub mod context;
pub mod cmd_handler;
pub mod scanner;
pub mod svc_table;
pub mod timer;

use std::process;
use std::thread;

use std::error::Error;
use std::fmt::Debug;
use std::time::Duration;

use tokio_core::reactor::Core as TokioCore;

use config::usage;

use config::ApplicationConfig;
use context::{ApplicationContext, ConnectionState};

use cmd_handler::{Command, CommandChannel};

use net::arrow;

use net::arrow::{ArrowError, ErrorKind};

use utils::logger::Logger;

/// Connectionn retry timeout.
const RETRY_TIMEOUT: f64 = 60.0;

/// Get maximum duration of the pairing mode.
const PAIRING_MODE_TIMEOUT: f64 = 1200.0;

/// Unwrap a given result (if possible) or print the error message and exit
/// the process printing application usage.
fn result_or_usage<T, E>(res: Result<T, E>) -> T
    where E: Error + Debug {
    match res {
        Ok(res)  => res,
        Err(err) => {
            println!("ERROR: {}\n", err);
            usage(1);
        }
    }
}

/// Arrow Client main thread.
///
/// This function ensures maintaining connection with a remote Arrow Service.
fn arrow_thread(mut app_context: ApplicationContext, cmd_channel: CommandChannel) {
    let mut logger = app_context.get_logger();

    let diagnostic_mode = app_context.get_diagnostic_mode();
    let addr = app_context.get_arrow_service_address();

    let t = time::precise_time_s();

    let mut pairing_mode_timeout = t + 1200.0;
    let mut cur_addr = addr.clone();
    let mut last_attempt;

    loop {
        log_info!(logger, "connecting to remote Arrow Service {}", cur_addr);

        last_attempt = time::precise_time_s();

        app_context.set_connection_state(ConnectionState::Connected);

        let res = arrow::connect(
            app_context.clone(),
            cmd_channel.clone(),
            &cur_addr);

        pairing_mode_timeout = get_pairing_mode_timeout(
            &res,
            last_attempt,
            pairing_mode_timeout);

        if diagnostic_mode {
            diagnose_connection_result(&res);
        }

        match res {
            Ok(addr) => cur_addr = addr,
            Err(err) => {
                log_warn!(logger, "{}", err.description());

                let cstate = match err.kind() {
                    ErrorKind::Unauthorized => ConnectionState::Unauthorized,
                    _                       => ConnectionState::Disconnected,
                };

                app_context.set_connection_state(cstate);

                let t = get_next_retry_timeout(
                    err,
                    last_attempt,
                    pairing_mode_timeout);

                if t > 0.5 {
                    log_info!(logger, "retrying in {:.3} seconds", t);
                    thread::sleep(Duration::from_millis((t * 1000.0) as u64));
                }

                cur_addr = addr.to_string();
            }
        }
    }
}

/// Get new timeout for the pairing mode.
fn get_pairing_mode_timeout(
    connection_result: &Result<String, ArrowError>,
    last_connection_attempt: f64,
    current_timeout: f64) -> f64 {
    let t = time::precise_time_s();

    match connection_result {
        // We know the client is authorized, we can update the timeout.
        &Ok(_)        => t + PAIRING_MODE_TIMEOUT,
        &Err(ref err) => match err.kind() {
            // We don't update the timeout in case the client is unauthorized.
            ErrorKind::Unauthorized => current_timeout,
            // We don't know if the client is authorized but we assume it is
            // if the last connection was longer than RETRY_TIMEOUT seconds.
            _ => if (last_connection_attempt + RETRY_TIMEOUT) < t {
                t + PAIRING_MODE_TIMEOUT
            } else {
                current_timeout
            }
        }
    }
}

/// Get next reconnect timeout for the Arrow Client thread.
fn get_next_retry_timeout(
    connection_error: ArrowError,
    last_connection_attempt: f64,
    pairing_mode_timeout: f64) -> f64 {
    let t = time::precise_time_s();

    match connection_error.kind() {
        // the client is not authorized to access the service yet; check the
        // pairing mode timeout
        ErrorKind::Unauthorized => match pairing_mode_timeout {
            // retry every 10 seconds in the first 10 minutes since the first
            // "unauthorized" response
            timeout if t < (timeout - 600.0) => 10.0,
            // retry every 30 seconds after the first 10 minutes since the
            // first "unauthorized" response
            timeout if t < timeout => 30.0,
            // retry in 10 hours after the first 20 minutes since the first
            // "unauthorized" response
            _ => 36000.0
        },
        // set a very long retry timeout if the version of the Arrow Protocol
        // is not supported by either side
        ErrorKind::UnsupportedProtocolVersion => 36000.0,
        // in all other cases
        _ => RETRY_TIMEOUT + last_connection_attempt - time::precise_time_s()
    }
}

/// Diagnose a given connection result and exit with exit code 0 if the
/// connection was successful or the server responded with UNAUTHORIZED,
/// otherwise exit with exit code 1.
fn diagnose_connection_result(
    connection_result: &Result<String, ArrowError>) -> ! {
    match connection_result {
        &Ok(_)        => process::exit(0),
        &Err(ref err) => match err.kind() {
            ErrorKind::Unauthorized => process::exit(0),
            _ => process::exit(1)
        }
    }
}

/// Arrow Client main function.
fn main() {
    let mut core = TokioCore::new()
        .expect("unable to create an event loop");

    let handle = core.handle();

    let config = result_or_usage(
        ApplicationConfig::create());

    let context = ApplicationContext::new(config);

    // create command handler
    let (tx, rx) = cmd_handler::new(context.clone());

    let cmd_channel = tx.clone();

    // schedule periodic network scan
    let periodic_network_scan = context.get_timer()
        .create_periodic_task(
            Duration::from_secs(1),
            move || {
                tx.send(Command::PeriodicNetworkScan)
            }
        );

    handle.spawn(periodic_network_scan);

    // start the Arrow client thread
    thread::spawn(move || {
        arrow_thread(context, cmd_channel)
    });

    // run the command handler event loop
    core.run(rx)
        .unwrap_or_default();
}
<commit_msg>Add "application started" log message<commit_after>// Copyright 2015 click2stream, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Arrow Client definitions.

extern crate libc;
extern crate native_tls;
extern crate openssl;
extern crate time;
extern crate uuid;

#[macro_use]
extern crate json;

extern crate bytes;

#[macro_use]
extern crate futures;

extern crate tokio_core;
extern crate tokio_io;
extern crate tokio_timer;
extern crate tokio_tls;

pub mod futures_ex;

#[macro_use]
pub mod utils;

pub mod net;

pub mod config;
pub mod context;
pub mod cmd_handler;
pub mod scanner;
pub mod svc_table;
pub mod timer;

use std::process;
use std::thread;

use std::error::Error;
use std::fmt::Debug;
use std::time::Duration;

use tokio_core::reactor::Core as TokioCore;

use config::usage;

use config::ApplicationConfig;
use context::{ApplicationContext, ConnectionState};

use cmd_handler::{Command, CommandChannel};

use net::arrow;

use net::arrow::{ArrowError, ErrorKind};

use utils::logger::Logger;

/// Connectionn retry timeout.
const RETRY_TIMEOUT: f64 = 60.0;

/// Get maximum duration of the pairing mode.
const PAIRING_MODE_TIMEOUT: f64 = 1200.0;

/// Unwrap a given result (if possible) or print the error message and exit
/// the process printing application usage.
fn result_or_usage<T, E>(res: Result<T, E>) -> T
    where E: Error + Debug {
    match res {
        Ok(res)  => res,
        Err(err) => {
            println!("ERROR: {}\n", err);
            usage(1);
        }
    }
}

/// Arrow Client main thread.
///
/// This function ensures maintaining connection with a remote Arrow Service.
fn arrow_thread(mut app_context: ApplicationContext, cmd_channel: CommandChannel) {
    let mut logger = app_context.get_logger();

    let diagnostic_mode = app_context.get_diagnostic_mode();
    let addr = app_context.get_arrow_service_address();

    let t = time::precise_time_s();

    let mut pairing_mode_timeout = t + 1200.0;
    let mut cur_addr = addr.clone();
    let mut last_attempt;

    loop {
        log_info!(logger, "connecting to remote Arrow Service {}", cur_addr);

        last_attempt = time::precise_time_s();

        app_context.set_connection_state(ConnectionState::Connected);

        let res = arrow::connect(
            app_context.clone(),
            cmd_channel.clone(),
            &cur_addr);

        pairing_mode_timeout = get_pairing_mode_timeout(
            &res,
            last_attempt,
            pairing_mode_timeout);

        if diagnostic_mode {
            diagnose_connection_result(&res);
        }

        match res {
            Ok(addr) => cur_addr = addr,
            Err(err) => {
                log_warn!(logger, "{}", err.description());

                let cstate = match err.kind() {
                    ErrorKind::Unauthorized => ConnectionState::Unauthorized,
                    _                       => ConnectionState::Disconnected,
                };

                app_context.set_connection_state(cstate);

                let t = get_next_retry_timeout(
                    err,
                    last_attempt,
                    pairing_mode_timeout);

                if t > 0.5 {
                    log_info!(logger, "retrying in {:.3} seconds", t);
                    thread::sleep(Duration::from_millis((t * 1000.0) as u64));
                }

                cur_addr = addr.to_string();
            }
        }
    }
}

/// Get new timeout for the pairing mode.
fn get_pairing_mode_timeout(
    connection_result: &Result<String, ArrowError>,
    last_connection_attempt: f64,
    current_timeout: f64) -> f64 {
    let t = time::precise_time_s();

    match connection_result {
        // We know the client is authorized, we can update the timeout.
        &Ok(_)        => t + PAIRING_MODE_TIMEOUT,
        &Err(ref err) => match err.kind() {
            // We don't update the timeout in case the client is unauthorized.
            ErrorKind::Unauthorized => current_timeout,
            // We don't know if the client is authorized but we assume it is
            // if the last connection was longer than RETRY_TIMEOUT seconds.
            _ => if (last_connection_attempt + RETRY_TIMEOUT) < t {
                t + PAIRING_MODE_TIMEOUT
            } else {
                current_timeout
            }
        }
    }
}

/// Get next reconnect timeout for the Arrow Client thread.
fn get_next_retry_timeout(
    connection_error: ArrowError,
    last_connection_attempt: f64,
    pairing_mode_timeout: f64) -> f64 {
    let t = time::precise_time_s();

    match connection_error.kind() {
        // the client is not authorized to access the service yet; check the
        // pairing mode timeout
        ErrorKind::Unauthorized => match pairing_mode_timeout {
            // retry every 10 seconds in the first 10 minutes since the first
            // "unauthorized" response
            timeout if t < (timeout - 600.0) => 10.0,
            // retry every 30 seconds after the first 10 minutes since the
            // first "unauthorized" response
            timeout if t < timeout => 30.0,
            // retry in 10 hours after the first 20 minutes since the first
            // "unauthorized" response
            _ => 36000.0
        },
        // set a very long retry timeout if the version of the Arrow Protocol
        // is not supported by either side
        ErrorKind::UnsupportedProtocolVersion => 36000.0,
        // in all other cases
        _ => RETRY_TIMEOUT + last_connection_attempt - time::precise_time_s()
    }
}

/// Diagnose a given connection result and exit with exit code 0 if the
/// connection was successful or the server responded with UNAUTHORIZED,
/// otherwise exit with exit code 1.
fn diagnose_connection_result(
    connection_result: &Result<String, ArrowError>) -> ! {
    match connection_result {
        &Ok(_)        => process::exit(0),
        &Err(ref err) => match err.kind() {
            ErrorKind::Unauthorized => process::exit(0),
            _ => process::exit(1)
        }
    }
}

/// Arrow Client main function.
fn main() {
    let mut core = TokioCore::new()
        .expect("unable to create an event loop");

    let handle = core.handle();

    let config = result_or_usage(
        ApplicationConfig::create());

    let context = ApplicationContext::new(config);

    let mut logger = context.get_logger();

    log_info!(&mut logger,
        "application started (uuid: {}, mac: {})",
        context.get_arrow_uuid(), context.get_arrow_mac_address());

    // create command handler
    let (tx, rx) = cmd_handler::new(context.clone());

    let cmd_channel = tx.clone();

    // schedule periodic network scan
    let periodic_network_scan = context.get_timer()
        .create_periodic_task(
            Duration::from_secs(1),
            move || {
                tx.send(Command::PeriodicNetworkScan)
            }
        );

    handle.spawn(periodic_network_scan);

    // start the Arrow client thread
    thread::spawn(move || {
        arrow_thread(context, cmd_channel)
    });

    // run the command handler event loop
    core.run(rx)
        .unwrap_or_default();
}
<|endoftext|>


<commit_before>extern crate html5ever;
#[macro_use(qualname,ns,atom)]
extern crate string_cache;
#[macro_use(format_tendril)]
extern crate tendril;

use html5ever::rcdom::NodeEnum::{Element, Text};
use html5ever::rcdom::{Handle, RcDom};
use html5ever::serialize::{SerializeOpts, serialize};
use html5ever::tendril::TendrilSink;
use html5ever::tree_builder::{NodeOrText, TreeBuilderOpts, TreeSink};
use html5ever::{ParseOpts, parse_document};

use std::io;
use std::string::String;

enum Entry {
    Const(Handle),
    Enum(Handle),
    // Function(Handle),
    // Macro(Handle),
    Method(Handle),
    Module(Handle),
    Struct(Handle),
    Trait(Handle),
    Type(Handle),
}

fn main() {
    let opts = ParseOpts {
        tree_builder: TreeBuilderOpts { drop_doctype: true, ..Default::default() },
        ..Default::default()
    };

    let stdin = io::stdin();
    let mut dom = parse_document(RcDom::default(), opts)
                      .from_utf8()
                      .read_from(&mut stdin.lock())
                      .unwrap();

    let entries = find_entry_elements(&mut dom);

    // https://kapeli.com/docsets#tableofcontents
    // https://kapeli.com/docsets#supportedentrytypes
    for entry in entries {
        match entry {
            Entry::Method(e) => {
                add_dash_link_before_entry(&mut dom, &e, "method", "42")
            }
            _ => {}
        }
    }

    let mut bytes = vec![];
    serialize(&mut bytes, &dom.document, SerializeOpts::default()).unwrap();
    let result = String::from_utf8(bytes).unwrap();
    println!("{}", result);
}

fn add_dash_link_before_entry(dom: &mut RcDom, p: &Handle, entrytype: &str, entryname: &str) {
    let class_attr = html5ever::Attribute {
        name: qualname!("", "class"),
        value: format_tendril!("dashAnchor"),
    };

    let name_attr = html5ever::Attribute {
        name: qualname!("", "name"),
        // TODO: percent escape entryname
        value: format_tendril!("//apple_ref/cpp/{}/{}", entrytype, entryname),
    };

    let dash_link = dom.create_element(qualname!(html, "a"), vec![name_attr, class_attr.clone()]);
    let _ = dom.append_before_sibling(p.clone(), NodeOrText::AppendNode(dash_link));
}

fn find_entry_elements(dom: &mut RcDom) -> Vec<Entry> {
    let mut entries = vec![];
    walk_tree(&dom.document, &mut entries);
    entries
}

fn walk_tree(h: &Handle, entries: &mut Vec<Entry>) {
    let node = h.borrow();
    for e in node.children.iter() {
        if let Element(ref name, _, ref attrs) = e.borrow().node {
            let tag = &(*name.local.to_ascii_lowercase());
            if let Some(attr) = attrs.iter().find(|ref x| x.name == qualname!("", "class")) {
                match (tag, attr.clone().value.to_string().as_str()) {
                    ("h4", "method") => entries.push(Entry::Method(e.clone())),
                    ("section", "content constant") => entries.push(Entry::Const(e.clone())),
                    ("section", "content enum") => entries.push(Entry::Enum(e.clone())),
                    ("section", "content mod") => entries.push(Entry::Module(e.clone())),
                    ("section", "content struct") => entries.push(Entry::Struct(e.clone())),
                    ("section", "content trait") => entries.push(Entry::Trait(e.clone())),
                    ("section", "type") => entries.push(Entry::Type(e.clone())),
                    (_, _) => {}
                }
            }
        }
        walk_tree(e, entries);
    }
}

fn find_element_with_class(h: &Handle, c: &str) -> Option<Handle> {
    let node = h.borrow();
    for e in node.children.iter() {
        if let Element(_, _, ref attrs) = e.borrow().node {
            if let Some(attr) = attrs.iter().find(|ref x| x.name == qualname!("", "class")) {
                if attr.clone().value.to_string() == c {
                    return Some(e.clone());
                }
            }
        }
        if let Some(e) = find_element_with_class(e, c) {
            return Some(e);
        }
    }
    None
}

fn get_text(h: &Handle) -> Option<String> {
    let node = h.borrow();
    for e in node.children.iter() {
        if let Text(ref t) = e.borrow().node {
            return Some(t.to_string());
        }
    }

    None
}

fn extract_entry_name(e: &Entry) -> Option<String> {
    match *e {
        Entry::Const(ref c) => {
            if let Some(e) = find_element_with_class(c, "constant") {
                return get_text(&e);
            }
        }
        Entry::Enum(ref e) => {
            if let Some(e) = find_element_with_class(e, "enum") {
                return get_text(&e);
            }
        }
        _ => (),
    }
    None
}

#[cfg(test)]
mod tests {
    use html5ever::{ParseOpts, parse_document};
    use html5ever::rcdom::{Handle, RcDom};
    use tendril::TendrilSink;
    use html5ever::tree_builder::TreeBuilderOpts;

    fn dom_from_snippet(s: &str) -> Handle {
        let opts = ParseOpts {
            tree_builder: TreeBuilderOpts { drop_doctype: true, ..Default::default() },
            ..Default::default()
        };

        let dom = parse_document(RcDom::default(), opts)
                      .from_utf8()
                      .read_from(&mut format!("{}", s)
                                          .as_bytes())
                      .unwrap();

        dom.document
    }

    #[test]
    fn it_extracts_name_for_const_correctly() {
        // alloc::boxed::HEAP constant, entry name: HEAP
        let document_with_constant_section = dom_from_snippet(r##"

                alloc::<wbr>boxed::<wbr>HEAP

                   [−]

                [src]

        "##);

        let mut entries: Vec<super::Entry> = Vec::new();
        super::walk_tree(&document_with_constant_section, &mut entries);

        assert_eq!(entries.len(), 1);
        match entries[0] {
            super::Entry::Const(_) => assert!(true),
            _ => assert!(false),
        }

        match super::extract_entry_name(&entries[0]) {
            Some(s) => assert_eq!(s, *"HEAP".to_string()),
            None => assert_eq!(true, false),
        }
    }

    #[test]
    fn it_extracts_name_for_enum_correctly() {
        // collections::borrow::Cow, entry name: Cow
        let document_with_enum_section = dom_from_snippet(r##"

              Enum collections::<wbr>borrow::<wbr>Cow

                1.0.0

                   [−]

                [src]

        "##);

        let mut entries: Vec<super::Entry> = Vec::new();
        super::walk_tree(&document_with_enum_section, &mut entries);

        assert_eq!(entries.len(), 1);
        match entries[0] {
            super::Entry::Enum(_) => assert!(true),
            _ => assert!(false),
        }

        match super::extract_entry_name(&entries[0]) {
            Some(s) => assert_eq!(s, *"Cow".to_string()),
            None => assert_eq!(true, false),
        }
    }
}

// function:

Function std::<wbr>fs::<wbr>metadata1.0.0  [−]  [src]

// macro:
 //
std::<wbr>println!1.0.0  [−]  [src]

// method:
fn hash<H: Hasher>(&self, state: &mut H)[−]

// modules:

Crate collections  [−]  [src]

// struct:

Struct std::<wbr>io::<wbr>Bytes1.0.0  [−]  [src]

// trait:

Trait collections::<wbr>fmt::<wbr>Binary1.0.0  [−]  [src]

// type:
type Output = bool

<commit_msg>Implement entry name extraction for function and macro<commit_after>extern crate html5ever;
#[macro_use(qualname,ns,atom)]
extern crate string_cache;
#[macro_use(format_tendril)]
extern crate tendril;

use html5ever::rcdom::NodeEnum::{Element, Text};
use html5ever::rcdom::{Handle, RcDom};
use html5ever::serialize::{SerializeOpts, serialize};
use html5ever::tendril::TendrilSink;
use html5ever::tree_builder::{NodeOrText, TreeBuilderOpts, TreeSink};
use html5ever::{ParseOpts, parse_document};

use std::io;
use std::string::String;

enum Entry {
    Const(Handle),
    Enum(Handle),
    Function(Handle),
    Macro(Handle),
    Method(Handle),
    Module(Handle),
    Struct(Handle),
    Trait(Handle),
    Type(Handle),
}

fn main() {
    let opts = ParseOpts {
        tree_builder: TreeBuilderOpts { drop_doctype: true, ..Default::default() },
        ..Default::default()
    };

    let stdin = io::stdin();
    let mut dom = parse_document(RcDom::default(), opts)
                      .from_utf8()
                      .read_from(&mut stdin.lock())
                      .unwrap();

    let entries = find_entry_elements(&mut dom);

    // https://kapeli.com/docsets#tableofcontents
    // https://kapeli.com/docsets#supportedentrytypes
    for entry in entries {
        match entry {
            Entry::Method(e) => {
                add_dash_link_before_entry(&mut dom, &e, "method", "42")
            }
            _ => {}
        }
    }

    let mut bytes = vec![];
    serialize(&mut bytes, &dom.document, SerializeOpts::default()).unwrap();
    let result = String::from_utf8(bytes).unwrap();
    println!("{}", result);
}

fn add_dash_link_before_entry(dom: &mut RcDom, p: &Handle, entrytype: &str, entryname: &str) {
    let class_attr = html5ever::Attribute {
        name: qualname!("", "class"),
        value: format_tendril!("dashAnchor"),
    };

    let name_attr = html5ever::Attribute {
        name: qualname!("", "name"),
        // TODO: percent escape entryname
        value: format_tendril!("//apple_ref/cpp/{}/{}", entrytype, entryname),
    };

    let dash_link = dom.create_element(qualname!(html, "a"), vec![name_attr, class_attr.clone()]);
    let _ = dom.append_before_sibling(p.clone(), NodeOrText::AppendNode(dash_link));
}

fn find_entry_elements(dom: &mut RcDom) -> Vec<Entry> {
    let mut entries = vec![];
    walk_tree(&dom.document, &mut entries);
    entries
}

fn walk_tree(h: &Handle, entries: &mut Vec<Entry>) {
    let node = h.borrow();
    for e in node.children.iter() {
        if let Element(ref name, _, ref attrs) = e.borrow().node {
            let tag = &(*name.local.to_ascii_lowercase());
            if let Some(attr) = attrs.iter().find(|ref x| x.name == qualname!("", "class")) {
                match (tag, attr.clone().value.to_string().as_str()) {
                    ("h4", "method") => entries.push(Entry::Method(e.clone())),
                    ("section", "content constant") => entries.push(Entry::Const(e.clone())),
                    ("section", "content enum") => entries.push(Entry::Enum(e.clone())),
                    ("section", "content fn") => entries.push(Entry::Function(e.clone())),
                    ("section", "content macro") => entries.push(Entry::Macro(e.clone())),
                    ("section", "content mod") => entries.push(Entry::Module(e.clone())),
                    ("section", "content struct") => entries.push(Entry::Struct(e.clone())),
                    ("section", "content trait") => entries.push(Entry::Trait(e.clone())),
                    ("section", "type") => entries.push(Entry::Type(e.clone())),
                    (_, _) => {}
                }
            }
        }
        walk_tree(e, entries);
    }
}

fn find_element_with_class(h: &Handle, c: &str) -> Option<Handle> {
    let node = h.borrow();
    for e in node.children.iter() {
        if let Element(_, _, ref attrs) = e.borrow().node {
            if let Some(attr) = attrs.iter().find(|ref x| x.name == qualname!("", "class")) {
                if attr.clone().value.to_string() == c {
                    return Some(e.clone());
                }
            }
        }
        if let Some(e) = find_element_with_class(e, c) {
            return Some(e);
        }
    }
    None
}

fn get_text(h: &Handle) -> Option<String> {
    let node = h.borrow();
    for e in node.children.iter() {
        if let Text(ref t) = e.borrow().node {
            return Some(t.to_string());
        }
    }

    None
}

fn extract_entry_name(e: &Entry) -> Option<String> {
    match *e {
        Entry::Const(ref c) => {
            if let Some(e) = find_element_with_class(c, "constant") {
                return get_text(&e);
            }
        }
        Entry::Enum(ref e) => {
            if let Some(e) = find_element_with_class(e, "enum") {
                return get_text(&e);
            }
        }
        Entry::Function(ref e) => {
            if let Some(e) = find_element_with_class(e, "fn") {
                return get_text(&e);
            }
        }
        Entry::Macro(ref e) => {
            if let Some(e) = find_element_with_class(e, "macro") {
                return get_text(&e);
            }
        }
        _ => (),
    }
    None
}

#[cfg(test)]
mod tests {
    use html5ever::{ParseOpts, parse_document};
    use html5ever::rcdom::{Handle, RcDom};
    use tendril::TendrilSink;
    use html5ever::tree_builder::TreeBuilderOpts;

    fn dom_from_snippet(s: &str) -> Handle {
        let opts = ParseOpts {
            tree_builder: TreeBuilderOpts { drop_doctype: true, ..Default::default() },
            ..Default::default()
        };

        let dom = parse_document(RcDom::default(), opts)
                      .from_utf8()
                      .read_from(&mut format!("{}", s)
                                          .as_bytes())
                      .unwrap();

        dom.document
    }

    #[test]
    fn it_extracts_name_for_const_correctly() {
        // alloc::boxed::HEAP constant, entry name: HEAP
        let document_with_constant_section = dom_from_snippet(r##"

                alloc::<wbr>boxed::<wbr>HEAP

                   [−]

                [src]

        "##);

        let mut entries: Vec<super::Entry> = Vec::new();
        super::walk_tree(&document_with_constant_section, &mut entries);

        assert_eq!(entries.len(), 1);
        match entries[0] {
            super::Entry::Const(_) => assert!(true),
            _ => assert!(false),
        }

        match super::extract_entry_name(&entries[0]) {
            Some(s) => assert_eq!(s, *"HEAP".to_string()),
            None => assert_eq!(true, false),
        }
    }

    #[test]
    fn it_extracts_name_for_enum_correctly() {
        // collections::borrow::Cow, entry name: Cow
        let document_with_enum_section = dom_from_snippet(r##"

              Enum collections::<wbr>borrow::<wbr>Cow

                1.0.0

                   [−]

                [src]

        "##);

        let mut entries: Vec<super::Entry> = Vec::new();
        super::walk_tree(&document_with_enum_section, &mut entries);

        assert_eq!(entries.len(), 1);
        match entries[0] {
            super::Entry::Enum(_) => assert!(true),
            _ => assert!(false),
        }

        match super::extract_entry_name(&entries[0]) {
            Some(s) => assert_eq!(s, *"Cow".to_string()),
            None => assert_eq!(true, false),
        }
    }

    #[test]
    fn it_extracts_name_for_function_correctly() {
        // Function std::fs::metadata, entry name: metadata
        let document_with_function_section = dom_from_snippet(r##"

              Function std::<wbr>fs::<wbr>metadata

                1.0.0

                   [−]

                [src]

        "##);

        let mut entries: Vec<super::Entry> = Vec::new();
        super::walk_tree(&document_with_function_section, &mut entries);

        assert_eq!(entries.len(), 1);
        match entries[0] {
            super::Entry::Function(_) => assert!(true),
            _ => assert!(false),
        }

        match super::extract_entry_name(&entries[0]) {
            Some(s) => assert_eq!(s, *"metadata".to_string()),
            None => assert_eq!(true, false),
        }
    }

    #[test]
    fn it_extracts_name_for_macro_correctly() {
        // Macro std::println!, entry name: println!
        let document_with_macro_section = dom_from_snippet(r##"

              std::<wbr>println!

                1.0.0
                  [−]
                [src]

        "##);

        let mut entries: Vec<super::Entry> = Vec::new();
        super::walk_tree(&document_with_macro_section, &mut entries);

        assert_eq!(entries.len(), 1);
        match entries[0] {
            super::Entry::Macro(_) => assert!(true),
            _ => assert!(false),
        }

        match super::extract_entry_name(&entries[0]) {
            Some(s) => assert_eq!(s, *"println!".to_string()),
            None => assert_eq!(true, false),
        }
    }

// method:
fn hash<H: Hasher>(&self, state: &mut H)[−]

// modules:

Crate collections  [−]  [src]

// struct:

Struct std::<wbr>io::<wbr>Bytes1.0.0  [−]  [src]

// trait:

Trait collections::<wbr>fmt::<wbr>Binary1.0.0  [−]  [src]

// type:
type Output = bool

}
<|endoftext|>


<commit_before>// App

#[macro_use] extern crate clap;
#[macro_use] extern crate log;

extern crate q;

use q::gentle_panic::GentlePanic;

fn main() {
    let options = clap::App::new("q")
        .author("Sergey Arkhipov <nineseconds@yandex.ru>")
        .version(&crate_version!()[..])
        .about("q is a gentle way to grep using predefined regexp sets.")
        .after_help("Please find more documentation at https://github.com/9seconds/q.")
        .arg(
            clap::Arg::with_name("SAME_LINE")
                .help("Keep matches on the same line")
                .short("l")
                .long("same_line")
        ).arg(
            clap::Arg::with_name("DEBUG")
                .help("Run q in debug mode")
                .short("d")
                .long("debug")
        ).arg(
            clap::Arg::with_name("CASE_INSENSITIVE")
                .help("Use case insensitive regex versions.")
                .short("i")
                .long("case-insensitive")
        ).arg(
            clap::Arg::with_name("RULES_DIRECTORY")
                .help("Directory where rules could be found. By default it uses $XDG_CONFIG_HOME/q/rules")
                .short("-r")
                .long("rules")
                .takes_value(true)
        ).arg(
            clap::Arg::with_name("FILE")
                .help("File to process. Use '-' to read from stdin (default is stdin).")
                .short("-f")
                .long("file")
                .takes_value(true)
        ).arg(
            clap::Arg::with_name("RULES")
                .help("Regexp rules to apply to the stdin as a comma-separated list.")
                .index(1)
                .required(true)
        )
        .get_matches();

    q::logging::configure_logging(options.is_present("DEBUG"));

    let same_line = options.is_present("SAME_LINE");
    let case_insensitive = options.is_present("CASE_INSENSITIVE");
    let filename = options
        .value_of("FILE")
        .unwrap_or("-");

    let rules_directory = q::rules::get_rules_directory(options.value_of("RULES_DIRECTORY"))
        .get_or_die_with(70, "Cannot discover rules directory!");

    let rules = q::rules::get_rules(
        &rules_directory, options.value_of("RULES").unwrap(), case_insensitive
    )
    .get_or_die_with(70, "Cannot parse rules");

    info!("Options: filename={}, rules={:?}, same_line={}", &filename, &rules, same_line);

    let _ = q::process::process(&filename, &rules, same_line)
        .get_or_die_with(70, "Cannot process stream");
}
<commit_msg>Add exit codes as constants<commit_after>// App

#[macro_use] extern crate clap;
#[macro_use] extern crate log;

extern crate q;

use q::gentle_panic::GentlePanic;

const EX_CANNOT_FIND_RULES_DIRECTORY: i32 = 70;
const EX_CANNOT_PARSE_RULES: i32 = 71;
const EX_CANNOT_PROCESS_STREAM: i32 = 72;

fn main() {
    let options = clap::App::new("q")
        .author("Sergey Arkhipov <nineseconds@yandex.ru>")
        .version(&crate_version!()[..])
        .about("q is a gentle way to grep using predefined regexp sets.")
        .after_help("Please find more documentation at https://github.com/9seconds/q.")
        .arg(
            clap::Arg::with_name("SAME_LINE")
                .help("Keep matches on the same line")
                .short("l")
                .long("same_line")
        ).arg(
            clap::Arg::with_name("DEBUG")
                .help("Run q in debug mode")
                .short("d")
                .long("debug")
        ).arg(
            clap::Arg::with_name("CASE_INSENSITIVE")
                .help("Use case insensitive regex versions.")
                .short("i")
                .long("case-insensitive")
        ).arg(
            clap::Arg::with_name("RULES_DIRECTORY")
                .help("Directory where rules could be found. By default it uses $XDG_CONFIG_HOME/q/rules")
                .short("-r")
                .long("rules")
                .takes_value(true)
        ).arg(
            clap::Arg::with_name("FILE")
                .help("File to process. Use '-' to read from stdin (default is stdin).")
                .short("-f")
                .long("file")
                .takes_value(true)
        ).arg(
            clap::Arg::with_name("RULES")
                .help("Regexp rules to apply to the stdin as a comma-separated list.")
                .index(1)
                .required(true)
        )
        .get_matches();

    q::logging::configure_logging(options.is_present("DEBUG"));

    let same_line = options.is_present("SAME_LINE");
    let case_insensitive = options.is_present("CASE_INSENSITIVE");
    let filename = options
        .value_of("FILE")
        .unwrap_or("-");

    let rules_directory = q::rules::get_rules_directory(options.value_of("RULES_DIRECTORY"))
        .get_or_die_with(EX_CANNOT_FIND_RULES_DIRECTORY, "Cannot discover rules directory!");

    let rules = q::rules::get_rules(
        &rules_directory, options.value_of("RULES").unwrap(), case_insensitive
    )
    .get_or_die_with(EX_CANNOT_PARSE_RULES, "Cannot parse rules");

    info!("Options: filename={}, rules={:?}, same_line={}", &filename, &rules, same_line);

    let _ = q::process::process(&filename, &rules, same_line)
        .get_or_die_with(EX_CANNOT_PROCESS_STREAM, "Cannot process stream");
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![plugin(oak)]

extern crate oak_runtime;
use oak_runtime::*;

use std::io::prelude::*;
use std::fs::File;
use std::io::BufReader;

grammar! bailey {
    program
        = ((expr / decl / stmt) terminator?)*

    decl
        = class_decl
        / method_decl

    class_decl
        = kw_class constant class_body > class_decl

    class_body
        = lbracket method_decl* rbracket

    method_decl
        = kw_def identifier lparen param_list? rparen block > method_decl

    block
        = lbracket ((stmt / expr) terminator?)* rbracket > block

    stmt
        = if_stmt
        / while_stmt

    if_stmt
        = kw_if expr block (kw_else block)? > if_stmt

    while_stmt
        = kw_while expr block > while_stmt

    call_with_receiver
        = (receiver_expr ".")? identifier lparen arg_list? rparen

    call_no_receiver
        = identifier lparen arg_list? rparen

    assign_expr
        = identifier bind_op expr > assign_expr

    expr
        = spacing cond_expr

    cond_expr
        = add_expr (cond_expr_op add_expr)* > fold_left

    add_expr
        = mult_expr (add_expr_op mult_expr)* > fold_left

    mult_expr
        = primary_expr (mult_expr_op primary_expr)* > fold_left

    primary_expr
        = assign_expr
          / call_with_receiver > call_with_receiver
          / receiver_expr

    receiver_expr
        = call_no_receiver > call_no_receiver
        / literal
        / constant
        / identifier
        / lparen expr rparen

    arg_list
        = (expr ("," spacing expr)*) > arg_list

    param_list
        = (identifier ("," spacing identifier)*) > param_list

    digit = ["0-9"]
    identifier_char = ["A-Za-z0-9_"]

    identifier = !digit !keyword identifier_char+ spacing > identifier
    constant = !keyword !digit ["A-Z"]+ identifier_char+ spacing > constant

    literal
        = lit_string
        / lit_float
        / lit_int

    lit_int = digit+ spacing > lit_int
    lit_float = digit+ "." digit+ spacing > lit_float
    lit_string = (dbl_quot_lit_string / sng_quot_lit_string) spacing > lit_string

    dbl_quot_lit_string = dbl_quot (spacing_char / !dbl_quot .)* dbl_quot
    sng_quot_lit_string = sng_quot (spacing_char / !sng_quot .)* sng_quot

    spacing_char = [" \n\r\t"]
    spacing = spacing_char* -> ()

    kw_class = "class" spacing
    kw_def = "def" spacing
    kw_else = "else" spacing
    kw_false = "false" spacing
    kw_if = "if" spacing
    kw_nil = "nil" spacing
    kw_true = "true" spacing
    kw_while = "while" spacing

    keyword
        = kw_def
        / kw_if
        / kw_else
        / kw_class
        / kw_true
        / kw_false
        / kw_nil

    add_expr_op
        = add_op > add_bin_op
        / sub_op > sub_bin_op

    mult_expr_op
        = mul_op > mul_bin_op
        / div_op > div_bin_op

    cond_expr_op
        = lt_op > lt_bin_op /
        lte_op > lte_bin_op /
        gt_op > gt_bin_op /
        gte_op > gte_bin_op /
        ne_op > ne_bin_op /
        eq_op > eq_bin_op /
        andand_op > and_bin_op /
        oror_op > or_bin_op

    bind_op = "=" spacing
    add_op = "+" spacing
    sub_op = "-" spacing
    mul_op = "*" spacing
    div_op = "/" spacing
    lt_op = "<" spacing
    lte_op = "<=" spacing
    gt_op = ">" spacing
    gte_op = ">=" spacing
    ne_op = "!=" spacing
    eq_op = "==" spacing
    andand_op  = "&&" spacing
    oror_op = "||" spacing

    lparen = "(" spacing
    rparen = ")" spacing
    lbracket = "{" spacing
    rbracket = "}" spacing
    dbl_quot = "\"" spacing
    sng_quot = "'" spacing
    terminator = ";" spacing

    fn lit_int(raw_text: Vec<char>) -> PExpr {
        Box::new(IntegerLiteral(to_number(raw_text)))
    }

    // FIXME: Please remove this hacky wacky '.' thingy
    fn lit_float(integer: Vec<char>, fractional: Vec<char>) -> PExpr {
        let mut buf = integer;
        buf.push('.');
        Box::new(FloatLiteral(to_float(combine(buf, fractional))))
    }

    fn lit_string(raw_text: Vec<char>) -> PExpr {
        Box::new(StringLiteral(to_string(raw_text)))
    }

    fn constant(first: Vec<char>, rest: Vec<char>) -> PExpr {
        Box::new(Constant(to_string(combine(first, rest))))
    }

    fn identifier(raw_text: Vec<char>) -> PExpr {
        Box::new(Identifier(to_string(raw_text)))
    }

    fn add_bin_op() -> BinOp { Add }
    fn sub_bin_op() -> BinOp { Sub }
    fn mul_bin_op() -> BinOp { Mul }
    fn div_bin_op() -> BinOp { Div }
    fn lt_bin_op() -> BinOp { Lt }
    fn lte_bin_op() -> BinOp { Lte }
    fn gt_bin_op() -> BinOp { Gt }
    fn gte_bin_op() -> BinOp { Gte }
    fn ne_bin_op() -> BinOp { Ne }
    fn eq_bin_op() -> BinOp { Eq }
    fn and_bin_op() -> BinOp { And }
    fn or_bin_op() -> BinOp { Or }

    fn call_no_receiver(method: PExpr, args: Option<Vec<PExpr>>) -> PExpr {
        Box::new(Call(None, method, args))
    }

    fn call_with_receiver(receiver: Option<PExpr>, method: PExpr, args: Option<Vec<PExpr>>) -> PExpr {
        Box::new(Call(receiver, method, args))
    }

    fn assign_expr(identifier: PExpr, expr: PExpr) -> PExpr {
        Box::new(AssingExpr(identifier, expr))
    }

    fn class_decl(class: PExpr, methods: Vec<PExpr>) -> PExpr {
        Box::new(ClassDecl(class, methods))
    }

    fn method_decl(identifier: PExpr, params: Option<Vec<PExpr>>, body: Vec<PExpr>) -> PExpr {
        Box::new(MethodDecl(identifier, params, body))
    }

    fn block(exprs: Vec<(PExpr, Option<()>)>) -> Vec<PExpr> {
        exprs.into_iter().map(|expr| {
            match expr {
                (e, _) => e
            }
        }).collect()
    }

    fn if_stmt(condition: PExpr, block: Vec<PExpr>, else_block: Option<Vec<PExpr>>) -> PExpr {
        Box::new(IfStatement(condition, block, else_block))
    }

    fn while_stmt(condition: PExpr,  block: Vec<PExpr>) -> PExpr {
      Box::new(WhileStatement(condition, block))
    }

    fn arg_list(first: PExpr, mut rest: Vec<PExpr>) -> Vec<PExpr> {
        combine_one_with_many(first, rest)
    }

    fn param_list(first: PExpr, mut rest: Vec<PExpr>) -> Vec<PExpr> {
        combine_one_with_many(first, rest)
    }

    fn to_string(raw_text: Vec<char>) -> String {
        raw_text.into_iter().collect()
    }

    fn to_number(raw_text: Vec<char>) -> u32 {
        u32::from_str(&*to_string(raw_text)).unwrap()
    }

    fn to_float(raw_text: Vec<char>) -> f32 {
        f32::from_str(&*to_string(raw_text)).unwrap()
    }

    fn fold_left(head: PExpr, rest: Vec<(BinOp, PExpr)>) -> PExpr {
        rest.into_iter().fold(head,
          |accu, (op, expr)| Box::new(BinaryExpr(op, accu, expr)))
    }

    fn fold_right(front: Vec<(PExpr, BinOp)>, last: PExpr) -> PExpr {
        front.into_iter().rev().fold(last,
          |accu, (expr, op)| Box::new(BinaryExpr(op, expr, accu)))
    }

    fn combine<T: Clone>(left: Vec<T>, right: Vec<T>) -> Vec<T> {
        let mut result = left.clone();
        result.extend(right.into_iter());
        result
    }

    fn combine_one_with_many<T: Clone>(first: T, rest: Vec<T>) -> Vec<T> {
        let mut result = vec![first];
        combine(result, rest)
    }

    use std::str::FromStr;
    use self::Expression::*;
    use self::BinOp::*;

    pub type PExpr = Box<Expression>;

    #[derive(Debug, Clone)]
    pub enum Expression {
        Identifier(String),
        Constant(String),
        IntegerLiteral(u32),
        FloatLiteral(f32),
        StringLiteral(String),
        BinaryExpr(BinOp, PExpr, PExpr),
        AssingExpr(PExpr, PExpr),
        ClassDecl(PExpr, Vec<PExpr>),
        IfStatement(PExpr, Vec<PExpr>, Option<Vec<PExpr>>),
        MethodDecl(PExpr, Option<Vec<PExpr>>, Vec<PExpr>),
        WhileStatement(PExpr, Vec<PExpr>),
        Call(Option<PExpr>, PExpr, Option<Vec<PExpr>>),
        Block(Vec<PExpr>)
    }

    #[derive(Debug, Clone)]
    pub enum BinOp {
        Add, Sub, Mul, Div, Lt, Lte, Gt, Gte, Ne, Eq, And, Or
    }
}

fn main() {
    let f = File::open("./src/example.bly").ok().expect("failed to open file");
    let mut reader = BufReader::new(f);
    let mut contents = String::new();
    reader.read_to_string(&mut contents);
    println!("{:#?}", bailey::parse_program(contents.stream()).unwrap_data());
}
<commit_msg>I think I've nailed the chained calls.<commit_after>#![feature(plugin)]
#![plugin(oak)]

extern crate oak_runtime;
use oak_runtime::*;

use std::io::prelude::*;
use std::fs::File;
use std::io::BufReader;

grammar! bailey {
    program
        = ((expr / decl / stmt) terminator?)*

    decl
        = class_decl
        / method_decl

    class_decl
        = kw_class constant class_body > class_decl

    class_body
        = lbracket method_decl* rbracket

    method_decl
        = kw_def identifier lparen param_list? rparen block > method_decl

    block
        = lbracket ((stmt / expr) terminator?)* rbracket > block

    stmt
        = if_stmt
        / while_stmt

    if_stmt
        = kw_if expr block (kw_else block)? > if_stmt

    while_stmt
        = kw_while expr block > while_stmt

    call_no_receiver
        = identifier lparen arg_list? rparen

    call_with_receiver
        = receiver_expr ("." call_no_receiver)* > call_with_receiver

    assign_expr
        = identifier bind_op expr > assign_expr

    expr
        = spacing cond_expr

    cond_expr
        = add_expr (cond_expr_op add_expr)* > fold_left

    add_expr
        = mult_expr (add_expr_op mult_expr)* > fold_left

    mult_expr
        = primary_expr (mult_expr_op primary_expr)* > fold_left

    primary_expr
        = assign_expr
          / call_with_receiver
          / receiver_expr

    receiver_expr
        = call_no_receiver > call_no_receiver
        / literal
        / constant
        / identifier
        / lparen expr rparen

    arg_list
        = (expr ("," spacing expr)*) > arg_list

    param_list
        = (identifier ("," spacing identifier)*) > param_list

    digit = ["0-9"]
    identifier_char = ["A-Za-z0-9_"]

    identifier = !digit !keyword identifier_char+ spacing > identifier
    constant = !keyword !digit ["A-Z"]+ identifier_char+ spacing > constant

    literal
        = lit_string
        / lit_float
        / lit_int

    lit_int = digit+ spacing > lit_int
    lit_float = digit+ "." digit+ spacing > lit_float
    lit_string = (dbl_quot_lit_string / sng_quot_lit_string) spacing > lit_string

    dbl_quot_lit_string = dbl_quot (spacing_char / !dbl_quot .)* dbl_quot
    sng_quot_lit_string = sng_quot (spacing_char / !sng_quot .)* sng_quot

    spacing_char = [" \n\r\t"]
    spacing = spacing_char* -> ()

    kw_class = "class" spacing
    kw_def = "def" spacing
    kw_else = "else" spacing
    kw_false = "false" spacing
    kw_if = "if" spacing
    kw_nil = "nil" spacing
    kw_true = "true" spacing
    kw_while = "while" spacing

    keyword
        = kw_def
        / kw_if
        / kw_else
        / kw_class
        / kw_true
        / kw_false
        / kw_nil

    add_expr_op
        = add_op > add_bin_op
        / sub_op > sub_bin_op

    mult_expr_op
        = mul_op > mul_bin_op
        / div_op > div_bin_op

    cond_expr_op
        = lt_op > lt_bin_op /
        lte_op > lte_bin_op /
        gt_op > gt_bin_op /
        gte_op > gte_bin_op /
        ne_op > ne_bin_op /
        eq_op > eq_bin_op /
        andand_op > and_bin_op /
        oror_op > or_bin_op

    bind_op = "=" spacing
    add_op = "+" spacing
    sub_op = "-" spacing
    mul_op = "*" spacing
    div_op = "/" spacing
    lt_op = "<" spacing
    lte_op = "<=" spacing
    gt_op = ">" spacing
    gte_op = ">=" spacing
    ne_op = "!=" spacing
    eq_op = "==" spacing
    andand_op  = "&&" spacing
    oror_op = "||" spacing

    lparen = "(" spacing
    rparen = ")" spacing
    lbracket = "{" spacing
    rbracket = "}" spacing
    dbl_quot = "\"" spacing
    sng_quot = "'" spacing
    terminator = ";" spacing

    fn lit_int(raw_text: Vec<char>) -> PExpr {
        Box::new(IntegerLiteral(to_number(raw_text)))
    }

    // FIXME: Please remove this hacky wacky '.' thingy
    fn lit_float(integer: Vec<char>, fractional: Vec<char>) -> PExpr {
        let mut buf = integer;
        buf.push('.');
        Box::new(FloatLiteral(to_float(combine(buf, fractional))))
    }

    fn lit_string(raw_text: Vec<char>) -> PExpr {
        Box::new(StringLiteral(to_string(raw_text)))
    }

    fn constant(first: Vec<char>, rest: Vec<char>) -> PExpr {
        Box::new(Constant(to_string(combine(first, rest))))
    }

    fn identifier(raw_text: Vec<char>) -> PExpr {
        Box::new(Identifier(to_string(raw_text)))
    }

    fn add_bin_op() -> BinOp { Add }
    fn sub_bin_op() -> BinOp { Sub }
    fn mul_bin_op() -> BinOp { Mul }
    fn div_bin_op() -> BinOp { Div }
    fn lt_bin_op() -> BinOp { Lt }
    fn lte_bin_op() -> BinOp { Lte }
    fn gt_bin_op() -> BinOp { Gt }
    fn gte_bin_op() -> BinOp { Gte }
    fn ne_bin_op() -> BinOp { Ne }
    fn eq_bin_op() -> BinOp { Eq }
    fn and_bin_op() -> BinOp { And }
    fn or_bin_op() -> BinOp { Or }

    fn call_no_receiver(method: PExpr, args: Option<Vec<PExpr>>) -> PExpr {
        Box::new(Call(None, method, args))
    }

    fn call_with_receiver(receiver: PExpr, call: Vec<(PExpr, Option<Vec<PExpr>>)>) -> PExpr {
        call.into_iter().fold(receiver,
          |accu, (identifier, args)| Box::new(Call(Some(accu), identifier, args)))
    }

    fn assign_expr(identifier: PExpr, expr: PExpr) -> PExpr {
        Box::new(AssingExpr(identifier, expr))
    }

    fn class_decl(class: PExpr, methods: Vec<PExpr>) -> PExpr {
        Box::new(ClassDecl(class, methods))
    }

    fn method_decl(identifier: PExpr, params: Option<Vec<PExpr>>, body: Vec<PExpr>) -> PExpr {
        Box::new(MethodDecl(identifier, params, body))
    }

    fn block(exprs: Vec<(PExpr, Option<()>)>) -> Vec<PExpr> {
        exprs.into_iter().map(|expr| {
            match expr {
                (e, _) => e
            }
        }).collect()
    }

    fn if_stmt(condition: PExpr, block: Vec<PExpr>, else_block: Option<Vec<PExpr>>) -> PExpr {
        Box::new(IfStatement(condition, block, else_block))
    }

    fn while_stmt(condition: PExpr,  block: Vec<PExpr>) -> PExpr {
      Box::new(WhileStatement(condition, block))
    }

    fn arg_list(first: PExpr, mut rest: Vec<PExpr>) -> Vec<PExpr> {
        combine_one_with_many(first, rest)
    }

    fn param_list(first: PExpr, mut rest: Vec<PExpr>) -> Vec<PExpr> {
        combine_one_with_many(first, rest)
    }

    fn to_string(raw_text: Vec<char>) -> String {
        raw_text.into_iter().collect()
    }

    fn to_number(raw_text: Vec<char>) -> u32 {
        u32::from_str(&*to_string(raw_text)).unwrap()
    }

    fn to_float(raw_text: Vec<char>) -> f32 {
        f32::from_str(&*to_string(raw_text)).unwrap()
    }

    fn fold_left(head: PExpr, rest: Vec<(BinOp, PExpr)>) -> PExpr {
        rest.into_iter().fold(head,
          |accu, (op, expr)| Box::new(BinaryExpr(op, accu, expr)))
    }

    fn fold_right(front: Vec<(PExpr, BinOp)>, last: PExpr) -> PExpr {
        front.into_iter().rev().fold(last,
          |accu, (expr, op)| Box::new(BinaryExpr(op, expr, accu)))
    }

    fn combine<T: Clone>(left: Vec<T>, right: Vec<T>) -> Vec<T> {
        let mut result = left.clone();
        result.extend(right.into_iter());
        result
    }

    fn combine_one_with_many<T: Clone>(first: T, rest: Vec<T>) -> Vec<T> {
        let mut result = vec![first];
        combine(result, rest)
    }

    use std::str::FromStr;
    use self::Expression::*;
    use self::BinOp::*;

    pub type PExpr = Box<Expression>;

    #[derive(Debug, Clone)]
    pub enum Expression {
        Identifier(String),
        Constant(String),
        IntegerLiteral(u32),
        FloatLiteral(f32),
        StringLiteral(String),
        BinaryExpr(BinOp, PExpr, PExpr),
        AssingExpr(PExpr, PExpr),
        ClassDecl(PExpr, Vec<PExpr>),
        IfStatement(PExpr, Vec<PExpr>, Option<Vec<PExpr>>),
        MethodDecl(PExpr, Option<Vec<PExpr>>, Vec<PExpr>),
        WhileStatement(PExpr, Vec<PExpr>),
        Call(Option<PExpr>, PExpr, Option<Vec<PExpr>>),
        Block(Vec<PExpr>)
    }

    #[derive(Debug, Clone)]
    pub enum BinOp {
        Add, Sub, Mul, Div, Lt, Lte, Gt, Gte, Ne, Eq, And, Or
    }
}

fn main() {
    let f = File::open("./src/example.bly").ok().expect("failed to open file");
    let mut reader = BufReader::new(f);
    let mut contents = String::new();
    reader.read_to_string(&mut contents);
    println!("{:#?}", bailey::parse_program(contents.stream()).unwrap_data());
}
<|endoftext|>


<commit_before>extern crate ar;
extern crate clap;
extern crate libflate;
extern crate tar;
extern crate tempfile;
extern crate time as crates_time;
extern crate xz2;
extern crate zip;

use std::fs;
use std::io;
use std::time;

use std::rc::Rc;
use std::vec::Vec;

use clap::{Arg, App};

use libflate::gzip;
use tempfile::tempfile;

// magic:
use std::io::Seek;
use std::io::Write;

#[derive(Debug)]
struct Node<T> {
    next: Option<Rc<Node<T>>>,
    value: T,
}

impl<T: Clone> Node<T> {
    fn head(obj: T) -> Rc<Node<T>> {
        Rc::new(Node {
            next: None,
            value: obj,
        })
    }

    fn plus(what: &Rc<Node<T>>, obj: T) -> Rc<Node<T>> {
        Rc::new(Node {
            next: Some(what.clone()),
            value: obj
        })
    }

    fn to_vec(&self) -> Vec<T> {
        let mut ret = Vec::new();
        let mut val = self;
        loop {
            ret.push(val.value.clone());
            if let Some(ref next) = val.next {
                val = next;
            } else {
                break;
            }
        }
        ret.reverse();
        ret
    }
}

struct OutputTo {
    path: Rc<Node<String>>,
    size: u64,
    atime: u64,
    mtime: u64,
    ctime: u64,
    btime: u64,
}

impl OutputTo {
    fn warn(&self, msg: String) {
        writeln!(io::stderr(), "TODO: {}", msg).expect("stderr");
    }

    fn raw<F: io::Read>(&self, mut file: &mut F) -> io::Result<()> {
        let stdout = io::stdout();
        let mut stdout = stdout.lock();
        writeln!(stdout, "{:?} {} {} {} {} {}",
            self.path.to_vec(),
            self.size,
            self.atime, self.mtime, self.ctime, self.btime
        )?;

        io::copy(&mut file, &mut stdout).and_then(move |written|
            if written != self.size {
                Err(io::Error::new(
                    io::ErrorKind::UnexpectedEof,
                    format!("expecting to write {} but wrote {}", self.size, written)))
            } else {
                Ok(())
            })
    }

    fn from_file(path: &str, fd: &fs::File) -> io::Result<OutputTo> {
        let meta = fd.metadata()?;
        Ok(OutputTo {
            path: Node::head(path.to_string()),
            size: meta.len(),
            atime: meta.accessed().map(simple_time_sys)?,
            mtime: meta.modified().map(simple_time_sys)?,
            ctime: simple_time_ctime(&meta),
            btime: simple_time_btime(&meta)?,
        })
    }

    fn with_path(&self, path: &str) -> OutputTo {
        OutputTo {
            path: Node::plus(&self.path, path.to_string()),
            size: 0,
            atime: 0,
            mtime: 0,
            ctime: 0,
            btime: 0,
        }
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum FileType {
    GZip,
    Zip,
    Tar,
    Ar,
    BZip2,
    Xz,
    Other,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum Status {
    Done,
    Rewind,
}

fn read_octal(bytes: &[u8]) -> Option<u32> {
    let mut start = 0;
    while start < bytes.len() && b' ' == bytes[start] {
        start += 1;
    }

    let mut end = bytes.len() - 1;
    while end > start && (b' ' == bytes[end] || 0 == bytes[end]) {
        end -= 1;
    }

    if let Ok(string) = std::str::from_utf8(&bytes[start..(end+1)]) {
        if let Ok(val) = u32::from_str_radix(string, 8) {
            return Some(val);
        }
    }
    None
}

fn simple_time(dur: time::Duration) -> u64 {
    dur.as_secs().checked_mul(1_000_000_000)
        .map_or(0, |nanos| nanos + dur.subsec_nanos() as u64)
}

fn simple_time_sys(val: time::SystemTime) -> u64 {
    val.duration_since(time::UNIX_EPOCH).map(simple_time).unwrap_or(0)
}

fn simple_time_tm(val: crates_time::Tm) -> u64 {
    let timespec = val.to_timespec();
    simple_time(time::Duration::new(timespec.sec as u64, timespec.nsec as u32))
}

fn simple_time_btime(val: &fs::Metadata) -> io::Result<u64> {
    match val.created() {
        Ok(time) => Ok(simple_time_sys(time)),
        // "Other" is how "unsupported" is represented here; ew.
        Err(ref e) if e.kind() == io::ErrorKind::Other => Ok(0),
        Err(other) => Err(other),
    }
}

// TODO: I really feel this should be exposed by Rust, without cfg.
fn simple_time_ctime(val: &fs::Metadata) -> u64 {
    #[cfg(linux)] {
        use std::os::linux::fs::MetadataExt;
        let ctime: i64 = val.st_ctime();
        if ctime <= 0 {
            0
        } else {
            ctime as u64
        }
    }

    #[cfg(not(linux))] {
        0
    }
}

fn is_probably_tar(header: &[u8]) -> bool {
    if header.len() < 512 {
        return false;
    }

    if let Some(checksum) = read_octal(&header[148..156]) {
        let mut sum: u32 = (b' ' as u32) * 8;
        for i in 0..148 {
            sum += header[i] as u32;
        }
        for i in 156..512 {
            sum += header[i] as u32;
        }

        if checksum == sum {
            return true;
        }
    }

    return false;
}

fn identify<'a, T: io::BufRead + ?Sized>(fd: &mut T) -> io::Result<FileType> {
    let header = fd.fill_buf()?;
    if header.len() >= 20
        && 0x1f == header[0] && 0x8b == header[1] {
        Ok(FileType::GZip)
   } else if header.len() >= 152
        && b'P' == header[0] && b'K' == header[1]
        && 0x03 == header[2] && 0x04 == header[3] {
        Ok(FileType::Zip)
    } else if header.len() > 257 + 10
        && b'u' == header[257] && b's' == header[258]
        && b't' == header[259] && b'a' == header[260]
        && b'r' == header[261]
        && (
            (0 == header[262] && b'0' == header[263] && b'0' == header[264]) ||
            (b' ' == header[262] && b' ' == header[263] && 0 == header[264])
        ) {
        Ok(FileType::Tar)
    } else if header.len() > 8
        && b'!' == header[0] && b'<' == header[1]
        && b'a' == header[2] && b'r' == header[3]
        && b'c' == header[4] && b'h' == header[5]
        && b'>' == header[6] && b'\n' == header[7] {
        Ok(FileType::Ar)
    } else if header.len() > 40
        && b'B' == header[0] && b'Z' == header[1]
        && b'h' == header[2] && 0x31 == header[3]
        && 0x41 == header[4] && 0x59 == header[5]
        && 0x26 == header[6] {
        Ok(FileType::BZip2)
    } else if header.len() > 6
        && 0xfd == header[0] && b'7' == header[1]
        && b'z' == header[2] && b'X' == header[3]
        && b'Z' == header[4] && 0 == header[5] {
        Ok(FileType::Xz)
    } else if is_probably_tar(header) {
        Ok(FileType::Tar)
    } else {
        Ok(FileType::Other)
    }
}

trait Tee {
    fn normal(&mut self) -> &mut io::BufRead;
    fn abort(&mut self) -> io::Result<&io::BufRead>;
}

struct TempFileTee<'a> {
    fp: Box<io::BufRead + 'a>,
    tmp: fs::File,
}

impl<'a> TempFileTee<'a> {
    fn new<U: 'a + io::Read>(from: U) -> io::Result<TempFileTee<'a>> {
        Ok(TempFileTee {
            fp: Box::new(io::BufReader::new(from)),
            tmp: tempfile()?
        })
    }
}

impl<'a> Tee for TempFileTee<'a> {
    fn normal(&mut self) -> &mut io::BufRead {
        &mut self.fp
    }
    fn abort(&mut self) -> io::Result<&io::BufRead> {
        unimplemented!();
    }
}

struct BufReaderTee<T> {
    fp: Box<T>,
}

impl<T> BufReaderTee<T> {
    // TODO: return type?
    fn new<U: io::Read>(from: U) -> BufReaderTee<io::BufReader<U>> {
        BufReaderTee {
            fp: Box::new(io::BufReader::new(from))
        }
    }
}

impl<T> Tee for BufReaderTee<T>
    where T: io::BufRead + io::Seek
{
    fn normal(&mut self) -> &mut io::BufRead {
        &mut self.fp
    }

    fn abort(&mut self) -> io::Result<&io::BufRead> {
        self.fp.seek(io::SeekFrom::Start(0))?;
        Ok(self.normal())
    }
}

fn count_bytes<'a, R: io::Read>(mut fd: &mut R) -> io::Result<u64> {
    let mut block = [0u8; 4096];
    let mut count = 0u64;
    loop {
        let found = fd.read(&mut block[..])?;
        if 0 == found {
            break;
        }
        count += found as u64;
    }
    Ok(count)
}
//
//fn unpack_or_raw<F>(
//    fd: Box<Tee>,
//    output: &OutputTo,
//    fun: F) -> io::Result<Status>
//where F: FnOnce(Box<Tee>) -> io::Result<Box<Tee>>
//{
//    match fun(fd) {
//        Ok(inner) => unpack(inner.normal(), output),
//        Err(e) => {
//            output.warn(format!(
//                    "thought we could unpack '{:?}' but we couldn't: {}",
//                    output.path, e));
//            Ok(Status::Rewind)
//        }
//    }
//}

fn unpack<'a>(mut fd: Box<Tee + 'a>, output: &OutputTo) -> io::Result<Status> {
    match identify(fd.normal())? {
//        FileType::GZip => {
//            unpack_or_raw(fd, output,
//                |fd| gzip::Decoder::new(fd).map(
//                    |dec| Box::new(io::BufReader::new(dec)) as Box<Tee>))
//        },
//        FileType::Xz => {
//            unpack_or_raw(fd, output,
//                |fd| Ok(Box::new(io::BufReader::new(xz2::bufread::XzDecoder::new(fd)))))
//        },
        FileType::Ar if output.path.value.ends_with(".deb") => {
            let mut decoder = ar::Archive::new(fd.normal());
            while let Some(entry) = decoder.next_entry() {
                let entry = entry?;
                let new_output = output.with_path(entry.header().identifier());
                unpack(Box::new(TempFileTee::new(entry)?), &new_output)?;
            }
            Ok(Status::Done)
        },
        FileType::Tar => {
            let mut decoder = tar::Archive::new(fd.normal());
            for entry in decoder.entries()? {
                let entry = entry?;
                let new_output = output.with_path(entry.path()?.to_str().expect("valid utf-8"));
                unpack(Box::new(TempFileTee::new(entry)?), &new_output)?;
            }
            Ok(Status::Done)
        },
        FileType::Zip => {
            let mut temp = tempfile()?;
            io::copy(fd.normal(), &mut temp)?;
            let mut zip = zip::ZipArchive::new(temp)?;

            for i in 0..zip.len() {

                // well, this has gone rather poorly
                let mut new_output = output.with_path(zip.by_index(i)?.name());

                let file_result = {
                    let entry = zip.by_index(i)?;

                    new_output.size = entry.size();
                    new_output.mtime = simple_time_tm(entry.last_modified());
                    let reader = Box::new(TempFileTee::new(entry)?);
                    unpack(reader, &new_output)?
                };
                match file_result {
                    Status::Done => (),
                    Status::Rewind => {
                        // TODO: update output? ? handling? ???
                        new_output.raw(&mut zip.by_index(i)?)?
                    },
                };
            }

            Ok(Status::Done)
        },

        // TODO: unimplemented!()
        _ => {
            Ok(Status::Rewind)
        },
    }
}

fn output_raw(file: &mut fs::File, output: &OutputTo) -> io::Result<()> {
    file.seek(io::SeekFrom::Start(0))?;
    output.raw(file)?;
    Ok(())
}

fn process_real_path(path: &str) -> io::Result<()> {
    let mut file = fs::File::open(path).unwrap();
    let output = OutputTo::from_file(
        path,
        &file
    )?;

    let attempt = {
//        let read = BufReaderTee::new::<BufReaderTee<io::BufReader<fs::File>>>(file);
        let read = TempFileTee::new(&file)?;
        unpack(Box::new(read), &output)
    };

    match attempt {
        Ok(Status::Done) => Ok(()),
        Ok(Status::Rewind) => output_raw(&mut file, &output),
        Err(e) => Err(e)
    }
}

fn real_main() -> u8 {

    let matches = App::new("contentin")
                    .arg(Arg::with_name("to-tar")
                         .long("to-tar")
                         .help("emit a tar file")
                         .required(true))
                    .arg(Arg::with_name("INPUT")
                         .required(true)
                         .multiple(true))
                    .get_matches();

    for path in matches.values_of("INPUT").unwrap() {
        if let Err(e) = process_real_path(path) {
            if let Err(_) = writeln!(io::stderr(), "fatal: processing '{}': {}", path, e) {
                return 6;
            }

            return 4;
        }
    }
    return 0;
}

fn main() {
    std::process::exit(real_main() as i32)
}
<commit_msg>inline unpack_or_raw and make it borrowcheck again<commit_after>extern crate ar;
extern crate clap;
extern crate libflate;
extern crate tar;
extern crate tempfile;
extern crate time as crates_time;
extern crate xz2;
extern crate zip;

use std::fs;
use std::io;
use std::time;

use std::rc::Rc;
use std::vec::Vec;

use clap::{Arg, App};

use libflate::gzip;
use tempfile::tempfile;

// magic:
use std::io::Seek;
use std::io::Write;

#[derive(Debug)]
struct Node<T> {
    next: Option<Rc<Node<T>>>,
    value: T,
}

impl<T: Clone> Node<T> {
    fn head(obj: T) -> Rc<Node<T>> {
        Rc::new(Node {
            next: None,
            value: obj,
        })
    }

    fn plus(what: &Rc<Node<T>>, obj: T) -> Rc<Node<T>> {
        Rc::new(Node {
            next: Some(what.clone()),
            value: obj
        })
    }

    fn to_vec(&self) -> Vec<T> {
        let mut ret = Vec::new();
        let mut val = self;
        loop {
            ret.push(val.value.clone());
            if let Some(ref next) = val.next {
                val = next;
            } else {
                break;
            }
        }
        ret.reverse();
        ret
    }
}

struct OutputTo {
    path: Rc<Node<String>>,
    size: u64,
    atime: u64,
    mtime: u64,
    ctime: u64,
    btime: u64,
}

impl OutputTo {
    fn warn(&self, msg: String) {
        writeln!(io::stderr(), "TODO: {}", msg).expect("stderr");
    }

    fn raw<F: io::Read + ?Sized>(&self, mut file: &mut F) -> io::Result<()> {
        let stdout = io::stdout();
        let mut stdout = stdout.lock();
        writeln!(stdout, "{:?} {} {} {} {} {}",
            self.path.to_vec(),
            self.size,
            self.atime, self.mtime, self.ctime, self.btime
        )?;

        io::copy(&mut file, &mut stdout).and_then(move |written|
            if written != self.size {
                Err(io::Error::new(
                    io::ErrorKind::UnexpectedEof,
                    format!("expecting to write {} but wrote {}", self.size, written)))
            } else {
                Ok(())
            })
    }

    fn from_file(path: &str, fd: &fs::File) -> io::Result<OutputTo> {
        let meta = fd.metadata()?;
        Ok(OutputTo {
            path: Node::head(path.to_string()),
            size: meta.len(),
            atime: meta.accessed().map(simple_time_sys)?,
            mtime: meta.modified().map(simple_time_sys)?,
            ctime: simple_time_ctime(&meta),
            btime: simple_time_btime(&meta)?,
        })
    }

    fn with_path(&self, path: &str) -> OutputTo {
        OutputTo {
            path: Node::plus(&self.path, path.to_string()),
            size: 0,
            atime: 0,
            mtime: 0,
            ctime: 0,
            btime: 0,
        }
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum FileType {
    GZip,
    Zip,
    Tar,
    Ar,
    BZip2,
    Xz,
    Other,
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum Status {
    Done,
    Rewind,
}

fn read_octal(bytes: &[u8]) -> Option<u32> {
    let mut start = 0;
    while start < bytes.len() && b' ' == bytes[start] {
        start += 1;
    }

    let mut end = bytes.len() - 1;
    while end > start && (b' ' == bytes[end] || 0 == bytes[end]) {
        end -= 1;
    }

    if let Ok(string) = std::str::from_utf8(&bytes[start..(end+1)]) {
        if let Ok(val) = u32::from_str_radix(string, 8) {
            return Some(val);
        }
    }
    None
}

fn simple_time(dur: time::Duration) -> u64 {
    dur.as_secs().checked_mul(1_000_000_000)
        .map_or(0, |nanos| nanos + dur.subsec_nanos() as u64)
}

fn simple_time_sys(val: time::SystemTime) -> u64 {
    val.duration_since(time::UNIX_EPOCH).map(simple_time).unwrap_or(0)
}

fn simple_time_tm(val: crates_time::Tm) -> u64 {
    let timespec = val.to_timespec();
    simple_time(time::Duration::new(timespec.sec as u64, timespec.nsec as u32))
}

fn simple_time_btime(val: &fs::Metadata) -> io::Result<u64> {
    match val.created() {
        Ok(time) => Ok(simple_time_sys(time)),
        // "Other" is how "unsupported" is represented here; ew.
        Err(ref e) if e.kind() == io::ErrorKind::Other => Ok(0),
        Err(other) => Err(other),
    }
}

// TODO: I really feel this should be exposed by Rust, without cfg.
fn simple_time_ctime(val: &fs::Metadata) -> u64 {
    #[cfg(linux)] {
        use std::os::linux::fs::MetadataExt;
        let ctime: i64 = val.st_ctime();
        if ctime <= 0 {
            0
        } else {
            ctime as u64
        }
    }

    #[cfg(not(linux))] {
        0
    }
}

fn is_probably_tar(header: &[u8]) -> bool {
    if header.len() < 512 {
        return false;
    }

    if let Some(checksum) = read_octal(&header[148..156]) {
        let mut sum: u32 = (b' ' as u32) * 8;
        for i in 0..148 {
            sum += header[i] as u32;
        }
        for i in 156..512 {
            sum += header[i] as u32;
        }

        if checksum == sum {
            return true;
        }
    }

    return false;
}

fn identify<'a, T: io::BufRead + ?Sized>(fd: &mut T) -> io::Result<FileType> {
    let header = fd.fill_buf()?;
    if header.len() >= 20
        && 0x1f == header[0] && 0x8b == header[1] {
        Ok(FileType::GZip)
   } else if header.len() >= 152
        && b'P' == header[0] && b'K' == header[1]
        && 0x03 == header[2] && 0x04 == header[3] {
        Ok(FileType::Zip)
    } else if header.len() > 257 + 10
        && b'u' == header[257] && b's' == header[258]
        && b't' == header[259] && b'a' == header[260]
        && b'r' == header[261]
        && (
            (0 == header[262] && b'0' == header[263] && b'0' == header[264]) ||
            (b' ' == header[262] && b' ' == header[263] && 0 == header[264])
        ) {
        Ok(FileType::Tar)
    } else if header.len() > 8
        && b'!' == header[0] && b'<' == header[1]
        && b'a' == header[2] && b'r' == header[3]
        && b'c' == header[4] && b'h' == header[5]
        && b'>' == header[6] && b'\n' == header[7] {
        Ok(FileType::Ar)
    } else if header.len() > 40
        && b'B' == header[0] && b'Z' == header[1]
        && b'h' == header[2] && 0x31 == header[3]
        && 0x41 == header[4] && 0x59 == header[5]
        && 0x26 == header[6] {
        Ok(FileType::BZip2)
    } else if header.len() > 6
        && 0xfd == header[0] && b'7' == header[1]
        && b'z' == header[2] && b'X' == header[3]
        && b'Z' == header[4] && 0 == header[5] {
        Ok(FileType::Xz)
    } else if is_probably_tar(header) {
        Ok(FileType::Tar)
    } else {
        Ok(FileType::Other)
    }
}

trait Tee {
    fn normal(&mut self) -> &mut io::BufRead;
    fn abort(&mut self) -> io::Result<&mut io::BufRead>;
}

struct TempFileTee<'a> {
    fp: Box<io::BufRead + 'a>,
    tmp: fs::File,
}

impl<'a> TempFileTee<'a> {
    fn new<U: 'a + io::Read>(from: U) -> io::Result<TempFileTee<'a>> {
        Ok(TempFileTee {
            fp: Box::new(io::BufReader::new(from)),
            tmp: tempfile()?
        })
    }
}

impl<'a> Tee for TempFileTee<'a> {
    fn normal(&mut self) -> &mut io::BufRead {
        &mut self.fp
    }
    fn abort(&mut self) -> io::Result<&mut io::BufRead> {
        unimplemented!();
    }
}

struct BufReaderTee<T> {
    fp: Box<T>,
}

impl<T> BufReaderTee<T> {
    // TODO: return type?
    fn new<U: io::Read>(from: U) -> BufReaderTee<io::BufReader<U>> {
        BufReaderTee {
            fp: Box::new(io::BufReader::new(from))
        }
    }
}

impl<T> Tee for BufReaderTee<T>
    where T: io::BufRead + io::Seek
{
    fn normal(&mut self) -> &mut io::BufRead {
        &mut self.fp
    }

    fn abort(&mut self) -> io::Result<&mut io::BufRead> {
        self.fp.seek(io::SeekFrom::Start(0))?;
        Ok(self.normal())
    }
}

fn count_bytes<'a, R: io::Read>(mut fd: &mut R) -> io::Result<u64> {
    let mut block = [0u8; 4096];
    let mut count = 0u64;
    loop {
        let found = fd.read(&mut block[..])?;
        if 0 == found {
            break;
        }
        count += found as u64;
    }
    Ok(count)
}

fn unpack<'a>(mut fd: Box<Tee + 'a>, output: &OutputTo) -> io::Result<Status> {
    match identify(fd.normal())? {
        FileType::GZip => {
            let process_decoded = match gzip::Decoder::new(fd.normal()) {
                Ok(dec) => Ok(unpack(Box::new(TempFileTee::new(dec)?), output)?),
                Err(e) => Err(e),
            };

            match process_decoded {
                Ok(status) => Ok(status),
                Err(e) => {
                    output.warn(format!(
                        "thought we could unpack '{:?}' but we couldn't: {}",
                        output.path, e));
                    output.raw(fd.abort()?)?;
                    Ok(Status::Done)
                }
            }
        },
//        FileType::Xz => {
//            unpack_or_raw(fd, output,
//                |fd| Ok(Box::new(TempFileTee::new(xz2::bufread::XzDecoder::new(fd)))))
//        },
        FileType::Ar if output.path.value.ends_with(".deb") => {
            let mut decoder = ar::Archive::new(fd.normal());
            while let Some(entry) = decoder.next_entry() {
                let entry = entry?;
                let new_output = output.with_path(entry.header().identifier());
                unpack(Box::new(TempFileTee::new(entry)?), &new_output)?;
            }
            Ok(Status::Done)
        },
        FileType::Tar => {
            let mut decoder = tar::Archive::new(fd.normal());
            for entry in decoder.entries()? {
                let entry = entry?;
                let new_output = output.with_path(entry.path()?.to_str().expect("valid utf-8"));
                unpack(Box::new(TempFileTee::new(entry)?), &new_output)?;
            }
            Ok(Status::Done)
        },
        FileType::Zip => {
            let mut temp = tempfile()?;
            io::copy(fd.normal(), &mut temp)?;
            let mut zip = zip::ZipArchive::new(temp)?;

            for i in 0..zip.len() {

                // well, this has gone rather poorly
                let mut new_output = output.with_path(zip.by_index(i)?.name());

                let file_result = {
                    let entry = zip.by_index(i)?;

                    new_output.size = entry.size();
                    new_output.mtime = simple_time_tm(entry.last_modified());
                    let reader = Box::new(TempFileTee::new(entry)?);
                    unpack(reader, &new_output)?
                };
                match file_result {
                    Status::Done => (),
                    Status::Rewind => {
                        // TODO: update output? ? handling? ???
                        new_output.raw(&mut zip.by_index(i)?)?
                    },
                };
            }

            Ok(Status::Done)
        },

        // TODO: unimplemented!()
        _ => {
            Ok(Status::Rewind)
        },
    }
}

fn output_raw(file: &mut fs::File, output: &OutputTo) -> io::Result<()> {
    file.seek(io::SeekFrom::Start(0))?;
    output.raw(file)?;
    Ok(())
}

fn process_real_path(path: &str) -> io::Result<()> {
    let mut file = fs::File::open(path).unwrap();
    let output = OutputTo::from_file(
        path,
        &file
    )?;

    let attempt = {
//        let read = BufReaderTee::new::<BufReaderTee<io::BufReader<fs::File>>>(file);
        let read = TempFileTee::new(&file)?;
        unpack(Box::new(read), &output)
    };

    match attempt {
        Ok(Status::Done) => Ok(()),
        Ok(Status::Rewind) => output_raw(&mut file, &output),
        Err(e) => Err(e)
    }
}

fn real_main() -> u8 {

    let matches = App::new("contentin")
                    .arg(Arg::with_name("to-tar")
                         .long("to-tar")
                         .help("emit a tar file")
                         .required(true))
                    .arg(Arg::with_name("INPUT")
                         .required(true)
                         .multiple(true))
                    .get_matches();

    for path in matches.values_of("INPUT").unwrap() {
        if let Err(e) = process_real_path(path) {
            if let Err(_) = writeln!(io::stderr(), "fatal: processing '{}': {}", path, e) {
                return 6;
            }

            return 4;
        }
    }
    return 0;
}

fn main() {
    std::process::exit(real_main() as i32)
}
<|endoftext|>