<commit_before>extern crate mysql;
extern crate toml;
extern crate rustc_serialize;
#[macro_use]
extern crate lazy_static;

mod lib;

use lib::config;
use lib::downloader::DownloadDB;
use lib::downloader::Downloader;
use lib::DownloadError;
use lib::converter::Converter;

use std::fs::{remove_file,remove_dir};

static VERSION : &'static str = "0.2"; // String not valid
static SLEEP_MS: u32 = 5000;
static CODE_SUCCESS: i8 = 2;
static CODE_SUCCESS_WARNINGS: i8 = 3; // finished with warnings
static CODE_FAILED_INTERNAL: i8 = 10; // internal error
static CODE_FAILED_QUALITY: i8 = 11; // qualitz not available
static CODE_FAILED_UNAVAILABLE: i8 = 12; // source unavailable (private / removed)

lazy_static! {
    pub static ref CONFIG: config::Config = {
        println!("Starting yayd-backend v{}",&VERSION);
        config::init_config() //return
    };

}

//#[repr(i8)] broken, enum not usable as of #10292
// enum QueryCodes {
//     Waiting = 0,
//     InProgress = 1,
//     Finished = 2,
//     Failed = 3,
// }

enum Thing { String(String), Bool(bool), None }

fn main() {

    let pool = lib::db_connect(lib::mysql_options(), SLEEP_MS);

    let converter = Converter::new(&CONFIG.general.ffmpeg_bin,&CONFIG.general.mp3_quality , pool.clone());
    let mut print_pause = true;
    loop {
        if let Some(result) = lib::request_entry(& pool) {
            print_pause = true;
            let qid = result.qid.clone();                 //&QueryCodes::InProgress as i32
            lib::set_query_code(&mut pool.get_conn().unwrap(), &1, &result.qid).ok().expect("Failed to set query code!");
            lib::set_query_state(&pool.clone(),&qid, "started", false);

            let action_result: Result<Thing,DownloadError>;
            {
                let mut left_files: Vec<String> = Vec::with_capacity(2);
                if result.playlist {
                    action_result = handle_playlist(result, &converter,&mut left_files);
                }else{
                    action_result = handle_download(result, &None, &converter,&mut left_files);
                }

                if !left_files.is_empty() {
                    println!("cleaning up files");
                    for i in &left_files {
                        match remove_file(&i) {
                            Ok(_) => (println!("cleaning up {}",i)),
                            Err(e) => println!("unable to remove file '{}' {}",i,e),
                        }
                    }
                }
            }

            let code: i8 = match action_result {
                Ok(t) => {
                    match t {
                        Thing::Bool(b) => {
                            if b { CODE_SUCCESS_WARNINGS } else { CODE_SUCCESS }
                        }
                        _ => CODE_SUCCESS,
                    }
                },
                Err(e) => {
                    println!("Error: {:?}", e);
                    match e {
                        DownloadError::NotAvailable => CODE_FAILED_UNAVAILABLE,
                        DownloadError::ExtractorError => CODE_FAILED_UNAVAILABLE,
                        DownloadError::QualityNotAvailable => CODE_FAILED_QUALITY,
                        _ => {
                            let details = match e {
                                DownloadError::DBError(s) => s,
                                DownloadError::DownloadError(s) => s,
                                DownloadError::FFMPEGError(s) => s,
                                DownloadError::InternalError(s) => s,
                                _ => unreachable!(),
                            };
                            lib::add_query_status(&pool.clone(),&qid, &details);
                            CODE_FAILED_INTERNAL
                        },
                    }
                }
            };
            lib::set_query_code(&mut pool.get_conn().unwrap(), &code,&qid).ok().expect("Failed to set query code!");

            let state = match code {
                2 => "finished",
                10 => "internal error",
                11 => "quality not available",
                12 => "source unavailable",
                _ => "unknown"
            };

            lib::set_query_state(&pool.clone(),&qid, state, true);

        } else {
            if print_pause { println!("Pausing.."); print_pause = false; }
            std::thread::sleep_ms(SLEEP_MS);
        }
    }
}

///Download handler
///Used by the playlist/file handler to download one file
///Based on the quality it needs to download audio & video separately and convert them together
///In case of a playlist download it depends on the target download folder & if it should be bezipped
///In case of a DMCA we need to download the file via the socket connector,
///which will output a mp3, or if requested, the video but with the highest quality.
///Thus in case of a DMCA we can't pick a quality anymore.
///Also the filename depends on the socket output then.
///
///If it's a non-zipped single file, it's moved after a successful download, converted etc to the
///main folder from which it should be downloadable.
///The original non-ascii & url_encode'd name of the file is stored in the DB
fn handle_download<'a>(downl_db: DownloadDB, folder: &Option<String>, converter: &Converter, file_db: &mut Vec<String>) -> Result<Thing,DownloadError>{
    //update progress
    let is_zipped = match *folder {
        Some(_) => true,
        None => false,
    };
    if !downl_db.compress {
        lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 1, 0,false);
    }

    let download = Downloader::new(&downl_db, &CONFIG.general);
    //get filename, check for DMCA
    let mut dmca = false; // "succ." dmca -> file already downloaded

    let name = match download.get_file_name() { // get filename
        Ok(v) => v,
        Err(DownloadError::DMCAError) => { //now request via lib.. // k if( k == Err(DownloadError::DMCAError) )
            println!("DMCA error!");
            match download.lib_request_video(1,0) {
                Err(err) => { println!("Offliberty-call error {:?}", err); return Err(err); },
                Ok(v) => { dmca = true; v },
            }
        },
        Err(e) => { // unknown error / restricted source etc.. abort
            println!("Unknown error: {:?}", e);
            return Err(e);
        },
    };

    let name_http_valid = lib::format_file_name(&name, &download, &downl_db.qid);

    let file_path = lib::format_file_path(&downl_db.qid, folder.clone(), false);
    file_db.push(file_path.clone());
    let save_path = lib::format_save_path(folder.clone(),&name, &download, &downl_db.qid);

    println!("Filename: {}", name);

    let is_splitted_video = if dmca {
        false
    } else {
        lib::is_split_container(&downl_db.quality)
    };
    let convert_audio = CONFIG.extensions.mp3.contains(&downl_db.quality);

    let total_steps = if dmca {
        if download.is_audio() {
            3
        } else {
            2
        }
    } else if is_splitted_video {
        4
    } else if download.is_audio() {
        3
    } else {
        2
    };

    if !dmca {
        if !downl_db.compress {
            lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 2, total_steps,false);
        }

        //download first file, download audio raw source if specified or video
        try!(download.download_file(&file_path, convert_audio));

        if is_splitted_video {
            // download audio file & convert together
			if !downl_db.compress {
                lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 3, total_steps,false);
			}

            let audio_path = lib::format_file_path(&downl_db.qid, folder.clone(), true);
            file_db.push(audio_path.clone());

            println!("Downloading audio.. {}", audio_path);
            try!(download.download_file(&audio_path, true));

            if !downl_db.compress {
                lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 4, total_steps,false);
            }

            match converter.merge_files(&downl_db.qid,&file_path, &audio_path,&save_path, !downl_db.compress) {
                Err(e) => {println!("merge error: {:?}",e); return Err(e);},
                Ok(()) => {},
            }

            try!(remove_file(&audio_path));
            file_db.pop();

        }else{ // we're already done, only need to copy if it's not raw audio source
            if !download.is_audio() {
                try!(lib::move_file(&file_path, &save_path));
            }
        }
    }

    if download.is_audio(){ // if audio-> convert m4a to mp3, which converts directly to downl. dir
        if !downl_db.compress {
            lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, total_steps, total_steps, false);
        }
        try!(converter.extract_audio(&file_path, &save_path,convert_audio));
        try!(remove_file(&file_path));
    }else{
        if is_splitted_video {
            try!(remove_file(&file_path));
        }else{
            try!(lib::move_file(&file_path, &save_path));
        }
    }
    file_db.pop();

    if !is_zipped { // add file to list, except it's for zip-compression later (=folder set)
        lib::add_file_entry(&downl_db.pool.clone(), &downl_db.qid,&name_http_valid, &name);
    }
    if !downl_db.compress {
        lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, total_steps, total_steps, true);
    }

    if folder.is_some() {
        Ok(Thing::String(save_path))
    } else {
        Ok(Thing::None)
    }
}

///Handles a playlist request
///If zipping isn't requested the downloads will be split up,
///so for each video in the playlist an own query entry will be created
///if warnings occured (unavailable video etc) the return will be true
fn handle_playlist(mut downl_db: DownloadDB, converter: &Converter, file_db: &mut Vec<String>) -> Result<Thing, DownloadError>{
    let mut max_steps: i32 = if downl_db.compress { 4 } else { 3 };
    lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 1, max_steps,false);

    let pl_id: i64 = downl_db.qid;
    downl_db.update_folder(format!("{}/{}",&CONFIG.general.save_dir,pl_id));
    println!("Folder:  {}",downl_db.folder);

    let db_copy = downl_db.clone();
    let download = Downloader::new(&db_copy, &CONFIG.general);

    let mut playlist_name = String::new();
    if downl_db.compress {
        playlist_name = try!(download.get_playlist_name());
        playlist_name = lib::url_encode(&playlist_name);
        println!("pl name {}",playlist_name);
        try!(std::fs::create_dir(&downl_db.folder));
        file_db.push(downl_db.folder.clone());
    }
    lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 2, max_steps,false);
    println!("retriving playlist videos..");
    let file_ids: Vec<String> = try!(download.get_playlist_ids());

    let handler_folder = if downl_db.compress {
        Some(pl_id.to_string())
    }else {
        None
    };

    println!("got em");
    max_steps += file_ids.len() as i32;
    let mut current_step = 2;
    let mut warnings = false;
    let mut current_url: String;
    let mut failed_log: String = String::from("Following urls couldn't be downloaded: \n");
    // we'll store all files and delete em later, so we don't need rm -rf
    let mut file_delete_list: Vec<String> = Vec::with_capacity(if downl_db.compress { file_ids.len() } else { 0 });
    for id in file_ids.iter() {
        current_step += 1;
        if downl_db.compress {
            lib::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        }
        current_url = format!("https://www.youtube.com/watch?v={}",id);
        downl_db.update_video(current_url.clone(), current_step as i64);
        println!("id: {}",id);
        let db_copy = downl_db.clone();
        match handle_download(db_copy, &handler_folder, converter, file_db) {
            Err(e) => { println!("error downloading {}: {:?}",id,e);
                        failed_log.push_str(&format!("{} {:?}\n", current_url, e));
                        warnings = true;
            },
            Ok(e) => {  if downl_db.compress {
                            match e {
                                Thing::String(v) => file_delete_list.push(v),
                                _ => panic!("handle_download not returning a filename"),
                            }
                        }
            },
        }
    }

    if warnings {
        lib::add_query_status(&downl_db.pool.clone(),&pl_id, &failed_log);
    }

    println!("downloaded all videos");
    if downl_db.compress { // zip to file, add to db & remove all sources
        current_step += 1;
        lib::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        let zip_name = format!("{}.zip",playlist_name);
        let zip_file = format!("{}/{}",&CONFIG.general.download_dir,zip_name);
        println!("zip file: {} \n zip source {}",zip_file, &downl_db.folder);
        try!(lib::zip_folder(&downl_db.folder, &zip_file));
        lib::add_file_entry(&downl_db.pool.clone(), &pl_id,&zip_name, &playlist_name);

        current_step += 1;
        lib::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        try!(lib::delete_files(file_delete_list));
        try!(remove_dir(downl_db.folder));
        file_db.pop();
    }

    Ok(Thing::Bool(warnings))
}
<commit_msg>changed playlist cleanup to remove everything inside the folder, avoiding possible problems<commit_after>extern crate mysql;
extern crate toml;
extern crate rustc_serialize;
#[macro_use]
extern crate lazy_static;

mod lib;

use lib::config;
use lib::downloader::DownloadDB;
use lib::downloader::Downloader;
use lib::DownloadError;
use lib::converter::Converter;

use std::fs::{remove_file,remove_dir_all};

static VERSION : &'static str = "0.2"; // String not valid
static SLEEP_MS: u32 = 5000;
static CODE_SUCCESS: i8 = 2;
static CODE_SUCCESS_WARNINGS: i8 = 3; // finished with warnings
static CODE_FAILED_INTERNAL: i8 = 10; // internal error
static CODE_FAILED_QUALITY: i8 = 11; // qualitz not available
static CODE_FAILED_UNAVAILABLE: i8 = 12; // source unavailable (private / removed)

lazy_static! {
    pub static ref CONFIG: config::Config = {
        println!("Starting yayd-backend v{}",&VERSION);
        config::init_config() //return
    };

}

//#[repr(i8)] broken, enum not usable as of #10292
// enum QueryCodes {
//     Waiting = 0,
//     InProgress = 1,
//     Finished = 2,
//     Failed = 3,
// }

enum Thing { String(String), Bool(bool), None }

fn main() {

    let pool = lib::db_connect(lib::mysql_options(), SLEEP_MS);

    let converter = Converter::new(&CONFIG.general.ffmpeg_bin,&CONFIG.general.mp3_quality , pool.clone());
    let mut print_pause = true;
    loop {
        if let Some(result) = lib::request_entry(& pool) {
            print_pause = true;
            let qid = result.qid.clone();                 //&QueryCodes::InProgress as i32
            lib::set_query_code(&mut pool.get_conn().unwrap(), &1, &result.qid).ok().expect("Failed to set query code!");
            lib::set_query_state(&pool.clone(),&qid, "started", false);

            let action_result: Result<Thing,DownloadError>;
            {
                let mut left_files: Vec<String> = Vec::with_capacity(2);
                if result.playlist {
                    action_result = handle_playlist(result, &converter,&mut left_files);
                }else{
                    action_result = handle_download(result, &None, &converter,&mut left_files);
                }

                if !left_files.is_empty() {
                    println!("cleaning up files");
                    for i in &left_files {
                        match remove_file(&i) {
                            Ok(_) => (println!("cleaning up {}",i)),
                            Err(e) => println!("unable to remove file '{}' {}",i,e),
                        }
                    }
                }
            }

            let code: i8 = match action_result {
                Ok(t) => {
                    match t {
                        Thing::Bool(b) => {
                            if b { CODE_SUCCESS_WARNINGS } else { CODE_SUCCESS }
                        }
                        _ => CODE_SUCCESS,
                    }
                },
                Err(e) => {
                    println!("Error: {:?}", e);
                    match e {
                        DownloadError::NotAvailable => CODE_FAILED_UNAVAILABLE,
                        DownloadError::ExtractorError => CODE_FAILED_UNAVAILABLE,
                        DownloadError::QualityNotAvailable => CODE_FAILED_QUALITY,
                        _ => {
                            let details = match e {
                                DownloadError::DBError(s) => s,
                                DownloadError::DownloadError(s) => s,
                                DownloadError::FFMPEGError(s) => s,
                                DownloadError::InternalError(s) => s,
                                _ => unreachable!(),
                            };
                            lib::add_query_status(&pool.clone(),&qid, &details);
                            CODE_FAILED_INTERNAL
                        },
                    }
                }
            };
            lib::set_query_code(&mut pool.get_conn().unwrap(), &code,&qid).ok().expect("Failed to set query code!");

            let state = match code {
                2 => "finished",
                10 => "internal error",
                11 => "quality not available",
                12 => "source unavailable",
                _ => "unknown"
            };

            lib::set_query_state(&pool.clone(),&qid, state, true);

        } else {
            if print_pause { println!("Pausing.."); print_pause = false; }
            std::thread::sleep_ms(SLEEP_MS);
        }
    }
}

///Download handler
///Used by the playlist/file handler to download one file
///Based on the quality it needs to download audio & video separately and convert them together
///In case of a playlist download it depends on the target download folder & if it should be bezipped
///In case of a DMCA we need to download the file via the socket connector,
///which will output a mp3, or if requested, the video but with the highest quality.
///Thus in case of a DMCA we can't pick a quality anymore.
///Also the filename depends on the socket output then.
///
///If it's a non-zipped single file, it's moved after a successful download, converted etc to the
///main folder from which it should be downloadable.
///The original non-ascii & url_encode'd name of the file is stored in the DB
fn handle_download<'a>(downl_db: DownloadDB, folder: &Option<String>, converter: &Converter, file_db: &mut Vec<String>) -> Result<Thing,DownloadError>{
    //update progress
    let is_zipped = match *folder {
        Some(_) => true,
        None => false,
    };
    if !downl_db.compress {
        lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 1, 0,false);
    }

    let download = Downloader::new(&downl_db, &CONFIG.general);
    //get filename, check for DMCA
    let mut dmca = false; // "succ." dmca -> file already downloaded

    let name = match download.get_file_name() { // get filename
        Ok(v) => v,
        Err(DownloadError::DMCAError) => { //now request via lib.. // k if( k == Err(DownloadError::DMCAError) )
            println!("DMCA error!");
            match download.lib_request_video(1,0) {
                Err(err) => { println!("Offliberty-call error {:?}", err); return Err(err); },
                Ok(v) => { dmca = true; v },
            }
        },
        Err(e) => { // unknown error / restricted source etc.. abort
            println!("Unknown error: {:?}", e);
            return Err(e);
        },
    };

    let name_http_valid = lib::format_file_name(&name, &download, &downl_db.qid);

    let file_path = lib::format_file_path(&downl_db.qid, folder.clone(), false);
    file_db.push(file_path.clone());
    let save_path = lib::format_save_path(folder.clone(),&name, &download, &downl_db.qid);

    println!("Filename: {}", name);

    let is_splitted_video = if dmca {
        false
    } else {
        lib::is_split_container(&downl_db.quality)
    };
    let convert_audio = CONFIG.extensions.mp3.contains(&downl_db.quality);

    let total_steps = if dmca {
        if download.is_audio() {
            3
        } else {
            2
        }
    } else if is_splitted_video {
        4
    } else if download.is_audio() {
        3
    } else {
        2
    };

    if !dmca {
        if !downl_db.compress {
            lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 2, total_steps,false);
        }

        //download first file, download audio raw source if specified or video
        try!(download.download_file(&file_path, convert_audio));

        if is_splitted_video {
            // download audio file & convert together
			if !downl_db.compress {
                lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 3, total_steps,false);
			}

            let audio_path = lib::format_file_path(&downl_db.qid, folder.clone(), true);
            file_db.push(audio_path.clone());

            println!("Downloading audio.. {}", audio_path);
            try!(download.download_file(&audio_path, true));

            if !downl_db.compress {
                lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 4, total_steps,false);
            }

            match converter.merge_files(&downl_db.qid,&file_path, &audio_path,&save_path, !downl_db.compress) {
                Err(e) => {println!("merge error: {:?}",e); return Err(e);},
                Ok(()) => {},
            }

            try!(remove_file(&audio_path));
            file_db.pop();

        }else{ // we're already done, only need to copy if it's not raw audio source
            if !download.is_audio() {
                try!(lib::move_file(&file_path, &save_path));
            }
        }
    }

    if download.is_audio(){ // if audio-> convert m4a to mp3, which converts directly to downl. dir
        if !downl_db.compress {
            lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, total_steps, total_steps, false);
        }
        try!(converter.extract_audio(&file_path, &save_path,convert_audio));
        try!(remove_file(&file_path));
    }else{
        if is_splitted_video {
            try!(remove_file(&file_path));
        }else{
            try!(lib::move_file(&file_path, &save_path));
        }
    }
    file_db.pop();

    if !is_zipped { // add file to list, except it's for zip-compression later (=folder set)
        lib::add_file_entry(&downl_db.pool.clone(), &downl_db.qid,&name_http_valid, &name);
    }
    if !downl_db.compress {
        lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, total_steps, total_steps, true);
    }

    if folder.is_some() {
        Ok(Thing::String(save_path))
    } else {
        Ok(Thing::None)
    }
}

///Handles a playlist request
///If zipping isn't requested the downloads will be split up,
///so for each video in the playlist an own query entry will be created
///if warnings occured (unavailable video etc) the return will be true
fn handle_playlist(mut downl_db: DownloadDB, converter: &Converter, file_db: &mut Vec<String>) -> Result<Thing, DownloadError>{
    let mut max_steps: i32 = if downl_db.compress { 4 } else { 3 };
    lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 1, max_steps,false);

    let pl_id: i64 = downl_db.qid;
    downl_db.update_folder(format!("{}/{}",&CONFIG.general.save_dir,pl_id));
    println!("Folder:  {}",downl_db.folder);

    let db_copy = downl_db.clone();
    let download = Downloader::new(&db_copy, &CONFIG.general);

    let mut playlist_name = String::new();
    if downl_db.compress {
        playlist_name = try!(download.get_playlist_name());
        playlist_name = lib::url_encode(&playlist_name);
        println!("pl name {}",playlist_name);
        try!(std::fs::create_dir(&downl_db.folder));
        file_db.push(downl_db.folder.clone());
    }
    lib::update_steps(&downl_db.pool.clone(),&downl_db.qid, 2, max_steps,false);
    println!("retriving playlist videos..");
    let file_ids: Vec<String> = try!(download.get_playlist_ids());

    let handler_folder = if downl_db.compress {
        Some(pl_id.to_string())
    }else {
        None
    };

    println!("got em");
    max_steps += file_ids.len() as i32;
    let mut current_step = 2;
    let mut warnings = false;
    let mut current_url: String;
    let mut failed_log: String = String::from("Following urls couldn't be downloaded: \n");
    // we'll store all files and delete em later, so we don't need rm -rf
    let mut file_delete_list: Vec<String> = Vec::with_capacity(if downl_db.compress { file_ids.len() } else { 0 });
    for id in file_ids.iter() {
        current_step += 1;
        if downl_db.compress {
            lib::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        }
        current_url = format!("https://www.youtube.com/watch?v={}",id);
        downl_db.update_video(current_url.clone(), current_step as i64);
        println!("id: {}",id);
        let db_copy = downl_db.clone();
        match handle_download(db_copy, &handler_folder, converter, file_db) {
            Err(e) => { println!("error downloading {}: {:?}",id,e);
                        failed_log.push_str(&format!("{} {:?}\n", current_url, e));
                        warnings = true;
            },
            Ok(e) => {  if downl_db.compress {
                            match e {
                                Thing::String(v) => file_delete_list.push(v),
                                _ => panic!("handle_download not returning a filename"),
                            }
                        }
            },
        }
    }

    if warnings {
        lib::add_query_status(&downl_db.pool.clone(),&pl_id, &failed_log);
    }

    println!("downloaded all videos");
    if downl_db.compress { // zip to file, add to db & remove all sources
        current_step += 1;
        lib::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        let zip_name = format!("{}.zip",playlist_name);
        let zip_file = format!("{}/{}",&CONFIG.general.download_dir,zip_name);
        println!("zip file: {} \n zip source {}",zip_file, &downl_db.folder);
        try!(lib::zip_folder(&downl_db.folder, &zip_file));
        lib::add_file_entry(&downl_db.pool.clone(), &pl_id,&zip_name, &playlist_name);

        current_step += 1;
        lib::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        try!(lib::delete_files(file_delete_list));
        try!(remove_dir_all(downl_db.folder));
        file_db.pop();
    }

    Ok(Thing::Bool(warnings))
}
<|endoftext|>


<commit_before>#![crate_name = "limonite"]
#![crate_type = "bin"]

use std::env;
use std::old_io::BufferedReader;
use std::old_io::File;

use syntax::lexer::Lexer;
use syntax::parser::Parser;

pub mod syntax;

// ToDo: Fill this out with usage and version info?
fn display_info() {
    // Display stuff about limonite.
    println!("Usage: limonite [FILE]");
}

fn main() {
    let args: Vec<String> = env::args().map(|x| x.into_string().unwrap()).collect();

    if args.len() != 2 {
        return display_info();
    }

    let ref file_name = args[1];
    let path = Path::new(file_name);
    let file = match File::open(&path) {
        Ok(f)  => f,
        Err(e) => panic!("Failed to open file. File error: {}", e)
    };

    let input_string = BufferedReader::new(file).read_to_string().unwrap();

    let lexer = Lexer::new(&input_string[]);
    let mut parser = Parser::new(lexer);

    parser.parse();
}
<commit_msg>Fixed broken line in main.<commit_after>#![crate_name = "limonite"]
#![crate_type = "bin"]

use std::env;
use std::old_io::BufferedReader;
use std::old_io::File;

use syntax::lexer::Lexer;
use syntax::parser::Parser;

pub mod syntax;

// ToDo: Fill this out with usage and version info?
fn display_info() {
    // Display stuff about limonite.
    println!("Usage: limonite [FILE]");
}

fn main() {
    let args: Vec<String> = env::args().collect();

    if args.len() != 2 {
        return display_info();
    }

    let ref file_name = args[1];
    let path = Path::new(file_name);
    let file = match File::open(&path) {
        Ok(f)  => f,
        Err(e) => panic!("Failed to open file. File error: {}", e)
    };

    let input_string = BufferedReader::new(file).read_to_string().unwrap();

    let lexer = Lexer::new(&input_string[]);
    let mut parser = Parser::new(lexer);

    parser.parse();
}
<|endoftext|>


<commit_before>fn main() {
    let mut s = String::from("this is a string");
    takes_a_reference(&mut s);

    println!("{}", s);
}

fn takes_a_reference(s: &mut String) {
    s.push_str(" this is a pushed string");
}
<commit_msg>string slices<commit_after>fn main() {
    let s = String::from("this is a string");
    println!("{}", get_first_n_words(&s, 2));
}

fn get_first_n_words(s: &str, n: u32) -> &str {
    let bytes = s.as_bytes();
    let mut count = 0;
    for (i, &byte) in bytes.iter().enumerate() {
        if byte == b' ' {
            count += 1;
        }
        if count == n {
            return &s[..i];
        }
    }
    &s
}
<|endoftext|>


<commit_before>extern crate arguments;
extern crate temporary;

use std::collections::HashMap;
use std::fmt::Display;
use std::fs::File;
use std::io::{stderr, stdin, stdout, Read, Write};
use std::path::Path;
use std::process::Command;
use temporary::Directory;

const USAGE: &'static str = "
Usage: cite [options]

Options:
    --bib <FILE>       A bibliography file.
    --ref <NAME>       A reference name.
    --tex <FILE>       A template file.

    --help             Display this message.
";

const TEMPLATE: &'static str = r#"
\documentclass[journal]{IEEEtran}
\pagestyle{empty}
\renewcommand{\refname}{}

\begin{document}
\nocite{<reference>}
\bibliography{IEEEabrv,<bibliography>}
\bibliographystyle{IEEEtran}
\end{document}
"#;

macro_rules! ok(
    ($result:expr) => (match $result {
        Ok(result) => result,
        Err(error) => raise!(error),
    });
);

macro_rules! raise(
    ($error:expr) => (return Err(Box::new($error)));
    ($($arg:tt)*) => (raise!(format!($($arg)*)));
);

pub type Error = Box<Display>;
pub type Result<T> = std::result::Result<T, Error>;

fn main() {
    start().unwrap_or_else(|error| fail(error));
}

fn start() -> Result<()> {
    let arguments = match arguments::parse(std::env::args()) {
        Ok(arguments) => arguments,
        Err(error) => raise!(error),
    };

    if arguments.get::<bool>("help").unwrap_or(false) {
        help();
    }

    let root = ok!(Directory::new("cite"));

    let bibliography = match arguments.get::<String>("bib") {
        Some(bibliography) => bibliography,
        _ => try!(create_bibliography(&root)),
    };
    let reference = match arguments.get::<String>("ref") {
        Some(reference) => reference,
        _ => raise!("a reference name is required"),
    };

    match arguments.get::<String>("tex") {
        Some(ref template) => {
            let mut file = ok!(File::open(template));
            let mut template = String::new();
            ok!(file.read_to_string(&mut template));
            process(&template, &bibliography, &reference, &root)
        },
        _ => {
            process(TEMPLATE, &bibliography, &reference, &root)
        },
    }
}

fn process(template: &str, bibliography: &str, reference: &str, root: &Path) -> Result<()> {
    macro_rules! run(
        ($program:expr, $argument:expr) => ({
            let mut command = Command::new($program);
            command.arg($argument).current_dir(&root);
            let output = match command.output() {
                Ok(output) => output,
                Err(error) => raise!(format!("`{}` has failed: {}", $program, error)),
            };
            if !output.status.success() {
                let _ = stdout().write_all(&output.stdout);
                let _ = stderr().write_all(&output.stderr);
                raise!(format!("`{}` has failed", $program));
            }
        });
    );

    {
        let mut map = HashMap::new();
        map.insert("<bibliography>", bibliography);
        map.insert("<reference>", reference);
        let tex = replace(template, &map);

        let mut file = ok!(File::create(root.join("paper.tex")));
        ok!(file.write_all(tex.as_bytes()));
    }

    run!("latex", "paper.tex");
    run!("bibtex", "paper");
    run!("latex", "paper.tex");
    run!("latex", "paper.tex");
    run!("dvipdf", "paper.dvi");
    run!("pdftotext", "paper.pdf");

    let mut buffer = Vec::new();
    {
        let mut file = ok!(File::open(root.join("paper.txt")));
        ok!(file.read_to_end(&mut buffer));
    }

    let content = String::from_utf8_lossy(&buffer);
    println!("{}", content.trim());

    Ok(())
}

fn create_bibliography(root: &Path) -> Result<String> {
    println!("Paste a bibliography content and press Ctrl-D:");
    let mut content = String::new();
    ok!(stdin().read_to_string(&mut content));

    let mut file = ok!(File::create(root.join("paper.bib")));
    ok!(file.write_all(content.as_bytes()));

    Ok(String::from("paper.bib"))
}

fn replace(text: &str, map: &HashMap<&str, &str>) -> String {
    let mut text = text.to_string();
    for (key, value) in map {
        text = text.replace(key, value);
    }
    text
}

fn help() -> ! {
    println!("{}", USAGE.trim());
    std::process::exit(0);
}

fn fail(error: Error) -> ! {
    let message = format!("Error: {}.\n{}", &*error, USAGE);
    let _ = stderr().write_all(message.as_bytes());
    std::process::exit(1);
}
<commit_msg>Seek for a reference if not specified<commit_after>extern crate arguments;
extern crate temporary;

use std::collections::HashMap;
use std::fmt::Display;
use std::fs::File;
use std::io::{BufRead, BufReader, Read, Write, stderr, stdin, stdout};
use std::path::{Path, PathBuf};
use std::process::Command;
use temporary::Directory;

const USAGE: &'static str = "
Usage: cite [options]

Options:
    --bib <FILE>       A bibliography file.
    --ref <NAME>       A reference name.
    --tex <FILE>       A template file.

    --help             Display this message.
";

const TEMPLATE: &'static str = r#"
\documentclass[journal]{IEEEtran}
\pagestyle{empty}
\renewcommand{\refname}{}

\begin{document}
\nocite{<reference>}
\bibliography{IEEEabrv,<bibliography>}
\bibliographystyle{IEEEtran}
\end{document}
"#;

macro_rules! ok(
    ($result:expr) => (match $result {
        Ok(result) => result,
        Err(error) => raise!(error),
    });
);

macro_rules! raise(
    ($error:expr) => (return Err(Box::new($error)));
    ($($arg:tt)*) => (raise!(format!($($arg)*)));
);

pub type Error = Box<Display>;
pub type Result<T> = std::result::Result<T, Error>;

fn main() {
    start().unwrap_or_else(|error| fail(error));
}

fn start() -> Result<()> {
    let arguments = match arguments::parse(std::env::args()) {
        Ok(arguments) => arguments,
        Err(error) => raise!(error),
    };

    if arguments.get::<bool>("help").unwrap_or(false) {
        help();
    }

    let root = ok!(Directory::new("cite"));

    let bibliography = match arguments.get::<String>("bib") {
        Some(ref bibliography) => PathBuf::from(bibliography),
        _ => {
            let path = root.join("paper.bib").to_path_buf();
            try!(create_bibliography(&path));
            path
        },
    };
    let reference = match arguments.get::<String>("ref") {
        Some(reference) => reference,
        _ => try!(find_reference(&bibliography)),
    };

    match arguments.get::<String>("tex") {
        Some(ref template) => {
            let mut file = ok!(File::open(template));
            let mut template = String::new();
            ok!(file.read_to_string(&mut template));
            process(&template, &bibliography, &reference, &root)
        },
        _ => {
            process(TEMPLATE, &bibliography, &reference, &root)
        },
    }
}

fn process(template: &str, bibliography: &Path, reference: &str, root: &Path) -> Result<()> {
    macro_rules! run(
        ($program:expr, $argument:expr) => ({
            let mut command = Command::new($program);
            command.arg($argument).current_dir(&root);
            let output = match command.output() {
                Ok(output) => output,
                Err(error) => raise!(format!("`{}` has failed: {}", $program, error)),
            };
            if !output.status.success() {
                let _ = stdout().write_all(&output.stdout);
                let _ = stderr().write_all(&output.stderr);
                raise!(format!("`{}` has failed", $program));
            }
        });
    );

    {
        let mut map = HashMap::new();
        map.insert("<bibliography>", bibliography.to_str().unwrap());
        map.insert("<reference>", reference);
        let tex = replace(template, &map);

        let mut file = ok!(File::create(root.join("paper.tex")));
        ok!(file.write_all(tex.as_bytes()));
    }

    run!("latex", "paper.tex");
    run!("bibtex", "paper");
    run!("latex", "paper.tex");
    run!("latex", "paper.tex");
    run!("dvipdf", "paper.dvi");
    run!("pdftotext", "paper.pdf");

    let mut buffer = Vec::new();
    {
        let mut file = ok!(File::open(root.join("paper.txt")));
        ok!(file.read_to_end(&mut buffer));
    }

    let content = String::from_utf8_lossy(&buffer);
    println!("{}", content.trim());

    Ok(())
}

fn create_bibliography(path: &Path) -> Result<()> {
    println!("Paste a bibliography content and press Ctrl-D:");
    let mut content = String::new();
    ok!(stdin().read_to_string(&mut content));

    let mut file = ok!(File::create(path));
    ok!(file.write_all(content.as_bytes()));

    Ok(())
}

fn find_reference(path: &Path) -> Result<String> {
    let file = ok!(File::open(path));
    let file = BufReader::new(file);
    for line in file.lines() {
        let line = ok!(line);
        if let Some(reference) = detect_reference(&line) {
            return Ok(reference);
        }
    }
    raise!("failed to find a reference; try specifying one explicitly");
}

fn detect_reference(line: &str) -> Option<String> {
    let line = line.trim();
    if !line.starts_with('@') {
        return None;
    }
    let i = match line.find('{') {
        Some(i) => i + 1,
        _ => return None,
    };
    let j = match line.find(',') {
        Some(j) => j,
        _ => line.len(),
    };
    if i >= j {
        return None
    }
    Some(String::from((&line[i..j]).trim()))
}

fn replace(text: &str, map: &HashMap<&str, &str>) -> String {
    let mut text = text.to_string();
    for (key, value) in map {
        text = text.replace(key, value);
    }
    text
}

fn help() -> ! {
    println!("{}", USAGE.trim());
    std::process::exit(0);
}

fn fail(error: Error) -> ! {
    let message = format!("Error: {}.\n{}", &*error, USAGE);
    let _ = stderr().write_all(message.as_bytes());
    std::process::exit(1);
}
<|endoftext|>


<commit_before>// Copyright 2015 Robert Helmer <rhelmer@rhelmer.org>. See the LICENSE
// file at the top-level directory of this distribution.

/**
  * Symbolapi - a microservice which accepts lists of symbol+addresses, and returns
  * a list of symbolicated functions.
  *
  * The goal is to take an HTTP JSON request such as:
  * {"stacks":[
  *   [
  *     [0,11723767],
  *     [1, 65802]
  *    ]
  *  ],
  *  "memoryMap":[
  *    ["xul.pdb","44E4EC8C2F41492B9369D6B9A059577C2"],
  *    ["wntdll.pdb","D74F79EB1F8D4A45ABCD2F476CCABACC2"]
  *  ],
  *  "version":4
  * }
  *
  * The symbolapi service then downloads the corresponding symbol files (e.g. from S3), and returns
  * the function names for the corresponding addresses (in the "stacks" array) and returns JSON
  * such as:
  * {"symbolicatedStacks": [
  *   [
  *     "XREMain::XRE_mainRun() (in xul.pdb)",
  *     "KiUserCallbackDispatcher (in wntdll.pdb)"]
  *   ],
  *   "knownModules": [true, true]
  * }
  */

extern crate breakpad_symbols;
extern crate hyper;
#[macro_use]
extern crate log;
extern crate log4rs;
extern crate rustc_serialize;
extern crate toml;

use std::collections::HashMap;
use std::fs::{File, create_dir_all};
use std::io::{Read, Write};
use std::path::{PathBuf};
use std::thread;

use breakpad_symbols::{Symbolizer, SimpleSymbolSupplier};
use hyper::Client;
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use rustc_serialize::json;

/**
  * Incoming JSON request format.
  */
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcDecodable)]
pub struct SymbolRequest {
    memoryMap: Vec<(String,String)>,
    // index, offset
    stacks: Vec<Vec<(i8,u64)>>,
    version: u8,
}

/**
  * Outgoing JSON response format.
  */
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcEncodable)]
pub struct SymbolResponse {
    symbolicatedStacks: Vec<Vec<String>>,
    knownModules: Vec<bool>,
}

fn main() {
    log4rs::init_file("config/log.toml", Default::default()).unwrap();

    let address = "0.0.0.0:5000";

    info!("Listening on {}", address);
    Server::http(address).unwrap().handle(server).unwrap();
}

/**
  * Receives single HTTP requests and demuxes to symbols file fetches from S3 bucket.
  */
fn server(mut req: Request, mut res: Response) {
    info!("incoming connection from {}", req.remote_addr);

    match req.method {
        hyper::Post => {
            let mut res = res.start().unwrap();

            let mut buffer = String::new();
            req.read_to_string(&mut buffer).unwrap();
            debug!("raw POST: {:?}", &buffer);

            let decoded: SymbolRequest = json::decode(&buffer).unwrap();

            debug!("decoded memoryMap: {:?}", decoded.memoryMap);
            debug!("decoded stacks: {:?}", decoded.stacks);
            debug!("decoded version: {}", decoded.version);

            let symbol_url = get_config("symbol_urls.public");

            // stacks come in as an array, turn into hashmap
            let mut stack_map: HashMap<i8, Vec<u64>> = HashMap::new();

            debug!("decoded.stacks: {:?}", decoded.stacks);

            let stacks = decoded.stacks[0].clone();
            for stack in &stacks {
                let (index, offset) = *stack;

                let mut offsets = vec!();
                if stack_map.contains_key(&index) {
                    offsets = stack_map.get(&index).unwrap().clone();
                }
                offsets.push(offset);
                stack_map.insert(index, offsets);
            }

            debug!("stack_map: {:?}", stack_map);

            // FIXME limit the number of possible threads
            // TODO maybe push these into a queue and have a thread pool service the queue?
            let symbol_response = client(symbol_url, decoded.memoryMap, stack_map.clone());
            res.write_all(symbol_response.as_bytes()).unwrap();

            res.end().unwrap();
        },
        hyper::Get => {
            let mut res = res.start().unwrap();
            res.write_all("symbolapi, see github.com/rhelmer/symbolapi".as_bytes()).unwrap();
        },
        _ => { *res.status_mut() = StatusCode::MethodNotAllowed },
    }

    debug!("finished serving request");
}

/**
  * Creates multiple client connections and aggregates result.
  */
fn client(url: String, memory_map: Vec<(String,String)>, stack_map: HashMap<i8, Vec<u64>>) -> String {
    let mut handles = vec![];
    let mut counter: i8 = 0;
    for (debug_file, debug_id) in memory_map {
        let debug_file_name = debug_file.clone();
        let pdb = debug_file.find(".pdb").unwrap();
        let (symbol_name, _) = debug_file.split_at(pdb);
        let symbol_file = format!("{}.sym", symbol_name);
        let this_url = format!("{}/{}/{}/{}", url, debug_file, debug_id, symbol_file);

        let mut symbol_path = PathBuf::new();
        // TODO get from config
        symbol_path.push("testdata/symbols");

        let mut full_symbol_path = symbol_path.clone();
        full_symbol_path.push(&debug_file);
        full_symbol_path.push(&debug_id);
        full_symbol_path.push(&symbol_file);

        let stack_map_copy = stack_map.clone();

        create_dir_all(&full_symbol_path.parent().unwrap()).unwrap();

        let supplier = SimpleSymbolSupplier::new(vec!(symbol_path.clone()));

        // TODO most of the time is probably spent waiting on I/O, maybe async would be more appropriate?
        // TODO decide min/max possible threads, possibly based on number of cores?
        handles.push(thread::spawn(move || {

            let symbolizer = Symbolizer::new(supplier);
            // FIXME use Arc<Mutex<File>> to prevent concurrent writes
            // TODO only write contents if server version newer
            if !&full_symbol_path.exists() {

                let mut body = String::new();
                let c = Client::new();
                let mut res = c.get(&this_url).send().unwrap();
                res.read_to_string(&mut body).unwrap();

                let mut f = File::create(&full_symbol_path).unwrap();
                f.write_all(body.as_bytes()).unwrap();
            }

            debug!("symbol_path: {:?}", &symbol_path);
            // FIXME this is having problems with CRLF

            let mut symbols = vec!();
            for stacks in stack_map_copy.get(&counter) {
                for address in stacks {
                    debug!("attempt to symbolicate: {} for: {:?}", *address, &symbol_path);
                    match symbolizer.get_symbol_at_address(&debug_file_name, &debug_id, *address) {
                        Some(x) => symbols.push(x),
                        // return the address rather than function name if symbol not found
                        None => symbols.push(format!("0x{:x}", address)),
                    }
                }
            }
            debug!("debug_file_name: {:?}, symbols: {:?}", debug_file_name, symbols);

            (debug_file_name, symbols)
        }));

        counter += 1;
    }

    let mut result = SymbolResponse {
        symbolicatedStacks: vec!(),
        knownModules: vec!(),
    };

    for handle in handles {
        let (debug_file_name, symbols) = handle.join().unwrap();

        for symbol in symbols {
            result.symbolicatedStacks.push(
                vec!(format!("{} (in {}))", symbol, debug_file_name))
            );
        }
        result.knownModules.push(true);
    }

    json::encode(&result).unwrap()
}

/**
  * Returns individual values from the configuration file.
  */
fn get_config(value_name: &str) -> String {
    // TODO move to actual file, static str for the moment
    let toml: &'static str = r#"
        [symbol_urls]
        public = "https://s3-us-west-2.amazonaws.com/org.mozilla.crash-stats.symbols-public/v1"
    "#;
    // TODO support multiple URLs
    let value: toml::Value = toml.parse().unwrap();

    value.lookup(value_name).unwrap().as_str().unwrap().to_string()
}
<commit_msg>use correct response<commit_after>// Copyright 2015 Robert Helmer <rhelmer@rhelmer.org>. See the LICENSE
// file at the top-level directory of this distribution.

/**
  * Symbolapi - a microservice which accepts lists of symbol+addresses, and returns
  * a list of symbolicated functions.
  *
  * The goal is to take an HTTP JSON request such as:
  * {"stacks":[
  *   [
  *     [0,11723767],
  *     [1, 65802]
  *    ]
  *  ],
  *  "memoryMap":[
  *    ["xul.pdb","44E4EC8C2F41492B9369D6B9A059577C2"],
  *    ["wntdll.pdb","D74F79EB1F8D4A45ABCD2F476CCABACC2"]
  *  ],
  *  "version":4
  * }
  *
  * The symbolapi service then downloads the corresponding symbol files (e.g. from S3), and returns
  * the function names for the corresponding addresses (in the "stacks" array) and returns JSON
  * such as:
  * {"symbolicatedStacks": [
  *   [
  *     "XREMain::XRE_mainRun() (in xul.pdb)",
  *     "KiUserCallbackDispatcher (in wntdll.pdb)"]
  *   ],
  *   "knownModules": [true, true]
  * }
  */

extern crate breakpad_symbols;
extern crate hyper;
#[macro_use]
extern crate log;
extern crate log4rs;
extern crate rustc_serialize;
extern crate toml;

use std::collections::HashMap;
use std::fs::{File, create_dir_all};
use std::io::{Read, Write};
use std::path::{PathBuf};
use std::thread;

use breakpad_symbols::{Symbolizer, SimpleSymbolSupplier};
use hyper::Client;
use hyper::server::{Server, Request, Response};
use hyper::status::StatusCode;
use rustc_serialize::json;

/**
  * Incoming JSON request format.
  */
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcDecodable)]
pub struct SymbolRequest {
    memoryMap: Vec<(String,String)>,
    // index, offset
    stacks: Vec<Vec<(i8,u64)>>,
    version: u8,
}

/**
  * Outgoing JSON response format.
  */
// required JSON keys are non-snakecase
#[allow(non_snake_case)]
#[derive(RustcEncodable)]
pub struct SymbolResponse {
    symbolicatedStacks: Vec<String>,
    knownModules: Vec<bool>,
}

fn main() {
    log4rs::init_file("config/log.toml", Default::default()).unwrap();

    let address = "0.0.0.0:5000";

    info!("Listening on {}", address);
    Server::http(address).unwrap().handle(server).unwrap();
}

/**
  * Receives single HTTP requests and demuxes to symbols file fetches from S3 bucket.
  */
fn server(mut req: Request, mut res: Response) {
    info!("incoming connection from {}", req.remote_addr);

    match req.method {
        hyper::Post => {
            let mut res = res.start().unwrap();

            let mut buffer = String::new();
            req.read_to_string(&mut buffer).unwrap();
            debug!("raw POST: {:?}", &buffer);

            let decoded: SymbolRequest = json::decode(&buffer).unwrap();

            debug!("decoded memoryMap: {:?}", decoded.memoryMap);
            debug!("decoded stacks: {:?}", decoded.stacks);
            debug!("decoded version: {}", decoded.version);

            let symbol_url = get_config("symbol_urls.public");

            // stacks come in as an array, turn into hashmap
            let mut stack_map: HashMap<i8, Vec<u64>> = HashMap::new();

            debug!("decoded.stacks: {:?}", decoded.stacks);

            let stacks = decoded.stacks[0].clone();
            for stack in &stacks {
                let (index, offset) = *stack;

                let mut offsets = vec!();
                if stack_map.contains_key(&index) {
                    offsets = stack_map.get(&index).unwrap().clone();
                }
                offsets.push(offset);
                stack_map.insert(index, offsets);
            }

            debug!("stack_map: {:?}", stack_map);

            // FIXME limit the number of possible threads
            // TODO maybe push these into a queue and have a thread pool service the queue?
            let symbol_response = client(symbol_url, decoded.memoryMap, stack_map.clone());
            res.write_all(symbol_response.as_bytes()).unwrap();

            res.end().unwrap();
        },
        hyper::Get => {
            let mut res = res.start().unwrap();
            res.write_all("symbolapi, see github.com/rhelmer/symbolapi".as_bytes()).unwrap();
        },
        _ => { *res.status_mut() = StatusCode::MethodNotAllowed },
    }

    debug!("finished serving request");
}

/**
  * Creates multiple client connections and aggregates result.
  */
fn client(url: String, memory_map: Vec<(String,String)>, stack_map: HashMap<i8, Vec<u64>>) -> String {
    let mut handles = vec![];
    let mut counter: i8 = 0;
    for (debug_file, debug_id) in memory_map {
        let debug_file_name = debug_file.clone();
        let pdb = debug_file.find(".pdb").unwrap();
        let (symbol_name, _) = debug_file.split_at(pdb);
        let symbol_file = format!("{}.sym", symbol_name);
        let this_url = format!("{}/{}/{}/{}", url, debug_file, debug_id, symbol_file);

        let mut symbol_path = PathBuf::new();
        // TODO get from config
        symbol_path.push("testdata/symbols");

        let mut full_symbol_path = symbol_path.clone();
        full_symbol_path.push(&debug_file);
        full_symbol_path.push(&debug_id);
        full_symbol_path.push(&symbol_file);

        let stack_map_copy = stack_map.clone();

        create_dir_all(&full_symbol_path.parent().unwrap()).unwrap();

        let supplier = SimpleSymbolSupplier::new(vec!(symbol_path.clone()));

        // TODO most of the time is probably spent waiting on I/O, maybe async would be more appropriate?
        // TODO decide min/max possible threads, possibly based on number of cores?
        handles.push(thread::spawn(move || {

            let symbolizer = Symbolizer::new(supplier);
            // FIXME use Arc<Mutex<File>> to prevent concurrent writes
            // TODO only write contents if server version newer
            if !&full_symbol_path.exists() {

                let mut body = String::new();
                let c = Client::new();
                let mut res = c.get(&this_url).send().unwrap();
                res.read_to_string(&mut body).unwrap();

                let mut f = File::create(&full_symbol_path).unwrap();
                f.write_all(body.as_bytes()).unwrap();
            }

            debug!("symbol_path: {:?}", &symbol_path);
            // FIXME this is having problems with CRLF

            let mut symbols = vec!();
            for stacks in stack_map_copy.get(&counter) {
                for address in stacks {
                    debug!("attempt to symbolicate: {} for: {:?}", *address, &symbol_path);
                    match symbolizer.get_symbol_at_address(&debug_file_name, &debug_id, *address) {
                        Some(x) => symbols.push(x),
                        // return the address rather than function name if symbol not found
                        None => symbols.push(format!("0x{:x}", address)),
                    }
                }
            }
            debug!("debug_file_name: {:?}, symbols: {:?}", debug_file_name, symbols);

            (debug_file_name, symbols)
        }));

        counter += 1;
    }

    let mut result = SymbolResponse {
        symbolicatedStacks: vec!(),
        knownModules: vec!(),
    };

    for handle in handles {
        let (debug_file_name, symbols) = handle.join().unwrap();

        for symbol in symbols {
            result.symbolicatedStacks.push(
                format!("{} (in {}))", symbol, debug_file_name)
            );
        }
        result.knownModules.push(true);
    }

    json::encode(&result).unwrap()
}

/**
  * Returns individual values from the configuration file.
  */
fn get_config(value_name: &str) -> String {
    // TODO move to actual file, static str for the moment
    let toml: &'static str = r#"
        [symbol_urls]
        public = "https://s3-us-west-2.amazonaws.com/org.mozilla.crash-stats.symbols-public/v1"
    "#;
    // TODO support multiple URLs
    let value: toml::Value = toml.parse().unwrap();

    value.lookup(value_name).unwrap().as_str().unwrap().to_string()
}
<|endoftext|>


<commit_before>extern crate bufstream;

use std::net::{TcpListener, TcpStream};
use std::io::{Write, BufRead, Read};
use bufstream::BufStream;
use std::error::Error;
use std::process::{Command, Stdio};

fn main(){
    let listener = TcpListener::bind("127.0.0.1:8000").unwrap();
    // accept connections and process them, spawning a new thread for each one
    for stream in listener.incoming() {
        match stream {
            Err(why) => println!("accept err {}", why.description()),
            Ok(stream) => {
                handler(&mut BufStream::new(stream));
            }
        }
    }
}

fn echo_err(why: std::io::Error, s: &mut BufStream<TcpStream>)-> std::io::Result<()>{
    s.write_all(why.description().as_bytes());
    s.write_all(b"\n")
}

fn echo_str(str: std::string::String, s: &mut BufStream<TcpStream>) -> std::io::Result<()>{
    //s.write_all(str.as_bytes()).and_then(s.write_all(b"\n").and_then(s.flush()));
    s.write_all(str.as_bytes());
    s.write_all(b"\n");
    s.flush()
}

fn handler(s: &mut BufStream<TcpStream>){
    // Until the user does not request to close.
    loop{
        let _ = s.write(b"> ");
        let _ = s.flush();
        let mut cmd = std::string::String::new();
        s.read_line(&mut cmd).unwrap();
        let cmd_splt : Vec<_> = cmd.as_str().split_whitespace().collect();
        let mut args = Vec::new();
        for i in 1..cmd_splt.len() {
            args.push(cmd_splt[i]);
        }
        let main_cmd = cmd_splt[0];
        // Here goes the command processing.
        match main_cmd{
            "exit" => break,
            _ => match Command::new(main_cmd).args(&args).stdout(Stdio::piped()).spawn() {
                Err(why) => {
                    match echo_err(why, s){
                        Err(why) => println!("could not send error {:?}", why),
                        Ok(_) =>{}
                    }
                },
                Ok(process) => {
                    let mut stde = String::new();
                    match process.stderr {
                        Some(mut err) => {
                            match err.read_to_string(&mut stde) {
                                Err(why) => match echo_err(why, s){
                                    Err(why) => println!("could not send error {:?}", why),
                                    Ok(_) =>{}
                                },
                                Ok(_) => match echo_str(stde, s){
                                    Err(why) => println!("{:?}", why),
                                    Ok(_) =>{}
                                },
                            };
                        },
                        None => {
                            let mut stdo = String::new();
                            match process.stdout.unwrap().read_to_string(&mut stdo) {
                                Err(why) => match echo_err(why, s){
                                    Err(why) => println!("could not send error {:?}", why),
                                    Ok(_) =>{}
                                },
                                Ok(_) => match echo_str(stdo, s){
                                    Err(why) => println!("{:?}", why),
                                    Ok(_) =>{}
                                },
                            }
                        }
                    }
                },
            }
        }
    }
}
<commit_msg>Implements #2<commit_after>#![feature(loop_break_value)]
extern crate bufstream;

use std::net::{TcpListener, TcpStream};
use std::io::{Write, BufRead, Read};
use bufstream::BufStream;
use std::error::Error;
use std::process::{Command, Stdio};

fn main(){
    let listener = TcpListener::bind("127.0.0.1:8000").unwrap();
    // accept connections and process them, spawning a new thread for each one
    for stream in listener.incoming() {
        match stream {
            Err(why) => println!("accept err {}", why.description()),
            Ok(stream) => {
                if handler(&mut BufStream::new(stream)){
                    break
                }
            }
        }
    }
}

fn echo_err(why: std::io::Error, s: &mut BufStream<TcpStream>)-> std::io::Result<()>{
    try!(s.write_all(why.description().as_bytes()));
    try!(s.write_all(b"\n"));
    Ok(())
}

fn echo_str(str: std::string::String, s: &mut BufStream<TcpStream>) -> std::io::Result<()>{
    try!(s.write_all(str.as_bytes()));
    try!(s.write_all(b"\n"));
    try!(s.flush());
    Ok(())
}

fn handler(s: &mut BufStream<TcpStream>) -> bool{
    // Until the user does not request to close.
    loop{
        let _ = s.write(b"> ");
        let _ = s.flush();
        let mut cmd = std::string::String::new();
        s.read_line(&mut cmd).unwrap();
        let cmd_splt : Vec<_> = cmd.as_str().split_whitespace().collect();
        let mut args = Vec::new();
        for i in 1..cmd_splt.len() {
            args.push(cmd_splt[i]);
        }
        let main_cmd = cmd_splt[0];
        // Here goes the command processing.
        match main_cmd{
            "exit" => return false,
            "kill" => return true,
            "cfgport" => {},
            "cfgpwd" => {},
            _ => match Command::new(main_cmd).args(&args).stdout(Stdio::piped()).spawn() {
                Err(why) => {
                    match echo_err(why, s){
                        Err(why) => println!("could not send error {:?}", why),
                        Ok(_) =>{}
                    }
                },
                Ok(process) => {
                    let mut stde = String::new();
                    match process.stderr {
                        Some(mut err) => {
                            match err.read_to_string(&mut stde) {
                                Err(why) => match echo_err(why, s){
                                    Err(why) => println!("could not send error {:?}", why),
                                    Ok(_) =>{}
                                },
                                Ok(_) => match echo_str(stde, s){
                                    Err(why) => println!("{:?}", why),
                                    Ok(_) =>{}
                                },
                            };
                        },
                        None => {
                            let mut stdo = String::new();
                            match process.stdout.unwrap().read_to_string(&mut stdo) {
                                Err(why) => match echo_err(why, s){
                                    Err(why) => println!("could not send error {:?}", why),
                                    Ok(_) =>{}
                                },
                                Ok(_) => match echo_str(stdo, s){
                                    Err(why) => println!("{:?}", why),
                                    Ok(_) =>{}
                                },
                            }
                        }
                    }
                },
            }
        }
    }
}
<|endoftext|>


<commit_before>/// An example of a client/server agnostic WebSocket that takes input from stdin and sends that
/// input to all other peers.
///
/// For example, to create a network like this:
///
/// 3013 ---- 3012 ---- 3014
///   \        |
///    \       |
///     \      |
///      \     |
///       \    |
///        \   |
///         \  |
///          3015
///
/// Run these commands in separate processes
/// ./peer2peer
/// ./peer2peer --server localhost:3013 ws://localhost:3012
/// ./peer2peer --server localhost:3014 ws://localhost:3012
/// ./peer2peer --server localhost:3015 ws://localhost:3012 ws://localhost:3013
///
/// Stdin on 3012 will be sent to all other peers
/// Stdin on 3013 will be sent to 3012 and 3015
/// Stdin on 3014 will be sent to 3012 only
/// Stdin on 3015 will be sent to 3012 and 2013

extern crate ws;
extern crate url;
extern crate clap;
extern crate env_logger;
extern crate rustc_serialize;
extern crate bincode;
#[macro_use] extern crate log;

pub mod handler;
pub mod message;

use message::PeerMessage;
use handler::MessageFactory;

use std::collections::hash_map::HashMap;
use std::clone::Clone;
use std::io;
use std::io::prelude::*;
use std::thread;
use std::sync::{Arc, Mutex};

use bincode::rustc_serialize::encode;

use clap::{App, Arg};

fn main() {
    // Setup logging
    env_logger::init().unwrap();

    // Parse command line arguments
    let matches = App::new("Simple Peer 2 Peer")
        .version("1.0")
        .author("Jonathan Almeida <hello@jonalmeida.com>")
        .about("Connect to other peers and listen for incoming connections.")
        .arg(Arg::with_name("server")
             .short("s")
             .long("server")
             .value_name("SERVER")
             .help("Set the address to listen for new connections. (default: localhost:3012"))
        .arg(Arg::with_name("PEER")
             .help("A WebSocket URL to attempt to connect to at start.")
             .multiple(true))
        .get_matches();

    // Get address of this peer
    let my_addr = matches.value_of("server").unwrap_or("localhost:3012");
    let sending_addr = String::from(my_addr);

    let mut clock = HashMap::new();
    clock.insert(sending_addr.clone(), 0u32);
    let clocks = Arc::new(Mutex::new(clock));
    let connecting_clocks = clocks.clone();
    let address_copy = sending_addr.clone();

    // Create simple websocket that just prints out messages
    let mut me  = ws::WebSocket::new(
        MessageFactory::build(connecting_clocks.clone())
            .me(address_copy.clone().as_str())
        ).unwrap();

    // Get a sender for ALL connections to the websocket
    let broacaster = me.broadcaster();

    // Setup thread for listening to stdin and sending messages to connections
    let input = thread::spawn(move || {
        let stdin = io::stdin();
        for line in stdin.lock().lines() {

            // Incrementing clock before sending
            let clock_clone = clocks.clone();
            let mut clock_lock = clock_clone.lock().unwrap();
            *clock_lock.get_mut(&sending_addr.clone()).unwrap() += 1;

            // Constructing new message
            let message =
                PeerMessage {
                    sender: sending_addr.clone(),
                    clocks: clock_lock.clone(),
                    message: line.unwrap()
                };

            // Encoding into bytes
            let encoded: Vec<u8> = encode(&message, bincode::SizeLimit::Infinite).unwrap();

            // Sending bytes to all connected clients
            broacaster.send(&*encoded.into_boxed_slice()).unwrap();
        }
    });

    // Connect to any existing peers specified on the cli
    if let Some(peers) = matches.values_of("PEER") {
        for peer in peers {
            me.connect(url::Url::parse(peer).unwrap()).unwrap();
        }
    }

    // Run the websocket
    me.listen(my_addr).unwrap();
    input.join().unwrap();

}

/*
fn message_checking(local_clocks: &mut MutexGuard<HashMap<String, u32>>, incoming_message: PeerMessage) {
    let local_clock = local_clocks;
    let message_clocks = incoming_message.clocks.clone();
    for (key, val) in message_clocks.iter() {
        info!("Looking at key: {}", key);
        //let key_clone = key.clone();
        let sender_clone = incoming_message.sender.clone();
        if !message_clocks.contains_key(&key.clone()) {
            local_clock.insert(String::from(key.clone()), 0u32);
        }
        if key.clone().into_boxed_str() == sender_clone.clone().into_boxed_str() {
            //local_clocks.get(&key_clone).unwrap()
            info!("Looking at val: {}", *val);
            if *val != *local_clock.get(&sender_clone.clone()).unwrap() + 1 {
                info!("Buffering.. message: {}", incoming_message.message);
                // buffer.push_back
                ()
            }
        } else {
            if *val == *local_clock.get(&sender_clone.clone()).unwrap() {
                //buffer.flush
                continue;
            } else {
                //buffer.push();
                ()
            }
            //approve_new_messages();
        }
    }
    for (key, val) in message_clocks.iter() {
        local_clock.insert(key.clone(), val.clone());
    }
    info!("Peer {} with clocks: {:?} got message: {}",
            incoming_message.sender, incoming_message.clocks, incoming_message.message);
}
*/
<commit_msg>Removed commented dead code<commit_after>/// An example of a client/server agnostic WebSocket that takes input from stdin and sends that
/// input to all other peers.
///
/// For example, to create a network like this:
///
/// 3013 ---- 3012 ---- 3014
///   \        |
///    \       |
///     \      |
///      \     |
///       \    |
///        \   |
///         \  |
///          3015
///
/// Run these commands in separate processes
/// ./peer2peer
/// ./peer2peer --server localhost:3013 ws://localhost:3012
/// ./peer2peer --server localhost:3014 ws://localhost:3012
/// ./peer2peer --server localhost:3015 ws://localhost:3012 ws://localhost:3013
///
/// Stdin on 3012 will be sent to all other peers
/// Stdin on 3013 will be sent to 3012 and 3015
/// Stdin on 3014 will be sent to 3012 only
/// Stdin on 3015 will be sent to 3012 and 2013

extern crate ws;
extern crate url;
extern crate clap;
extern crate env_logger;
extern crate rustc_serialize;
extern crate bincode;
#[macro_use] extern crate log;

pub mod handler;
pub mod message;

use message::PeerMessage;
use handler::MessageFactory;

use std::collections::hash_map::HashMap;
use std::clone::Clone;
use std::io;
use std::io::prelude::*;
use std::thread;
use std::sync::{Arc, Mutex};

use bincode::rustc_serialize::encode;

use clap::{App, Arg};

fn main() {
    // Setup logging
    env_logger::init().unwrap();

    // Parse command line arguments
    let matches = App::new("Simple Peer 2 Peer")
        .version("1.0")
        .author("Jonathan Almeida <hello@jonalmeida.com>")
        .about("Connect to other peers and listen for incoming connections.")
        .arg(Arg::with_name("server")
             .short("s")
             .long("server")
             .value_name("SERVER")
             .help("Set the address to listen for new connections. (default: localhost:3012"))
        .arg(Arg::with_name("PEER")
             .help("A WebSocket URL to attempt to connect to at start.")
             .multiple(true))
        .get_matches();

    // Get address of this peer
    let my_addr = matches.value_of("server").unwrap_or("localhost:3012");
    let sending_addr = String::from(my_addr);

    let mut clock = HashMap::new();
    clock.insert(sending_addr.clone(), 0u32);
    let clocks = Arc::new(Mutex::new(clock));
    let connecting_clocks = clocks.clone();
    let address_copy = sending_addr.clone();

    // Create simple websocket that just prints out messages
    let mut me  = ws::WebSocket::new(
        MessageFactory::build(connecting_clocks.clone())
            .me(address_copy.clone().as_str())
        ).unwrap();

    // Get a sender for ALL connections to the websocket
    let broacaster = me.broadcaster();

    // Setup thread for listening to stdin and sending messages to connections
    let input = thread::spawn(move || {
        let stdin = io::stdin();
        for line in stdin.lock().lines() {

            // Incrementing clock before sending
            let clock_clone = clocks.clone();
            let mut clock_lock = clock_clone.lock().unwrap();
            *clock_lock.get_mut(&sending_addr.clone()).unwrap() += 1;

            // Constructing new message
            let message =
                PeerMessage {
                    sender: sending_addr.clone(),
                    clocks: clock_lock.clone(),
                    message: line.unwrap()
                };

            // Encoding into bytes
            let encoded: Vec<u8> = encode(&message, bincode::SizeLimit::Infinite).unwrap();

            // Sending bytes to all connected clients
            broacaster.send(&*encoded.into_boxed_slice()).unwrap();
        }
    });

    // Connect to any existing peers specified on the cli
    if let Some(peers) = matches.values_of("PEER") {
        for peer in peers {
            me.connect(url::Url::parse(peer).unwrap()).unwrap();
        }
    }

    // Run the websocket
    me.listen(my_addr).unwrap();
    input.join().unwrap();

}

<|endoftext|>


<commit_before>extern crate rustc_serialize;
extern crate docopt;

use std::env;
use docopt::Docopt;

// Write the Docopt usage string.
static USAGE: &'static str = "
Usage:
    rpath (-h | --help)
    rpath list
    rpath remove <location>
    rpath insert <location> <path>
    rpath replace <location> <path>

Options:
    -h --help     Show this screen.
    -v            Verbose.
";

#[derive(RustcDecodable, Debug)]
struct Args {
    cmd_remove: bool,
    cmd_list: bool,
    cmd_insert: bool,
    cmd_replace: bool,
    arg_location: Option<usize>,
    arg_path: Option<String>,
    flag_v: bool
}

fn print_path(path: &str) {
    println!("Location: Path");
    for (i, part) in path.split(":").enumerate() {
        println!("{}: {}", i, part)
    }
}

fn main() {
    let args: Args = Docopt::new(USAGE).and_then(|d| d.decode()).unwrap_or_else(|e| e.exit());
    let path = env::var("PATH").unwrap();
    let mut parts: Vec<&str> = path.split(":").collect();
    match args {
        Args { arg_location: Some(loc), .. } if loc >= parts.len() =>
            println!("Error: location is not valid. For a list of locations, run: rpath list"),
        Args { cmd_list: true, .. } => print_path(&path),
        Args { cmd_remove: true, arg_location: Some(loc), .. } => {
            parts.remove(loc);
            print!("{}", parts.connect(":"))
        },
        // Note that this handles both insert/replace
        Args { cmd_replace: is_replace, arg_location: Some(loc), arg_path: Some(ref ipath), .. } => {
            if is_replace {
                parts.remove(loc);
            }
            parts.insert(loc, ipath);
            print!("{}", parts.connect(":"))
        },
        _ => println!("Must have forgotten an option..."),
    }
}
<commit_msg>Add nicer path formatting<commit_after>extern crate rustc_serialize;
extern crate docopt;

use std::env;
use docopt::Docopt;

// Write the Docopt usage string.
static USAGE: &'static str = "
Usage:
    rpath (-h | --help)
    rpath list
    rpath remove <location>
    rpath insert <location> <path>
    rpath replace <location> <path>

Options:
    -h --help     Show this screen.
    -v            Verbose.
";

#[derive(RustcDecodable, Debug)]
struct Args {
    cmd_remove: bool,
    cmd_list: bool,
    cmd_insert: bool,
    cmd_replace: bool,
    arg_location: Option<usize>,
    arg_path: Option<String>,
    flag_v: bool
}

fn print_path(path: &str) {
    println!("Location:  Path");
    for (i, part) in path.split(":").enumerate() {
        println!("{:>8}:  {}", i, part)
    }
}

fn main() {
    let args: Args = Docopt::new(USAGE).and_then(|d| d.decode()).unwrap_or_else(|e| e.exit());
    let path = env::var("PATH").unwrap();
    let mut parts: Vec<&str> = path.split(":").collect();
    match args {
        Args { arg_location: Some(loc), .. } if loc >= parts.len() =>
            println!("Error: location is not valid. For a list of locations, run: rpath list"),
        Args { cmd_list: true, .. } => print_path(&path),
        Args { cmd_remove: true, arg_location: Some(loc), .. } => {
            parts.remove(loc);
            print!("{}", parts.connect(":"))
        },
        // Note that this handles both insert/replace
        Args { cmd_replace: is_replace, arg_location: Some(loc), arg_path: Some(ref ipath), .. } => {
            if is_replace {
                parts.remove(loc);
            }
            parts.insert(loc, ipath);
            print!("{}", parts.connect(":"))
        },
        _ => println!("Must have forgotten an option..."),
    }
}
<|endoftext|>


<commit_before>#![cfg_attr(feature = "deny-warnings", deny(warnings))]

use rustc_tools_util::VersionInfo;
use std::env;
use std::path::PathBuf;
use std::process::{self, Command};
use std::ffi::OsString;

const CARGO_CLIPPY_HELP: &str = r#"Checks a package to catch common mistakes and improve your Rust code.

Usage:
    cargo clippy [options] [--] [<opts>...]

Common options:
    -h, --help               Print this message
    -V, --version            Print version info and exit

Other options are the same as `cargo check`.

To allow or deny a lint from the command line you can use `cargo clippy --`
with:

    -W --warn OPT       Set lint warnings
    -A --allow OPT      Set lint allowed
    -D --deny OPT       Set lint denied
    -F --forbid OPT     Set lint forbidden

You can use tool lints to allow or deny lints from your code, eg.:

    #[allow(clippy::needless_lifetimes)]
"#;

fn show_help() {
    println!("{}", CARGO_CLIPPY_HELP);
}

fn show_version() {
    let version_info = rustc_tools_util::get_version_info!();
    println!("{}", version_info);
}

pub fn main() {
    // Check for version and help flags even when invoked as 'cargo-clippy'
    if env::args().any(|a| a == "--help" || a == "-h") {
        show_help();
        return;
    }

    if env::args().any(|a| a == "--version" || a == "-V") {
        show_version();
        return;
    }

    if let Err(code) = process(env::args().skip(2)) {
        process::exit(code);
    }
}

struct ClippyCmd {
    unstable_options: bool,
    cmd: &'static str,
    args: Vec<String>,
    clippy_args: String
}

impl ClippyCmd
{
    fn new<I>(mut old_args: I) -> Self
    where
        I: Iterator<Item = String>,
    {
        let mut cmd = "check";
        let mut unstable_options = false;
        let mut args = vec![];

        for arg in old_args.by_ref() {
            match arg.as_str() {
                "--fix" => {
                    cmd = "fix";
                    continue;
                }
                "--" => break,
                // Cover -Zunstable-options and -Z unstable-options
                s if s.ends_with("unstable-options") => unstable_options = true,
                _ => {}
            }

            args.push(arg);
        }

        if cmd == "fix" && !unstable_options {
            panic!("Usage of `--fix` requires `-Z unstable-options`");
        }

        // Run the dogfood tests directly on nightly cargo. This is required due
        // to a bug in rustup.rs when running cargo on custom toolchains. See issue #3118.
        if env::var_os("CLIPPY_DOGFOOD").is_some() && cfg!(windows) {
            args.insert(0, "+nightly".to_string());
        }

        let clippy_args: String =
            old_args
            .map(|arg| format!("{}__CLIPPY_HACKERY__", arg))
            .collect();

        ClippyCmd {
            unstable_options,
            cmd,
            args,
            clippy_args,
        }
    }

    fn path_env(&self) -> &'static str {
        if self.unstable_options {
            "RUSTC_WORKSPACE_WRAPPER"
        } else {
            "RUSTC_WRAPPER"
        }
    }

    fn path(&self) -> PathBuf {
        let mut path = env::current_exe()
            .expect("current executable path invalid")
            .with_file_name("clippy-driver");

        if cfg!(windows) {
            path.set_extension("exe");
        }

        path
    }

    fn target_dir() -> Option<(&'static str, OsString)> {
        env::var_os("CLIPPY_DOGFOOD")
            .map(|_| {
                env::var_os("CARGO_MANIFEST_DIR").map_or_else(
                    || std::ffi::OsString::from("clippy_dogfood"),
                    |d| {
                        std::path::PathBuf::from(d)
                            .join("target")
                            .join("dogfood")
                            .into_os_string()
                    },
                )
            })
            .map(|p| ("CARGO_TARGET_DIR", p))
    }

    fn to_std_cmd(self) -> Command {
        let mut cmd = Command::new("cargo");

        cmd.env(self.path_env(), self.path())
            .envs(ClippyCmd::target_dir())
            .env("CLIPPY_ARGS", self.clippy_args)
            .arg(self.cmd)
            .args(&self.args);

        cmd
    }
}

fn process<I>(old_args: I) -> Result<(), i32>
where
    I: Iterator<Item = String>,
{
    let cmd = ClippyCmd::new(old_args);

    let mut cmd = cmd.to_std_cmd();

    let exit_status = cmd
        .spawn()
        .expect("could not run cargo")
        .wait()
        .expect("failed to wait for cargo?");

    if exit_status.success() {
        Ok(())
    } else {
        Err(exit_status.code().unwrap_or(-1))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    #[should_panic]
    fn fix_without_unstable() {
        let args = "cargo clippy --fix".split_whitespace().map(ToString::to_string);
        let _ = ClippyCmd::new(args);
    }

    #[test]
    fn fix_unstable() {
        let args = "cargo clippy --fix -Zunstable-options".split_whitespace().map(ToString::to_string);
        let cmd = ClippyCmd::new(args);
        assert_eq!("fix", cmd.cmd);
        assert_eq!("RUSTC_WORKSPACE_WRAPPER", cmd.path_env());
        assert!(cmd.args.iter().find(|arg| arg.ends_with("unstable-options")).is_some());
    }

    #[test]
    fn check() {
        let args = "cargo clippy".split_whitespace().map(ToString::to_string);
        let cmd = ClippyCmd::new(args);
        assert_eq!("check", cmd.cmd);
        assert_eq!("RUSTC_WRAPPER", cmd.path_env());
    }

    #[test]
    fn check_unstable() {
        let args = "cargo clippy -Zunstable-options".split_whitespace().map(ToString::to_string);
        let cmd = ClippyCmd::new(args);
        assert_eq!("check", cmd.cmd);
        assert_eq!("RUSTC_WORKSPACE_WRAPPER", cmd.path_env());
    }
}
<commit_msg>rename field<commit_after>#![cfg_attr(feature = "deny-warnings", deny(warnings))]

use rustc_tools_util::VersionInfo;
use std::env;
use std::path::PathBuf;
use std::process::{self, Command};
use std::ffi::OsString;

const CARGO_CLIPPY_HELP: &str = r#"Checks a package to catch common mistakes and improve your Rust code.

Usage:
    cargo clippy [options] [--] [<opts>...]

Common options:
    -h, --help               Print this message
    -V, --version            Print version info and exit

Other options are the same as `cargo check`.

To allow or deny a lint from the command line you can use `cargo clippy --`
with:

    -W --warn OPT       Set lint warnings
    -A --allow OPT      Set lint allowed
    -D --deny OPT       Set lint denied
    -F --forbid OPT     Set lint forbidden

You can use tool lints to allow or deny lints from your code, eg.:

    #[allow(clippy::needless_lifetimes)]
"#;

fn show_help() {
    println!("{}", CARGO_CLIPPY_HELP);
}

fn show_version() {
    let version_info = rustc_tools_util::get_version_info!();
    println!("{}", version_info);
}

pub fn main() {
    // Check for version and help flags even when invoked as 'cargo-clippy'
    if env::args().any(|a| a == "--help" || a == "-h") {
        show_help();
        return;
    }

    if env::args().any(|a| a == "--version" || a == "-V") {
        show_version();
        return;
    }

    if let Err(code) = process(env::args().skip(2)) {
        process::exit(code);
    }
}

struct ClippyCmd {
    unstable_options: bool,
    cargo_subcommand: &'static str,
    args: Vec<String>,
    clippy_args: String
}

impl ClippyCmd
{
    fn new<I>(mut old_args: I) -> Self
    where
        I: Iterator<Item = String>,
    {
        let mut cargo_subcommand = "check";
        let mut unstable_options = false;
        let mut args = vec![];

        for arg in old_args.by_ref() {
            match arg.as_str() {
                "--fix" => {
                    cargo_subcommand = "fix";
                    continue;
                }
                "--" => break,
                // Cover -Zunstable-options and -Z unstable-options
                s if s.ends_with("unstable-options") => unstable_options = true,
                _ => {}
            }

            args.push(arg);
        }

        if cargo_subcommand == "fix" && !unstable_options {
            panic!("Usage of `--fix` requires `-Z unstable-options`");
        }

        // Run the dogfood tests directly on nightly cargo. This is required due
        // to a bug in rustup.rs when running cargo on custom toolchains. See issue #3118.
        if env::var_os("CLIPPY_DOGFOOD").is_some() && cfg!(windows) {
            args.insert(0, "+nightly".to_string());
        }

        let clippy_args: String =
            old_args
            .map(|arg| format!("{}__CLIPPY_HACKERY__", arg))
            .collect();

        ClippyCmd {
            unstable_options,
            cargo_subcommand,
            args,
            clippy_args,
        }
    }

    fn path_env(&self) -> &'static str {
        if self.unstable_options {
            "RUSTC_WORKSPACE_WRAPPER"
        } else {
            "RUSTC_WRAPPER"
        }
    }

    fn path(&self) -> PathBuf {
        let mut path = env::current_exe()
            .expect("current executable path invalid")
            .with_file_name("clippy-driver");

        if cfg!(windows) {
            path.set_extension("exe");
        }

        path
    }

    fn target_dir() -> Option<(&'static str, OsString)> {
        env::var_os("CLIPPY_DOGFOOD")
            .map(|_| {
                env::var_os("CARGO_MANIFEST_DIR").map_or_else(
                    || std::ffi::OsString::from("clippy_dogfood"),
                    |d| {
                        std::path::PathBuf::from(d)
                            .join("target")
                            .join("dogfood")
                            .into_os_string()
                    },
                )
            })
            .map(|p| ("CARGO_TARGET_DIR", p))
    }

    fn to_std_cmd(self) -> Command {
        let mut cmd = Command::new("cargo");

        cmd.env(self.path_env(), self.path())
            .envs(ClippyCmd::target_dir())
            .env("CLIPPY_ARGS", self.clippy_args)
            .arg(self.cargo_subcommand)
            .args(&self.args);

        cmd
    }
}

fn process<I>(old_args: I) -> Result<(), i32>
where
    I: Iterator<Item = String>,
{
    let cmd = ClippyCmd::new(old_args);

    let mut cmd = cmd.to_std_cmd();

    let exit_status = cmd
        .spawn()
        .expect("could not run cargo")
        .wait()
        .expect("failed to wait for cargo?");

    if exit_status.success() {
        Ok(())
    } else {
        Err(exit_status.code().unwrap_or(-1))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    #[should_panic]
    fn fix_without_unstable() {
        let args = "cargo clippy --fix".split_whitespace().map(ToString::to_string);
        let _ = ClippyCmd::new(args);
    }

    #[test]
    fn fix_unstable() {
        let args = "cargo clippy --fix -Zunstable-options".split_whitespace().map(ToString::to_string);
        let cmd = ClippyCmd::new(args);
        assert_eq!("fix", cmd.cargo_subcommand);
        assert_eq!("RUSTC_WORKSPACE_WRAPPER", cmd.path_env());
        assert!(cmd.args.iter().find(|arg| arg.ends_with("unstable-options")).is_some());
    }

    #[test]
    fn check() {
        let args = "cargo clippy".split_whitespace().map(ToString::to_string);
        let cmd = ClippyCmd::new(args);
        assert_eq!("check", cmd.cargo_subcommand);
        assert_eq!("RUSTC_WRAPPER", cmd.path_env());
    }

    #[test]
    fn check_unstable() {
        let args = "cargo clippy -Zunstable-options".split_whitespace().map(ToString::to_string);
        let cmd = ClippyCmd::new(args);
        assert_eq!("check", cmd.cargo_subcommand);
        assert_eq!("RUSTC_WORKSPACE_WRAPPER", cmd.path_env());
    }
}
<|endoftext|>


<commit_before>extern crate chrono;
extern crate clap;
extern crate hyper;
extern crate libc;
extern crate multirust;
extern crate rust_install;

extern crate rust_bisect;

use std::process;

use chrono::NaiveDate;
use clap::{App, AppSettings, Arg};
use hyper::client::Client;
use rust_install::dist::ToolchainDesc;

use rust_bisect::{Result, least_satisfying};

const NIGHTLY: &'static str = "nightly";

fn nightly(date: NaiveDate) -> String {
    format!("{}-{}", NIGHTLY, date)
}

fn list_available_nightlies(dist_root: &str,
                            from: NaiveDate,
                            to: NaiveDate)
                            -> Result<Vec<NaiveDate>> {
    assert!(from < to, "`from` must be less than `to`");
    let client = Client::new();
    let mut nightlies = Vec::with_capacity((to - from).num_days() as usize);
    let mut date = from;
    while date < to {
        let desc = ToolchainDesc::from_str(&nightly(date)).expect("should always parse");
        let resp = try!(client.head(&desc.manifest_url(dist_root)).send());
        // TODO: ensure failures are 404
        if resp.status.is_success() {
            nightlies.push(date);
        }
        date = date.succ();
    }
    Ok(nightlies)
}

fn run_rust_bisect() -> Result<i32> {
    fn validate_version(s: String) -> std::result::Result<(), String> {
        let ret = ToolchainDesc::from_str(&s);
        match ret {
            Some(ref desc) if desc.channel == NIGHTLY && desc.date.is_some() => Ok(()),
            Some(_) => Err(String::from("can only bisect on dated nightlies")),
            None => Err(String::from(format!("invalid version: {}", s))),
        }
    }
    let matches = App::new("rust-bisect")
                      .author("Kamal Marhubi <kamal@marhubi.com>")
                      .about("Find the Rust nightly that that changed some behavior")
                      .setting(AppSettings::TrailingVarArg)
                      .usage("rust-bisect [FLAGS] --bad <VERSION> --good <VERSION> <COMMAND> \
                              [ARGS...]")
                      .arg(Arg::with_name("good")
                               .long("good")
                               .takes_value(true)
                               .value_name("VERSION")
                               .help("A known good nightly release")
                               .validator(validate_version)
                               .required(true))
                      .arg(Arg::with_name("bad")
                               .long("bad")
                               .takes_value(true)
                               .value_name("VERSION")
                               .help("A known bad nightly release")
                               .validator(validate_version)
                               .required(true))
                      .arg(Arg::with_name("COMMAND")
                               .index(1)
                               .help("The command to run")
                               .required(true))
                      .arg(Arg::with_name("ARGS")
                               .index(2)
                               .multiple(true)
                               .help("Arguments for COMMAND"))
                      .get_matches();

    let cfg = try!(multirust::Cfg::from_env(rust_install::notify::SharedNotifyHandler::none()));

    let good = matches.value_of("good").expect("clap didn't respect required arg `good`");
    let good = ToolchainDesc::from_str(good).expect("clap validator misbehaved for `good`");

    let bad = matches.value_of("bad").expect("clap didn't respect required arg `bad`");
    let bad = ToolchainDesc::from_str(bad).expect("clap validator misbehaved for `bad`");

    let good_date: NaiveDate = try!(good.date
                                        .expect("clap validator misbehaved for `good`")
                                        .parse());
    let bad_date: NaiveDate = try!(bad.date
                                      .expect("clap validator misbehaved for `bad`")
                                      .parse());

    let cmd = matches.value_of_os("COMMAND")
                     .expect("clap didn't respect required arg `COMMAND`");
    let args: Vec<_> = matches.values_of_os("ARGS")
                              .map(|args| args.collect())
                              .unwrap_or(Vec::new());

    let nightlies = try!(list_available_nightlies(&*cfg.dist_root_url, good_date, bad_date));

    let idx = least_satisfying(&nightlies[..], |date| {
        let version = nightly(*date);
        let toolchain = cfg.get_toolchain(&version, false).expect("could not get toolchain");

        toolchain.install_from_dist_if_not_installed().expect("could not install toolchain");

        let mut cmd = toolchain.create_command(cmd).expect("could not create command");
        cmd.args(&args);
        let res = cmd.status().expect("could not run command").success();

        println!("command {} at {}",
                 if res {
                     "succeeded"
                 } else {
                     "failed"
                 },
                 version);
        !res
    });

    println!("first failing nightly: {}", nightly(nightlies[idx]));
    Ok(libc::EXIT_SUCCESS)
}

fn main() {
    process::exit(run_rust_bisect().expect("something went wrong"));
}
<commit_msg>cli: Improve logging<commit_after>extern crate chrono;
extern crate clap;
extern crate hyper;
extern crate libc;
extern crate multirust;
extern crate rust_install;

extern crate rust_bisect;

use std::process;

use chrono::NaiveDate;
use clap::{App, AppSettings, Arg};
use hyper::client::Client;
use rust_install::dist::ToolchainDesc;

use rust_bisect::{Result, least_satisfying};

const NIGHTLY: &'static str = "nightly";

fn nightly(date: NaiveDate) -> String {
    format!("{}-{}", NIGHTLY, date)
}

fn list_available_nightlies(dist_root: &str,
                            from: NaiveDate,
                            to: NaiveDate)
                            -> Result<Vec<NaiveDate>> {
    assert!(from < to, "`from` must be less than `to`");
    println!("finding available nightlies between {} and {}", from, to);
    let client = Client::new();
    let mut nightlies = Vec::with_capacity((to - from).num_days() as usize);
    let mut date = from;
    while date < to {
        let desc = ToolchainDesc::from_str(&nightly(date)).expect("should always parse");
        let resp = try!(client.head(&desc.manifest_url(dist_root)).send());
        // TODO: ensure failures are 404
        if resp.status.is_success() {
            nightlies.push(date);
        }
        date = date.succ();
    }
    println!("found {} nightlies", nightlies.len());
    Ok(nightlies)
}

fn run_rust_bisect() -> Result<i32> {
    fn validate_version(s: String) -> std::result::Result<(), String> {
        let ret = ToolchainDesc::from_str(&s);
        match ret {
            Some(ref desc) if desc.channel == NIGHTLY && desc.date.is_some() => Ok(()),
            Some(_) => Err(String::from("can only bisect on dated nightlies")),
            None => Err(String::from(format!("invalid version: {}", s))),
        }
    }
    let matches = App::new("rust-bisect")
                      .author("Kamal Marhubi <kamal@marhubi.com>")
                      .about("Find the Rust nightly that that changed some behavior")
                      .setting(AppSettings::TrailingVarArg)
                      .usage("rust-bisect [FLAGS] --bad <VERSION> --good <VERSION> <COMMAND> \
                              [ARGS...]")
                      .arg(Arg::with_name("good")
                               .long("good")
                               .takes_value(true)
                               .value_name("VERSION")
                               .help("A known good nightly release")
                               .validator(validate_version)
                               .required(true))
                      .arg(Arg::with_name("bad")
                               .long("bad")
                               .takes_value(true)
                               .value_name("VERSION")
                               .help("A known bad nightly release")
                               .validator(validate_version)
                               .required(true))
                      .arg(Arg::with_name("COMMAND")
                               .index(1)
                               .help("The command to run")
                               .required(true))
                      .arg(Arg::with_name("ARGS")
                               .index(2)
                               .multiple(true)
                               .help("Arguments for COMMAND"))
                      .get_matches();

    let cfg = try!(multirust::Cfg::from_env(rust_install::notify::SharedNotifyHandler::none()));

    let good = matches.value_of("good").expect("clap didn't respect required arg `good`");
    let good = ToolchainDesc::from_str(good).expect("clap validator misbehaved for `good`");

    let bad = matches.value_of("bad").expect("clap didn't respect required arg `bad`");
    let bad = ToolchainDesc::from_str(bad).expect("clap validator misbehaved for `bad`");

    let good_date: NaiveDate = try!(good.date
                                        .expect("clap validator misbehaved for `good`")
                                        .parse());
    let bad_date: NaiveDate = try!(bad.date
                                      .expect("clap validator misbehaved for `bad`")
                                      .parse());

    let cmd = matches.value_of_os("COMMAND")
                     .expect("clap didn't respect required arg `COMMAND`");
    let args: Vec<_> = matches.values_of_os("ARGS")
                              .map(|args| args.collect())
                              .unwrap_or(Vec::new());

    let nightlies = try!(list_available_nightlies(&*cfg.dist_root_url, good_date, bad_date));
    println!("bisecting across {} nightlies (about {} steps)",
             nightlies.len(),
             nightlies.len().next_power_of_two().trailing_zeros());

    let idx = least_satisfying(&nightlies[..], |date| {
        let version = nightly(*date);
        println!("testing with {}", version);

        let toolchain = cfg.get_toolchain(&version, false).expect("could not get toolchain");
        toolchain.install_from_dist_if_not_installed().expect("could not install toolchain");

        let mut cmd = toolchain.create_command(cmd).expect("could not create command");
        cmd.args(&args);
        let res = cmd.status().expect("could not run command").success();

        println!("command {} with {}",
                 if res {
                     "succeeded"
                 } else {
                     "failed"
                 },
                 version);
        !res
    });

    println!("{} is the first failing nightly", nightly(nightlies[idx]));
    Ok(libc::EXIT_SUCCESS)
}

fn main() {
    process::exit(run_rust_bisect().expect("something went wrong"));
}
<|endoftext|>