<commit_before>#![crate_type = "bin"]

extern crate getopts;

use std::collections::HashMap;
use std::collections::hash_map::Entry::{Occupied,Vacant};
use std::env;
use std::fs::File;
use std::io::{BufRead,BufReader};
use std::path::Path;

use getopts::Options;

type Hash = u64;
type Count = u32;
type Words = HashMap<Hash, Count>;
type Clusters = HashMap<String, Count>;

fn main() {
    let mut opts = Options::new();
    opts.optopt("c", "cluster-threshold", "cluster appearance threshold for diplay", "CLUSTERTHRESHOLD");
    opts.optflag("r", "rare", "display only clusters below CLUSTERTHRESHOLD");
    opts.optopt("w", "word-threshold", "minimum frequency of a word to be considered for a cluster", "WORDTHRESHOLD");
    opts.optopt("", "max-line-length", "discard lines longer than this many characters", "MAX_LINE_LENGTH");

    let matches = opts.parse(env::args().skip(1)).unwrap_or_else(|f| panic!(f.to_string()));

    let cluster_threshold: Count = match matches.opt_str("c") {
        Some(ct) => { ct.parse().unwrap() }
        None => { 1000 }
    };

    let word_threshold: Count = match matches.opt_str("w") {
        Some(wt) => { wt.parse().unwrap() }
        None => { 1000 }
    };

    let max_line_length: usize = match matches.opt_str("max-line-length") {
        Some(len) => len.parse().unwrap(),
        None => 1000,
    };

    let show_rare = matches.opt_present("r");

    let inputs: Vec<&Path> = matches.free.iter().map(|f| Path::new(f)).collect();

    let word_freq = calc_word_freq(&inputs, max_line_length);
    println!("found {} unique words", word_freq.len());

    let clusters = calc_clusters(&inputs, &word_freq, word_threshold, max_line_length);
    println!("found {} clusters", clusters.len());

    for (k,v) in clusters.iter() {
        if show_rare == (*v < cluster_threshold) {
            println!("{}\t{}", v, k);
        }
    }
}

fn calc_word_freq(paths: &Vec<&Path>, max_line_length: usize) -> Words {
    let mut word_freq: HashMap<Hash, Count> = HashMap::new();

    for path in paths.iter() {
        let file = File::open(path);
        let reader = BufReader::new(file.unwrap());

        for l in reader.lines() {
            let line = l.ok().unwrap();
            if max_line_length > 0 && line.len() > max_line_length {
                continue;
            }

            for w in line.split(char::is_whitespace).filter(|s| !s.is_empty()) {
                let hash = fnv1a(w);
                match word_freq.entry(hash) {
                    Vacant(wf) => { wf.insert(1); },
                    Occupied(mut wf) => { *wf.get_mut() += 1; },
                };
            }
        }
    }

    word_freq
}

fn calc_clusters(paths: &Vec<&Path>, word_freq: &Words, word_threshold: Count, max_line_length: usize) -> Clusters {
    let mut clusters: HashMap<String, Count> = HashMap::new();

    for path in paths.iter() {
        let file = File::open(path);
        let reader = BufReader::new(file.unwrap());

        for l in reader.lines() {
            let line = l.ok().unwrap();
            if max_line_length > 0 && line.len() > max_line_length {
                continue;
            }

            let cluster = clusterify(line, word_freq, word_threshold);
            if !cluster.is_empty() {
                match clusters.entry(cluster) {
                    Vacant(c) => { c.insert(1); },
                    Occupied(mut c) => { *c.get_mut() += 1; },
                };
            }
        }
    }

    clusters
}

fn clusterify(line: String, word_freq: &Words, word_threshold: Count) -> String {
    let words: Vec<&str> = line.split(char::is_whitespace).filter(|s| !s.is_empty()).map({ |w|
        if word_freq[&fnv1a(w)] < word_threshold {
            "*"
        } else {
            w
        }
    }).collect();

    words.join(" ")
}

const FNV_PRIME_64: Hash = 1099511628211;
const FNV1_OFFSET_BASIS_64: Hash = 14695981039346656037;

fn fnv1a(s: &str) -> Hash {
    let mut hash = FNV1_OFFSET_BASIS_64;
    for b in s.as_bytes().iter() {
        hash = (hash ^ *b as Hash) * FNV_PRIME_64;
    }
    hash
}
<commit_msg>Better error messages and code formatting<commit_after>#![crate_type = "bin"]

extern crate getopts;

use std::collections::HashMap;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use std::env;
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::iter::Iterator;
use std::path::Path;

use getopts::Options;

type Hash = u64;
type Count = u32;
type Words = HashMap<Hash, Count>;
type Clusters = HashMap<String, Count>;

fn main() {
    let mut opts = Options::new();
    opts.optopt("c",
                "cluster-threshold",
                "cluster appearance threshold for diplay (1000)",
                "CLUSTERTHRESHOLD");
    opts.optflag("r", "rare", "display only clusters below CLUSTERTHRESHOLD");
    opts.optopt("w",
                "word-threshold",
                "minimum frequency of a word to be considered for a cluster (1000)",
                "WORDTHRESHOLD");
    opts.optopt("",
                "max-line-length",
                "discard lines longer than this many characters (1000)",
                "MAX_LINE_LENGTH");

    let matches = opts.parse(env::args().skip(1)).expect("Parsing args");

    let cluster_threshold: Count = match matches.opt_str("c") {
        Some(ct) => ct.parse().expect("cluster-threshold"),
        None => 1000,
    };

    let word_threshold: Count = match matches.opt_str("w") {
        Some(wt) => wt.parse().expect("word-threshold"),
        None => 1000,
    };

    let max_line_length: usize = match matches.opt_str("max-line-length") {
        Some(len) => len.parse().expect("max-line-length"),
        None => 1000,
    };

    let show_rare = matches.opt_present("r");

    let inputs: Vec<&Path> = matches.free
                                    .iter()
                                    .map(|p| Path::new(p))
                                    .collect();

    let word_freq = calc_word_freq(&inputs, max_line_length);
    println!("found {} unique words", word_freq.len());

    let clusters = calc_clusters(&inputs, &word_freq, word_threshold, max_line_length);
    println!("found {} clusters", clusters.len());

    for (k, v) in clusters.iter() {
        if show_rare == (*v < cluster_threshold) {
            println!("{}\t{}", v, k);
        }
    }
}

fn calc_word_freq(paths: &Vec<&Path>, max_line_length: usize) -> Words {
    let mut word_freq: HashMap<Hash, Count> = HashMap::new();

    for path in paths {
        let file = File::open(path);
        let reader = BufReader::new(file.expect(&format!("Opening file {}",
                                                         path.to_string_lossy())));

        for l in reader.lines() {
            let line = l.ok().expect("Reading input");
            if max_line_length > 0 && line.len() > max_line_length {
                continue;
            }

            for w in line.split(char::is_whitespace).filter(|s| !s.is_empty()) {
                let hash = fnv1a(w);
                match word_freq.entry(hash) {
                    Vacant(wf) => {
                        wf.insert(1);
                    }
                    Occupied(mut wf) => {
                        *wf.get_mut() += 1;
                    }
                };
            }
        }
    }

    word_freq
}

fn calc_clusters(paths: &Vec<&Path>,
                 word_freq: &Words,
                 word_threshold: Count,
                 max_line_length: usize)
                 -> Clusters {
    let mut clusters: HashMap<String, Count> = HashMap::new();

    for path in paths {
        let file = File::open(path);
        let reader = BufReader::new(file.expect(&format!("Opening file {}",
                                                         path.to_string_lossy())));

        for l in reader.lines() {
            let line = l.ok().expect("Reading input");
            if max_line_length > 0 && line.len() > max_line_length {
                continue;
            }

            let cluster = clusterify(line, word_freq, word_threshold);
            if !cluster.is_empty() {
                match clusters.entry(cluster) {
                    Vacant(c) => {
                        c.insert(1);
                    }
                    Occupied(mut c) => {
                        *c.get_mut() += 1;
                    }
                };
            }
        }
    }

    clusters
}

fn clusterify(line: String, word_freq: &Words, word_threshold: Count) -> String {
    let words: Vec<&str> = line.split(char::is_whitespace)
                               .filter(|s| !s.is_empty())
                               .map({
                                   |w| {
                                       if word_freq[&fnv1a(w)] < word_threshold {
                                           "*"
                                       } else {
                                           w
                                       }
                                   }
                               })
                               .collect();

    words.join(" ")
}

const FNV_PRIME_64: Hash = 1099511628211;
const FNV1_OFFSET_BASIS_64: Hash = 14695981039346656037;

#[inline]
fn fnv1a(s: &str) -> Hash {
    let mut hash = FNV1_OFFSET_BASIS_64;
    for b in s.as_bytes().iter() {
        hash = (hash ^ *b as Hash) * FNV_PRIME_64;
    }
    hash
}
<|endoftext|>


<commit_before>extern crate mysql;
extern crate toml;
extern crate rustc_serialize;
#[macro_use]
extern crate log;
extern crate log4rs;
#[macro_use]
extern crate lazy_static;

mod lib;

use lib::config;
use lib::db;
use lib::logger;
use lib::downloader::DownloadDB;
use lib::downloader::Downloader;
use lib::DownloadError;
use lib::converter::Converter;

use std::fs::{remove_file,remove_dir_all};

static VERSION : &'static str = "0.4"; // String not valid
static CONFIG_PATH : &'static str = "config.cfg";
static LOG_CONFIG: &'static str = "log.conf";
static LOG_PATTERN: &'static str = "%d{%d-%m-%Y %H:%M:%S}\t[%l]\t%f:%L \t%m";
static SLEEP_MS: u32 = 5000;
static CODE_SUCCESS: i8 = 2;
static CODE_SUCCESS_WARNINGS: i8 = 3; // finished with warnings
static CODE_FAILED_INTERNAL: i8 = 10; // internal error
static CODE_FAILED_QUALITY: i8 = 11; // qualitz not available
static CODE_FAILED_UNAVAILABLE: i8 = 12; // source unavailable (private / removed)
const TYPE_YT_VIDEO: i16 = 0;
const TYPE_YT_PL: i16 = 1;
const TYPE_TWITCH: i16 = 2;

lazy_static! {
    pub static ref CONFIG: config::Config = {
        println!("Starting yayd-backend v{}",&VERSION);
        config::init_config() //return
    };

}

//#[repr(i8)] broken, enum not usable as of #10292
// enum QueryCodes {
//     Waiting = 0,
//     InProgress = 1,
//     Finished = 2,
//     Failed = 3,
// }

enum Thing { String(String), Bool(bool), None }

fn main() {
    logger::initialize();

    let pool = db::db_connect(db::mysql_options(), SLEEP_MS);

    let converter = Converter::new(&CONFIG.general.ffmpeg_bin_dir,&CONFIG.general.mp3_quality , pool.clone());
    let mut print_pause = true;
    loop {
        if let Some(result) = db::request_entry(& pool) {
            print_pause = true;
            let qid = result.qid.clone();                 //&QueryCodes::InProgress as i32
            db::set_query_code(&mut pool.get_conn().unwrap(), &1, &result.qid).ok().expect("Failed to set query code!");
            db::set_query_state(&pool.clone(),&qid, "started", false);
            let action_result: Result<Thing,DownloadError>;
            {
                let mut left_files: Vec<String> = Vec::with_capacity(2);
                if result.playlist {
                    action_result = handle_playlist(result, &converter,&mut left_files);
                }else{
                    action_result = handle_download(result, &None, &converter,&mut left_files);
                }

                if !left_files.is_empty() {
                    trace!("cleaning up files");
                    for i in &left_files {
                        match remove_file(&i) {
                            Ok(_) => (trace!("cleaning up {}",i)),
                            Err(e) => warn!("unable to remove file '{}' {}",i,e),
                        }
                    }
                }
            }

            let code: i8 = match action_result {
                Ok(t) => {
                    match t {
                        Thing::Bool(b) => {
                            if b { CODE_SUCCESS_WARNINGS } else { CODE_SUCCESS }
                        }
                        _ => CODE_SUCCESS,
                    }
                },
                Err(e) => {
                    warn!("Error: {:?}", e);
                    match e {
                        DownloadError::NotAvailable => CODE_FAILED_UNAVAILABLE,
                        DownloadError::ExtractorError => CODE_FAILED_UNAVAILABLE,
                        DownloadError::QualityNotAvailable => CODE_FAILED_QUALITY,
                        _ => {
                            let details = match e {
                                DownloadError::DBError(s) => s,
                                DownloadError::DownloadError(s) => s,
                                DownloadError::FFMPEGError(s) => s,
                                DownloadError::InternalError(s) => s,
                                _ => unreachable!(),
                            };
                            db::add_query_status(&pool.clone(),&qid, &details);
                            CODE_FAILED_INTERNAL
                        },
                    }
                }
            };
            db::set_query_code(&mut pool.get_conn().unwrap(), &code,&qid).ok().expect("Failed to set query code!");
			db::set_null_state(&pool.clone(), &qid);

        } else {
            if print_pause { debug!("Pausing.."); print_pause = false; }
            std::thread::sleep_ms(SLEEP_MS);
        }
    }
}

/// Download handler
/// Used by the playlist/file handler to download one file
/// Based on the quality it needs to download audio & video separately and convert them together
/// In case of a playlist download it depends on the target download folder & if it should be bezipped
/// In case of a DMCA we need to download the file via the socket connector,
/// which will output a mp3, or if requested, the video but with the highest quality.
/// Thus in case of a DMCA we can't pick a quality anymore.
/// Also the filename depends on the socket output then.
///
/// If it's a non-zipped single file, it's moved after a successful download, converted etc to the
/// main folder from which it should be downloadable.
/// The original non-ascii & url_encode'd name of the file is stored in the DB
fn handle_download<'a>(downl_db: DownloadDB, folder: &Option<String>, converter: &Converter, file_db: &mut Vec<String>) -> Result<Thing,DownloadError>{
    //update progress
    let is_zipped = match *folder {
        Some(_) => true,
        None => false,
    };
    if !downl_db.compress {
        db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 1, 0,false);
    }

    let download = Downloader::new(&downl_db, &CONFIG.general);
    //get filename, check for DMCA
    let mut dmca = false; // "succ." dmca -> file already downloaded

    let temp_path = lib::format_file_path(&downl_db.qid, folder.clone(), false);
    let name = match download.get_file_name() { // get filename
        Ok(v) => v,
        Err(DownloadError::DMCAError) => { //now request via lib.. // k if( k == Err(DownloadError::DMCAError) )
            info!("DMCA error!");
            if CONFIG.general.lib_use {
                match download.lib_request_video(1,0, &temp_path) {
                    Err(err) => { warn!("lib-call error {:?}", err); return Err(err); },
                    Ok(v) => { dmca = true; v },
                }
            } else {
                return Err(DownloadError::NotAvailable);
            }
        },
        Err(e) => { // unknown error / restricted source etc.. abort
            error!("Unknown error: {:?}", e);
            return Err(e);
        },
    };

    file_db.push(temp_path.clone());
    let save_path = lib::format_save_path(folder.clone(),&name, lib::get_file_ext(&download));

    trace!("Filename: {}", name);

    let is_splitted_video = if dmca {
        false
    } else {
        lib::is_split_container(&downl_db.quality, &downl_db.source_type)
    };
    let convert_audio = CONFIG.extensions.mp3.contains(&downl_db.quality);

    let total_steps = if dmca {
        if download.is_audio() {
            3
        } else {
            2
        }
    } else if is_splitted_video {
        4
    } else if download.is_audio() {
        3
    } else {
        2
    };

    if !dmca { // on dmca the called lib handles splitt videos, only audio conversion has to handled by us
        if !downl_db.compress {
            db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 2, total_steps,false);
        }

        //download first file, download audio raw source if specified or video
        match download.download_file(&temp_path, convert_audio) {
            Err(e) => {file_db.pop(); return Err(e); },
            Ok(_) => (),
        }

        if is_splitted_video {
            debug!("splitted video");
            // download audio file & convert together
            if !downl_db.compress {
                db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 3, total_steps,false);
            }

            let audio_path = lib::format_file_path(&downl_db.qid, folder.clone(), true);

            trace!("Downloading audio.. {}", audio_path);
            try!(download.download_file(&audio_path, true));
            file_db.push(audio_path.clone());

            if !downl_db.compress {
                db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 4, total_steps,false);
            }

            match converter.merge_files(&downl_db.qid,&temp_path, &audio_path,&save_path.to_string_lossy(), !downl_db.compress) {
                Err(e) => {println!("merge error: {:?}",e); return Err(e);},
                Ok(()) => {},
            }

            try!(remove_file(&audio_path));
            file_db.pop();
        }
    }

    if !is_splitted_video { // if it's no splitted video & no audio we're moving the file, regardless of an earlier DMCA (lib call)
        debug!("no split container");
        if !download.is_audio() {
            try!(lib::move_file(&temp_path, &save_path));
        }
    }

    if download.is_audio(){ // if audio-> convert m4a to mp3 or extract m4a, directly to output file
    	debug!("is audio file");
        if !downl_db.compress {
            db::update_steps(&downl_db.pool.clone(),&downl_db.qid, total_steps, total_steps, false);
        }
        try!(converter.extract_audio(&temp_path, &save_path.to_string_lossy(), convert_audio));
        try!(remove_file(&temp_path));
    }else{
        debug!("no audio");
        if is_splitted_video {
            try!(remove_file(&temp_path));
        }
    }
    file_db.pop();

    if !is_zipped { // add file to list, except it's for zip-compression later (=folder set)
        try!(db::add_file_entry(&downl_db.pool.clone(), &downl_db.qid,&save_path.file_name().unwrap().to_string_lossy(), &name));
    }
    if !downl_db.compress {
        db::update_steps(&downl_db.pool.clone(),&downl_db.qid, total_steps, total_steps, true);
    }

    if folder.is_some() {
        Ok(Thing::String(save_path.to_string_lossy().into_owned()))
    } else {
        Ok(Thing::None)
    }
}

/// Handles a playlist request
/// If zipping isn't requested the downloads will be split up,
/// so for each video in the playlist an own query entry will be created
/// if warnings occured (unavailable video etc) the return will be true
fn handle_playlist(mut downl_db: DownloadDB, converter: &Converter, file_db: &mut Vec<String>) -> Result<Thing, DownloadError>{
    let mut max_steps: i32 = if downl_db.compress { 4 } else { 3 };
    db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 1, max_steps,false);

    let pl_id: i64 = downl_db.qid;
    downl_db.update_folder(format!("{}/{}",&CONFIG.general.temp_dir,pl_id));
    trace!("Folder:  {}",downl_db.folder);

    let db_copy = downl_db.clone();
    let download = Downloader::new(&db_copy, &CONFIG.general);

    let mut playlist_name = String::new();
    if downl_db.compress {
        playlist_name = try!(download.get_playlist_name());
        println!("pl name {}",playlist_name);
        try!(std::fs::create_dir(&downl_db.folder));
        file_db.push(downl_db.folder.clone());
    }
    db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 2, max_steps,false);
    trace!("retriving playlist videos..");
    let file_ids: Vec<String> = try!(download.get_playlist_ids());

    let handler_folder = if downl_db.compress {
        Some(pl_id.to_string())
    }else {
        None
    };

    trace!("got em");
    max_steps += file_ids.len() as i32;
    let mut current_step = 2;
    let mut warnings = false;
    let mut current_url: String;
    let mut failed_log: String = String::from("Following urls couldn't be downloaded: \n");
    // we'll store all files and delete em later, so we don't need rm -rf
    let mut file_delete_list: Vec<String> = Vec::with_capacity(if downl_db.compress { file_ids.len() } else { 0 });
    for id in file_ids.iter() {
        current_step += 1;
        if downl_db.compress {
            db::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        }
        current_url = format!("https://www.youtube.com/watch?v={}",id);
        downl_db.update_video(current_url.clone(), current_step as i64);
        debug!("id: {}",id);
        let db_copy = downl_db.clone();
        match handle_download(db_copy, &handler_folder, converter, file_db) {
            Err(e) => { warn!("error downloading {}: {:?}",id,e);
                        failed_log.push_str(&format!("{} {:?}\n", current_url, e));
                        warnings = true;
            },
            Ok(e) => {  if downl_db.compress {
                            match e {
                                Thing::String(v) => file_delete_list.push(v),
                                _ => {error!("handle_download not returning a filename"); panic!();},
                            }
                        }
            },
        }
    }

    if warnings {
        db::add_query_status(&downl_db.pool.clone(),&pl_id, &failed_log);
    }

    trace!("downloaded all videos");
    if downl_db.compress { // zip to file, add to db & remove all sources
        current_step += 1;
        db::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        let zip_name = format!("{}.zip",playlist_name);
        let zip_file = lib::format_save_path(None, &lib::url_sanitize(&playlist_name),"zip");
        trace!("zip file: {} \n zip source {}",zip_file.to_string_lossy(), &downl_db.folder);
        try!(lib::zip_folder(&downl_db.folder, zip_file));
        try!(db::add_file_entry(&downl_db.pool.clone(), &pl_id,&zip_name, &playlist_name));

        current_step += 1;
        db::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        try!(lib::delete_files(file_delete_list));
        try!(remove_dir_all(downl_db.folder));
        file_db.pop();
    }

    Ok(Thing::Bool(warnings))
}
<commit_msg>changed from copy to mut ref<commit_after>extern crate mysql;
extern crate toml;
extern crate rustc_serialize;
#[macro_use]
extern crate log;
extern crate log4rs;
#[macro_use]
extern crate lazy_static;

mod lib;

use lib::config;
use lib::db;
use lib::logger;
use lib::downloader::DownloadDB;
use lib::downloader::Downloader;
use lib::DownloadError;
use lib::converter::Converter;

use std::fs::{remove_file,remove_dir_all};

static VERSION : &'static str = "0.4"; // String not valid
static CONFIG_PATH : &'static str = "config.cfg";
static LOG_CONFIG: &'static str = "log.conf";
static LOG_PATTERN: &'static str = "%d{%d-%m-%Y %H:%M:%S}\t[%l]\t%f:%L \t%m";
static SLEEP_MS: u32 = 5000;
static CODE_SUCCESS: i8 = 2;
static CODE_SUCCESS_WARNINGS: i8 = 3; // finished with warnings
static CODE_FAILED_INTERNAL: i8 = 10; // internal error
static CODE_FAILED_QUALITY: i8 = 11; // qualitz not available
static CODE_FAILED_UNAVAILABLE: i8 = 12; // source unavailable (private / removed)
const TYPE_YT_VIDEO: i16 = 0;
const TYPE_YT_PL: i16 = 1;
const TYPE_TWITCH: i16 = 2;

lazy_static! {
    pub static ref CONFIG: config::Config = {
        println!("Starting yayd-backend v{}",&VERSION);
        config::init_config() //return
    };

}

//#[repr(i8)] broken, enum not usable as of #10292
// enum QueryCodes {
//     Waiting = 0,
//     InProgress = 1,
//     Finished = 2,
//     Failed = 3,
// }

enum Thing { String(String), Bool(bool), None }

fn main() {
    logger::initialize();

    let pool = db::db_connect(db::mysql_options(), SLEEP_MS);

    let converter = Converter::new(&CONFIG.general.ffmpeg_bin_dir,&CONFIG.general.mp3_quality , pool.clone());
    let mut print_pause = true;
    loop {
        if let Some(mut result) = db::request_entry(& pool) {
            print_pause = true;
            let qid = result.qid.clone();                 //&QueryCodes::InProgress as i32
            db::set_query_code(&mut pool.get_conn().unwrap(), &1, &result.qid).ok().expect("Failed to set query code!");
            db::set_query_state(&pool.clone(),&qid, "started", false);
            let action_result: Result<Thing,DownloadError>;
            {
                let mut left_files: Vec<String> = Vec::with_capacity(2);
                if result.playlist {
                    action_result = handle_playlist(& mut result, &converter,&mut left_files);
                }else{
                    action_result = handle_download(& result, &None, &converter,&mut left_files);
                }

                if !left_files.is_empty() {
                    trace!("cleaning up files");
                    for i in &left_files {
                        match remove_file(&i) {
                            Ok(_) => (trace!("cleaning up {}",i)),
                            Err(e) => warn!("unable to remove file '{}' {}",i,e),
                        }
                    }
                }
            }

            let code: i8 = match action_result {
                Ok(t) => {
                    match t {
                        Thing::Bool(b) => {
                            if b { CODE_SUCCESS_WARNINGS } else { CODE_SUCCESS }
                        }
                        _ => CODE_SUCCESS,
                    }
                },
                Err(e) => {
                    warn!("Error: {:?}", e);
                    match e {
                        DownloadError::NotAvailable => CODE_FAILED_UNAVAILABLE,
                        DownloadError::ExtractorError => CODE_FAILED_UNAVAILABLE,
                        DownloadError::QualityNotAvailable => CODE_FAILED_QUALITY,
                        _ => {
                            let details = match e {
                                DownloadError::DBError(s) => s,
                                DownloadError::DownloadError(s) => s,
                                DownloadError::FFMPEGError(s) => s,
                                DownloadError::InternalError(s) => s,
                                _ => unreachable!(),
                            };
                            db::add_query_status(&pool.clone(),&qid, &details);
                            CODE_FAILED_INTERNAL
                        },
                    }
                }
            };
            db::set_query_code(&mut pool.get_conn().unwrap(), &code,&qid).ok().expect("Failed to set query code!");
			db::set_null_state(&pool.clone(), &qid);

        } else {
            if print_pause { debug!("Pausing.."); print_pause = false; }
            std::thread::sleep_ms(SLEEP_MS);
        }
    }
}

/// Download handler
/// Used by the playlist/file handler to download one file
/// Based on the quality it needs to download audio & video separately and convert them together
/// In case of a playlist download it depends on the target download folder & if it should be bezipped
/// In case of a DMCA we need to download the file via the socket connector,
/// which will output a mp3, or if requested, the video but with the highest quality.
/// Thus in case of a DMCA we can't pick a quality anymore.
/// Also the filename depends on the socket output then.
///
/// If it's a non-zipped single file, it's moved after a successful download, converted etc to the
/// main folder from which it should be downloadable.
/// The original non-ascii & url_encode'd name of the file is stored in the DB
fn handle_download<'a>(downl_db: &DownloadDB, folder: &Option<String>, converter: &Converter, file_db: &mut Vec<String>) -> Result<Thing,DownloadError>{
    //update progress
    let is_zipped = match *folder {
        Some(_) => true,
        None => false,
    };
    if !downl_db.compress {
        db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 1, 0,false);
    }

    let download = Downloader::new(&downl_db, &CONFIG.general);
    //get filename, check for DMCA
    let mut dmca = false; // "succ." dmca -> file already downloaded

    let temp_path = lib::format_file_path(&downl_db.qid, folder.clone(), false);
    let name = match download.get_file_name() { // get filename
        Ok(v) => v,
        Err(DownloadError::DMCAError) => { //now request via lib.. // k if( k == Err(DownloadError::DMCAError) )
            info!("DMCA error!");
            if CONFIG.general.lib_use {
                match download.lib_request_video(1,0, &temp_path) {
                    Err(err) => { warn!("lib-call error {:?}", err); return Err(err); },
                    Ok(v) => { dmca = true; v },
                }
            } else {
                return Err(DownloadError::NotAvailable);
            }
        },
        Err(e) => { // unknown error / restricted source etc.. abort
            error!("Unknown error: {:?}", e);
            return Err(e);
        },
    };

    file_db.push(temp_path.clone());
    let save_path = lib::format_save_path(folder.clone(),&name, lib::get_file_ext(&download));

    trace!("Filename: {}", name);

    let is_splitted_video = if dmca {
        false
    } else {
        lib::is_split_container(&downl_db.quality, &downl_db.source_type)
    };
    let convert_audio = CONFIG.extensions.mp3.contains(&downl_db.quality);

    let total_steps = if dmca {
        if download.is_audio() {
            3
        } else {
            2
        }
    } else if is_splitted_video {
        4
    } else if download.is_audio() {
        3
    } else {
        2
    };

    if !dmca { // on dmca the called lib handles splitt videos, only audio conversion has to handled by us
        if !downl_db.compress {
            db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 2, total_steps,false);
        }

        //download first file, download audio raw source if specified or video
        match download.download_file(&temp_path, convert_audio) {
            Err(e) => {file_db.pop(); return Err(e); },
            Ok(_) => (),
        }

        if is_splitted_video {
            debug!("splitted video");
            // download audio file & convert together
            if !downl_db.compress {
                db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 3, total_steps,false);
            }

            let audio_path = lib::format_file_path(&downl_db.qid, folder.clone(), true);

            trace!("Downloading audio.. {}", audio_path);
            try!(download.download_file(&audio_path, true));
            file_db.push(audio_path.clone());

            if !downl_db.compress {
                db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 4, total_steps,false);
            }

            match converter.merge_files(&downl_db.qid,&temp_path, &audio_path,&save_path.to_string_lossy(), !downl_db.compress) {
                Err(e) => {println!("merge error: {:?}",e); return Err(e);},
                Ok(()) => {},
            }

            try!(remove_file(&audio_path));
            file_db.pop();
        }
    }

    if !is_splitted_video { // if it's no splitted video & no audio we're moving the file, regardless of an earlier DMCA (lib call)
        debug!("no split container");
        if !download.is_audio() {
            try!(lib::move_file(&temp_path, &save_path));
        }
    }

    if download.is_audio(){ // if audio-> convert m4a to mp3 or extract m4a, directly to output file
    	debug!("is audio file");
        if !downl_db.compress {
            db::update_steps(&downl_db.pool.clone(),&downl_db.qid, total_steps, total_steps, false);
        }
        try!(converter.extract_audio(&temp_path, &save_path.to_string_lossy(), convert_audio));
        try!(remove_file(&temp_path));
    }else{
        debug!("no audio");
        if is_splitted_video {
            try!(remove_file(&temp_path));
        }
    }
    file_db.pop();

    if !is_zipped { // add file to list, except it's for zip-compression later (=folder set)
        try!(db::add_file_entry(&downl_db.pool.clone(), &downl_db.qid,&save_path.file_name().unwrap().to_string_lossy(), &name));
    }
    if !downl_db.compress {
        db::update_steps(&downl_db.pool.clone(),&downl_db.qid, total_steps, total_steps, true);
    }

    if folder.is_some() {
        Ok(Thing::String(save_path.to_string_lossy().into_owned()))
    } else {
        Ok(Thing::None)
    }
}

/// Handles a playlist request
/// If zipping isn't requested the downloads will be split up,
/// so for each video in the playlist an own query entry will be created
/// if warnings occured (unavailable video etc) the return will be true
fn handle_playlist(mut downl_db: & mut DownloadDB, converter: &Converter, file_db: &mut Vec<String>) -> Result<Thing, DownloadError>{
    let mut max_steps: i32 = if downl_db.compress { 4 } else { 3 };
    db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 1, max_steps,false);

    let pl_id: i64 = downl_db.qid;
    downl_db.update_folder(format!("{}/{}",&CONFIG.general.temp_dir,pl_id));
    trace!("Folder:  {}",downl_db.folder);

    let db_copy = downl_db.clone();
    let download = Downloader::new(&db_copy, &CONFIG.general);

    let mut playlist_name = String::new();
    if downl_db.compress {
        playlist_name = try!(download.get_playlist_name());
        println!("pl name {}",playlist_name);
        try!(std::fs::create_dir(&downl_db.folder));
        file_db.push(downl_db.folder.clone());
    }
    db::update_steps(&downl_db.pool.clone(),&downl_db.qid, 2, max_steps,false);
    trace!("retriving playlist videos..");
    let file_ids: Vec<String> = try!(download.get_playlist_ids());

    let handler_folder = if downl_db.compress {
        Some(pl_id.to_string())
    }else {
        None
    };

    trace!("got em");
    max_steps += file_ids.len() as i32;
    let mut current_step = 2;
    let mut warnings = false;
    let mut current_url: String;
    let mut failed_log: String = String::from("Following urls couldn't be downloaded: \n");
    // we'll store all files and delete em later, so we don't need rm -rf
    let mut file_delete_list: Vec<String> = Vec::with_capacity(if downl_db.compress { file_ids.len() } else { 0 });
    for id in file_ids.iter() {
        current_step += 1;
        if downl_db.compress {
            db::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        }
        current_url = format!("https://www.youtube.com/watch?v={}",id);
        downl_db.update_video(current_url.clone(), current_step as i64);
        debug!("id: {}",id);
        match handle_download(downl_db, &handler_folder, converter, file_db) {
            Err(e) => { warn!("error downloading {}: {:?}",id,e);
                        failed_log.push_str(&format!("{} {:?}\n", current_url, e));
                        warnings = true;
            },
            Ok(e) => {  if downl_db.compress {
                            match e {
                                Thing::String(v) => file_delete_list.push(v),
                                _ => {error!("handle_download not returning a filename"); panic!();},
                            }
                        }
            },
        }
    }

    if warnings {
        db::add_query_status(&downl_db.pool.clone(),&pl_id, &failed_log);
    }

    trace!("downloaded all videos");
    if downl_db.compress { // zip to file, add to db & remove all sources
        current_step += 1;
        db::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        let zip_name = format!("{}.zip",playlist_name);
        let zip_file = lib::format_save_path(None, &lib::url_sanitize(&playlist_name),"zip");
        trace!("zip file: {} \n zip source {}",zip_file.to_string_lossy(), &downl_db.folder);
        try!(lib::zip_folder(&downl_db.folder, zip_file));
        try!(db::add_file_entry(&downl_db.pool.clone(), &pl_id,&zip_name, &playlist_name));

        current_step += 1;
        db::update_steps(&downl_db.pool.clone(),&pl_id, current_step, max_steps,false);
        try!(lib::delete_files(file_delete_list));
        try!(remove_dir_all(&downl_db.folder));
        file_db.pop();
    }

    Ok(Thing::Bool(warnings))
}
<|endoftext|>


<commit_before>use std::io::prelude::*;
use std::io::BufReader;
use std::fs::File;

fn load_types() -> Result<Vec<String>, std::io::Error> {
    let f = File::open("/usr/share/mime/types")?;
    let reader = BufReader::new(f);
    let mut out = Vec::new();

    for line in reader.lines() {
        //println!("{}", line.ok().expect("Could not read line"));
        out.push(line.ok().expect("Could not read line"));
    }

    Ok(out)

}

fn main() {
    match load_types() {
        Err(why) => panic!("{:?}", why),
        Ok(ftypes) => for ftype in &ftypes {
            println!("{}", ftype);
        },
    }
}
<commit_msg>Now reads subclasses file too<commit_after>use std::io::prelude::*;
use std::io::BufReader;
use std::fs::File;

fn load_types() -> Result<Vec<String>, std::io::Error> {
    let f = File::open("/usr/share/mime/types")?;
    let reader = BufReader::new(f);
    let mut out = Vec::new();

    for line in reader.lines() {
        out.push(line?);
    }

    Ok(out)

}

fn load_relations() -> Result<Vec<(String, String)>, std::io::Error> {
    let f = File::open("/usr/share/mime/subclasses")?;
    let reader = BufReader::new(f);
    let mut out = Vec::new();

    for line in reader.lines() {
        let line_raw = line?;
        let parent = line_raw.split_whitespace().nth(0).unwrap_or("").to_string();
        let child = line_raw.split_whitespace().nth(1).unwrap_or("").to_string();
        out.push((parent, child));
    }

    Ok(out)

}

fn main() {

    match load_relations() {
        Err(why) => panic!("{:?}", why),
        Ok(ftypes) => for ftype in &ftypes {
            println!("{} -> {}", ftype.0, ftype.1);
        },
    };

    /*match load_types() {
        Err(why) => panic!("{:?}", why),
        Ok(ftypes) => for ftype in &ftypes {
            println!("{}", ftype);
        },
    };*/
}
<|endoftext|>


<commit_before>mod logger;
mod toml_file;
mod git;
mod changelog;
mod commit_analyzer;

extern crate rustc_serialize;
extern crate toml;
extern crate regex;
extern crate semver;
extern crate docopt;
extern crate git2_commit;
extern crate git2;
extern crate clog;

use docopt::Docopt;
use commit_analyzer::CommitType;

const USAGE: &'static str = "
semantic.rs 🚀

Usage:
  semantic-rs [options]
  semantic-rs --version

Options:
  -h --help              Show this screen.
  --version              Show version.
  -p PATH, --path=PATH   Specifies the repository path. [default: .]
  -n, --dry-run          Run without writing, committing, pushing or publishing anything.
";

#[derive(Debug, RustcDecodable)]
struct Args {
    flag_path: String,
    flag_dry_run: bool,
}

fn version_bump(version: &Version, bump: CommitType) -> Option<Version> {
    let mut version = version.clone();
    match bump {
        CommitType::Unknown => return None,
        CommitType::Patch => version.increment_patch(),
        CommitType::Minor => version.increment_minor(),
        CommitType::Major => version.increment_major(),
    }

    Some(version)
}

use std::process;
use semver::Version;

fn main() {
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());

    println!("semantic.rs 🚀");

    logger::stdout("Analyzing your repository");
    let repository_path = &args.flag_path;

    match git2::Repository::open(repository_path) {
        Ok(_) => { },
        Err(e) => {
            logger::stderr(format!("Could not open the git repository: {:?}", e));
            process::exit(1);
        }
    };

    let version = match toml_file::read_from_file(repository_path) {
        Ok(toml) => toml,
        Err(e) => {
            logger::stderr(format!("Reading `Cargo.toml` failed: {:?}", e));
            process::exit(1);
        }
    };

    let version = Version::parse(&version).expect("Not a valid version");
    logger::stdout(format!("Current version: {}", version.to_string()));

    logger::stdout("Analyzing commits");

    let bump = git::version_bump_since_latest(repository_path);
    logger::stdout(format!("Commits analyzed. Bump will be {:?}", bump));

    let new_version = match version_bump(&version, bump) {
        Some(new_version) => new_version.to_string(),
        None => {
            logger::stdout("No version bump. Nothing to do.");
            process::exit(0);
        }
    };

    logger::stdout(format!("New version: {}", new_version));
    match toml_file::write_new_version(repository_path, &new_version) {
        Ok(_)    => { },
        Err(err) => {
            logger::stderr(format!("Writing `Cargo.toml` failed: {:?}", err));
            process::exit(1);
        }
    }

    logger::stdout(format!("Writing Changelog"));
    match changelog::write(repository_path, &version.to_string(), &new_version.to_string()) {
        Ok(_)    => { },
        Err(err) => {
            logger::stderr(format!("Writing Changelog failed: {:?}", err));
            process::exit(1);
        }
    }

    match git::commit_files(repository_path, &new_version) {
        Ok(_)    => { },
        Err(err) => {
            logger::stderr(format!("Committing `Cargo.toml` failed: {:?}", err));
            process::exit(1);
        }
    }

    logger::stdout("Creating annotated git tag");
    let tag_message = match changelog::generate(repository_path, &version.to_string(), &new_version) {
        Ok(msg) => msg,
        Err(err) => {
            logger::stderr(format!("Can't geneate changelog: {:?}", err));
            process::exit(1);
        }
    };

    let tag_name = format!("v{}", new_version);
    match git::tag(repository_path, &tag_name, &tag_message) {
        Ok(_) => { },
        Err(err) => {
            logger::stderr(format!("Failed to create git tag: {:?}", err));
            process::exit(1);
        }
    }
}
<commit_msg>feat: Show semantic-rs' version<commit_after>mod logger;
mod toml_file;
mod git;
mod changelog;
mod commit_analyzer;

extern crate rustc_serialize;
extern crate toml;
extern crate regex;
extern crate semver;
extern crate docopt;
extern crate git2_commit;
extern crate git2;
extern crate clog;

use docopt::Docopt;
use commit_analyzer::CommitType;

const VERSION: &'static str = env!("CARGO_PKG_VERSION");
const USAGE: &'static str = "
semantic.rs 🚀

Usage:
  semantic-rs [options]
  semantic-rs --version

Options:
  -h --help              Show this screen.
  --version              Show version.
  -p PATH, --path=PATH   Specifies the repository path. [default: .]
  -n, --dry-run          Run without writing, committing, pushing or publishing anything.
";

#[derive(Debug, RustcDecodable)]
struct Args {
    flag_path: String,
    flag_dry_run: bool,
    flag_version: bool,
}

fn version_bump(version: &Version, bump: CommitType) -> Option<Version> {
    let mut version = version.clone();
    match bump {
        CommitType::Unknown => return None,
        CommitType::Patch => version.increment_patch(),
        CommitType::Minor => version.increment_minor(),
        CommitType::Major => version.increment_major(),
    }

    Some(version)
}

use std::process;
use semver::Version;

fn main() {
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());

    if args.flag_version {
        println!("semantic.rs 🚀 -- v{}", VERSION);
        process::exit(0);
    }

    println!("semantic.rs 🚀");

    logger::stdout("Analyzing your repository");
    let repository_path = &args.flag_path;

    match git2::Repository::open(repository_path) {
        Ok(_) => { },
        Err(e) => {
            logger::stderr(format!("Could not open the git repository: {:?}", e));
            process::exit(1);
        }
    };

    let version = match toml_file::read_from_file(repository_path) {
        Ok(toml) => toml,
        Err(e) => {
            logger::stderr(format!("Reading `Cargo.toml` failed: {:?}", e));
            process::exit(1);
        }
    };

    let version = Version::parse(&version).expect("Not a valid version");
    logger::stdout(format!("Current version: {}", version.to_string()));

    logger::stdout("Analyzing commits");

    let bump = git::version_bump_since_latest(repository_path);
    logger::stdout(format!("Commits analyzed. Bump will be {:?}", bump));

    let new_version = match version_bump(&version, bump) {
        Some(new_version) => new_version.to_string(),
        None => {
            logger::stdout("No version bump. Nothing to do.");
            process::exit(0);
        }
    };

    logger::stdout(format!("New version: {}", new_version));
    match toml_file::write_new_version(repository_path, &new_version) {
        Ok(_)    => { },
        Err(err) => {
            logger::stderr(format!("Writing `Cargo.toml` failed: {:?}", err));
            process::exit(1);
        }
    }

    logger::stdout(format!("Writing Changelog"));
    match changelog::write(repository_path, &version.to_string(), &new_version.to_string()) {
        Ok(_)    => { },
        Err(err) => {
            logger::stderr(format!("Writing Changelog failed: {:?}", err));
            process::exit(1);
        }
    }

    match git::commit_files(repository_path, &new_version) {
        Ok(_)    => { },
        Err(err) => {
            logger::stderr(format!("Committing `Cargo.toml` failed: {:?}", err));
            process::exit(1);
        }
    }

    logger::stdout("Creating annotated git tag");
    let tag_message = match changelog::generate(repository_path, &version.to_string(), &new_version) {
        Ok(msg) => msg,
        Err(err) => {
            logger::stderr(format!("Can't geneate changelog: {:?}", err));
            process::exit(1);
        }
    };

    let tag_name = format!("v{}", new_version);
    match git::tag(repository_path, &tag_name, &tag_message) {
        Ok(_) => { },
        Err(err) => {
            logger::stderr(format!("Failed to create git tag: {:?}", err));
            process::exit(1);
        }
    }
}
<|endoftext|>


<commit_before>#![feature(os)]
#![feature(core)]
#![feature(collections)]
#![feature(unicode)]
#![feature(str_char)]

extern crate git2;
extern crate chrono;
extern crate getopts;
extern crate unicode;
extern crate docopt;
extern crate rustc_serialize;
extern crate collections;
extern crate core;

use std::path::Path;
use docopt::Docopt;

#[derive(RustcDecodable)]
pub struct Args {
    arg_path: String
}

#[cfg(not(test))]
fn main() {
    const USAGE: &'static str = "
usage: gitstat [options] <path>
Options:
-h, --help show this message
";
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());

    // println!("Total count of commits in '{}' is: {}", path.display(), stat.total);
    match gitstat::run(&args) {
        Ok(()) => {}
        Err(e) => println!("error: {}", e),
    }
}

mod gitstat {
    use git2;
    use git2::{Repository,Commit,Oid,Signature,Time,Tree,Object,ObjectType};
    use std::collections::HashMap;
    use std::collections::hash_map::Entry;
    use std::sync::{Mutex, Arc};
    use std::path::Path;
    use collections::vec::IntoIter;
    use core::iter::IntoIterator;
    use std::ops::Add;
    use Args;
    use chrono::datetime::DateTime;
    use chrono::naive::datetime::NaiveDateTime;
    use chrono::{Timelike, Datelike};
    use std::cmp;

    struct Files {
        files: Vec<String>
    }

    impl Files {

        pub fn new(repo: &Repository, head: &Tree) -> Result<Files, git2::Error> {
            let mut vec = Vec::new();

            for entry in head.iter() {
                let name: String = format!("{}", entry.name().unwrap());

                match entry.kind() {
                    Some(ObjectType::Tree) => {
                        let object: Object = try!(entry.to_object(repo));
                        let subfolder = try!(Files::new(repo, &object.as_tree().unwrap()));
                        vec.push_all(&subfolder.files);
                    },
                    Some(ObjectType::Blob) => vec.push(name),
                    _ => {}
                }
            }

            Ok(Files {
                files: vec
            })
        }

        pub fn len(&self) -> usize {
            self.files.len()
        }
    }

    impl IntoIterator for Files {
        type Item = String;
        type IntoIter = IntoIter<String>;

        fn into_iter(self) -> IntoIter<String> {
            self.files.into_iter()
        }
    }

    pub fn run(args: &Args) -> Result<(), git2::Error> {
        let path = Path::new(&args.arg_path);
        let repo = try!(Repository::open(path));

        let authors: HashMap<String, usize> = try!(self::get_authors(&repo));

        // iterate over everything.
        for (name, count) in authors.iter() {
            println!("{}: {}", *name, *count);
        }

        Ok(())
    }

    /// Helper method for gets HEAD commit of given git repository
    fn get_head_commit(repo: &Box<Repository>) -> Option<Commit> {
        repo.head().ok()
            .and_then(|h| h.target())
            .and_then(|oid| repo.find_commit(oid).ok())
    }

    fn get_authors(repo: &Repository) -> Result<HashMap<String, usize>, git2::Error> {
        let mut heatmap = [0; 24*7];
        let mut authors: HashMap<String, usize> = HashMap::new();
        let mut revwalk = try!(repo.revwalk());
        // let mut files_number = Vec::new();

        revwalk.push_head();
        revwalk.set_sorting(git2::SORT_TOPOLOGICAL);
        // let mutex = Mutex::new(repo);

        for oid in revwalk {
            let commit = try!(repo.find_commit(oid));

            let uniq_name: String = get_uniq_name(&commit.author());
            let (weekday, hour) = get_heatmat_coords(&commit.time());
            let tree = try!(commit.tree());
            // let files = try!(Files::new(&repo, &tree));

            // files_number.push(files.len());

            heatmap[(weekday * 24 + hour) as usize] += 1;
            // println!("{} {}", time.seconds(), time.offset_minutes());

            match authors.entry(uniq_name) {
                Entry::Vacant(entry) => entry.insert(1),
                Entry::Occupied(mut entry) => {
                    *entry.get_mut() += 1;
                    entry.into_mut()
                }
            };
        }

        // find max
        let mut max: u32 = 0;
        for count in heatmap.iter() {
            max = cmp::max(*count, max);
        }

        let arts = ['.', '▪', '◾', '◼', '⬛'];

        print!(" ");
        for i in 0..24 {
            print!("{:3}", i);
        }
        println!("");
        for i in 0..24*7 {
            if i % 24 == 0 {
                print!("{}: ", i / 24);
            }
            print!("{:3}", arts[(heatmap[i] as f32 / max as f32 * (arts.len() - 1) as f32) as usize]);
            // print!("{:3}", heatmap[i]);
            if (i + 1) % 24 == 0 {
                println!("");
            }
        }

        Ok(authors)
    }

    fn get_uniq_name(author: &Signature) -> String {
        format!("{} <{}>", author.name().unwrap(), author.email().unwrap())
    }

    fn get_heatmat_coords(time: &Time) -> (u32, u32) {
        let timestamp: NaiveDateTime = NaiveDateTime::from_timestamp(time.seconds(), 0);

        (timestamp.weekday().num_days_from_monday(), timestamp.hour())
    }
}

#[cfg(test)]
mod tests {
    use std::old_io::TempDir;
    use git2::Repository;
    use gitstat::run;

    // fn repo_init() -> (TempDir, Repository) {
    //     let td = TempDir::new("test").unwrap();
    //     let repo = Repository::init(td.path()).unwrap();
    //     {
    //         let mut config = repo.config().unwrap();
    //         config.set_str("user.name", "name").unwrap();
    //         config.set_str("user.email", "email").unwrap();
    //         let mut index = repo.index().unwrap();
    //         let id = index.write_tree().unwrap();
    //
    //         let tree = repo.find_tree(id).unwrap();
    //         let sig = repo.signature().unwrap();
    //         repo.commit(Some("HEAD"), &sig, &sig, "initial", &tree, []).unwrap();
    //     }
    //     (td, repo)
    // }
    //
    // #[test]
    // fn smoke_run() {
    //     let (td, _repo) = self::repo_init();
    //     let path = td.unwrap();
    //     let result = run(&path);
    //     assert!(result.is_ok());
    //     assert_eq!(result.unwrap().total, 1);
    // }
}
<commit_msg>build heatmap with respect to timezone<commit_after>#![feature(os)]
#![feature(core)]
#![feature(collections)]
#![feature(unicode)]
#![feature(str_char)]

extern crate git2;
extern crate chrono;
extern crate getopts;
extern crate unicode;
extern crate docopt;
extern crate rustc_serialize;
extern crate collections;
extern crate core;

use std::path::Path;
use docopt::Docopt;

#[derive(RustcDecodable)]
pub struct Args {
    arg_path: String
}

#[cfg(not(test))]
fn main() {
    const USAGE: &'static str = "
usage: gitstat [options] <path>
Options:
-h, --help show this message
";
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());

    // println!("Total count of commits in '{}' is: {}", path.display(), stat.total);
    match gitstat::run(&args) {
        Ok(()) => {}
        Err(e) => println!("error: {}", e),
    }
}

mod gitstat {
    use git2;
    use git2::{Repository,Commit,Oid,Signature,Time,Tree,Object,ObjectType};
    use std::collections::HashMap;
    use std::collections::hash_map::Entry;
    use std::sync::{Mutex, Arc};
    use std::path::Path;
    use collections::vec::IntoIter;
    use core::iter::IntoIterator;
    use std::ops::Add;
    use Args;
    use chrono::datetime::DateTime;
    use chrono::offset::fixed::FixedOffset;
    use chrono::offset::utc::UTC;
    use chrono::offset::TimeZone;
    use chrono::{Datelike, Timelike};
    use std::cmp;

    struct Files {
        files: Vec<String>
    }

    impl Files {

        pub fn new(repo: &Repository, head: &Tree) -> Result<Files, git2::Error> {
            let mut vec = Vec::new();

            for entry in head.iter() {
                let name: String = format!("{}", entry.name().unwrap());

                match entry.kind() {
                    Some(ObjectType::Tree) => {
                        let object: Object = try!(entry.to_object(repo));
                        let subfolder = try!(Files::new(repo, &object.as_tree().unwrap()));
                        vec.push_all(&subfolder.files);
                    },
                    Some(ObjectType::Blob) => vec.push(name),
                    _ => {}
                }
            }

            Ok(Files {
                files: vec
            })
        }

        pub fn len(&self) -> usize {
            self.files.len()
        }
    }

    impl IntoIterator for Files {
        type Item = String;
        type IntoIter = IntoIter<String>;

        fn into_iter(self) -> IntoIter<String> {
            self.files.into_iter()
        }
    }

    pub fn run(args: &Args) -> Result<(), git2::Error> {
        let path = Path::new(&args.arg_path);
        let repo = try!(Repository::open(path));

        let authors: HashMap<String, usize> = try!(self::get_authors(&repo));

        // iterate over everything.
        for (name, count) in authors.iter() {
            println!("{}: {}", *name, *count);
        }

        Ok(())
    }

    /// Helper method for gets HEAD commit of given git repository
    fn get_head_commit(repo: &Box<Repository>) -> Option<Commit> {
        repo.head().ok()
            .and_then(|h| h.target())
            .and_then(|oid| repo.find_commit(oid).ok())
    }

    fn get_authors(repo: &Repository) -> Result<HashMap<String, usize>, git2::Error> {
        let mut heatmap = [0; 24*7];
        let mut authors: HashMap<String, usize> = HashMap::new();
        let mut revwalk = try!(repo.revwalk());
        // let mut files_number = Vec::new();

        revwalk.push_head();
        revwalk.set_sorting(git2::SORT_TOPOLOGICAL);
        // let mutex = Mutex::new(repo);

        for oid in revwalk {
            let commit = try!(repo.find_commit(oid));

            let uniq_name: String = get_uniq_name(&commit.author());
            let (weekday, hour) = get_heatmat_coords(&commit.time());
            let tree = try!(commit.tree());
            // let files = try!(Files::new(&repo, &tree));

            // files_number.push(files.len());

            heatmap[(weekday * 24 + hour) as usize] += 1;
            // println!("{} {}", time.seconds(), time.offset_minutes());

            match authors.entry(uniq_name) {
                Entry::Vacant(entry) => entry.insert(1),
                Entry::Occupied(mut entry) => {
                    *entry.get_mut() += 1;
                    entry.into_mut()
                }
            };
        }

        // find max
        let mut max: u32 = 0;
        // TODO: sort this
        for count in heatmap.iter() {
            max = cmp::max(*count, max);
        }

        let arts = ['.', '▪', '◾', '◼', '⬛'];

        print!(" ");
        for i in 0..24 {
            print!("{:3}", i);
        }
        println!("");
        for i in 0..24*7 {
            if i % 24 == 0 {
                print!("{}: ", i / 24);
            }
            print!("{:3}", arts[(heatmap[i] as f32 / max as f32 * (arts.len() - 1) as f32) as usize]);
            // print!("{:3}", heatmap[i]);
            if (i + 1) % 24 == 0 {
                println!("");
            }
        }

        Ok(authors)
    }

    fn get_uniq_name(author: &Signature) -> String {
        format!("{} <{}>", author.name().unwrap(), author.email().unwrap())
    }

    fn get_heatmat_coords(time: &Time) -> (u32, u32) {
        let timestamp = UTC.timestamp(time.seconds(), 0)
            .with_timezone(&FixedOffset::east(time.offset_minutes() * 60));

        (timestamp.weekday().num_days_from_monday(), timestamp.hour())
    }
}

#[cfg(test)]
mod tests {
    use std::old_io::TempDir;
    use git2::Repository;
    use gitstat::run;

    // fn repo_init() -> (TempDir, Repository) {
    //     let td = TempDir::new("test").unwrap();
    //     let repo = Repository::init(td.path()).unwrap();
    //     {
    //         let mut config = repo.config().unwrap();
    //         config.set_str("user.name", "name").unwrap();
    //         config.set_str("user.email", "email").unwrap();
    //         let mut index = repo.index().unwrap();
    //         let id = index.write_tree().unwrap();
    //
    //         let tree = repo.find_tree(id).unwrap();
    //         let sig = repo.signature().unwrap();
    //         repo.commit(Some("HEAD"), &sig, &sig, "initial", &tree, []).unwrap();
    //     }
    //     (td, repo)
    // }
    //
    // #[test]
    // fn smoke_run() {
    //     let (td, _repo) = self::repo_init();
    //     let path = td.unwrap();
    //     let result = run(&path);
    //     assert!(result.is_ok());
    //     assert_eq!(result.unwrap().total, 1);
    // }
}
<|endoftext|>


<commit_before>extern crate piston_window;

use piston_window::*;

fn main() {
    let window: PistonWindow = WindowSettings::new(
            "colors",
            [600, 600]
        )
        .exit_on_esc(true)
        .build()
        .unwrap();

    let target_color = [1.0, 0.0, 0.0, 1.0];

    for e in window {
        e.draw_2d(|c, g| {
           clear(target_color, g);

           let playing_surface = Ellipse::new([1.0, 1.0, 1.0, 1.0]);
           playing_surface.draw([50.0, 50.0, 500.0, 500.0], &c.draw_state, c.transform, g);

           let candidate1 = Ellipse::new(target_color);
           candidate1.draw([300.0, 300.0, 100.0, 100.0], &c.draw_state, c.transform, g);

           let candidate2 = Ellipse::new([0.0, 1.0, 0.0, 1.0]);
           candidate2.draw([100.0, 200.0, 100.0, 100.0], &c.draw_state, c.transform, g);

           let candidate3 = Ellipse::new([0.0, 0.0, 1.0, 1.0]);
           candidate3.draw([300.0, 100.0, 100.0, 100.0], &c.draw_state, c.transform, g);
        });
    }
}
<commit_msg>Keep track of mouse position and do stuff on click<commit_after>extern crate piston_window;

use piston_window::*;

fn main() {
    let window: PistonWindow = WindowSettings::new(
            "colors",
            [600, 600]
        )
        .exit_on_esc(true)
        .build()
        .unwrap();

    let target_color = [1.0, 0.0, 0.0, 1.0];
    let mut mouse_position = (0.0, 0.0);

    for e in window {
        e.draw_2d(|c, g| {
           clear(target_color, g);

           let playing_surface = Ellipse::new([1.0, 1.0, 1.0, 1.0]);
           playing_surface.draw([50.0, 50.0, 500.0, 500.0], &c.draw_state, c.transform, g);

           let candidate1 = Ellipse::new(target_color);
           candidate1.draw([300.0, 300.0, 100.0, 100.0], &c.draw_state, c.transform, g);

           let candidate2 = Ellipse::new([0.0, 1.0, 0.0, 1.0]);
           candidate2.draw([100.0, 200.0, 100.0, 100.0], &c.draw_state, c.transform, g);

           let candidate3 = Ellipse::new([0.0, 0.0, 1.0, 1.0]);
           candidate3.draw([300.0, 100.0, 100.0, 100.0], &c.draw_state, c.transform, g);
        });
        e.mouse_cursor(|x, y| {
            mouse_position = (x, y);
        });
        if let Some(button) = e.press_args() {
            if button == Button::Mouse(MouseButton::Left) {
                println!("press {:?}", mouse_position);
            }
        };

    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate glium;
extern crate image;

use std::io::Cursor;
use std::collections::HashMap;
use glium::{DisplayBuild, Surface};

mod atlas;
mod run; // loop.rs

#[derive(Copy, Clone)]
struct Vertex {
    position: [f32; 2],
    tex_coords: [f32; 2],
}

implement_vertex!(Vertex, position, tex_coords);

/// insert all atlas textures into vertices
/// returns attachment (name, index) defining 4 consecutive vertices
fn load_atlas(atlas_src: &str, width: u16, height: u16, vertices: &mut Vec<Vertex>)
    -> HashMap<String, u32>
{
    let (width, height) = (width as f32, height as f32);

    // helper closure to convert atlas texture coordinates into vertex texture's
    let to_tex_coords = |x: u16, y: u16| {
        let (x, y) = (x as f32, y as f32);
        [2.0 * x / width - 1.0, 1.0 - 2.0 * y / height]
    };

    // iterates over atlas textures and convert it to centered rectangle (4 vertices)
    let mut textures = HashMap::new();
    let mut n = shapes.len() as u32;
    for (name, t) in atlas::Atlas::from_file(atlas_src).into_iter() {
        let tex0 = to_tex_coords(t.xy.0,            t.xy.1 + t.size.1);
        let tex1 = to_tex_coords(t.xy.0 + t.size.0, t.xy.1 + t.size.1);
        let tex2 = to_tex_coords(t.xy.0 + t.size.0, t.xy.1);
        let tex3 = to_tex_coords(t.xy.0,            t.xy.1);
        // get 4 vertices defining rectangle texture, centered per default
        vertices.push(Vertex { pos: [0; 2], tex_coords: tex0 });
        vertices.push(Vertex { pos: [0; 2], tex_coords: tex1 });
        vertices.push(Vertex { pos: [0; 2], tex_coords: tex2 });
        vertices.push(Vertex { pos: [0; 2], tex_coords: tex3 });
        textures.insert(name, n);
        n += 4;
    }
    textures
}

fn draw_identity(texture_names: &HashMap<String, u32>, vertices: &mut Vec<Vertex>) -> Vec<u32> {

    let tex_to_pos = |t: &[f32; 2]| {
        [2.0 * t[0] - 1.0, 2.0 * t[1] - 1.0]
    };

    // Change all default position to texture position
    for &mut v in vertices {
        let pos = tex_to_pos(v.tex_coords);
        v.position = pos;
    }

    // build shapes by manipulating indices
    // show all shapes for now
    let mut indices = Vec::with_capacity(texture_names.len() * 6);
    for n in texture_names.values() {
        indices.extend([n, n + 1, n + 2, n + 2, n + 3, n].iter());
    }

    indices

}

fn main() {

    // TODO: parse program arguments
    let atlas_src = "/home/johann/projects/spine_render/spineboy.atlas";
    let atlas_img = "../spineboy.png";

    // prepare glium objects
    let image = image::load(Cursor::new(&include_bytes!(atlas_img)[..]), image::PNG).unwrap();
    let (width, height) = image.dimensions().expect("couldn't read texture dimension");
    let display = glium::glutin::WindowBuilder::new().build_glium().unwrap();
    let texture = glium::texture::Texture2d::new(&display, image).unwrap();

    // load textures
    let mut vertices: Vec<Vertex> = Vec::new();
    let texture_names = load_atlas(atlas_src, width, height, &mut vertices);

    // draw identity (for test)
    let indices = draw_identity(&texture_names, &mut vertices);

    let vertex_buffer = glium::VertexBuffer::new(&display, &vertices).unwrap();
    let indices = glium::IndexBuffer::new(&display,
                                          glium::index::PrimitiveType::TrianglesList,
                                          &indices).unwrap();

    // opengl vertex program
    let vertex_shader_src = r#"
        #version 140

        in vec2 position;
        in vec2 tex_coords;
        out vec2 v_tex_coords;
        uniform mat4 matrix;

        void main() {
            v_tex_coords = tex_coords;
            gl_Position = matrix * vec4(position, 0.0, 1.0);
        }
    "#;

    // opengl fragment program
    let fragment_shader_src = r#"
        #version 140

        in vec2 v_tex_coords;
        out vec4 color;
        uniform sampler2D tex;

        void main() {
            color = texture(tex, v_tex_coords);
        }
    "#;

    let program = glium::Program::from_source(&display,
                                              vertex_shader_src,
                                              fragment_shader_src,
                                              None).unwrap();

    let mut t = -0.5;

    // the main loop, supposed to run with a constant FPS
    run::start_loop(|| {

        // we update `t`
        // t += 0.002;
        // if t > 0.5 {
        //     t = -0.5;
        // }

        let mut target = display.draw();
        target.clear_color(0.0, 0.0, 1.0, 0.0);

        let uniforms = uniform! {
            matrix: [
                [1.0, 0.0, 0.0, 0.0],
                [0.0, 1.0, 0.0, 0.0],
                [0.0, 0.0, 1.0, 0.0],
                [ t , 0.0, 0.0, 1.0f32],
            ],
            tex: &texture,
        };

        target.draw(&vertex_buffer, &indices, &program, &uniforms,
                    &Default::default()).unwrap();
        target.finish().unwrap();

        for ev in display.poll_events() {
            match ev {
                glium::glutin::Event::Closed => return,
                _ => ()
            }
        }

        support::Action::Continue
    });
}
<commit_msg>fix width/height<commit_after>#[macro_use]
extern crate glium;
extern crate image;

use std::io::Cursor;
use std::collections::HashMap;
use glium::{DisplayBuild, Surface};

mod atlas;
mod run; // loop.rs

#[derive(Copy, Clone)]
struct Vertex {
    position: [f32; 2],
    tex_coords: [f32; 2],
}

implement_vertex!(Vertex, position, tex_coords);

/// insert all atlas textures into vertices
/// returns attachment (name, index) defining 4 consecutive vertices
fn load_atlas(atlas_src: &str, width: u16, height: u16, vertices: &mut Vec<Vertex>)
    -> HashMap<String, u32>
{
    let (width, height) = (width as f32, height as f32);

    // helper closure to convert atlas texture coordinates into vertex texture's
    let to_tex_coords = |x: u16, y: u16| {
        let (x, y) = (x as f32, y as f32);
        [2.0 * x / width - 1.0, 1.0 - 2.0 * y / height]
    };

    // iterates over atlas textures and convert it to centered rectangle (4 vertices)
    let mut textures = HashMap::new();
    let mut n = shapes.len() as u32;
    for (name, t) in atlas::Atlas::from_file(atlas_src).into_iter() {
        let tex0 = to_tex_coords(t.xy.0,            t.xy.1 + t.size.1);
        let tex1 = to_tex_coords(t.xy.0 + t.size.0, t.xy.1 + t.size.1);
        let tex2 = to_tex_coords(t.xy.0 + t.size.0, t.xy.1);
        let tex3 = to_tex_coords(t.xy.0,            t.xy.1);
        // get 4 vertices defining rectangle texture, centered per default
        vertices.push(Vertex { pos: [0; 2], tex_coords: tex0 });
        vertices.push(Vertex { pos: [0; 2], tex_coords: tex1 });
        vertices.push(Vertex { pos: [0; 2], tex_coords: tex2 });
        vertices.push(Vertex { pos: [0; 2], tex_coords: tex3 });
        textures.insert(name, n);
        n += 4;
    }
    textures
}

fn draw_identity(texture_names: &HashMap<String, u32>, vertices: &mut Vec<Vertex>) -> Vec<u32> {

    let tex_to_pos = |t: &[f32; 2]| {
        [2.0 * t[0] - 1.0, 2.0 * t[1] - 1.0]
    };

    // Change all default position to texture position
    for &mut v in vertices {
        let pos = tex_to_pos(v.tex_coords);
        v.position = pos;
    }

    // build shapes by manipulating indices
    // show all shapes for now
    let mut indices = Vec::with_capacity(texture_names.len() * 6);
    for n in texture_names.values() {
        indices.extend([n, n + 1, n + 2, n + 2, n + 3, n].iter());
    }

    indices

}

fn main() {

    // TODO: parse program arguments
    let atlas_src = "/home/johann/projects/spine_render/spineboy.atlas";
    let atlas_img = "../spineboy.png";

    // prepare glium objects
    let image = image::load(Cursor::new(&include_bytes!(atlas_img)[..]), image::PNG).unwrap();
    let (width, height) = match image {
        DynamicImage::ImageRgba8(rgba) => (rgba.width(), rgba.height),
        image => {
            let rgba = image.to_rgba();
            (rgba.width(), rgba.height)
        }
    };
    let display = glium::glutin::WindowBuilder::new().build_glium().unwrap();
    let texture = glium::texture::Texture2d::new(&display, image).unwrap();

    // load textures
    let mut vertices: Vec<Vertex> = Vec::new();
    let texture_names = load_atlas(atlas_src, width, height, &mut vertices);

    // draw identity (for test)
    let indices = draw_identity(&texture_names, &mut vertices);

    let vertex_buffer = glium::VertexBuffer::new(&display, &vertices).unwrap();
    let indices = glium::IndexBuffer::new(&display,
                                          glium::index::PrimitiveType::TrianglesList,
                                          &indices).unwrap();

    // opengl vertex program
    let vertex_shader_src = r#"
        #version 140

        in vec2 position;
        in vec2 tex_coords;
        out vec2 v_tex_coords;
        uniform mat4 matrix;

        void main() {
            v_tex_coords = tex_coords;
            gl_Position = matrix * vec4(position, 0.0, 1.0);
        }
    "#;

    // opengl fragment program
    let fragment_shader_src = r#"
        #version 140

        in vec2 v_tex_coords;
        out vec4 color;
        uniform sampler2D tex;

        void main() {
            color = texture(tex, v_tex_coords);
        }
    "#;

    let program = glium::Program::from_source(&display,
                                              vertex_shader_src,
                                              fragment_shader_src,
                                              None).unwrap();

    let mut t = -0.5;

    // the main loop, supposed to run with a constant FPS
    run::start_loop(|| {

        // we update `t`
        // t += 0.002;
        // if t > 0.5 {
        //     t = -0.5;
        // }

        let mut target = display.draw();
        target.clear_color(0.0, 0.0, 1.0, 0.0);

        let uniforms = uniform! {
            matrix: [
                [1.0, 0.0, 0.0, 0.0],
                [0.0, 1.0, 0.0, 0.0],
                [0.0, 0.0, 1.0, 0.0],
                [ t , 0.0, 0.0, 1.0f32],
            ],
            tex: &texture,
        };

        target.draw(&vertex_buffer, &indices, &program, &uniforms,
                    &Default::default()).unwrap();
        target.finish().unwrap();

        for ev in display.poll_events() {
            match ev {
                glium::glutin::Event::Closed => return,
                _ => ()
            }
        }

        support::Action::Continue
    });
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate lazy_static;
extern crate regex;
extern crate hyper;
extern crate hubcaps;
extern crate afterparty;
extern crate serde_json;
extern crate serde;
extern crate rustc_serialize;
extern crate toml;
#[macro_use]
extern crate log;
extern crate env_logger;

use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::env;
use std::io::Read;

use hyper::{Client, Server, Url};
use hyper::header::{Headers, Authorization, Bearer, UserAgent};
use hubcaps::{Credentials, Github};
use afterparty::*;
use serde_json::Value;
use regex::Regex;
use toml::decode_str;

#[derive(RustcEncodable, RustcDecodable)]
struct RepoData {
    pub qa_user: String,
    pub qa_flags: Vec<String>,
}

#[derive(RustcEncodable, RustcDecodable)]
struct ConfigData {
    pub api_root: String,
    pub repos: HashMap<String, RepoData>,
}

macro_rules! panic_log {
    ($($arg:tt)*) => ( {
        error!($($arg)*);
        panic!()
    } );
}

trait ExpectLog {
    type Item;
    fn expect_log(self, &str) -> Self::Item;
}

impl<T> ExpectLog for Option<T> {
    type Item = T;
    fn expect_log(self, msg: &str) -> T {
        match self {
            None => panic_log!("{}", msg),
            Some(s) => s,
        }
    }
}

impl<T, E> ExpectLog for Result<T, E>
    where E: std::fmt::Display
{
    type Item = T;
    fn expect_log(self, msg: &str) -> T {
        match self {
            Err(err) => panic_log!("{} {}", msg, err),
            Ok(s) => s,
        }
    }
}

fn read_config_file(file_name: &str) -> ConfigData {
    let mut config_file = File::open(file_name).expect("Couldn't open config file");
    let mut str = String::new();
    config_file.read_to_string(&mut str).expect("Couldn't read config file");
    let toml: Option<ConfigData> = decode_str(&str);
    match toml {
        Some(cfg) => cfg,
        _ => panic!("Couldn't decode config file"),
    }
}

#[test]
fn it_reads_toml() {
    let config_data = read_config_file("test-data/test.toml");
    assert_eq!(config_data.api_root, "github.ibm.com/api/v3");
    let steve = config_data.repos.get("lkgrele/steve").unwrap();
    assert_eq!(steve.qa_user, "leif");
    assert_eq!(steve.qa_flags, ["qa", "test"]);
    let raul = config_data.repos.get("raul/robot").unwrap();
    assert_eq!(raul.qa_user, "raul");
    assert_eq!(raul.qa_flags, ["qa"]);
}

fn main() {
    env_logger::init().unwrap();

    let mut hub = Hub::new();
    hub.handle("pull_request", |delivery: &Delivery| {
        match delivery.payload {
            Event::PullRequest { ref pull_request, ref repository, .. } => {
                if pull_request.merged {
                    handle_pr(&pull_request.commits_url, &repository.full_name)
                }
            }
            _ => info!("Recived a request that wasn't a pull-request webhook"),
        }
    });

    match Server::http("0.0.0.0:0") {
        Err(err) => error!("ERROR: failed to start server, the error was: {}", err),
        Ok(server) => {
            match server.handle(hub) {
                Err(err) => error!("ERROR starting handler, the error was: {}", err),
                Ok(server) => info!("Successfully started server"),
            }
        }
    }
}

fn handle_pr(commits_url: &str, repository: &str) {
    let config_data = read_config_file(".steve");
    let repo_config = config_data.repos.get(&repository.to_owned());
    match env::var_os("STEVE_GITHUB_TOKEN") {
        None => {
            error!("Missing github token, STEVE_GITHUB_TOKEN is not defined!");
            panic!()
        }
        Some(tok) => {
            match tok.into_string() {
                Err(_) => {
                    error!("Github token contains illegal characters!");
                    panic!()
                }
                Ok(auth_token) => {
                    match repo_config {
                        Some(config) => {
                            update_issues(&commits_url,
                                          &auth_token,
                                          &config,
                                          &config_data.api_root,
                                          &repository)
                        }
                        None => info!("No config found for repo {}, skipping", repository),
                    }
                }
            }
        }
    }
}

fn update_issues(commits_url: &str,
                 auth_token: &str,
                 config: &RepoData,
                 api_root: &str,
                 repository: &str) {
    let client = Client::new();
    let mut headers = Headers::new();
    headers.set(Authorization(Bearer { token: auth_token.to_owned() }));
    headers.set(UserAgent("steve".to_owned()));
    let commits = client.get(commits_url).headers(headers).send().unwrap();
    assert_eq!(commits.status, hyper::Ok); //replace with actual error handling

    match serde_json::from_reader(commits) {
        Ok(json) => {
            match parse_commit_data(&json) {
                Ok(messages) => {
                    let mut issues = HashSet::new();
                    for message in messages {
                        get_issues(&mut issues, &message)
                    }

                    for issue in issues {
                        update_issue(&client, auth_token, issue, config, &repository, api_root);
                    }
                }
                Err(err) => error!("{:?}", err),
            }
        }
        Err(err) => error!("{:?}", err),
    }
}

#[test]
fn it_parses_commit_data() {
    let file = File::open("test-data/test-messages.json").expect("Couldn't open message file");
    let json = serde_json::from_reader(file);
    assert!(json.is_ok());
    let json = json.unwrap();
    let messages = parse_commit_data(&json);
    assert!(messages.is_ok());
    let messages = messages.unwrap();
    assert_eq!(messages[0], "message1");
    assert_eq!(messages[1], "swordfish");
}

fn parse_commit_data<'a>(json: &'a Value) -> Result<Vec<&'a str>, &str> {
    let arr = try!(json.as_array().ok_or("expected array as top level object"));
    arr.iter()
        .map(|obj| {
            obj.pointer("/commit/message")
                .ok_or("missing message field")
                .and_then(|m| m.as_string().ok_or("message field has wrong type"))
        })
        .collect()
}

fn update_issue(client: &Client,
                auth_token: &str,
                issue_num: u64,
                config: &RepoData,
                repository: &str,
                api_root: &str) {
    let github = Github::host(api_root,
                              "steve",
                              client,
                              Credentials::Token(auth_token.to_owned()));
    let (owner, repo_name) = match repository.split("/").collect::<Vec<_>>() {
        strs => (strs[0], strs[1]),
    };
    let _ = github.repo(repo_name, owner)
        .issue(issue_num)
        .labels()
        .add(config.qa_flags.iter().map(String::as_str).collect());
    let mut headers = Headers::new();
    headers.set(Authorization(Bearer { token: auth_token.to_owned() }));
    headers.set(UserAgent("steve".to_owned()));
    let update_json = format!("{{\"assignee\":\"{}\"}}", config.qa_user);
    let url = Url::parse(&format!("{}/repos/{}/issues/{}", api_root, repository, issue_num))
        .expect("bad url");
    let res = client.patch(url)
        .body(&update_json)
        .headers(headers.to_owned())
        .send()
        .unwrap();
    assert_eq!(res.status, hyper::Ok);
}

#[test]
fn it_finds_issue_numbers() {
    let message = r"This is a commit message
        needs qa: #72 should get caught
        needs QA #333 should also get caught
        but #44 won't, and needs QA: #33 #45
        only gets #33. Duplicates are discarded:
        needs qa #72";
    let mut expected: HashSet<u64> = HashSet::new();
    expected.insert(72);
    expected.insert(333);
    expected.insert(33);
    let mut numbers: HashSet<u64> = HashSet::new();
    get_issues(&mut numbers, message);
    assert_eq!(numbers, expected);
}

fn get_issues(numbers: &mut HashSet<u64>, message: &str) {
    lazy_static! {
        static ref RE: Regex = Regex::new(r"(?i)needs\sqa:?\s#(\d+)").unwrap();
    }
    for num in RE.captures_iter(message).filter_map(|cap| cap.at(1)) {
        numbers.insert(num.parse::<u64>().unwrap());
    }
}
<commit_msg>Add logging to read_config_file().<commit_after>#[macro_use]
extern crate lazy_static;
extern crate regex;
extern crate hyper;
extern crate hubcaps;
extern crate afterparty;
extern crate serde_json;
extern crate serde;
extern crate rustc_serialize;
extern crate toml;
#[macro_use]
extern crate log;
extern crate env_logger;

use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::env;
use std::io::Read;

use hyper::{Client, Server, Url};
use hyper::header::{Headers, Authorization, Bearer, UserAgent};
use hubcaps::{Credentials, Github};
use afterparty::*;
use serde_json::Value;
use regex::Regex;
use toml::decode_str;

#[derive(RustcEncodable, RustcDecodable)]
struct RepoData {
    pub qa_user: String,
    pub qa_flags: Vec<String>,
}

#[derive(RustcEncodable, RustcDecodable)]
struct ConfigData {
    pub api_root: String,
    pub repos: HashMap<String, RepoData>,
}

macro_rules! panic_log {
    ($($arg:tt)*) => ( {
        error!($($arg)*);
        panic!()
    } );
}

trait ExpectLog {
    type Item;
    fn expect_log(self, &str) -> Self::Item;
}

impl<T> ExpectLog for Option<T> {
    type Item = T;
    fn expect_log(self, msg: &str) -> T {
        match self {
            None => panic_log!("{}", msg),
            Some(s) => s,
        }
    }
}

impl<T, E> ExpectLog for Result<T, E>
    where E: std::fmt::Display
{
    type Item = T;
    fn expect_log(self, msg: &str) -> T {
        match self {
            Err(err) => panic_log!("{} {}", msg, err),
            Ok(s) => s,
        }
    }
}

fn read_config_file(file_name: &str) -> ConfigData {
    let mut config_file = File::open(file_name)
        .expect_log(&format!("Couldn't open config file {}", file_name));
    let mut str = String::new();
    config_file.read_to_string(&mut str)
        .expect_log(&format!("Couldn't read config file {}", file_name));
    let toml: Option<ConfigData> = decode_str(&str);
    toml.expect_log(&format!("Couldn't decode config file {}", file_name))
}

#[test]
fn it_reads_toml() {
    env_logger::init().unwrap();
    let config_data = read_config_file("test-data/test.toml");
    assert_eq!(config_data.api_root, "github.ibm.com/api/v3");
    let steve = config_data.repos.get("lkgrele/steve").unwrap();
    assert_eq!(steve.qa_user, "leif");
    assert_eq!(steve.qa_flags, ["qa", "test"]);
    let raul = config_data.repos.get("raul/robot").unwrap();
    assert_eq!(raul.qa_user, "raul");
    assert_eq!(raul.qa_flags, ["qa"]);
}

fn main() {
    env_logger::init().unwrap();

    let mut hub = Hub::new();
    hub.handle("pull_request", |delivery: &Delivery| {
        match delivery.payload {
            Event::PullRequest { ref pull_request, ref repository, .. } => {
                if pull_request.merged {
                    handle_pr(&pull_request.commits_url, &repository.full_name)
                }
            }
            _ => info!("Recived a request that wasn't a pull-request webhook"),
        }
    });

    match Server::http("0.0.0.0:0") {
        Err(err) => error!("ERROR: failed to start server, the error was: {}", err),
        Ok(server) => {
            match server.handle(hub) {
                Err(err) => error!("ERROR starting handler, the error was: {}", err),
                Ok(server) => info!("Successfully started server"),
            }
        }
    }
}

fn handle_pr(commits_url: &str, repository: &str) {
    let config_data = read_config_file(".steve");
    let repo_config = config_data.repos.get(&repository.to_owned());
    match env::var_os("STEVE_GITHUB_TOKEN") {
        None => {
            error!("Missing github token, STEVE_GITHUB_TOKEN is not defined!");
            panic!()
        }
        Some(tok) => {
            match tok.into_string() {
                Err(_) => {
                    error!("Github token contains illegal characters!");
                    panic!()
                }
                Ok(auth_token) => {
                    match repo_config {
                        Some(config) => {
                            update_issues(&commits_url,
                                          &auth_token,
                                          &config,
                                          &config_data.api_root,
                                          &repository)
                        }
                        None => info!("No config found for repo {}, skipping", repository),
                    }
                }
            }
        }
    }
}

fn update_issues(commits_url: &str,
                 auth_token: &str,
                 config: &RepoData,
                 api_root: &str,
                 repository: &str) {
    let client = Client::new();
    let mut headers = Headers::new();
    headers.set(Authorization(Bearer { token: auth_token.to_owned() }));
    headers.set(UserAgent("steve".to_owned()));
    let commits = client.get(commits_url).headers(headers).send().unwrap();
    assert_eq!(commits.status, hyper::Ok); //replace with actual error handling

    match serde_json::from_reader(commits) {
        Ok(json) => {
            match parse_commit_data(&json) {
                Ok(messages) => {
                    let mut issues = HashSet::new();
                    for message in messages {
                        get_issues(&mut issues, &message)
                    }

                    for issue in issues {
                        update_issue(&client, auth_token, issue, config, &repository, api_root);
                    }
                }
                Err(err) => error!("{:?}", err),
            }
        }
        Err(err) => error!("{:?}", err),
    }
}

#[test]
fn it_parses_commit_data() {
    let file = File::open("test-data/test-messages.json").expect("Couldn't open message file");
    let json = serde_json::from_reader(file);
    assert!(json.is_ok());
    let json = json.unwrap();
    let messages = parse_commit_data(&json);
    assert!(messages.is_ok());
    let messages = messages.unwrap();
    assert_eq!(messages[0], "message1");
    assert_eq!(messages[1], "swordfish");
}

fn parse_commit_data<'a>(json: &'a Value) -> Result<Vec<&'a str>, &str> {
    let arr = try!(json.as_array().ok_or("expected array as top level object"));
    arr.iter()
        .map(|obj| {
            obj.pointer("/commit/message")
                .ok_or("missing message field")
                .and_then(|m| m.as_string().ok_or("message field has wrong type"))
        })
        .collect()
}

fn update_issue(client: &Client,
                auth_token: &str,
                issue_num: u64,
                config: &RepoData,
                repository: &str,
                api_root: &str) {
    let github = Github::host(api_root,
                              "steve",
                              client,
                              Credentials::Token(auth_token.to_owned()));
    let (owner, repo_name) = match repository.split("/").collect::<Vec<_>>() {
        strs => (strs[0], strs[1]),
    };
    let _ = github.repo(repo_name, owner)
        .issue(issue_num)
        .labels()
        .add(config.qa_flags.iter().map(String::as_str).collect());
    let mut headers = Headers::new();
    headers.set(Authorization(Bearer { token: auth_token.to_owned() }));
    headers.set(UserAgent("steve".to_owned()));
    let update_json = format!("{{\"assignee\":\"{}\"}}", config.qa_user);
    let url = Url::parse(&format!("{}/repos/{}/issues/{}", api_root, repository, issue_num))
        .expect("bad url");
    let res = client.patch(url)
        .body(&update_json)
        .headers(headers.to_owned())
        .send()
        .unwrap();
    assert_eq!(res.status, hyper::Ok);
}

#[test]
fn it_finds_issue_numbers() {
    let message = r"This is a commit message
        needs qa: #72 should get caught
        needs QA #333 should also get caught
        but #44 won't, and needs QA: #33 #45
        only gets #33. Duplicates are discarded:
        needs qa #72";
    let mut expected: HashSet<u64> = HashSet::new();
    expected.insert(72);
    expected.insert(333);
    expected.insert(33);
    let mut numbers: HashSet<u64> = HashSet::new();
    get_issues(&mut numbers, message);
    assert_eq!(numbers, expected);
}

fn get_issues(numbers: &mut HashSet<u64>, message: &str) {
    lazy_static! {
        static ref RE: Regex = Regex::new(r"(?i)needs\sqa:?\s#(\d+)").unwrap();
    }
    for num in RE.captures_iter(message).filter_map(|cap| cap.at(1)) {
        numbers.insert(num.parse::<u64>().unwrap());
    }
}
<|endoftext|>