<commit_before>use std::fs;
use std::env;
use std::fs::File;
use std::io::Read;
use std::hash::Hasher;
use std::path::PathBuf;
use std::collections::HashMap;
use std::collections::BTreeSet;
use std::os::unix::fs::MetadataExt;
use std::collections::hash_map::DefaultHasher;

#[macro_use]
extern crate log;
extern crate env_logger;

extern crate serde;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;

#[derive(Serialize, Deserialize)]
struct FileEntry {
    size : u64,
    hash : u64,
    files : Vec<String>,
}

fn collect_files(dir: &String, files: &mut HashMap<u64, Vec<String>>,
                 inodes: &mut BTreeSet<(u64, u64)>) {

    let entries = match fs::read_dir(dir) {
        Ok(x) => x,
        _ => {
            warn!("{}: invalid directory", dir);
            return
        },
    };

    for path in  entries.filter( |x| x.is_ok() )
                        .map( |x| x.unwrap().path() )
                        .collect::<Vec<PathBuf>>() {
        if let (Some(path_str), Ok(metadata)) =
                (path.to_str(), path.symlink_metadata()) {
            let ft = metadata.file_type();
            if ft.is_symlink() {
                continue;
            } else if ft.is_file() {
                let inode = metadata.ino();
                let device = metadata.dev();
                if inodes.contains(&(inode, device)) {
                    continue;
                }
                inodes.insert((inode, device));
                match metadata.len() {
                    0 => continue, //Ignore empty files.
                    i => files.entry(i).or_insert_with(Vec::new)
                          .push(String::from(path_str)),
                };
            } else if ft.is_dir() {
                collect_files(&(String::from(path_str)), files, inodes);
            }
        } else {
            warn!("Failed to retrieve metadata for {}", path.to_str().unwrap());
        }
    }
}

fn find_duplicates(duplicates: &mut HashMap<u64, Vec<String>>,
                   vec: &Vec<String>, thorough: bool ) {
    for file_entry in vec {
        if let Ok(mut file) = File::open(file_entry) {
            let mut hash: u64 = 0;
            let mut hasher = DefaultHasher::new();
            if thorough {
                let mut contents: Vec<u8> = Vec::new();
                if let Ok(_) = file.read_to_end(&mut contents) {
                    hasher.write(&contents[..]);
                    hash = hasher.finish();
                } else {
                    warn!("Failed to read {}", file_entry);
                }
            } else {
                let mut buf: [u8; 1024] = [0; 1024];
                if let Ok(nbytes) = file.read(&mut buf) {
                    hasher.write(&buf[..nbytes as usize]);
                    hash = hasher.finish();
                } else {
                    warn!("Failed to read {} chunk", file_entry);
                }
            }
            if hash != 0 {
                duplicates.entry(hash).or_insert_with(Vec::new)
                    .push(file_entry.to_string());
            }
        } else {
            warn!("Failed to open file {}", file_entry);
        }
    }
}

fn print_duplicates(vec: &Vec<String>, cluster: i32, key: u64, hash: u64,
                    json_list: &mut Vec<FileEntry>, json_output: bool) {
    if json_output {
        json_list.push( FileEntry {
            size  : key,
            hash  : hash,
            files : vec.to_vec(),
        });
    } else {
        println!("{} files in cluster {} (size: {}, digest: {})",
                 vec.len(), cluster, key, hash);
        // for_each becomes stable v1.22.0 onwards. Should uncomment then.
        //vec.iter().for_each(|f| println!("{}", f.path));
        for file_entry in vec {
            println!("{}", file_entry);
        }
    }
}

fn exhaustive_search(emap: &mut HashMap<Vec<u8>, Vec<String>>,
                     vec: &Vec<String>) {
    for file_entry in vec {
        if let Ok(mut file) = File::open(file_entry) {
            let mut contents: Vec<u8> = Vec::new();
            if let Ok(_) = file.read_to_end(&mut contents) {
                emap.entry(contents).or_insert_with(Vec::new)
                    .push(file_entry.to_string());
            } else {
                warn!("Failed to read {}", file_entry);
            }
        } else {
            warn!("Failed to open {}", file_entry);
        }
    }
}

fn main() {
    env_logger::init().unwrap();

    let mut args: Vec<String> = env::args().collect();

    let binary_name = args.remove(0);

    let mut hmap: HashMap<u64, Vec<String> > = HashMap::new();
    let mut bset: BTreeSet<(u64, u64)> = BTreeSet::new();

    let mut thorough = false;
    let mut exhaustive = false;
    let mut json_output = false;
    let mut print_usage = true;

    for dir in &args[..] {
        match dir.as_ref() {
            "-t" => {
                thorough = true;
                continue;
            },
            "-e" => {
                thorough = false;
                exhaustive = false;
                continue;
            },
            "-j" => {
                json_output = true;
                continue;
            },
            _ => {
                print_usage = false;
                collect_files(dir, &mut hmap, &mut bset);
            },
        }
    }

    if print_usage {
        println!("Usage: {} [-t] [-e] [-j] <dir1> [dir2 [dir3 ...]]",
                 binary_name);
        ::std::process::exit(0);
    }

    // Get rid of all the single entries.
    hmap.retain(|_, v| v.len() >= 2);

    let mut cluster = 1;

    let mut json_list : Vec<FileEntry> = Vec::new();
    for (key, val) in hmap {
        let mut duplicates: HashMap<u64, Vec<String> > = HashMap::new();
        find_duplicates(&mut duplicates, &val, thorough);

        for (hash, vec) in duplicates {
            if vec.len() >= 2 {
                if exhaustive {
                    let mut emap: HashMap<Vec<u8>, Vec<String>> = HashMap::new();
                    exhaustive_search(&mut emap, &vec);
                    for (_, v) in emap {
                        if v.len() >= 2 {
                            print_duplicates(&v, cluster, key, hash,
                                             &mut json_list, json_output);
                            cluster += 1;
                        }
                    }
                } else {
                    print_duplicates(&vec, cluster, key, hash, &mut json_list,
                                     json_output);
                    cluster += 1;
                }
            }
        }
    }
    if json_output {
        println!("{}", serde_json::to_string_pretty(&json_list).unwrap());
    }
}
<commit_msg>Some more refactoring<commit_after>use std::fs;
use std::env;
use std::fs::File;
use std::io::Read;
use std::hash::Hasher;
use std::path::PathBuf;
use std::collections::HashMap;
use std::collections::BTreeSet;
use std::os::unix::fs::MetadataExt;
use std::collections::hash_map::DefaultHasher;

#[macro_use]
extern crate log;
extern crate env_logger;

extern crate serde;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;

#[derive(Serialize, Deserialize)]
struct FileEntry {
    size : u64,
    hash : u64,
    files : Vec<String>,
}

fn collect_files(dir: &String, files: &mut HashMap<u64, Vec<String>>,
                 inodes: &mut BTreeSet<(u64, u64)>) {

    let entries = match fs::read_dir(dir) {
        Ok(x) => x,
        _ => {
            warn!("{}: invalid directory", dir);
            return
        },
    };

    for path in  entries.filter( |x| x.is_ok() )
                        .map( |x| x.unwrap().path() )
                        .collect::<Vec<PathBuf>>() {
        if let (Some(path_str), Ok(metadata)) =
                (path.to_str(), path.symlink_metadata()) {
            let ft = metadata.file_type();
            if ft.is_symlink() {
                continue;
            } else if ft.is_file() {
                let inode = metadata.ino();
                let device = metadata.dev();
                if inodes.contains(&(inode, device)) {
                    continue;
                }
                inodes.insert((inode, device));
                match metadata.len() {
                    0 => continue, //Ignore empty files.
                    i => files.entry(i).or_insert_with(Vec::new)
                          .push(String::from(path_str)),
                };
            } else if ft.is_dir() {
                collect_files(&(String::from(path_str)), files, inodes);
            }
        } else {
            warn!("Failed to retrieve metadata for {}", path.to_str().unwrap());
        }
    }
}

fn find_duplicates(duplicates: &mut HashMap<u64, Vec<String>>,
                   vec: &Vec<String>, thorough: bool ) {
    for file_entry in vec {
        if let Ok(mut file) = File::open(file_entry) {
            let mut hash: u64 = 0;
            let mut hasher = DefaultHasher::new();
            if thorough {
                let mut contents: Vec<u8> = Vec::new();
                if let Ok(_) = file.read_to_end(&mut contents) {
                    hasher.write(&contents[..]);
                    hash = hasher.finish();
                } else {
                    warn!("Failed to read {}", file_entry);
                }
            } else {
                let mut buf: [u8; 1024] = [0; 1024];
                if let Ok(nbytes) = file.read(&mut buf) {
                    hasher.write(&buf[..nbytes as usize]);
                    hash = hasher.finish();
                } else {
                    warn!("Failed to read {} chunk", file_entry);
                }
            }
            if hash != 0 {
                duplicates.entry(hash).or_insert_with(Vec::new)
                    .push(file_entry.to_string());
            }
        } else {
            warn!("Failed to open file {}", file_entry);
        }
    }
}

fn print_duplicates(files: &Vec<String>, cluster: i32, size: u64, hash: u64,
                    json_list: &mut Vec<FileEntry>, json_output: bool) {
    if json_output {
        json_list.push( FileEntry {
            size,
            hash,
            files: files.to_vec(),
       });
    } else {
        println!("{} files in cluster {} (size: {}, digest: {})",
                 files.len(), cluster, size, hash);
        // for_each becomes stable v1.22.0 onwards. Should uncomment then.
        //files.iter().for_each(|f| println!("{}", f.path));
        for file in files {
            println!("{}", file);
        }
    }
}

fn exhaustive_search(emap: &mut HashMap<Vec<u8>, Vec<String>>,
                     vec: &Vec<String>) {
    for file_entry in vec {
        if let Ok(mut file) = File::open(file_entry) {
            let mut contents: Vec<u8> = Vec::new();
            if let Ok(_) = file.read_to_end(&mut contents) {
                emap.entry(contents).or_insert_with(Vec::new)
                    .push(file_entry.to_string());
            } else {
                warn!("Failed to read {}", file_entry);
            }
        } else {
            warn!("Failed to open {}", file_entry);
        }
    }
}

fn main() {
    env_logger::init().unwrap();

    let mut args: Vec<String> = env::args().collect();

    let binary_name = args.remove(0);

    let mut files: HashMap<u64, Vec<String> > = HashMap::new();

    let mut thorough = false;
    let mut exhaustive = false;
    let mut json_output = false;
    let mut print_usage = true;

    {
        let mut inodes: BTreeSet<(u64, u64)> = BTreeSet::new();
        for dir in &args[..] {
            match dir.as_ref() {
                "-t" => {
                    thorough = true;
                    continue;
                },
                "-e" => {
                    thorough = false;
                    exhaustive = false;
                    continue;
                },
                "-j" => {
                    json_output = true;
                    continue;
                },
                _ => {
                    print_usage = false;
                    collect_files(dir, &mut files, &mut inodes);
                },
            }
        }
    }

    if print_usage {
        println!("Usage: {} [-t] [-e] [-j] <dir1> [dir2 [dir3 ...]]",
                 binary_name);
        ::std::process::exit(0);
    }

    // Get rid of all the single entries.
    files.retain(|_, v| v.len() >= 2);

    let mut cluster = 1;

    let mut json : Vec<FileEntry> = Vec::new();
    for (key, val) in files {
        let mut duplicates: HashMap<u64, Vec<String> > = HashMap::new();
        find_duplicates(&mut duplicates, &val, thorough);

        for (hash, vec) in duplicates {
            if vec.len() >= 2 {
                if exhaustive {
                    let mut emap: HashMap<Vec<u8>, Vec<String>> = HashMap::new();
                    exhaustive_search(&mut emap, &vec);
                    for (_, v) in emap {
                        if v.len() >= 2 {
                            print_duplicates(&v, cluster, key, hash,
                                             &mut json, json_output);
                            cluster += 1;
                        }
                    }
                } else {
                    print_duplicates(&vec, cluster, key, hash, &mut json,
                                     json_output);
                    cluster += 1;
                }
            }
        }
    }
    if json_output {
        println!("{}", serde_json::to_string_pretty(&json).unwrap());
    }
}
<|endoftext|>


<commit_before>#[macro_use] extern crate log;
extern crate env_logger;

use std::io::prelude::*;
use std::io;
use std::fs::File;
use std::collections::HashMap;
use std::string::String;
use std::vec::Vec;
use std::path::Path;
use std::net::{TcpListener, TcpStream};
use std::thread;
use std::result;
use std::fs;
use std::sync::{Arc, Mutex};
use std::env;
use std::error;
use std::env::Args;

enum Mode {
    NONE,
    CLIENT,
    SERVER
}

struct CacheError {
    reason: String
}

impl CacheError{
    pub fn with_reason( reason: String ) -> CacheError {
        return CacheError{
            reason: reason.clone()
        }
    }
}

struct CacheOperation {
    commands: Vec<char>,
    value: String
}

impl CacheOperation {
    fn new_from_string(cache_op_str: &String) -> CacheOperation {
        let mut cache_op = CacheOperation {
            commands: Vec::new(),
            value: String::new()
        };
        let mut idx = 0;
        let chars_iter = cache_op_str.chars();
        for c in chars_iter {
            idx += 1;
            if c == '$' {
                cache_op.value = String::from(&cache_op_str[idx..]);
                break;
            }
            cache_op.commands.push(c);
        }
        return cache_op;
    }
}

struct Cache {
    map_internal: HashMap<String, Vec<u8>>
}

impl Cache {
    fn new() -> Cache {
        return Cache {
            map_internal: HashMap::new()
        }
    }

    fn cache_item(&mut self, key: String, val: Vec<u8>) -> Result<(), CacheError>{
        debug!("Writing to cache: key={:?} val={:?}", key, val);
        fs::create_dir("_cache");
        let mut f: File = File::create(format!("_cache/{}", key)).unwrap();
        f.write(&val.as_slice());
        f.flush();
        self.map_internal.insert(key, val);
        return Ok(());
    }

    fn read_item(&self, key: String) -> Option<Box<Vec<u8>>> {
        let mut res: Option<Box<Vec<u8>>> = None;
        if self.map_internal.contains_key(&key) {
            let res_opt = self.map_internal.get(&key);
            match res_opt {
                Some(vec_ref) => res = Some(Box::new(vec_ref.clone())),
                None => res = None
            }
        } else {
            let mut buf: Vec<u8> = Vec::new();
            let mut f_res = match File::open(&key) {
                Err(why) => res = None,
                Ok(mut file) => {
                    file.read_to_end(&mut buf);
                    res = Some(Box::new(buf));
                }
            };
        }
        return res;
    }

    fn delete_item(&mut self, key: String) -> Result<(), CacheError>{
        if self.map_internal.contains_key(&key) {
            self.map_internal.remove(&key);
        }
        let path = format!("_cache/{}", key);
        if Path::new(&path).exists() {
            match fs::remove_file(path) {
                Ok(x) => x,
                Err(e) => return Err(CacheError::with_reason(format!("Failed to delete file for key: {}", key)))
            }
        }
        return Ok(())
    }
}

fn string_from_stream( mut stream: &mut TcpStream ) -> String {
    stream.set_nodelay(true);
    let mut buf_arr: [u8; 64] = [0; 64];
    stream.read(&mut buf_arr);

    let mut size_str = String::new();
    let mut buf_size: usize = 0;
    for i in 0..64 {
        buf_size += 1;
        if buf_arr[i] == '|' as u8 {
            break;
        }
        size_str.push(buf_arr[i as usize] as char);
    }

    let upper_idx: usize = size_str.parse::<i32>().unwrap() as usize;
    let msg_buf = buf_arr;
    let mut buf_temp: Vec<u8> = msg_buf.to_vec();
    let mut buf: Vec<u8> = buf_temp.drain(buf_size..upper_idx + buf_size).collect();

    stream.flush().unwrap();

    let buf_str: String = String::from_utf8(buf).unwrap();
    return buf_str;
}

fn handle_client(mut stream: &mut TcpStream, mut cache: &Arc<Mutex<Cache>>) -> Result<(), CacheError>{
    let buf_str: String = string_from_stream( stream );
    if buf_str.len() > 0 {
        let cache_op: CacheOperation = CacheOperation::new_from_string(&buf_str);

        debug!("cmds = {:?}, value = {}", cache_op.commands, cache_op.value);

        if cache_op.commands.len() > 0 {
            let prim_cmd: char = cache_op.commands[0];
            return match prim_cmd {
                'W' => write_stream_str_to_cache(cache_op.value, cache),
                'R' => read_value_from_cache(cache_op.value, cache, &mut stream),
                'D' => delete_value_from_cache(cache_op.value, cache),
                _ => Err(CacheError::with_reason(format!("Invalid cache command {:?}", prim_cmd)))
            }
        }
    } else {
        panic!("Empty cache operation");
    }
    return Ok(());
}

fn read_value_from_cache(key: String, cache_mtx: &Arc<Mutex<Cache>>, stream: &mut TcpStream) -> Result<(), CacheError> {
    let cache = cache_mtx.lock().unwrap();
    let cache_opt: Option<Box<Vec<u8>>> = cache.read_item(key);
    match cache_opt {
        Some(boxed_val) => {
            let mut val:Vec<u8> = *boxed_val;
            debug!("Writing to stream {:?}", val);
            let mut sized_val:Vec<u8> = String::from(format!("{}|", val.len())).into_bytes();
            sized_val.append(val.as_mut());
            stream.write( &sized_val.as_slice() );
            stream.flush();
            return Ok(());
        },
        None => {
            stream.write(&[65, 65, 65, 65]);
            stream.flush();
            return Err(CacheError::with_reason(String::from("Could not read from cache")));
        }
    }
}

fn write_stream_str_to_cache(stream_str: String, cache_mtx: &Arc<Mutex<Cache>>) -> Result<(), CacheError>{
    let mut key: String = String::new();
    let mut val: String = String::new();
    let mut idx = 0;
    let mut chars_iter = stream_str.chars();
    for c in chars_iter {
        idx += 1;
        if c == ':' {
            val = String::from(&stream_str[idx..]);
            break;
        }
        key.push(c);
    }
    let bytes = val.into_bytes();
    let mut cache = cache_mtx.lock().unwrap();
    return cache.cache_item(key, bytes);
}

fn delete_value_from_cache(key: String, cache_mtx: &Arc<Mutex<Cache>>) -> Result<(), CacheError>{
    let mut cache = cache_mtx.lock().unwrap();
    return cache.delete_item(key);
}

fn launch_client( ip:String, port:String ){
    info!("Connection to {}:{}", ip, port);

    match TcpStream::connect(format!("{}:{}", ip, port).as_str()){
        Ok( mut stream ) => {
            loop {
                 let stdin = io::stdin();
                 for line_res in stdin.lock().lines() {
                    let mut line:String = line_res.unwrap();
                    if line.starts_with("write "){
                        let key_val = String::from(&line["write ".len()..line.len()]);
                        let delim_idx = key_val.find(" ").unwrap();
                        let key = &key_val[0..delim_idx];
                        let val = &key_val[delim_idx + 1..key_val.len()];
                        let sized_val = String::from( format!("W${}:{}", key, val));
                        let formatted = String::from(format!("{}|{}", sized_val.len(), sized_val));
                        stream.write(formatted.as_bytes());
                        stream.flush();
                    }else if line.starts_with("read "){
                        let key = String::from(&line["read ".len()..line.len()]);
                        let cmd_val = String::from( format!("R${}", key));
                        let sized_val = String::from( format!("{}|{}", cmd_val.len(), cmd_val));
                        stream.write( String::from(sized_val).as_bytes());
                        stream.flush();
                        let val:String = string_from_stream( &mut stream );
                        println!("{}", val);
                        io::stdout().flush();
                    }else if line.starts_with("delete "){

                    }
                }
            }
        },
        Err(e) => {
            panic!("Failed to connect to server");
        }
    }

}

fn launch_server( ip:String, port:String ){

    info!("Starting on {}:{}", ip, port);

    let listener: TcpListener = TcpListener::bind( format!( "{}:{}", ip, port).as_str() ).unwrap();

    let cache = Arc::new(Mutex::new(Cache::new()));

    loop{
        // accept connections and process them, spawning a new thread for each one
        for stream in listener.incoming() {
            let cache: Arc<Mutex<Cache>> = cache.clone();
            match stream {
                Ok(mut stream) => {
                    thread::spawn(move || {
                        info!("Incoming connection: {:?}", stream.peer_addr());
                        io::stdout().flush().unwrap();
                        loop {
                            handle_client(&mut stream, &cache);
                        }
                    });
                }
                Err(e) => { error! {"Incoming connection failed"} }
            }
        }
    }
}

fn main() {
    env_logger::init().unwrap();

    let mut args: Args = env::args();

    let mut ip:String = String::from("127.0.0.1");
    let mut port:String = String::from("8080");

    let mut mode:Mode = Mode::NONE;

    loop{
        match args.next().as_ref() {
            Some(opt)=>{
                match opt.as_ref(){
                    "server" => {
                        mode = Mode::SERVER;
                    },
                    "client" => {
                        mode = Mode::CLIENT;
                    }
                    "-port" => {
                        match args.next(){
                            Some(x) => port = x,
                            None => break
                        }
                    },
                    "-ip" =>{
                        match args.next(){
                            Some(x) => ip = x,
                            None => break
                        }
                    }
                    _ => continue
                }
            }
            None => break
        }
    }

    match mode {
        Mode::CLIENT => launch_client(ip, port),
        Mode::SERVER => launch_server(ip, port),
        Mode::NONE => panic!("Mode must be specified")
    }
}

<commit_msg>More client work<commit_after>#[macro_use] extern crate log;
extern crate env_logger;

use std::io::prelude::*;
use std::io;
use std::fs::File;
use std::collections::HashMap;
use std::string::String;
use std::vec::Vec;
use std::path::Path;
use std::net::{TcpListener, TcpStream};
use std::thread;
use std::result;
use std::fs;
use std::sync::{Arc, Mutex};
use std::env;
use std::error;
use std::env::Args;

enum Mode {
    NONE,
    CLIENT,
    SERVER
}

struct CacheError {
    reason: String
}

impl CacheError{
    pub fn with_reason( reason: String ) -> CacheError {
        return CacheError{
            reason: reason.clone()
        }
    }
}

struct CacheOperation {
    commands: Vec<char>,
    value: String
}

impl CacheOperation {
    fn new_from_string(cache_op_str: &String) -> CacheOperation {
        let mut cache_op = CacheOperation {
            commands: Vec::new(),
            value: String::new()
        };
        let mut idx = 0;
        let chars_iter = cache_op_str.chars();
        for c in chars_iter {
            idx += 1;
            if c == '$' {
                cache_op.value = String::from(&cache_op_str[idx..]);
                break;
            }
            cache_op.commands.push(c);
        }
        return cache_op;
    }
}

struct Cache {
    map_internal: HashMap<String, Vec<u8>>
}

impl Cache {
    fn new() -> Cache {
        return Cache {
            map_internal: HashMap::new()
        }
    }

    fn cache_item(&mut self, key: String, val: Vec<u8>) -> Result<(), CacheError>{
        debug!("Writing to cache: key={:?} val={:?}", key, val);
        fs::create_dir("_cache");
        let mut f: File = File::create(format!("_cache/{}", key)).unwrap();
        f.write(&val.as_slice());
        f.flush();
        self.map_internal.insert(key, val);
        return Ok(());
    }

    fn read_item(&self, key: String) -> Option<Box<Vec<u8>>> {
        let mut res: Option<Box<Vec<u8>>> = None;
        if self.map_internal.contains_key(&key) {
            let res_opt = self.map_internal.get(&key);
            match res_opt {
                Some(vec_ref) => res = Some(Box::new(vec_ref.clone())),
                None => res = None
            }
        } else {
            let mut buf: Vec<u8> = Vec::new();
            let mut f_res = match File::open(&key) {
                Err(why) => res = None,
                Ok(mut file) => {
                    file.read_to_end(&mut buf);
                    res = Some(Box::new(buf));
                }
            };
        }
        return res;
    }

    fn delete_item(&mut self, key: String) -> Result<(), CacheError>{
        if self.map_internal.contains_key(&key) {
            self.map_internal.remove(&key);
        }
        let path = format!("_cache/{}", key);
        if Path::new(&path).exists() {
            match fs::remove_file(path) {
                Ok(x) => x,
                Err(e) => return Err(CacheError::with_reason(format!("Failed to delete file for key: {}", key)))
            }
        }
        return Ok(())
    }
}

fn string_from_stream( stream: &mut TcpStream ) -> String {
    stream.set_nodelay(true);
    let mut buf_arr: [u8; 64] = [0; 64];
    stream.read(&mut buf_arr);

    let mut size_str = String::new();
    let mut buf_size: usize = 0;
    for i in 0..64 {
        buf_size += 1;
        if buf_arr[i] == '|' as u8 {
            break;
        }
        size_str.push(buf_arr[i as usize] as char);
    }

    let upper_idx: usize = size_str.parse::<i32>().unwrap() as usize;
    let msg_buf = buf_arr;
    let mut buf_temp: Vec<u8> = msg_buf.to_vec();
    let mut buf: Vec<u8> = buf_temp.drain(buf_size..upper_idx + buf_size).collect();

    stream.flush().unwrap();

    let buf_str: String = String::from_utf8(buf).unwrap();
    return buf_str;
}

fn handle_client(mut stream: &mut TcpStream, mut cache: &Arc<Mutex<Cache>>) -> Result<(), CacheError>{
    let buf_str: String = string_from_stream( stream );
    if buf_str.len() > 0 {
        let cache_op: CacheOperation = CacheOperation::new_from_string(&buf_str);

        debug!("cmds = {:?}, value = {}", cache_op.commands, cache_op.value);

        if cache_op.commands.len() > 0 {
            let prim_cmd: char = cache_op.commands[0];
            return match prim_cmd {
                'W' => write_stream_str_to_cache(cache_op.value, cache),
                'R' => read_value_from_cache(cache_op.value, cache, &mut stream),
                'D' => delete_value_from_cache(cache_op.value, cache),
                _ => Err(CacheError::with_reason(format!("Invalid cache command {:?}", prim_cmd)))
            }
        }
    } else {
        panic!("Empty cache operation");
    }
    return Ok(());
}

fn read_value_from_cache(key: String, cache_mtx: &Arc<Mutex<Cache>>, mut stream: &mut TcpStream) -> Result<(), CacheError> {
    let cache = cache_mtx. lock().unwrap();
    let cache_opt: Option<Box<Vec<u8>>> = cache.read_item(key);
    match cache_opt {
        Some(boxed_val) => {
            let mut val:Vec<u8> = *boxed_val;
            debug!("Writing to stream {:?}", val);
            write_str_to_stream_with_size( &mut stream, String::from_utf8( val ).unwrap());
            return Ok(());
        },
        None => {
            write_str_to_stream_with_size( &mut stream, String::from("Invalid Key"));
            return Err(CacheError::with_reason(String::from("Could not read from cache")));
        }
    }
}

fn write_stream_str_to_cache(stream_str: String, cache_mtx: &Arc<Mutex<Cache>>) -> Result<(), CacheError>{
    let mut key: String = String::new();
    let mut val: String = String::new();
    let mut idx = 0;
    let mut chars_iter = stream_str.chars();
    for c in chars_iter {
        idx += 1;
        if c == ':' {
            val = String::from(&stream_str[idx..]);
            break;
        }
        key.push(c);
    }
    let bytes = val.into_bytes();
    let mut cache = cache_mtx.lock().unwrap();
    return cache.cache_item(key, bytes);
}

fn delete_value_from_cache(key: String, cache_mtx: &Arc<Mutex<Cache>>) -> Result<(), CacheError>{
    let mut cache = cache_mtx.lock().unwrap();
    return cache.delete_item(key);
}

fn write_str_to_stream_with_size( stream:&mut TcpStream, value:String ){
    let sized_val = String::from( format!("{}|{}", value.len(), value));
    stream.write( String::from(sized_val).as_bytes());
    stream.flush();
}

fn launch_client( ip:String, port:String ){
    info!("Connection to {}:{}", ip, port);

    match TcpStream::connect(format!("{}:{}", ip, port).as_str()){
        Ok( mut stream ) => {
            loop {
                 let stdin = io::stdin();
                 for line_res in stdin.lock().lines() {
                    let mut line:String = line_res.unwrap();
                    if line.starts_with("write "){
                        let key_val = String::from(&line["write ".len()..line.len()]);
                        let delim_idx = key_val.find(" ").unwrap();
                        let key = &key_val[0..delim_idx];
                        let val = &key_val[delim_idx + 1..key_val.len()];
                        let sized_val = String::from( format!("W${}:{}", key, val));
                        write_str_to_stream_with_size( &mut stream, sized_val );
                    }else if line.starts_with("read "){
                        let key = String::from(&line["read ".len()..line.len()]);
                        let cmd_val = String::from( format!("R${}", key));
                        write_str_to_stream_with_size( &mut stream, cmd_val );
                        let val:String = string_from_stream( &mut stream );
                        println!("{}", val);
                        io::stdout().flush();
                    }else if line.starts_with("delete "){
                        let key = String::from(&line["delete ".len()..line.len()]);
                        let cmd_val = String::from( format!("D${}", key));
                        write_str_to_stream_with_size( &mut stream, cmd_val )
                    }
                }
            }
        },
        Err(e) => {
            panic!("Failed to connect to server");
        }
    }
}

fn launch_server( ip:String, port:String ){

    info!("Starting on {}:{}", ip, port);

    let listener: TcpListener = TcpListener::bind( format!( "{}:{}", ip, port).as_str() ).unwrap();

    let cache = Arc::new(Mutex::new(Cache::new()));

    loop{
        // accept connections and process them, spawning a new thread for each one
        for stream in listener.incoming() {
            let cache: Arc<Mutex<Cache>> = cache.clone();
            match stream {
                Ok(mut stream) => {
                    thread::spawn(move || {
                        info!("Incoming connection: {:?}", stream.peer_addr());
                        io::stdout().flush().unwrap();
                        loop {
                            handle_client(&mut stream, &cache);
                        }
                    });
                }
                Err(e) => { error! {"Incoming connection failed"} }
            }
        }
    }
}

fn main() {
    env_logger::init().unwrap();

    let mut args: Args = env::args();

    let mut ip:String = String::from("127.0.0.1");
    let mut port:String = String::from("8080");

    let mut mode:Mode = Mode::NONE;

    loop{
        match args.next().as_ref() {
            Some(opt)=>{
                match opt.as_ref(){
                    "server" => {
                        mode = Mode::SERVER;
                    },
                    "client" => {
                        mode = Mode::CLIENT;
                    }
                    "-port" => {
                        match args.next(){
                            Some(x) => port = x,
                            None => break
                        }
                    },
                    "-ip" =>{
                        match args.next(){
                            Some(x) => ip = x,
                            None => break
                        }
                    }
                    _ => continue
                }
            }
            None => break
        }
    }

    match mode {
        Mode::CLIENT => launch_client(ip, port),
        Mode::SERVER => launch_server(ip, port),
        Mode::NONE   => panic!("Mode must be specified")
    }
}

<|endoftext|>


<commit_before>extern crate docopt;
extern crate env_logger;
extern crate iron;
#[macro_use]
extern crate log;
extern crate portier_broker;
#[macro_use(router)]
extern crate router;
extern crate rustc_serialize;

use portier_broker as broker;
use docopt::Docopt;
use iron::Iron;
use std::str::FromStr;
use std::sync::Arc;

/// Defines the program's version, as set by Cargo at compile time.
const VERSION: &'static str = env!("CARGO_PKG_VERSION");

/// Defines the program's usage string.
///
/// [Docopt](http://docopt.org) parses this and generates a custom argv parser.
const USAGE: &'static str = r#"
Portier Broker

Usage:
  portier_broker CONFIG
  portier_broker --version
  portier_broker --help

Options:
  --version       Print version information and exit
  --help          Print this help message and exit
"#;

/// Holds parsed command line parameters.
#[derive(RustcDecodable)]
#[allow(non_snake_case)]
struct Args {
    arg_CONFIG: String,
}

/// The `main()` method. Will loop forever to serve HTTP requests.
fn main() {
    env_logger::init().unwrap();
    let args: Args = Docopt::new(USAGE)
                         .and_then(|d| d.version(Some(VERSION.to_string())).decode())
                         .unwrap_or_else(|e| e.exit());

    // Read the configuration from the provided file.
    let app = Arc::new(
        broker::AppConfig::from_json_file(&args.arg_CONFIG).unwrap()
    );

    let router = router!{
        // Human-targeted endpoints
        get "/" => broker::WelcomeHandler::new(&app),
        get "/.well-known/*" => broker::WellKnownHandler::new(&app),
        get "/confirm" => broker::ConfirmHandler::new(&app),

        // OpenID Connect provider endpoints
        get "/.well-known/openid-configuration" =>
               broker::OIDConfigHandler::new(&app),
        get "/keys.json" => broker::KeysHandler::new(&app),
        get "/auth" => broker::AuthHandler::new(&app),
        post "/auth" => broker::AuthHandler::new(&app),

        // OpenID Connect relying party endpoints
        get "/callback" => broker::CallbackHandler::new(&app),
    };

    let ipaddr = std::net::IpAddr::from_str(&app.listen_ip).unwrap();
    let socket = std::net::SocketAddr::new(ipaddr, app.listen_port);
    info!("listening on http://{}", socket);

    Iron::new(router).http(socket).unwrap();
}
<commit_msg>Fix docopt typo<commit_after>extern crate docopt;
extern crate env_logger;
extern crate iron;
#[macro_use]
extern crate log;
extern crate portier_broker;
#[macro_use(router)]
extern crate router;
extern crate rustc_serialize;

use portier_broker as broker;
use docopt::Docopt;
use iron::Iron;
use std::str::FromStr;
use std::sync::Arc;

/// Defines the program's version, as set by Cargo at compile time.
const VERSION: &'static str = env!("CARGO_PKG_VERSION");

/// Defines the program's usage string.
///
/// [Docopt](http://docopt.org) parses this and generates a custom argv parser.
const USAGE: &'static str = r#"
Portier Broker

Usage:
  portier-broker CONFIG
  portier-broker --version
  portier-broker --help

Options:
  --version       Print version information and exit
  --help          Print this help message and exit
"#;

/// Holds parsed command line parameters.
#[derive(RustcDecodable)]
#[allow(non_snake_case)]
struct Args {
    arg_CONFIG: String,
}

/// The `main()` method. Will loop forever to serve HTTP requests.
fn main() {
    env_logger::init().unwrap();
    let args: Args = Docopt::new(USAGE)
                         .and_then(|d| d.version(Some(VERSION.to_string())).decode())
                         .unwrap_or_else(|e| e.exit());

    // Read the configuration from the provided file.
    let app = Arc::new(
        broker::AppConfig::from_json_file(&args.arg_CONFIG).unwrap()
    );

    let router = router!{
        // Human-targeted endpoints
        get "/" => broker::WelcomeHandler::new(&app),
        get "/.well-known/*" => broker::WellKnownHandler::new(&app),
        get "/confirm" => broker::ConfirmHandler::new(&app),

        // OpenID Connect provider endpoints
        get "/.well-known/openid-configuration" =>
               broker::OIDConfigHandler::new(&app),
        get "/keys.json" => broker::KeysHandler::new(&app),
        get "/auth" => broker::AuthHandler::new(&app),
        post "/auth" => broker::AuthHandler::new(&app),

        // OpenID Connect relying party endpoints
        get "/callback" => broker::CallbackHandler::new(&app),
    };

    let ipaddr = std::net::IpAddr::from_str(&app.listen_ip).unwrap();
    let socket = std::net::SocketAddr::new(ipaddr, app.listen_port);
    info!("listening on http://{}", socket);

    Iron::new(router).http(socket).unwrap();
}
<|endoftext|>


<commit_before>use std::io;
use std::io::BufReader;
use std::io::BufRead;
use std::collections::HashMap;

mod request;
mod turn;
mod region;
mod superregion;

use request::Request;
use turn::Turn;
use region::Region;
use superregion::SuperRegion;

fn main()
{
	std::env::set_var("RUST_BACKTRACE","1");
	let input = io::stdin();
	let input = BufReader::new(input);
	let input = input.lines();
	let input = input
		.map(Result::unwrap)
		.filter(|s|s.len()>0)
		.map(|s|s.parse::<Request>())
		.map(Result::unwrap);

	let mut name_you = String::new();
	let mut name_other = String::new();

	let mut sup_regions: HashMap<usize,SuperRegion> = HashMap::new();
	let mut regions: HashMap<usize,Region> = HashMap::new();

	let mut armies_left = 0;

	for req in input
	{
		use request::Request::*;
		match req
		{
			ListSuperRegions(sregions) =>
			{
				sup_regions.extend(sregions.iter().map(|sr|(sr.id,sr.clone())));
			},
			ListRegions(regs) =>
			{
				regions.extend(regs.iter().map(|r|(r.id,r.clone())));
			},
			ListNeighbours(relations) =>
			{
				for (id,neighbours) in relations
				{
					regions.get_mut(&id).unwrap().neighbours.extend(neighbours.iter());
					for n in neighbours
					{
						regions.get_mut(&n).unwrap().neighbours.push(id);
					}
				}
			},
			RequestStartingRegions(_avail) =>
			{
				println!("give me randomly");
			},
			SettingNameYou(name) => name_you = name,
			SettingNameOther(name) => name_other = name,
			SettingArmies(count) =>
			{
				armies_left = count;
			},
			UpdateMap(updates) =>
			{
				for u in updates
				{
					let mut reg = regions.get_mut(&u.0).unwrap();
					reg.player = u.1;
					reg.count = u.2;
				}
			},
			TurnOther(_turns) =>
			{
				//TODO: what the hell?
			},
			TurnPlace =>
			{
				let mut v = Vec::new();
				{
					let regs = regions.values()
						.filter(|r|r.player == name_you)
						.filter(|r|r.neighbours.iter()
							.map(|o|regions.get(o))
							.map(Option::unwrap)
							.filter(|o|o.player == name_other)
							.any(|o|o.count >= r.count)
						).collect::<Vec<_>>();

					if armies_left > regs.len()
					{
						let count = if regs.len() > 0
						{
							armies_left/regs.len()
						}
						else
						{
							1
						};
						for r in regs.iter()
						{
							v.push(
								Turn::Place
								{
									name: name_you.clone(),
									region: r.id,
									count: count,
								}
							);
							armies_left -= count;
						}
						while armies_left > 0
						{
							for r in regions.values()
								.filter(|r|r.player == name_you)
							{
								if armies_left <= 0
								{
									break;
								}
								for n in regions.get(&r.id).unwrap().neighbours.iter()
								{
									if regions.get(&n).unwrap().player != name_you
									{
										v.push(
											Turn::Place
											{
												name: name_you.clone(),
												region: r.id,
												count: 1,
											}
										);
										armies_left -= 1;
										break;
									}
								}
							}
						}
					}
					if v.len() <= 0
					{
						v.push(Turn::Noop);
					}
					let v = v.iter()
						.filter(|t|
							if let &&Turn::Place{count,..} = t
							{
								count > 0
							}
							else
							{
								true
							}
						)
						.map(|t|format!("{}",t))
						.collect::<Vec<_>>();
					println!("{}",v.join(","));
				}
				for r in v.iter()
				{
					if let &Turn::Place{region,count,..} = r
					{
						regions.get_mut(&region).unwrap().count += count;
					}
				}
			},
			TurnArmies =>
			{
				let mut v = Vec::new();
				let regs = regions.values()
					.filter(|r|r.player == name_you)
					.filter_map(|r|
					{
						let x =r.neighbours.iter()
						.map(|o|regions.get(o))
						.map(Option::unwrap)
						.filter(|o|o.player != name_you)
						.filter(|o|o.count*5 <= r.count*2)
						.next()
						.map(|o|o.id);
						if let Some(x) = x
						{
							Some(((r.id,(r.count*2)/3),x))
						}
						else
						{
							None
						}
					})
					.collect::<Vec<_>>();
				for ((r,count),n) in regs
				{
					v.push(
						Turn::Turn
						{
							name: name_you.clone(),
							source: r,
							target: n,
							count: count,
						}
					);
				}
				if v.len() <= 0
				{
					v.push(Turn::Noop);
				}
				let v = v.iter().map(|t|format!("{}",t)).collect::<Vec<_>>();
				println!("{}",v.join(","));
			},
		}
	}
}

<commit_msg>fix halting<commit_after>use std::io;
use std::io::BufReader;
use std::io::BufRead;
use std::collections::HashMap;

mod request;
mod turn;
mod region;
mod superregion;

use request::Request;
use turn::Turn;
use region::Region;
use superregion::SuperRegion;

fn main()
{
	std::env::set_var("RUST_BACKTRACE","1");
	let input = io::stdin();
	let input = BufReader::new(input);
	let input = input.lines();
	let input = input
		.map(Result::unwrap)
		.filter(|s|s.len()>0)
		.map(|s|s.parse::<Request>())
		.map(Result::unwrap);

	let mut name_you = String::new();
	let mut name_other = String::new();

	let mut sup_regions: HashMap<usize,SuperRegion> = HashMap::new();
	let mut regions: HashMap<usize,Region> = HashMap::new();

	let mut armies_left = 0;

	for req in input
	{
		use request::Request::*;
		match req
		{
			ListSuperRegions(sregions) =>
			{
				sup_regions.extend(sregions.iter().map(|sr|(sr.id,sr.clone())));
			},
			ListRegions(regs) =>
			{
				regions.extend(regs.iter().map(|r|(r.id,r.clone())));
			},
			ListNeighbours(relations) =>
			{
				for (id,neighbours) in relations
				{
					regions.get_mut(&id).unwrap().neighbours.extend(neighbours.iter());
					for n in neighbours
					{
						regions.get_mut(&n).unwrap().neighbours.push(id);
					}
				}
			},
			RequestStartingRegions(_avail) =>
			{
				println!("give me randomly");
			},
			SettingNameYou(name) => name_you = name,
			SettingNameOther(name) => name_other = name,
			SettingArmies(count) =>
			{
				armies_left = count;
			},
			UpdateMap(updates) =>
			{
				for u in updates
				{
					let mut reg = regions.get_mut(&u.0).unwrap();
					reg.player = u.1;
					reg.count = u.2;
				}
			},
			TurnOther(_turns) =>
			{
				//TODO: what the hell?
			},
			TurnPlace =>
			{
				let mut v = Vec::new();
				{
					let regs = regions.values()
						.filter(|r|r.player == name_you)
						.filter(|r|r.neighbours.iter()
							.map(|o|regions.get(o))
							.map(Option::unwrap)
							.filter(|o|o.player == name_other)
							.any(|o|o.count >= r.count)
						)
						.take(3)
						.collect::<Vec<_>>();

					let count = if regs.len() > 0
					{
						armies_left/regs.len()
					}
					else
					{
						1
					};
					for r in regs.iter()
					{
						v.push(
							Turn::Place
							{
								name: name_you.clone(),
								region: r.id,
								count: count,
							}
						);
						armies_left -= count;
					}
					while armies_left > 0
					{
						for r in regions.values()
							.filter(|r|r.player == name_you)
						{
							if armies_left <= 0
							{
								break;
							}
							for n in regions.get(&r.id).unwrap().neighbours.iter()
							{
								if regions.get(&n).unwrap().player != name_you
								{
									v.push(
										Turn::Place
										{
											name: name_you.clone(),
											region: r.id,
											count: 1,
										}
									);
									armies_left -= 1;
									break;
								}
							}
						}
					}
				}
				if v.len() <= 0
				{
					v.push(Turn::Noop);
				}
				{
					let v = v.iter()
						.filter(|t|
							if let &&Turn::Place{count,..} = t
							{
								count > 0
							}
							else
							{
								true
							}
						)
						.map(|t|format!("{}",t))
						.collect::<Vec<_>>();
					println!("{}",v.join(","));
				}
				for r in v.iter()
				{
					if let &Turn::Place{region,count,..} = r
					{
						regions.get_mut(&region).unwrap().count += count;
					}
				}
			},
			TurnArmies =>
			{
				let mut v = Vec::new();
				let regs = regions.values()
					.filter(|r|r.player == name_you)
					.filter_map(|r|
					{
						let x =r.neighbours.iter()
						.map(|o|regions.get(o))
						.map(Option::unwrap)
						.filter(|o|o.player != name_you)
						.filter(|o|o.count*5 <= r.count*2)
						.next()
						.map(|o|o.id);
						if let Some(x) = x
						{
							Some(((r.id,(r.count*2)/3),x))
						}
						else
						{
							None
						}
					})
					.collect::<Vec<_>>();
				for ((r,count),n) in regs
				{
					v.push(
						Turn::Turn
						{
							name: name_you.clone(),
							source: r,
							target: n,
							count: count,
						}
					);
				}
				if v.len() <= 0
				{
					v.push(Turn::Noop);
				}
				let v = v.iter().map(|t|format!("{}",t)).collect::<Vec<_>>();
				println!("{}",v.join(","));
			},
		}
	}
}

<|endoftext|>


<commit_before>extern crate time;

use std::env;
use std::f32;
use std::ops::{Add, Sub, Mul, Div};
use std::fs::File;
use std::io::Read;

use time::PreciseTime;

#[derive(Copy, Clone, Debug)]
struct Measure {
    x: f32,
    y: f32,
}

impl Measure {
	pub fn new(x: f32, y: f32) -> Measure {
		Measure { x: x, y: y }
	}

	pub fn dist(&self, to: &Self) -> f32 {
		let a = self.x - to.x;
		let b = self.y - to.y;
		f32::sqrt(a * a + b * b)
	}
}

impl Add for Measure {
	type Output = Measure;

	fn add(self, rhs: Measure) -> Measure {
		Measure::new(self.x + rhs.x, self.y + rhs.y)
	}
}

impl Sub for Measure {
	type Output = Measure;

	fn sub(self, rhs: Measure) -> Measure {
		Measure::new(self.x - rhs.x, self.y - rhs.y)
	}
}

impl Mul<f32> for Measure {
    type Output = Measure;

    fn mul(self, rhs: f32) -> Measure {
        Measure::new(self.x * rhs, self.y * rhs)
    }
}

impl Div<f32> for Measure {
    type Output = Measure;

    fn div(self, rhs: f32) -> Measure {
        Measure::new(self.x / rhs, self.y / rhs)
    }
}

#[test]
fn test_measure() {
    let a = Measure::new(1.0, 3.0);
    let b = Measure::new(4.0, 5.0);
    let add = a + b;
    assert!(add.x == 5.0 && add.y == 8.0);
    let dif = a - b;
    assert!(dif.x == -3.0 && dif.y == -2.0);
    let mlt = a * 3.0;
    assert!(mlt.x == 3.0 && mlt.y == 9.0);
    let div = b / 2.0;
    assert!(div.x == 2.0 && div.y == 2.5);
}

#[derive(Clone, Debug)]
struct Class {
    mean: Measure,
    n: usize,
}

impl Class {
	pub fn new(m: &Measure) -> Class {
		Class { mean: m.clone(), n: 1 }
	}

	pub fn append(&mut self, m: &Measure) {
		self.mean = ( self.mean * self.n as f32 + *m ) / ( self.n + 1 ) as f32;
		self.n += 1;
	}

	pub fn merge(&mut self, m: &Class) {
		self.mean = (self.mean * self.n as f32 + m.mean * m.n as f32) / (self.n + m.n) as f32;
        self.n += m.n;
	}
}

#[test]
fn test_class() {
    let mut cls = Class::new(&Measure::new(1.0, 2.0));
    assert!(cls.mean.x == 1.0 && cls.mean.y == 2.0);
    assert!(cls.n == 1);
    cls.append(&Measure::new(3.0, 4.0));
    assert!(cls.mean.x == 2.0 && cls.mean.y == 3.0);
    assert!(cls.n == 2);
}

#[derive(Debug)]
struct Classifer {
    list: Vec<Box<Class>>,
    ncls_dist: f32,
}

impl Classifer {
	pub fn new(mdist: f32) -> Classifer {
		Classifer { ncls_dist: mdist, list: Vec::new() }
	}

	pub fn classificate(&mut self, m: &Measure) {
		if self.list.len() > 0 {
			let mut min_dist = std::f32::MAX;
			let mut near_cls = self.list[0].clone();

			for i in self.list.iter() {
				let d = m.dist(&i.mean);
				if d < min_dist {
					min_dist = d;
					near_cls = i.clone();
				}
			}

			if min_dist < self.ncls_dist {
				near_cls.append(m);
			} else {
				self.list.push(Box::new(Class::new(m)));
			}
		} else {
			self.list.push(Box::new(Class::new(m)));
		}
	}

	pub fn merge_classes(&mut self) {
		let mut uniq: Vec<Box<Class>> = Vec::new();

		for cls in self.list.iter() {
			let mut is_uniq = true;

			for trg in uniq.iter_mut() {
				if cls.mean.dist(&trg.mean) < self.ncls_dist {
                    trg.merge(cls);
                    is_uniq = false;
                }

				if !is_uniq {
					break;
				}
			}

			if is_uniq {
				uniq.push(cls.clone());
			}
		}

		self.list = uniq;
	}
}

fn read_measures_from_file(path: &str) -> Vec<Measure> {
	let mut res = Vec::new();

	let mut file = File::open(path).unwrap();
	let mut text = String::new();
	assert!(file.read_to_string(&mut text).is_ok());

	for line in text.lines() {
		let values: Vec<&str> = line.split_whitespace().take(2).collect();
		let x = values[0].parse::<f32>().unwrap();
		let y = values[1].parse::<f32>().unwrap();
		res.push(Measure::new(x, y));
	}

	return res;
}

fn main() {
	let mut args = env::args();
    let measures = read_measures_from_file( &args.nth(1).expect("Need argument: `input file`") );

	let start = PreciseTime::now();

	let mut clsf = Classifer::new(3.0);

	for i in measures.iter() {
		clsf.classificate(i);
	}

	if cfg!(feature = "merge") {
		clsf.merge_classes();
	}

	let duration = start.to(PreciseTime::now());
	println!("work time: {}", duration);

	for i in clsf.list.iter() {
		println!("[{}, {}]: {}", i.mean.x, i.mean.y, i.n);
	}
}
<commit_msg>Change tabs to spaces<commit_after>extern crate time;

use std::env;
use std::f32;
use std::ops::{Add, Sub, Mul, Div};
use std::fs::File;
use std::io::Read;

use time::PreciseTime;

#[derive(Copy, Clone, Debug)]
struct Measure {
    x: f32,
    y: f32,
}

impl Measure {
    pub fn new(x: f32, y: f32) -> Measure {
        Measure { x: x, y: y }
    }

    pub fn dist(&self, to: &Self) -> f32 {
        let a = self.x - to.x;
        let b = self.y - to.y;
        f32::sqrt(a * a + b * b)
    }
}

impl Add for Measure {
    type Output = Measure;

    fn add(self, rhs: Measure) -> Measure {
        Measure::new(self.x + rhs.x, self.y + rhs.y)
    }
}

impl Sub for Measure {
    type Output = Measure;

    fn sub(self, rhs: Measure) -> Measure {
        Measure::new(self.x - rhs.x, self.y - rhs.y)
    }
}

impl Mul<f32> for Measure {
    type Output = Measure;

    fn mul(self, rhs: f32) -> Measure {
        Measure::new(self.x * rhs, self.y * rhs)
    }
}

impl Div<f32> for Measure {
    type Output = Measure;

    fn div(self, rhs: f32) -> Measure {
        Measure::new(self.x / rhs, self.y / rhs)
    }
}

#[test]
fn test_measure() {
    let a = Measure::new(1.0, 3.0);
    let b = Measure::new(4.0, 5.0);
    let add = a + b;
    assert!(add.x == 5.0 && add.y == 8.0);
    let dif = a - b;
    assert!(dif.x == -3.0 && dif.y == -2.0);
    let mlt = a * 3.0;
    assert!(mlt.x == 3.0 && mlt.y == 9.0);
    let div = b / 2.0;
    assert!(div.x == 2.0 && div.y == 2.5);
}

#[derive(Clone, Debug)]
struct Class {
    mean: Measure,
    n: usize,
}

impl Class {
    pub fn new(m: &Measure) -> Class {
        Class { mean: m.clone(), n: 1 }
    }

    pub fn append(&mut self, m: &Measure) {
        self.mean = ( self.mean * self.n as f32 + *m ) / ( self.n + 1 ) as f32;
        self.n += 1;
    }

    pub fn merge(&mut self, m: &Class) {
        self.mean = (self.mean * self.n as f32 + m.mean * m.n as f32) / (self.n + m.n) as f32;
        self.n += m.n;
    }
}

#[test]
fn test_class() {
    let mut cls = Class::new(&Measure::new(1.0, 2.0));
    assert!(cls.mean.x == 1.0 && cls.mean.y == 2.0);
    assert!(cls.n == 1);
    cls.append(&Measure::new(3.0, 4.0));
    assert!(cls.mean.x == 2.0 && cls.mean.y == 3.0);
    assert!(cls.n == 2);
}

#[derive(Debug)]
struct Classifer {
    list: Vec<Box<Class>>,
    ncls_dist: f32,
}

impl Classifer {
    pub fn new(mdist: f32) -> Classifer {
        Classifer { ncls_dist: mdist, list: Vec::new() }
    }

    pub fn classificate(&mut self, m: &Measure) {
        if self.list.len() > 0 {
            let mut min_dist = std::f32::MAX;
            let mut near_cls = self.list[0].clone();

            for i in self.list.iter() {
                let d = m.dist(&i.mean);
                if d < min_dist {
                    min_dist = d;
                    near_cls = i.clone();
                }
            }

            if min_dist < self.ncls_dist {
                near_cls.append(m);
            } else {
                self.list.push(Box::new(Class::new(m)));
            }
        } else {
            self.list.push(Box::new(Class::new(m)));
        }
    }

    pub fn merge_classes(&mut self) {
        let mut uniq: Vec<Box<Class>> = Vec::new();

        for cls in self.list.iter() {
            let mut is_uniq = true;

            for trg in uniq.iter_mut() {
                if cls.mean.dist(&trg.mean) < self.ncls_dist {
                    trg.merge(cls);
                    is_uniq = false;
                }

                if !is_uniq {
                    break;
                }
            }

            if is_uniq {
                uniq.push(cls.clone());
            }
        }

        self.list = uniq;
    }
}

fn read_measures_from_file(path: &str) -> Vec<Measure> {
    let mut res = Vec::new();

    let mut file = File::open(path).unwrap();
    let mut text = String::new();
    assert!(file.read_to_string(&mut text).is_ok());

    for line in text.lines() {
        let values: Vec<&str> = line.split_whitespace().take(2).collect();
        let x = values[0].parse::<f32>().unwrap();
        let y = values[1].parse::<f32>().unwrap();
        res.push(Measure::new(x, y));
    }

    return res;
}

fn main() {
    let mut args = env::args();
    let measures = read_measures_from_file( &args.nth(1).expect("Need argument: `input file`") );

    let start = PreciseTime::now();

    let mut clsf = Classifer::new(3.0);

    for i in measures.iter() {
        clsf.classificate(i);
    }

    if cfg!(feature = "merge") {
        clsf.merge_classes();
    }

    let duration = start.to(PreciseTime::now());
    println!("work time: {}", duration);

    for i in clsf.list.iter() {
        println!("[{}, {}]: {}", i.mean.x, i.mean.y, i.n);
    }
}
<|endoftext|>


<commit_before>pub trait IteratorCloneEach<'a, E, It> where E: Clone, It: Iterator<&'a E> {
    /**
Creates an iterator which will clone each element of the input iterator.
    */
    fn clone_each(self) -> CloneItems<It>;
}

impl<'a, E, It> IteratorCloneEach<'a, E, It> for It where E: Clone, It: Iterator<&'a E> {
    fn clone_each(self) -> CloneItems<It> {
        CloneItems {
            iter: self,
        }
    }
}

pub struct CloneItems<It> {
    iter: It,
}

impl<'a, E, It> Iterator<E> for CloneItems<It> where E: 'a+Clone, It: Iterator<&'a E> {
    fn next(&mut self) -> Option<E> {
        match self.iter.next() {
            None => None,
            Some(e) => Some(e.clone())
        }
    }
}

#[test]
fn test_clone_each() {
    let it: Vec<int> = vec![1, 2, 3];
    let mut it = it.iter().clone_each();
    assert_eq!(it.next(), Some(1));
    assert_eq!(it.next(), Some(2));
    assert_eq!(it.next(), Some(3));
    assert_eq!(it.next(), None);
}

pub trait IteratorFoldl<E> {
    /**
Folds the elements of the iterator together, from left to right, using `f`.

Returns `None` if the iterator is empty.
    */
    fn foldl(self, f: |E, E| -> E) -> Option<E>;
}

impl<It, E> IteratorFoldl<E> for It where It: Iterator<E> {
    fn foldl(mut self, f: |E, E| -> E) -> Option<E> {
        let first = match self.next() {
            None => return None,
            Some(e) => e
        };

        Some(self.fold(first, f))
    }
}

#[test]
fn test_foldl() {
    let vs = vec!["a", "b", "c"];
    let vs = vs.into_iter().map(|e| e.into_string());
    assert_eq!(Some("((a, b), c)".into_string()), vs.foldl(|a,b| format!("({}, {})", a, b)));
}

pub trait IteratorFoldr<E> {
    /**
Folds the elements of the iterator together, from right to left, using `f`.

Returns `None` if the iterator is empty.
    */
    fn foldr(self, f: |E, E| -> E) -> Option<E>;
}

impl<It, E> IteratorFoldr<E> for It where It: DoubleEndedIterator<E> {
    fn foldr(mut self, f: |E, E| -> E) -> Option<E> {
        let mut last = match self.next_back() {
            None => return None,
            Some(e) => e
        };

        loop {
            match self.next_back() {
                None => break,
                Some(e) => last = f(e, last)
            }
        }

        Some(last)
    }
}

#[test]
fn test_foldr() {
    let vs = vec!["a", "b", "c"];
    let vs = vs.into_iter().map(|e| e.into_string());
    assert_eq!(Some("(a, (b, c))".into_string()), vs.foldr(|a,b| format!("({}, {})", a, b)));
}
<commit_msg>Added stride.<commit_after>pub trait IteratorCloneEach<'a, E, It> where E: Clone, It: Iterator<&'a E> {
    /**
Creates an iterator which will clone each element of the input iterator.
    */
    fn clone_each(self) -> CloneItems<It>;
}

impl<'a, E, It> IteratorCloneEach<'a, E, It> for It where E: Clone, It: Iterator<&'a E> {
    fn clone_each(self) -> CloneItems<It> {
        CloneItems {
            iter: self,
        }
    }
}

pub struct CloneItems<It> {
    iter: It,
}

impl<'a, E, It> Iterator<E> for CloneItems<It> where E: 'a+Clone, It: Iterator<&'a E> {
    fn next(&mut self) -> Option<E> {
        match self.iter.next() {
            None => None,
            Some(e) => Some(e.clone())
        }
    }
}

#[test]
fn test_clone_each() {
    let it: Vec<int> = vec![1, 2, 3];
    let mut it = it.iter().clone_each();
    assert_eq!(it.next(), Some(1));
    assert_eq!(it.next(), Some(2));
    assert_eq!(it.next(), Some(3));
    assert_eq!(it.next(), None);
}

pub trait IteratorFoldl<E> {
    /**
Folds the elements of the iterator together, from left to right, using `f`.

Returns `None` if the iterator is empty.
    */
    fn foldl(self, f: |E, E| -> E) -> Option<E>;
}

impl<It, E> IteratorFoldl<E> for It where It: Iterator<E> {
    fn foldl(mut self, f: |E, E| -> E) -> Option<E> {
        let first = match self.next() {
            None => return None,
            Some(e) => e
        };

        Some(self.fold(first, f))
    }
}

#[test]
fn test_foldl() {
    let vs = vec!["a", "b", "c"];
    let vs = vs.into_iter().map(|e| e.into_string());
    assert_eq!(Some("((a, b), c)".into_string()), vs.foldl(|a,b| format!("({}, {})", a, b)));
}

pub trait IteratorFoldr<E> {
    /**
Folds the elements of the iterator together, from right to left, using `f`.

Returns `None` if the iterator is empty.
    */
    fn foldr(self, f: |E, E| -> E) -> Option<E>;
}

impl<It, E> IteratorFoldr<E> for It where It: DoubleEndedIterator<E> {
    fn foldr(mut self, f: |E, E| -> E) -> Option<E> {
        let mut last = match self.next_back() {
            None => return None,
            Some(e) => e
        };

        loop {
            match self.next_back() {
                None => break,
                Some(e) => last = f(e, last)
            }
        }

        Some(last)
    }
}

#[test]
fn test_foldr() {
    let vs = vec!["a", "b", "c"];
    let vs = vs.into_iter().map(|e| e.into_string());
    assert_eq!(Some("(a, (b, c))".into_string()), vs.foldr(|a,b| format!("({}, {})", a, b)));
}

pub trait IteratorStride<E, It> where It: Iterator<E> {
    /**
Creates an iterator which yields every `n`th element of the input iterator, including the first.
    */
    fn stride(self, n: uint) -> StrideItems<It>;
}

impl<E, It> IteratorStride<E, It> for It where It: Iterator<E> {
    fn stride(self, n: uint) -> StrideItems<It> {
        StrideItems {
            iter: self,
            stride: n
        }
    }
}

pub struct StrideItems<It> {
    iter: It,
    stride: uint,
}

impl<E, It> Iterator<E> for StrideItems<It> where It: Iterator<E> {
    fn next(&mut self) -> Option<E> {
        let v = match self.iter.next() {
            Some(v) => v,
            None => return None
        };

        for _ in range(0, self.stride - 1) {
            match self.iter.next() {
                None => break,
                _ => ()
            }
        }

        Some(v)
    }
}

#[test]
fn test_stride() {
    let v = vec![0i, 1, 2, 3, 4, 5, 6, 7, 8, 9];
    let mut it = v.iter().clone_each().stride(2);
    assert_eq!(it.next(), Some(0));
    assert_eq!(it.next(), Some(2));
    assert_eq!(it.next(), Some(4));
    assert_eq!(it.next(), Some(6));
    assert_eq!(it.next(), Some(8));
    assert_eq!(it.next(), None);
}
<|endoftext|>


<commit_before>use std::io::prelude::*;
use std::fs::File;

use cpu::Cpu;
use util;

#[derive(Clone, Debug)]
pub struct Node {
	id: usize,
	cpus: Vec<Cpu>,
	cpumask: Vec<bool>,
	mem_free: u64,
	mem_total: u64,
	mem_used: u64,
	hugepages_2m_free: usize,
	hugepages_2m_total: usize,
	hugepages_1g_free: usize,
	hugepages_1g_total: usize,
}

impl Node {
	pub fn new(id: usize) -> Result<Node, &'static str> {
		let node = Node {
			id: id,
			cpus: Vec::new(),
			cpumask: Vec::new(),
			mem_free: 0, // TODO: implement
			mem_total: 0, // TODO: implement
			mem_used: 0, // TODO: implement
			hugepages_1g_total: 0,
			hugepages_1g_free: 0,
			hugepages_2m_total: 0,
			hugepages_2m_free: 0,
		};
		node.init()
	}

	fn init(mut self) -> Result<Self, &'static str> {
		if let Ok(cpumask) = util::bitmask_from_hex_file(format!("/sys/devices/system/node/node{}/cpumap", self.id)) {
			self.cpumask = cpumask.clone();
			for i in 0..cpumask.len() {
				if cpumask[i] {
					let id = 2u32.pow(i as u32) as usize;
					if let Ok(mut cpu) = Cpu::new(id) {
						cpu.set_node_id(self.id);
						self.cpus.push(cpu);
					} else {
						break;
					}
				}
			}
			self.mem_init();
			Ok(self)
		} else {
			Err("no node found")
		}
	}

	fn mem_init(&mut self) {
		let path = format!("/sys/devices/system/node/node{}/meminfo", self.id);
		if let Ok(mut f) = File::open(&path) {
			let mut s = String::new();
			if let Ok(_) = f.read_to_string(&mut s) {
				for l in s.lines() {
					let tokens: Vec<&str> = l.split_whitespace().collect();
					match tokens[2] {
						"MemTotal:" => {
							self.mem_total = tokens[3].parse().unwrap_or(0);
						},
						"MemUsed:" => {
							self.mem_used = tokens[3].parse().unwrap_or(0);
						},
						"MemFree:" => {
							self.mem_free = tokens[3].parse().unwrap_or(0);
						},
						_ => {}
					}
				}
			}
		}
		self.hugepages_2m_free = util::usize_from_file(format!("/sys/devices/system/node/node{}/hugepages/hugepages-2048kB/free_hugepages", self.id)).unwrap_or(0);
		self.hugepages_2m_total = util::usize_from_file(format!("/sys/devices/system/node/node{}/hugepages/hugepages-2048kB/nr_hugepages", self.id)).unwrap_or(0);
		self.hugepages_1g_free = util::usize_from_file(format!("/sys/devices/system/node/node{}/hugepages/hugepages-1048576kB/free_hugepages", self.id)).unwrap_or(0);
		self.hugepages_1g_total = util::usize_from_file(format!("/sys/devices/system/node/node{}/hugepages/hugepages-1048576kB/nr_hugepages", self.id)).unwrap_or(0);
	}

	pub fn id(&self) -> usize {
		self.id
	}

	pub fn cpus(&self) -> &Vec<Cpu> {
		&self.cpus
	}

	pub fn cpumask(&self) -> &Vec<bool> {
		&self.cpumask
	}

	pub fn mem_free(&self) -> u64 {
		self.mem_free
	}

	pub fn mem_total(&self) -> u64 {
		self.mem_total
	}

	pub fn mem_used(&self) -> u64 {
		self.mem_used
	}

	pub fn hugepages_2m_free(&self) -> usize {
		self.hugepages_2m_free
	}

	pub fn hugepages_2m_total(&self) -> usize {
		self.hugepages_2m_total
	}

	pub fn hugepages_1g_free(&self) -> usize {
		self.hugepages_1g_free
	}

	pub fn hugepages_1g_total(&self) -> usize {
		self.hugepages_1g_total
	}
}
<commit_msg>bugfix in cpu population<commit_after>use std::io::prelude::*;
use std::fs::File;

use cpu::Cpu;
use util;

#[derive(Clone, Debug)]
pub struct Node {
	id: usize,
	cpus: Vec<Cpu>,
	cpumask: Vec<bool>,
	mem_free: u64,
	mem_total: u64,
	mem_used: u64,
	hugepages_2m_free: usize,
	hugepages_2m_total: usize,
	hugepages_1g_free: usize,
	hugepages_1g_total: usize,
}

impl Node {
	pub fn new(id: usize) -> Result<Node, &'static str> {
		let node = Node {
			id: id,
			cpus: Vec::new(),
			cpumask: Vec::new(),
			mem_free: 0, // TODO: implement
			mem_total: 0, // TODO: implement
			mem_used: 0, // TODO: implement
			hugepages_1g_total: 0,
			hugepages_1g_free: 0,
			hugepages_2m_total: 0,
			hugepages_2m_free: 0,
		};
		node.init()
	}

	fn init(mut self) -> Result<Self, &'static str> {
		if let Ok(cpumask) = util::bitmask_from_hex_file(format!("/sys/devices/system/node/node{}/cpumap", self.id)) {
			self.cpumask = cpumask.clone();
			for i in 0..cpumask.len() {
				if cpumask[i] {
					let id = 2u32.pow(i as u32) as usize - 1;
					if let Ok(mut cpu) = Cpu::new(id) {
						cpu.set_node_id(self.id);
						self.cpus.push(cpu);
					} else {
						break;
					}
				}
			}
			self.mem_init();
			Ok(self)
		} else {
			Err("no node found")
		}
	}

	fn mem_init(&mut self) {
		let path = format!("/sys/devices/system/node/node{}/meminfo", self.id);
		if let Ok(mut f) = File::open(&path) {
			let mut s = String::new();
			if let Ok(_) = f.read_to_string(&mut s) {
				for l in s.lines() {
					let tokens: Vec<&str> = l.split_whitespace().collect();
					match tokens[2] {
						"MemTotal:" => {
							self.mem_total = tokens[3].parse().unwrap_or(0);
						},
						"MemUsed:" => {
							self.mem_used = tokens[3].parse().unwrap_or(0);
						},
						"MemFree:" => {
							self.mem_free = tokens[3].parse().unwrap_or(0);
						},
						_ => {}
					}
				}
			}
		}
		self.hugepages_2m_free = util::usize_from_file(format!("/sys/devices/system/node/node{}/hugepages/hugepages-2048kB/free_hugepages", self.id)).unwrap_or(0);
		self.hugepages_2m_total = util::usize_from_file(format!("/sys/devices/system/node/node{}/hugepages/hugepages-2048kB/nr_hugepages", self.id)).unwrap_or(0);
		self.hugepages_1g_free = util::usize_from_file(format!("/sys/devices/system/node/node{}/hugepages/hugepages-1048576kB/free_hugepages", self.id)).unwrap_or(0);
		self.hugepages_1g_total = util::usize_from_file(format!("/sys/devices/system/node/node{}/hugepages/hugepages-1048576kB/nr_hugepages", self.id)).unwrap_or(0);
	}

	pub fn id(&self) -> usize {
		self.id
	}

	pub fn cpus(&self) -> &Vec<Cpu> {
		&self.cpus
	}

	pub fn cpumask(&self) -> &Vec<bool> {
		&self.cpumask
	}

	pub fn mem_free(&self) -> u64 {
		self.mem_free
	}

	pub fn mem_total(&self) -> u64 {
		self.mem_total
	}

	pub fn mem_used(&self) -> u64 {
		self.mem_used
	}

	pub fn hugepages_2m_free(&self) -> usize {
		self.hugepages_2m_free
	}

	pub fn hugepages_2m_total(&self) -> usize {
		self.hugepages_2m_total
	}

	pub fn hugepages_1g_free(&self) -> usize {
		self.hugepages_1g_free
	}

	pub fn hugepages_1g_total(&self) -> usize {
		self.hugepages_1g_total
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Alex Crawford
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#[macro_use]
extern crate clap;
extern crate hyper;
extern crate pulldown_cmark;
extern crate url;
extern crate walkdir;

use clap::{Arg, App};
use hyper::client::Client;
use hyper::header::UserAgent;
use hyper::status::StatusCode;
use pulldown_cmark::{Event, OPTION_ENABLE_TABLES, Parser, Tag};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs::File;
use std::io::{Read, Write};
use std::path::Path;
use std::process::exit;
use url::{ParseError, Url};
use walkdir::WalkDir;

macro_rules! fail {
    ($($print:expr),+) => {
        {
            let _ = writeln!(&mut std::io::stderr(), $($print),+);
            exit(2);
        }
    };
}

macro_rules! print_issue {
    ($flag:expr, $($print:expr),+) => {
        {
            println!($($print),+);
            $flag = true;
        }
    };
}

struct State<'a> {
    skip_http: bool,
    code_block: bool,
    last_text: String,
    directory: &'a Path,
}

enum LinkError {
    PathAbsolute,
    PathNonExistant,
    HttpStatus(StatusCode),
    HttpError(hyper::error::Error),
    UrlMalformed(ParseError),
}

fn main() {
    let matches = App::new("Marker")
        .version(crate_version!())
        .arg(Arg::with_name("root")
            .short("r")
            .long("root")
            .help("The path to the root of the documentation to be checked")
            .takes_value(true))
        .arg(Arg::with_name("skip-http")
            .long("skip-http")
            .help("Skip validation of HTTP[S] URLs"))
        .get_matches();

    let skip_http = matches.is_present("skip-http");
    let root = Path::new(matches.value_of("root").unwrap_or("."));
    let mut links = HashMap::new();
    let mut found_issue = false;

    for entry in WalkDir::new(root).into_iter().filter_map(|entry| {
        let entry = entry.unwrap_or_else(|err| {
            fail!("Failed to walk directory: {}", err);
        });
        if entry.file_type().is_file() && entry.path().extension() == Some(OsStr::new("md")) {
            Some(entry)
        } else {
            None
        }
    }) {
        let contents = {
            let mut file = File::open(entry.path()).unwrap_or_else(|err| {
                fail!("Failed to open file ({}): {}", entry.path().display(), err);
            });
            let mut text = String::new();
            if let Err(err) = file.read_to_string(&mut text) {
                fail!("Failed to read file ({}): {}", entry.path().display(), err);
            };
            text
        };

        let newlines = contents.match_indices('\n').map(|(i, _)| i).collect::<Vec<_>>();

        let mut state = State {
            skip_http: skip_http,
            code_block: false,
            last_text: String::new(),
            directory: entry.path().parent().expect("non-root path"),
        };
        let mut parser = Parser::new_ext(&contents, OPTION_ENABLE_TABLES);
        while let Some(event) = parser.next() {
            let line = newlines.iter().take_while(|i| i <= &&parser.get_offset()).count() + 1;
            match event {
                Event::Text(ref text) => {
                    state.last_text = text.to_string();
                    if let Some(reference) = try_reference(&state, text) {
                        print_issue!(found_issue,
                                     "Found broken reference ({}) in {}:{}",
                                     reference,
                                     entry.path().display(),
                                     line)
                    }
                }
                Event::End(Tag::Link(dest, _)) => {
                    match check_link(&mut state, &mut links, &dest) {
                        Ok(()) => {}
                        Err(LinkError::PathAbsolute) => {
                            print_issue!(found_issue,
                                         "Found absolute path    ({} -> {}) at {}:{}",
                                         state.last_text,
                                         dest,
                                         entry.path().display(),
                                         line)
                        }
                        Err(LinkError::PathNonExistant) => {
                            print_issue!(found_issue,
                                         "Found broken path      ({} -> {}) at {}:{}",
                                         state.last_text,
                                         dest,
                                         entry.path().display(),
                                         line)
                        }
                        Err(LinkError::HttpStatus(status)) => {
                            print_issue!(found_issue,
                                         "Found broken url       ({} -> {}) at {}:{} : {}",
                                         state.last_text,
                                         dest,
                                         entry.path().display(),
                                         line,
                                         status)
                        }
                        Err(LinkError::HttpError(err)) => {
                            print_issue!(found_issue,
                                         "HTTP failure           ({} -> {}) at {}:{} : {}",
                                         state.last_text,
                                         dest,
                                         entry.path().display(),
                                         line,
                                         err);
                        }
                        Err(LinkError::UrlMalformed(err)) => {
                            print_issue!(found_issue,
                                         "Found malformed URL    ({} -> {}) at {}:{} : {}",
                                         state.last_text,
                                         dest,
                                         entry.path().display(),
                                         line,
                                         err);
                        }
                    }
                }
                Event::Start(Tag::Code) |
                Event::Start(Tag::CodeBlock(_)) => state.code_block = true,
                Event::End(Tag::Code) |
                Event::End(Tag::CodeBlock(_)) => state.code_block = false,
                _ => {}
            }
        }
    }

    if found_issue {
        exit(1)
    }
}

fn try_reference<'a>(state: &State, text: &'a str) -> Option<&'a str> {
    if !state.code_block && text.chars().next() == Some('[') && text.chars().last() == Some(']') {
        Some(&text[1..(text.len() - 1)])
    } else {
        None
    }
}

fn check_link(state: &mut State,
              links: &mut HashMap<Url, StatusCode>,
              dest: &str)
              -> Result<(), LinkError> {
    match (Url::parse(&dest), state.skip_http) {
        (Ok(url), false) => check_url(links, url),
        (Ok(_), true) => Ok(()),
        (Err(ParseError::RelativeUrlWithoutBase), _) => check_path(state, dest),
        (Err(err), _) => Err(LinkError::UrlMalformed(err)),
    }
}

fn check_url(links: &mut HashMap<Url, StatusCode>, url: Url) -> Result<(), LinkError> {
    if url.scheme() != "http" && url.scheme() != "https" {
        return Ok(());
    }

    let client = Client::new();
    let agent = UserAgent(format!("marker/{}", crate_version!()));

    let status = if let Some(status) = links.remove(&url) {
        status
    } else {
        let res = client.head(url.clone()).header(agent.clone()).send().and_then(|resp| {
            if resp.status == StatusCode::MethodNotAllowed {
                client.get(url.clone()).header(agent.clone()).send()
            } else {
                Ok(resp)
            }
        });
        match res {
            Ok(resp) => resp.status,
            Err(err) => return Err(LinkError::HttpError(err)),
        }
    };
    links.insert(url, status);
    match status {
        StatusCode::Ok => Ok(()),
        status => Err(LinkError::HttpStatus(status)),
    }
}

fn check_path(state: &mut State, path: &str) -> Result<(), LinkError> {
    let path = Path::new(OsStr::new(path.split('#').next().expect("string")));
    if path.is_absolute() {
        Err(LinkError::PathAbsolute)
    } else if !state.directory.join(path).exists() {
        Err(LinkError::PathNonExistant)
    } else {
        Ok(())
    }
}
<commit_msg>main: fix reported line for links that start line<commit_after>// Copyright 2016 Alex Crawford
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#[macro_use]
extern crate clap;
extern crate hyper;
extern crate pulldown_cmark;
extern crate url;
extern crate walkdir;

use clap::{Arg, App};
use hyper::client::Client;
use hyper::header::UserAgent;
use hyper::status::StatusCode;
use pulldown_cmark::{Event, OPTION_ENABLE_TABLES, Parser, Tag};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs::File;
use std::io::{Read, Write};
use std::path::Path;
use std::process::exit;
use url::{ParseError, Url};
use walkdir::WalkDir;

macro_rules! fail {
    ($($print:expr),+) => {
        {
            let _ = writeln!(&mut std::io::stderr(), $($print),+);
            exit(2);
        }
    };
}

macro_rules! print_issue {
    ($flag:expr, $($print:expr),+) => {
        {
            println!($($print),+);
            $flag = true;
        }
    };
}

struct State<'a> {
    skip_http: bool,
    code_block: bool,
    last_text: String,
    directory: &'a Path,
}

enum LinkError {
    PathAbsolute,
    PathNonExistant,
    HttpStatus(StatusCode),
    HttpError(hyper::error::Error),
    UrlMalformed(ParseError),
}

fn main() {
    let matches = App::new("Marker")
        .version(crate_version!())
        .arg(Arg::with_name("root")
            .short("r")
            .long("root")
            .help("The path to the root of the documentation to be checked")
            .takes_value(true))
        .arg(Arg::with_name("skip-http")
            .long("skip-http")
            .help("Skip validation of HTTP[S] URLs"))
        .get_matches();

    let skip_http = matches.is_present("skip-http");
    let root = Path::new(matches.value_of("root").unwrap_or("."));
    let mut links = HashMap::new();
    let mut found_issue = false;

    for entry in WalkDir::new(root).into_iter().filter_map(|entry| {
        let entry = entry.unwrap_or_else(|err| {
            fail!("Failed to walk directory: {}", err);
        });
        if entry.file_type().is_file() && entry.path().extension() == Some(OsStr::new("md")) {
            Some(entry)
        } else {
            None
        }
    }) {
        let contents = {
            let mut file = File::open(entry.path()).unwrap_or_else(|err| {
                fail!("Failed to open file ({}): {}", entry.path().display(), err);
            });
            let mut text = String::new();
            if let Err(err) = file.read_to_string(&mut text) {
                fail!("Failed to read file ({}): {}", entry.path().display(), err);
            };
            text
        };

        let newlines = contents.match_indices('\n').map(|(i, _)| i).collect::<Vec<_>>();

        let mut state = State {
            skip_http: skip_http,
            code_block: false,
            last_text: String::new(),
            directory: entry.path().parent().expect("non-root path"),
        };
        let mut parser = Parser::new_ext(&contents, OPTION_ENABLE_TABLES);
        while let Some(event) = parser.next() {
            let line = newlines.iter().take_while(|i| i < &&parser.get_offset()).count() + 1;
            match event {
                Event::Text(ref text) => {
                    state.last_text = text.to_string();
                    if let Some(reference) = try_reference(&state, text) {
                        print_issue!(found_issue,
                                     "Found broken reference ({}) in {}:{}",
                                     reference,
                                     entry.path().display(),
                                     line)
                    }
                }
                Event::End(Tag::Link(dest, _)) => {
                    match check_link(&mut state, &mut links, &dest) {
                        Ok(()) => {}
                        Err(LinkError::PathAbsolute) => {
                            print_issue!(found_issue,
                                         "Found absolute path    ({} -> {}) at {}:{}",
                                         state.last_text,
                                         dest,
                                         entry.path().display(),
                                         line)
                        }
                        Err(LinkError::PathNonExistant) => {
                            print_issue!(found_issue,
                                         "Found broken path      ({} -> {}) at {}:{}",
                                         state.last_text,
                                         dest,
                                         entry.path().display(),
                                         line)
                        }
                        Err(LinkError::HttpStatus(status)) => {
                            print_issue!(found_issue,
                                         "Found broken url       ({} -> {}) at {}:{} : {}",
                                         state.last_text,
                                         dest,
                                         entry.path().display(),
                                         line,
                                         status)
                        }
                        Err(LinkError::HttpError(err)) => {
                            print_issue!(found_issue,
                                         "HTTP failure           ({} -> {}) at {}:{} : {}",
                                         state.last_text,
                                         dest,
                                         entry.path().display(),
                                         line,
                                         err);
                        }
                        Err(LinkError::UrlMalformed(err)) => {
                            print_issue!(found_issue,
                                         "Found malformed URL    ({} -> {}) at {}:{} : {}",
                                         state.last_text,
                                         dest,
                                         entry.path().display(),
                                         line,
                                         err);
                        }
                    }
                }
                Event::Start(Tag::Code) |
                Event::Start(Tag::CodeBlock(_)) => state.code_block = true,
                Event::End(Tag::Code) |
                Event::End(Tag::CodeBlock(_)) => state.code_block = false,
                _ => {}
            }
        }
    }

    if found_issue {
        exit(1)
    }
}

fn try_reference<'a>(state: &State, text: &'a str) -> Option<&'a str> {
    if !state.code_block && text.chars().next() == Some('[') && text.chars().last() == Some(']') {
        Some(&text[1..(text.len() - 1)])
    } else {
        None
    }
}

fn check_link(state: &mut State,
              links: &mut HashMap<Url, StatusCode>,
              dest: &str)
              -> Result<(), LinkError> {
    match (Url::parse(&dest), state.skip_http) {
        (Ok(url), false) => check_url(links, url),
        (Ok(_), true) => Ok(()),
        (Err(ParseError::RelativeUrlWithoutBase), _) => check_path(state, dest),
        (Err(err), _) => Err(LinkError::UrlMalformed(err)),
    }
}

fn check_url(links: &mut HashMap<Url, StatusCode>, url: Url) -> Result<(), LinkError> {
    if url.scheme() != "http" && url.scheme() != "https" {
        return Ok(());
    }

    let client = Client::new();
    let agent = UserAgent(format!("marker/{}", crate_version!()));

    let status = if let Some(status) = links.remove(&url) {
        status
    } else {
        let res = client.head(url.clone()).header(agent.clone()).send().and_then(|resp| {
            if resp.status == StatusCode::MethodNotAllowed {
                client.get(url.clone()).header(agent.clone()).send()
            } else {
                Ok(resp)
            }
        });
        match res {
            Ok(resp) => resp.status,
            Err(err) => return Err(LinkError::HttpError(err)),
        }
    };
    links.insert(url, status);
    match status {
        StatusCode::Ok => Ok(()),
        status => Err(LinkError::HttpStatus(status)),
    }
}

fn check_path(state: &mut State, path: &str) -> Result<(), LinkError> {
    let path = Path::new(OsStr::new(path.split('#').next().expect("string")));
    if path.is_absolute() {
        Err(LinkError::PathAbsolute)
    } else if !state.directory.join(path).exists() {
        Err(LinkError::PathNonExistant)
    } else {
        Ok(())
    }
}
<|endoftext|>