<commit_before>extern crate crossbeam;
extern crate walkdir;

pub mod single_threaded;

use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::fs;
use self::crossbeam::sync::chase_lev;
use self::walkdir::WalkDir;
use regex::{RegexSet, Regex};
use std::path::PathBuf;
use colored::*;
use std::process::exit;
use utils::size;
use std::thread;
use error::*;
use types::FileSize;
use std::path::Path;

pub use walk_parallel::single_threaded::*;

/// Enum for messaging between workers/stealers
pub enum Status<T> {
    Done,
    Data(T),
}

/// The 'Walk' struct contains all the information we need to traverse a directory.
#[derive(Debug)]
pub struct Walk {
    pub path: PathBuf,
    gitignore: Option<RegexSet>,
    hgignore: Option<RegexSet>,
    darcs_boring: Option<RegexSet>,
    excludes: Option<Regex>,
    max_depth: Option<u8>,
    threshold: Option<u64>,
    start_depth: usize,
    nproc: usize,
    show_files: bool,
    get_blocks: bool,
    follow_symlinks: bool,
    artifacts_only: bool,
}

impl Walk {
    /// function to make output from a 'Walk', using one thread. It also takes an 'Arc<AtomicU64>'
    /// and will add the relevant directory sizes to it.
    pub fn print_dir(w: Walk, total: Arc<AtomicU64>) -> () {

        let excludes = match w.excludes {
            Some(ref x) => Some(x),
            _ => None,
        };

        let v = if excludes.is_some() || w.artifacts_only {
            read_all(
                &w.path,
                (w.start_depth as u8),
                w.max_depth,
                excludes,
                &w.gitignore,
                false,
                w.artifacts_only,
            )
        } else {
            read_all_fast(&w.path, (w.start_depth as u8), w.max_depth)
        };

        let subdir_size = v.file_size.get();

        total.fetch_add(subdir_size, Ordering::Relaxed);

        let mut to_print = if let Some(m) = w.threshold {
            subdir_size > m
        } else {
            true
        };

        if let Some(0) = w.max_depth {
            to_print = false;
        }

        if to_print {
            // filter by depth
            let mut v_filtered = v.filtered(w.threshold, !w.show_files, w.max_depth);

            v_filtered.display_tree(w.path);
        }
    }

    /// set the maximum depth to display
    pub fn set_depth(&mut self, d: u8) -> () {
        self.max_depth = Some(d);
    }

    /// set the regex for excludes
    pub fn set_regex(&mut self, r: Regex) -> () {
        self.excludes = Some(r);
    }

    /// set the minumum file size
    pub fn set_threshold(&mut self, n: u64) -> () {
        self.threshold = Some(n);
    }

    /// include files when printing
    pub fn with_files(&mut self) -> () {
        self.show_files = true;
    }

    /// include files when printing
    pub fn blocks(&mut self) -> () {
        self.get_blocks = true;
    }

    /// include files when printing
    pub fn artifacts_only(&mut self) -> () {
        self.artifacts_only = true;
    }

    fn get_proc(&self) -> usize {
        self.nproc
    }

    /// Create a new 'Walk' from a 'PathBuf' and the number
    /// of processor cores to be used.
    pub fn new(p: PathBuf, n: usize) -> Walk {
        Walk {
            path: p,
            gitignore: None,
            hgignore: None,
            darcs_boring: None,
            excludes: None,
            max_depth: None,
            threshold: None,
            start_depth: 0,
            nproc: n,
            show_files: false,
            get_blocks: false,
            follow_symlinks: false,
            artifacts_only: false,
        }
    }

    fn bump_depth(&mut self) -> () {
        self.start_depth += 1;
    }

    /// This takes a 'Walk' and a 'Worker<Status<Walk>>' and executes the walk *in parallel*,
    /// creating new work for each subdirectory. It's not the most efficient concurrency
    /// imaginable, but it's fast and easy-ish to use. It *also* takes in an 'Arc<AtomicU64>',
    /// which it updates with any file sizes in the directory.
    pub fn push_subdir(
        w: &Walk,
        worker: &mut chase_lev::Worker<Status<Walk>>,
        total: Arc<AtomicU64>,
    ) {

        let in_paths = &w.path;

        // fill up queue + print out files
        if let Ok(paths) = fs::read_dir(in_paths) {

            // iterate over all the entries in the directory
            for p in paths {
                let val = match p {
                    Ok(x) => x,
                    _ => {
                        eprintln!("{}: path error at {:?}.", "Error".red(), p);
                        exit(0x0001)
                    }
                };

                let exclude_check = if let Some(ref x) = w.excludes {
                    if let Some(r) = val.path().into_os_string().to_str() {
                        !x.is_match(r)
                    } else {
                        eprintln!(
                            "{}: ignoring invalid unicode at: {:?}",
                            "Warning".yellow(),
                            val.path().display()
                        );
                        true
                    }
                } else {
                    true
                };

                if exclude_check {
                    match val.file_type() {
                        Ok(t) => {
                            if t.is_dir() {
                                let mut new_path = w.path.to_owned();
                                new_path.push(val.file_name());
                                let mut new_walk = Walk::new(new_path, w.get_proc());
                                if w.excludes.is_some() {
                                    new_walk.set_regex(w.excludes.clone().unwrap());
                                }
                                if w.show_files {
                                    new_walk.with_files();
                                }
                                new_walk.bump_depth();
                                if let Some(d) = w.max_depth {
                                    new_walk.set_depth(d);
                                }
                                if let Some(b) = w.threshold {
                                    new_walk.set_threshold(b);
                                }
                                worker.push(Status::Data(new_walk)); // pass a vector of Arc's to do 2-level traversals?
                            } else if t.is_file() {
                                if let Ok(l) = val.metadata() {
                                    let size = size(&l, w.get_blocks); // l.len();
                                    total.fetch_add(size, Ordering::Relaxed);
                                    if w.show_files && size != 0 {
                                        let to_formatted = format!("{}", FileSize::new(size));
                                        println!(
                                            "{}\t {}",
                                            &to_formatted.green(),
                                            val.path().display()
                                        );
                                    }
                                } else {
                                    eprintln!(
                                        "{}: could not find filesize for file at {}.",
                                        "Warning".yellow(),
                                        val.path().display()
                                    );
                                }
                            }
                        }
                        _ => {
                            eprintln!(
                                "{}: could not determine file type for: {}",
                                "Warning".yellow(),
                                val.path().display()
                            )
                        }
                    }
                }
            }

            // send "done" messages to all the workers
            let iter = 0..(w.get_proc());
            iter.map(|_| worker.push(Status::Done)).count();

        }
        // if we can't read the directory contents, figure out why
        // 1: check the path exists
        else if !in_paths.exists() {
            eprintln!(
                "{}: path '{}' does not exist, or you do not have permission to enter.",
                "Error".red(),
                &in_paths.display()
            );
        }
        // 2: check the path is actually a directory
        else if !in_paths.is_dir() {
            if w.artifacts_only {
                eprintln!(
                    "{}: {} is not a directory; not searching for artifacts",
                    "Warning".yellow(),
                    &in_paths.display()
                );
            }

            if let Ok(l) = in_paths.metadata() {
                let size = size(&l, w.get_blocks); // l.len();
                let to_formatted = format!("{}", FileSize::new(size));
                println!("{}\t {}", &to_formatted.green(), in_paths.display());
            } else {
                panic!("{}", Internal::IoError);
            }
        }
        // 3: otherwise, give a warning about permissions
        else {
            eprintln!(
                "{}: permission denied for directory: {}",
                "Warning".yellow(),
                &in_paths.display()
            );
        }

    }
}

fn latex_log<P: AsRef<Path>>(p: P) -> bool {

    lazy_static! {
        static ref LOG: Regex =
            Regex::new(r"\.log$")
            .unwrap();
    }

    if LOG.is_match(&p.as_ref().to_string_lossy().to_string()) {
        let mut parent = (&p.as_ref())
            .parent()
            .unwrap()
            .to_string_lossy()
            .to_string();
        parent.push_str("/*.tex");
        glob_exists(&parent)
    } else {
        false
    }

}

// TODO figure out why the unwrap_or is failing?
// FIXME take optional reference to a regex
pub fn clean_project_dirs<P: AsRef<Path>>(p: P, exclude: Option<Regex>, vimtags: bool) -> () {

    lazy_static! {
        static ref REGEX: Regex =
            Regex::new(r"(\.(a|la|lo|o|ll|keter|bc|dyn_o|d|rlib|crate|hi|hc|dyn_hi|S|jsexe|webapp|js\.externs|ibc|toc|aux|fdb_latexmk|fls|egg-info|whl|js_a|js_hi|jld|ji|js_o|so.*|dump-.*|vmb|crx|orig|elmo|elmi|hspec-failures|pyc|mod|go\.(v|teak|xmldef)|p_hi|p_o|prof|hide-cache|ghc\.environment\..*-\d.\d.\d|tix|synctex\.gz)|tags)$")
            .unwrap();
    }

    for dir in WalkDir::new(p)
        .into_iter()
        .filter_map(|e| e.ok())
        .filter(|p| {
            exclude.clone().map(|e| {
                e.is_match(&p.path().to_string_lossy().to_string())
            }) != Some(false)
        })
        .filter(|p| {
            REGEX.is_match(&p.path().to_string_lossy().to_string()) ||
                is_project_dir(
                    &p.path().to_string_lossy().to_string(),
                    &p.path()
                        .file_name()
                        .map(|x| x.to_string_lossy().to_string())
                        .unwrap_or("".to_string()),
                ) || latex_log(&p.path()) ||
                (vimtags && ((&p.path().to_string_lossy().to_string()).ends_with("tags"))) ||
                ({
                     let x = &p.path().to_string_lossy().to_string();
                     x.ends_with("flxg_stats.txt")
                 })
        })
    {
        if dir.file_type().is_file() {
            fs::remove_file(dir.path()).unwrap_or(());
        } else if dir.file_type().is_dir() {
            fs::remove_dir_all(dir.path()).unwrap_or(());
        }
    }
}

/// Given a 'Walk' struct, traverse it concurrently and print out any relevant outputs.
/// Currently, this only works for a depth of two, which is probably bad.
pub fn print_parallel(w: Walk) -> () {

    // initialize the total at 0 and create a reference to it
    let val = AtomicU64::new(0);
    let arc = Arc::new(val);
    let arc_producer = arc.clone();
    let arc_child = arc.clone();
    let path_display = w.path.clone();

    // set up worker & stealer
    let (mut worker, stealer): (chase_lev::Worker<Status<Walk>>,
                                chase_lev::Stealer<Status<Walk>>) = chase_lev::deque();

    // set up our iterator for the workers
    let iter = 0..(&w.get_proc() - 1);

    let spawn_workers = (&w.path).is_file();

    // create the producer in another thread
    let child_producer = thread::spawn(move || {

        let arc_local = arc_producer.clone();

        // assign work to everyone
        Walk::push_subdir(&w, &mut worker, arc_local.clone());

        // start popping off values in the worker's thread
        loop {
            if let Some(p) = worker.try_pop() {
                match p {
                    Status::Data(d) => Walk::print_dir(d, arc_local.clone()),
                    _ => break,
                };
            }
        }

    });

    // create a vector of thread handles so that it doesn't execute
    // everything sequentially.
    let mut threads = Vec::new();

    // set up as many workers as we have threads
    for _ in iter {

        // create a new stealer
        let stealer_clone = stealer.clone();

        let arc_local = arc_child.clone();

        // run the stealer in a new thread
        let child_consumer = thread::spawn(move || loop {

            if let chase_lev::Steal::Data(p) = stealer_clone.steal() {
                match p {
                    Status::Data(d) => Walk::print_dir(d, arc_local.clone()),
                    _ => break,
                };
            }
        });

        threads.push(child_consumer);

    }

    // join the child producer to the main thread
    let _ = child_producer.join();

    // join the workers to the main thread
    let _ = threads
        .into_iter()
        .map(|v| {
            let result = v.join();
            if let Ok(exit) = result {
                exit
            } else if let Err(e) = result {
                panic!("{:?}", e)
            }
        })
        .count();

    // get the total size
    let m = arc.load(Ordering::SeqCst); // TODO - check if this works with Relaxed?
    let size = FileSize::new(m);

    // print directory total.
    if size != FileSize::new(0) {
        let to_formatted = format!("{}", size);
        println!("{}\t {}", &to_formatted.green(), path_display.display());
    }

}
<commit_msg>don't remove all tags<commit_after>extern crate crossbeam;
extern crate walkdir;

pub mod single_threaded;

use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::fs;
use self::crossbeam::sync::chase_lev;
use self::walkdir::WalkDir;
use regex::{RegexSet, Regex};
use std::path::PathBuf;
use colored::*;
use std::process::exit;
use utils::size;
use std::thread;
use error::*;
use types::FileSize;
use std::path::Path;

pub use walk_parallel::single_threaded::*;

/// Enum for messaging between workers/stealers
pub enum Status<T> {
    Done,
    Data(T),
}

/// The 'Walk' struct contains all the information we need to traverse a directory.
#[derive(Debug)]
pub struct Walk {
    pub path: PathBuf,
    gitignore: Option<RegexSet>,
    hgignore: Option<RegexSet>,
    darcs_boring: Option<RegexSet>,
    excludes: Option<Regex>,
    max_depth: Option<u8>,
    threshold: Option<u64>,
    start_depth: usize,
    nproc: usize,
    show_files: bool,
    get_blocks: bool,
    follow_symlinks: bool,
    artifacts_only: bool,
}

impl Walk {
    /// function to make output from a 'Walk', using one thread. It also takes an 'Arc<AtomicU64>'
    /// and will add the relevant directory sizes to it.
    pub fn print_dir(w: Walk, total: Arc<AtomicU64>) -> () {

        let excludes = match w.excludes {
            Some(ref x) => Some(x),
            _ => None,
        };

        let v = if excludes.is_some() || w.artifacts_only {
            read_all(
                &w.path,
                (w.start_depth as u8),
                w.max_depth,
                excludes,
                &w.gitignore,
                false,
                w.artifacts_only,
            )
        } else {
            read_all_fast(&w.path, (w.start_depth as u8), w.max_depth)
        };

        let subdir_size = v.file_size.get();

        total.fetch_add(subdir_size, Ordering::Relaxed);

        let mut to_print = if let Some(m) = w.threshold {
            subdir_size > m
        } else {
            true
        };

        if let Some(0) = w.max_depth {
            to_print = false;
        }

        if to_print {
            // filter by depth
            let mut v_filtered = v.filtered(w.threshold, !w.show_files, w.max_depth);

            v_filtered.display_tree(w.path);
        }
    }

    /// set the maximum depth to display
    pub fn set_depth(&mut self, d: u8) -> () {
        self.max_depth = Some(d);
    }

    /// set the regex for excludes
    pub fn set_regex(&mut self, r: Regex) -> () {
        self.excludes = Some(r);
    }

    /// set the minumum file size
    pub fn set_threshold(&mut self, n: u64) -> () {
        self.threshold = Some(n);
    }

    /// include files when printing
    pub fn with_files(&mut self) -> () {
        self.show_files = true;
    }

    /// include files when printing
    pub fn blocks(&mut self) -> () {
        self.get_blocks = true;
    }

    /// include files when printing
    pub fn artifacts_only(&mut self) -> () {
        self.artifacts_only = true;
    }

    fn get_proc(&self) -> usize {
        self.nproc
    }

    /// Create a new 'Walk' from a 'PathBuf' and the number
    /// of processor cores to be used.
    pub fn new(p: PathBuf, n: usize) -> Walk {
        Walk {
            path: p,
            gitignore: None,
            hgignore: None,
            darcs_boring: None,
            excludes: None,
            max_depth: None,
            threshold: None,
            start_depth: 0,
            nproc: n,
            show_files: false,
            get_blocks: false,
            follow_symlinks: false,
            artifacts_only: false,
        }
    }

    fn bump_depth(&mut self) -> () {
        self.start_depth += 1;
    }

    /// This takes a 'Walk' and a 'Worker<Status<Walk>>' and executes the walk *in parallel*,
    /// creating new work for each subdirectory. It's not the most efficient concurrency
    /// imaginable, but it's fast and easy-ish to use. It *also* takes in an 'Arc<AtomicU64>',
    /// which it updates with any file sizes in the directory.
    pub fn push_subdir(
        w: &Walk,
        worker: &mut chase_lev::Worker<Status<Walk>>,
        total: Arc<AtomicU64>,
    ) {

        let in_paths = &w.path;

        // fill up queue + print out files
        if let Ok(paths) = fs::read_dir(in_paths) {

            // iterate over all the entries in the directory
            for p in paths {
                let val = match p {
                    Ok(x) => x,
                    _ => {
                        eprintln!("{}: path error at {:?}.", "Error".red(), p);
                        exit(0x0001)
                    }
                };

                let exclude_check = if let Some(ref x) = w.excludes {
                    if let Some(r) = val.path().into_os_string().to_str() {
                        !x.is_match(r)
                    } else {
                        eprintln!(
                            "{}: ignoring invalid unicode at: {:?}",
                            "Warning".yellow(),
                            val.path().display()
                        );
                        true
                    }
                } else {
                    true
                };

                if exclude_check {
                    match val.file_type() {
                        Ok(t) => {
                            if t.is_dir() {
                                let mut new_path = w.path.to_owned();
                                new_path.push(val.file_name());
                                let mut new_walk = Walk::new(new_path, w.get_proc());
                                if w.excludes.is_some() {
                                    new_walk.set_regex(w.excludes.clone().unwrap());
                                }
                                if w.show_files {
                                    new_walk.with_files();
                                }
                                new_walk.bump_depth();
                                if let Some(d) = w.max_depth {
                                    new_walk.set_depth(d);
                                }
                                if let Some(b) = w.threshold {
                                    new_walk.set_threshold(b);
                                }
                                worker.push(Status::Data(new_walk)); // pass a vector of Arc's to do 2-level traversals?
                            } else if t.is_file() {
                                if let Ok(l) = val.metadata() {
                                    let size = size(&l, w.get_blocks); // l.len();
                                    total.fetch_add(size, Ordering::Relaxed);
                                    if w.show_files && size != 0 {
                                        let to_formatted = format!("{}", FileSize::new(size));
                                        println!(
                                            "{}\t {}",
                                            &to_formatted.green(),
                                            val.path().display()
                                        );
                                    }
                                } else {
                                    eprintln!(
                                        "{}: could not find filesize for file at {}.",
                                        "Warning".yellow(),
                                        val.path().display()
                                    );
                                }
                            }
                        }
                        _ => {
                            eprintln!(
                                "{}: could not determine file type for: {}",
                                "Warning".yellow(),
                                val.path().display()
                            )
                        }
                    }
                }
            }

            // send "done" messages to all the workers
            let iter = 0..(w.get_proc());
            iter.map(|_| worker.push(Status::Done)).count();

        }
        // if we can't read the directory contents, figure out why
        // 1: check the path exists
        else if !in_paths.exists() {
            eprintln!(
                "{}: path '{}' does not exist, or you do not have permission to enter.",
                "Error".red(),
                &in_paths.display()
            );
        }
        // 2: check the path is actually a directory
        else if !in_paths.is_dir() {
            if w.artifacts_only {
                eprintln!(
                    "{}: {} is not a directory; not searching for artifacts",
                    "Warning".yellow(),
                    &in_paths.display()
                );
            }

            if let Ok(l) = in_paths.metadata() {
                let size = size(&l, w.get_blocks); // l.len();
                let to_formatted = format!("{}", FileSize::new(size));
                println!("{}\t {}", &to_formatted.green(), in_paths.display());
            } else {
                panic!("{}", Internal::IoError);
            }
        }
        // 3: otherwise, give a warning about permissions
        else {
            eprintln!(
                "{}: permission denied for directory: {}",
                "Warning".yellow(),
                &in_paths.display()
            );
        }

    }
}

fn latex_log<P: AsRef<Path>>(p: P) -> bool {

    lazy_static! {
        static ref LOG: Regex =
            Regex::new(r"\.log$")
            .unwrap();
    }

    if LOG.is_match(&p.as_ref().to_string_lossy().to_string()) {
        let mut parent = (&p.as_ref())
            .parent()
            .unwrap()
            .to_string_lossy()
            .to_string();
        parent.push_str("/*.tex");
        glob_exists(&parent)
    } else {
        false
    }

}

// TODO figure out why the unwrap_or is failing?
// FIXME take optional reference to a regex
pub fn clean_project_dirs<P: AsRef<Path>>(p: P, exclude: Option<Regex>, vimtags: bool) -> () {

    lazy_static! {
        static ref REGEX: Regex =
            Regex::new(r"(\.(a|la|lo|o|ll|keter|bc|dyn_o|d|rlib|crate|hi|hc|dyn_hi|S|jsexe|webapp|js\.externs|ibc|toc|aux|fdb_latexmk|fls|egg-info|whl|js_a|js_hi|jld|ji|js_o|so.*|dump-.*|vmb|crx|orig|elmo|elmi|hspec-failures|pyc|mod|go\.(v|teak|xmldef)|p_hi|p_o|prof|hide-cache|ghc\.environment\..*-\d.\d.\d|tix|synctex\.gz)|tags)$")
            .unwrap();
    }

    for dir in WalkDir::new(p)
        .into_iter()
        .filter_map(|e| e.ok())
        .filter(|p| {
            exclude.clone().map(|e| {
                e.is_match(&p.path().to_string_lossy().to_string())
            }) != Some(false)
        })
        .filter(|p| {
            REGEX.is_match(&p.path().to_string_lossy().to_string()) ||
                is_project_dir(
                    &p.path().to_string_lossy().to_string(),
                    &p.path()
                        .file_name()
                        .map(|x| x.to_string_lossy().to_string())
                        .unwrap_or("".to_string()),
                ) || latex_log(&p.path()) ||
                (vimtags && ((&p.path().to_string_lossy().to_string()).ends_with("/tags"))) ||
                ({
                     let x = &p.path().to_string_lossy().to_string();
                     x.ends_with("/flxg_stats.txt")
                 })
        })
    {
        if dir.file_type().is_file() {
            fs::remove_file(dir.path()).unwrap_or(());
        } else if dir.file_type().is_dir() {
            fs::remove_dir_all(dir.path()).unwrap_or(());
        }
    }
}

/// Given a 'Walk' struct, traverse it concurrently and print out any relevant outputs.
/// Currently, this only works for a depth of two, which is probably bad.
pub fn print_parallel(w: Walk) -> () {

    // initialize the total at 0 and create a reference to it
    let val = AtomicU64::new(0);
    let arc = Arc::new(val);
    let arc_producer = arc.clone();
    let arc_child = arc.clone();
    let path_display = w.path.clone();

    // set up worker & stealer
    let (mut worker, stealer): (chase_lev::Worker<Status<Walk>>,
                                chase_lev::Stealer<Status<Walk>>) = chase_lev::deque();

    // set up our iterator for the workers
    let iter = 0..(&w.get_proc() - 1);

    let spawn_workers = (&w.path).is_file();

    // create the producer in another thread
    let child_producer = thread::spawn(move || {

        let arc_local = arc_producer.clone();

        // assign work to everyone
        Walk::push_subdir(&w, &mut worker, arc_local.clone());

        // start popping off values in the worker's thread
        loop {
            if let Some(p) = worker.try_pop() {
                match p {
                    Status::Data(d) => Walk::print_dir(d, arc_local.clone()),
                    _ => break,
                };
            }
        }

    });

    // create a vector of thread handles so that it doesn't execute
    // everything sequentially.
    let mut threads = Vec::new();

    // set up as many workers as we have threads
    for _ in iter {

        // create a new stealer
        let stealer_clone = stealer.clone();

        let arc_local = arc_child.clone();

        // run the stealer in a new thread
        let child_consumer = thread::spawn(move || loop {

            if let chase_lev::Steal::Data(p) = stealer_clone.steal() {
                match p {
                    Status::Data(d) => Walk::print_dir(d, arc_local.clone()),
                    _ => break,
                };
            }
        });

        threads.push(child_consumer);

    }

    // join the child producer to the main thread
    let _ = child_producer.join();

    // join the workers to the main thread
    let _ = threads
        .into_iter()
        .map(|v| {
            let result = v.join();
            if let Ok(exit) = result {
                exit
            } else if let Err(e) = result {
                panic!("{:?}", e)
            }
        })
        .count();

    // get the total size
    let m = arc.load(Ordering::SeqCst); // TODO - check if this works with Relaxed?
    let size = FileSize::new(m);

    // print directory total.
    if size != FileSize::new(0) {
        let to_formatted = format!("{}", size);
        println!("{}\t {}", &to_formatted.green(), path_display.display());
    }

}
<|endoftext|>


<commit_before>//
// Copyright (c) Pirmin Kalberer. All rights reserved.
// Licensed under the MIT License. See LICENSE file in the project root for full license information.
//

use crate::core::config::ApplicationCfg;
use crate::mvt_service::MvtService;
use crate::runtime_config::{config_from_args, service_from_args};
use crate::static_files::StaticFiles;
use actix_cors::Cors;
use actix_files as fs;
use actix_rt;
use actix_web::dev::BodyEncoding;
use actix_web::http::{header, ContentEncoding};
use actix_web::middleware::Compress;
use actix_web::{middleware, web, App, HttpRequest, HttpResponse, HttpServer, Result};
use clap::ArgMatches;
use log::Level;
use num_cpus;
use open;
use std::collections::HashMap;
use std::str;
use std::str::FromStr;

static DINO: &'static str = "             xxxxxxxxx
        xxxxxxxxxxxxxxxxxxxxxxxx
      xxxxxxxxxxxxxxxxxxxxxxxxxxxx
     xxxxxxxxxxxxxxxxxxxxxxxxx xxxx
     xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
    xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
    xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
   xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx xxxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx  xxxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx   xxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx   xxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx     xxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx      x
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxx    xxxxxxxxxxx
xxxxxxxxxxxxxx                   xxxxxx
xxxxxxxxxxxx
xxxxxxxxxxx
xxxxxxxxxx
xxxxxxxxx
xxxxxxx
xxxxxx
xxxxxxx";

async fn mvt_metadata(service: web::Data<MvtService>) -> Result<HttpResponse> {
    let json = service.get_mvt_metadata().unwrap();
    Ok(HttpResponse::Ok().json(json))
}

/// Font list for Maputnik
async fn fontstacks() -> Result<HttpResponse> {
    Ok(HttpResponse::Ok().json(["Roboto Medium", "Roboto Regular"]))
}

// Include method fonts() which returns HashMap with embedded font files
include!(concat!(env!("OUT_DIR"), "/fonts.rs"));

/// Fonts for Maputnik
/// Example: /fonts/Open%20Sans%20Regular,Arial%20Unicode%20MS%20Regular/0-255.pbf
async fn fonts_pbf(params: web::Path<(String, String)>) -> Result<HttpResponse> {
    let fontpbfs = fonts();
    let fontlist = &params.0;
    let range = &params.1;
    let mut fonts = fontlist.split(",").collect::<Vec<_>>();
    fonts.push("Roboto Regular"); // Fallback
    let mut resp = HttpResponse::NotFound().finish();
    for font in fonts {
        let key = format!("fonts/{}/{}.pbf", font.replace("%20", " "), range);
        debug!("Font lookup: {}", key);
        if let Some(pbf) = fontpbfs.get(&key as &str) {
            resp = HttpResponse::Ok()
                .content_type("application/x-protobuf")
                // data is already gzip compressed
                .encoding(ContentEncoding::Identity)
                .header(header::CONTENT_ENCODING, "gzip")
                .body(*pbf); // TODO: chunked response
            break;
        }
    }
    Ok(resp)
}

fn req_baseurl(req: &HttpRequest) -> String {
    let conninfo = req.connection_info();
    format!("{}://{}", conninfo.scheme(), conninfo.host())
}

async fn tileset_tilejson(
    service: web::Data<MvtService>,
    tileset: web::Path<String>,
    req: HttpRequest,
) -> Result<HttpResponse> {
    let json = service.get_tilejson(&req_baseurl(&req), &tileset).unwrap();
    Ok(HttpResponse::Ok().json(json))
}

async fn tileset_style_json(
    service: web::Data<MvtService>,
    tileset: web::Path<String>,
    req: HttpRequest,
) -> Result<HttpResponse> {
    let json = service.get_stylejson(&req_baseurl(&req), &tileset).unwrap();
    Ok(HttpResponse::Ok().json(json))
}

async fn tileset_metadata_json(
    service: web::Data<MvtService>,
    tileset: web::Path<String>,
) -> Result<HttpResponse> {
    let json = service.get_mbtiles_metadata(&tileset).unwrap();
    Ok(HttpResponse::Ok().json(json))
}

async fn tile_pbf(
    config: web::Data<ApplicationCfg>,
    service: web::Data<MvtService>,
    params: web::Path<(String, u8, u32, u32)>,
    req: HttpRequest,
) -> Result<HttpResponse> {
    let tileset = &params.0;
    let z = params.1;
    let x = params.2;
    let y = params.3;
    let gzip = req
        .headers()
        .get(header::ACCEPT_ENCODING)
        .and_then(|headerval| {
            headerval
                .to_str()
                .ok()
                .and_then(|headerstr| Some(headerstr.contains("gzip")))
        })
        .unwrap_or(false);
    let tile = service.tile_cached(tileset, x, y, z, gzip, None);
    let cache_max_age = config.webserver.cache_control_max_age.unwrap_or(300);

    let resp = if let Some(tile) = tile {
        HttpResponse::Ok()
            .content_type("application/x-protobuf")
            .if_true(gzip, |r| {
                // data is already gzip compressed
                r.encoding(ContentEncoding::Identity)
                    .header(header::CONTENT_ENCODING, "gzip");
            })
            .header(header::CACHE_CONTROL, format!("max-age={}", cache_max_age))
            .body(tile) // TODO: chunked response
    } else {
        HttpResponse::NoContent().finish()
    };
    Ok(resp)
}

lazy_static! {
    static ref STATIC_FILES: StaticFiles = StaticFiles::init();
}

async fn static_file_handler(req: HttpRequest) -> Result<HttpResponse> {
    let key = req.path()[1..].to_string();
    let resp = if let Some(ref content) = STATIC_FILES.content(None, key) {
        HttpResponse::Ok()
            .header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") // TOOD: use Actix middleware
            .content_type(content.1)
            .body(content.0) // TODO: chunked response
    } else {
        HttpResponse::NotFound().finish()
    };
    Ok(resp)
}

#[derive(Deserialize)]
struct DrilldownParams {
    minzoom: Option<u8>,
    maxzoom: Option<u8>,
    points: String, //x1,y1,x2,y2,..
}

async fn drilldown_handler(
    service: web::Data<MvtService>,
    params: web::Query<DrilldownParams>,
) -> Result<HttpResponse> {
    let tileset = None; // all tilesets
    let progress = false;
    let points: Vec<f64> = params
        .points
        .split(",")
        .map(|v| {
            v.parse()
                .expect("Error parsing 'point' as pair of float values")
            //FIXME: map_err(|_| error::ErrorInternalServerError("...")
        })
        .collect();
    let stats = service.drilldown(tileset, params.minzoom, params.maxzoom, points, progress);
    let json = stats.as_json().unwrap();
    Ok(HttpResponse::Ok().json(json))
}

#[actix_rt::main]
pub async fn webserver(args: ArgMatches<'static>) -> std::io::Result<()> {
    let config = config_from_args(&args);
    let host = config
        .webserver
        .bind
        .clone()
        .unwrap_or("127.0.0.1".to_string());
    let port = config.webserver.port.unwrap_or(6767);
    let bind_addr = format!("{}:{}", host, port);
    let workers = config.webserver.threads.unwrap_or(num_cpus::get() as u8);
    let mvt_viewer = config.service.mvt.viewer;
    let openbrowser =
        bool::from_str(args.value_of("openbrowser").unwrap_or("true")).unwrap_or(false);
    let static_dirs = config.webserver.static_.clone();

    let mut service = service_from_args(&config, &args);
    service.prepare_feature_queries();
    service.init_cache();

    let server = HttpServer::new(move || {
        let mut app = App::new()
            .data(config.clone())
            .data(service.clone())
            .wrap(middleware::Logger::new("%r %s %b %Dms %a"))
            .wrap(Compress::default())
            .wrap(
                Cors::new()
                    .send_wildcard()
                    .allowed_methods(vec!["GET"])
                    .finish(),
            )
            .service(web::resource("/index.json").route(web::get().to(mvt_metadata)))
            .service(web::resource("/fontstacks.json").route(web::get().to(fontstacks)))
            .service(web::resource("/fonts.json").route(web::get().to(fontstacks)))
            .service(web::resource("/fonts/{fonts}/{range}.pbf").route(web::get().to(fonts_pbf)));
        for static_dir in &static_dirs {
            let dir = &static_dir.dir;
            if std::path::Path::new(dir).is_dir() {
                info!("Serving static files from directory '{}'", dir);
                app = app.service(fs::Files::new(&static_dir.path, dir));
            } else {
                warn!("Static file directory '{}' not found", dir);
            }
        }
        app = app
            .service(
                web::resource("/{tileset}.style.json").route(web::get().to(tileset_style_json)),
            )
            .service(
                web::resource("/{tileset}/metadata.json")
                    .route(web::get().to(tileset_metadata_json)),
            )
            .service(web::resource("/{tileset}.json").route(web::get().to(tileset_tilejson)))
            .service(web::resource("/{tileset}/{z}/{x}/{y}.pbf").route(web::get().to(tile_pbf)));
        if mvt_viewer {
            app = app.service(web::resource("/drilldown").route(web::get().to(drilldown_handler)));
            app = app.default_service(web::to(static_file_handler));
        }
        app
    })
    .workers(workers as usize)
    .bind(&bind_addr)
    .expect("Can not start server on given IP/Port")
    .shutdown_timeout(3) // default: 30s
    .run();

    if log_enabled!(Level::Info) {
        println!("{}", DINO);
    }

    if openbrowser && mvt_viewer {
        let _res = open::that(format!("http://{}:{}", &host, port));
    }

    server.await
}
<commit_msg>Enabling HTTP HEAD on most resources<commit_after>//
// Copyright (c) Pirmin Kalberer. All rights reserved.
// Licensed under the MIT License. See LICENSE file in the project root for full license information.
//

use crate::core::config::ApplicationCfg;
use crate::mvt_service::MvtService;
use crate::runtime_config::{config_from_args, service_from_args};
use crate::static_files::StaticFiles;
use actix_cors::Cors;
use actix_files as fs;
use actix_rt;
use actix_web::dev::BodyEncoding;
use actix_web::http::{header, ContentEncoding};
use actix_web::middleware::Compress;
use actix_web::{middleware, web, App, HttpRequest, guard, HttpResponse, HttpServer, Result};
use clap::ArgMatches;
use log::Level;
use num_cpus;
use open;
use std::collections::HashMap;
use std::str;
use std::str::FromStr;

static DINO: &'static str = "             xxxxxxxxx
        xxxxxxxxxxxxxxxxxxxxxxxx
      xxxxxxxxxxxxxxxxxxxxxxxxxxxx
     xxxxxxxxxxxxxxxxxxxxxxxxx xxxx
     xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
    xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
    xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
   xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx xxxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx  xxxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx   xxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx   xxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx     xxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx      x
xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
xxxxxxxxxxxxxxxxxxxxxxxxxx    xxxxxxxxxxx
xxxxxxxxxxxxxx                   xxxxxx
xxxxxxxxxxxx
xxxxxxxxxxx
xxxxxxxxxx
xxxxxxxxx
xxxxxxx
xxxxxx
xxxxxxx";

async fn mvt_metadata(service: web::Data<MvtService>) -> Result<HttpResponse> {
    let json = service.get_mvt_metadata().unwrap();
    Ok(HttpResponse::Ok().json(json))
}

/// Font list for Maputnik
async fn fontstacks() -> Result<HttpResponse> {
    Ok(HttpResponse::Ok().json(["Roboto Medium", "Roboto Regular"]))
}

// Include method fonts() which returns HashMap with embedded font files
include!(concat!(env!("OUT_DIR"), "/fonts.rs"));

/// Fonts for Maputnik
/// Example: /fonts/Open%20Sans%20Regular,Arial%20Unicode%20MS%20Regular/0-255.pbf
async fn fonts_pbf(params: web::Path<(String, String)>) -> Result<HttpResponse> {
    let fontpbfs = fonts();
    let fontlist = &params.0;
    let range = &params.1;
    let mut fonts = fontlist.split(",").collect::<Vec<_>>();
    fonts.push("Roboto Regular"); // Fallback
    let mut resp = HttpResponse::NotFound().finish();
    for font in fonts {
        let key = format!("fonts/{}/{}.pbf", font.replace("%20", " "), range);
        debug!("Font lookup: {}", key);
        if let Some(pbf) = fontpbfs.get(&key as &str) {
            resp = HttpResponse::Ok()
                .content_type("application/x-protobuf")
                // data is already gzip compressed
                .encoding(ContentEncoding::Identity)
                .header(header::CONTENT_ENCODING, "gzip")
                .body(*pbf); // TODO: chunked response
            break;
        }
    }
    Ok(resp)
}

fn req_baseurl(req: &HttpRequest) -> String {
    let conninfo = req.connection_info();
    format!("{}://{}", conninfo.scheme(), conninfo.host())
}

async fn tileset_tilejson(
    service: web::Data<MvtService>,
    tileset: web::Path<String>,
    req: HttpRequest,
) -> Result<HttpResponse> {
    let json = service.get_tilejson(&req_baseurl(&req), &tileset).unwrap();
    Ok(HttpResponse::Ok().json(json))
}

async fn tileset_style_json(
    service: web::Data<MvtService>,
    tileset: web::Path<String>,
    req: HttpRequest,
) -> Result<HttpResponse> {
    let json = service.get_stylejson(&req_baseurl(&req), &tileset).unwrap();
    Ok(HttpResponse::Ok().json(json))
}

async fn tileset_metadata_json(
    service: web::Data<MvtService>,
    tileset: web::Path<String>,
) -> Result<HttpResponse> {
    let json = service.get_mbtiles_metadata(&tileset).unwrap();
    Ok(HttpResponse::Ok().json(json))
}

async fn tile_pbf(
    config: web::Data<ApplicationCfg>,
    service: web::Data<MvtService>,
    params: web::Path<(String, u8, u32, u32)>,
    req: HttpRequest,
) -> Result<HttpResponse> {
    let tileset = &params.0;
    let z = params.1;
    let x = params.2;
    let y = params.3;
    let gzip = req
        .headers()
        .get(header::ACCEPT_ENCODING)
        .and_then(|headerval| {
            headerval
                .to_str()
                .ok()
                .and_then(|headerstr| Some(headerstr.contains("gzip")))
        })
        .unwrap_or(false);
    let tile = service.tile_cached(tileset, x, y, z, gzip, None);
    let cache_max_age = config.webserver.cache_control_max_age.unwrap_or(300);

    let resp = if let Some(tile) = tile {
        HttpResponse::Ok()
            .content_type("application/x-protobuf")
            .if_true(gzip, |r| {
                // data is already gzip compressed
                r.encoding(ContentEncoding::Identity)
                    .header(header::CONTENT_ENCODING, "gzip");
            })
            .header(header::CACHE_CONTROL, format!("max-age={}", cache_max_age))
            .body(tile) // TODO: chunked response
    } else {
        HttpResponse::NoContent().finish()
    };
    Ok(resp)
}

lazy_static! {
    static ref STATIC_FILES: StaticFiles = StaticFiles::init();
}

async fn static_file_handler(req: HttpRequest) -> Result<HttpResponse> {
    let key = req.path()[1..].to_string();
    let resp = if let Some(ref content) = STATIC_FILES.content(None, key) {
        HttpResponse::Ok()
            .header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") // TOOD: use Actix middleware
            .content_type(content.1)
            .body(content.0) // TODO: chunked response
    } else {
        HttpResponse::NotFound().finish()
    };
    Ok(resp)
}

#[derive(Deserialize)]
struct DrilldownParams {
    minzoom: Option<u8>,
    maxzoom: Option<u8>,
    points: String, //x1,y1,x2,y2,..
}

async fn drilldown_handler(
    service: web::Data<MvtService>,
    params: web::Query<DrilldownParams>,
) -> Result<HttpResponse> {
    let tileset = None; // all tilesets
    let progress = false;
    let points: Vec<f64> = params
        .points
        .split(",")
        .map(|v| {
            v.parse()
                .expect("Error parsing 'point' as pair of float values")
            //FIXME: map_err(|_| error::ErrorInternalServerError("...")
        })
        .collect();
    let stats = service.drilldown(tileset, params.minzoom, params.maxzoom, points, progress);
    let json = stats.as_json().unwrap();
    Ok(HttpResponse::Ok().json(json))
}

#[actix_rt::main]
pub async fn webserver(args: ArgMatches<'static>) -> std::io::Result<()> {
    let config = config_from_args(&args);
    let host = config
        .webserver
        .bind
        .clone()
        .unwrap_or("127.0.0.1".to_string());
    let port = config.webserver.port.unwrap_or(6767);
    let bind_addr = format!("{}:{}", host, port);
    let workers = config.webserver.threads.unwrap_or(num_cpus::get() as u8);
    let mvt_viewer = config.service.mvt.viewer;
    let openbrowser =
        bool::from_str(args.value_of("openbrowser").unwrap_or("true")).unwrap_or(false);
    let static_dirs = config.webserver.static_.clone();

    let mut service = service_from_args(&config, &args);
    service.prepare_feature_queries();
    service.init_cache();

    let server = HttpServer::new(move || {
        let mut app = App::new()
            .data(config.clone())
            .data(service.clone())
            .wrap(middleware::Logger::new("%r %s %b %Dms %a"))
            .wrap(Compress::default())
            .wrap(
                Cors::new()
                    .send_wildcard()
                    .allowed_methods(vec!["GET"])
                    .finish(),
            )
            .service(web::resource("/index.json").route(web::route().guard(guard::Any(guard::Get()).or(guard::Head())).to(mvt_metadata)))
            .service(web::resource("/fontstacks.json").route(web::route().guard(guard::Any(guard::Get()).or(guard::Head())).to(fontstacks)))
            .service(web::resource("/fonts.json").route(web::route().guard(guard::Any(guard::Get()).or(guard::Head())).to(fontstacks)))
            .service(web::resource("/fonts/{fonts}/{range}.pbf").route(web::route().guard(guard::Any(guard::Get()).or(guard::Head())).to(fonts_pbf)));
        for static_dir in &static_dirs {
            let dir = &static_dir.dir;
            if std::path::Path::new(dir).is_dir() {
                info!("Serving static files from directory '{}'", dir);
                app = app.service(fs::Files::new(&static_dir.path, dir));
            } else {
                warn!("Static file directory '{}' not found", dir);
            }
        }
        app = app
            .service(
                web::resource("/{tileset}.style.json").route(web::route().guard(guard::Any(guard::Get()).or(guard::Head())).to(tileset_style_json)),
            )
            .service(
                web::resource("/{tileset}/metadata.json")
                    .route(web::route().guard(guard::Any(guard::Get()).or(guard::Head())).to(tileset_metadata_json)),
            )
            .service(web::resource("/{tileset}.json").route(web::route().guard(guard::Any(guard::Get()).or(guard::Head())).to(tileset_tilejson)))
            .service(web::resource("/{tileset}/{z}/{x}/{y}.pbf").route(web::route().guard(guard::Any(guard::Get()).or(guard::Head())).to(tile_pbf)));
        if mvt_viewer {
            app = app.service(web::resource("/drilldown").route(web::route().guard(guard::Any(guard::Get()).or(guard::Head())).to(drilldown_handler)));
            app = app.default_service(web::to(static_file_handler));
        }
        app
    })
    .workers(workers as usize)
    .bind(&bind_addr)
    .expect("Can not start server on given IP/Port")
    .shutdown_timeout(3) // default: 30s
    .run();

    if log_enabled!(Level::Info) {
        println!("{}", DINO);
    }

    if openbrowser && mvt_viewer {
        let _res = open::that(format!("http://{}:{}", &host, port));
    }

    server.await
}
<|endoftext|>


<commit_before>// This file is part of Twig (ported to Rust).
//
// For the copyright and license information, please view the LICENSE
// file that was distributed with this source code.

//! Twig Error Handling
//!
//! # Examples
//!
//! ```rust,macro_test
//! #[macro_use] extern crate twig; // Macros must be imported *explicitly*
//!
//! use std::fmt;
//! use twig::api::error::Traced;
//! use std::error::Error;
//!
//! // Implement a custom error code.
//! type MyTracedSimpleError = Traced<MySimpleError>;
//!
//! #[derive(Debug)]
//! enum MySimpleError {
//!     Critical,
//!     Recoverable
//! }
//!
//! impl Error for MySimpleError {
//!     fn description(&self) -> &str {
//!         match *self {
//!             MySimpleError::Critical => "Critical error.",
//!             MySimpleError::Recoverable => "Recoverable error."
//!         }
//!     }
//! }
//!
//! impl fmt::Display for MySimpleError {
//!     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
//!         write!(f, "{} With human readable details", self.description())
//!     }
//! }
//!
//! fn main() {
//!     // Create a twig error, wrapping this error code + code location.
//!     let result: Result<(), Traced<MySimpleError>> = traced_err!(MySimpleError::Critical);
//!
//!     if let Err(error) = result {
//!         assert!(error.to_string().starts_with("Critical error. With human readable details at <anon>:"));
//!     }
//! }
//! ```

#[macro_use] mod macros;
use std::fmt::{self, Debug, Display};
use std::error::Error; // use std Error-trait to improve cross-crate compatibility
use std::ops::{Deref, DerefMut};

/// Extension trait for std::error::Error
///
/// adds conversion to `Traced` error
pub trait ErrorExt: Error {
    /// Returns generic twig error for this error code.
    /// You must provide the location, where the error occured.
    fn at(self, location: Location) -> Traced<Self>
        where Self: Sized
    {
        Traced::new(self, location)
    }
}

impl<T> ErrorExt for T where T: Error {}

/// Record current state of complex objects
///
/// The purpose of a dump is to be wrapped in error codes instead of complex objects.
/// This abstraction allows to
/// * keep error codes free of any lifetimes
/// * maintain the ability of receiver of error codes to decide about verbosity
///
/// The Dump::Data type may not contain lifetimes.
/// In practice this means cloning all referenced data into the dump.
///
/// For a type `X: Dump` you can reference the associated dump type via `<X as Dump>::Data`.
pub trait Dump {
    type Data: Debug + Display + 'static;

    fn dump(&self) -> Self::Data;
}

/// Generic error with backtrace.
///
/// Wrapper around some error type `T` implementing `std::error::Error`.
/// * Adds support for a backtrace.
/// * Automatically derefs to the inner error.
pub struct Traced<T>
    where T: Error
{
    error: T,
    trace: Trace,
}

impl<T> Traced<T> where
    T: Error
{
    /// Create a new twig error out of some generic error code.
    ///
    /// # Examples
    ///
    /// ```
    /// # #[macro_use] extern crate twig;
    /// # fn main() {
    /// use twig::api::error;
    /// use std::env::VarError;
    ///
    /// error::Traced::new(VarError::NotPresent, loc!()); // shorthand: `trace!(VarError::NotPresent)`
    /// # }
    /// ```
    pub fn new(error: T, location: Location) -> Traced<T> {
        Traced {
            error: error,
            trace: Trace(vec![location]),
        }
    }

    /// Return the first location the error occured.
    pub fn location(&self) -> Option<&Location> {
        self.trace.first()
    }

    pub fn backtrace(&self) -> &[Location] {
        &self.trace
    }

    // should not name it `at`, because `Traced` derefs to `Error` which impl `ErrorExt::at()`
    pub fn trace<R>(self, loc: Location) -> Traced<R>
        where T: Into<R>,
              R: Error
    {
        let Traced { error, mut trace } = self;
        trace.push(loc);

        Traced {
            error: error.into(),
            trace: trace
        }
    }

    /// Creates an iterator to iterate along the error cause-chain.
    pub fn iter(&self) -> ErrorIter {
        ErrorIter {
            next: Some(&self.error),
        }
    }
}

impl<T> Deref for Traced<T> where
    T: Error
{
    type Target = T;

    fn deref(&self) -> &Self::Target {
        &self.error
    }
}

/// Iterator to iterate along the error cause-chain.
pub struct ErrorIter<'a> {
    next: Option<&'a Error>
}

impl<'a> Iterator for ErrorIter<'a> {
    type Item = &'a Error;

    fn next(&mut self) -> Option<Self::Item> {
        return match self.next {
            Some(err) => {
                self.next = err.cause();
                Some(err)
            }
            None => None,
        }
    }
}

impl<T> Debug for Traced<T> where
    T: Error
{
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        f.debug_struct("Traced")
            .field("error", &self.error)
            .field("at", &self.trace)
            .finish()
    }
}

impl<T> Display for Traced<T> where
    T: Error
{
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{error} at {trace}\n",
            error = self.error,
            trace = &self.trace)
    }
}

/// Trace wraps `Vec<Location>`  with specialized `Display` and `Debug` impls.
///
/// For everything else it just derefs to `Vec<Location>`.
pub struct Trace(Vec<Location>);

impl Deref for Trace {
    type Target = Vec<Location>;

    fn deref(&self) -> &Self::Target {
        let Trace(ref trace) = *self;

        trace
    }
}

impl DerefMut for Trace {
    fn deref_mut(&mut self) -> &mut Self::Target {
        let Trace(ref mut trace) = *self;

        trace
    }
}

impl Debug for Trace {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        self.iter().enumerate().fold(
            &mut f.debug_struct("Trace"),
            |s, i| s.field(&format!("#{}", i.0), i.1)
        ).finish()
    }
}

impl Display for Trace {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match self.first() {
            Some(location) => write!(f, "{:?}", location),
            None =>  write!(f, "unknown (no backtrace)")
        }
    }
}

/// Location in rust source code
///
/// Debug::fmt() output is formatted in a compact way: `"{filename}:{line}:{column}"`.
#[derive(PartialEq)]
pub struct Location {
    pub filename : &'static str,    // e.g. /src/lexer/job/state/shared_traits.rs
    pub line : u32,
    pub column : u32,
}

impl Debug for Location {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{filename}:{line}:{column}",
            filename = self.filename,
            line     = self.line,
            column   = self.column)
    }
}
<commit_msg>impl Error for Traced<T: Error><commit_after>// This file is part of Twig (ported to Rust).
//
// For the copyright and license information, please view the LICENSE
// file that was distributed with this source code.

//! Twig Error Handling
//!
//! # Examples
//!
//! ```rust,macro_test
//! #[macro_use] extern crate twig; // Macros must be imported *explicitly*
//!
//! use std::fmt;
//! use twig::api::error::Traced;
//! use std::error::Error;
//!
//! // Implement a custom error code.
//! type MyTracedSimpleError = Traced<MySimpleError>;
//!
//! #[derive(Debug)]
//! enum MySimpleError {
//!     Critical,
//!     Recoverable
//! }
//!
//! impl Error for MySimpleError {
//!     fn description(&self) -> &str {
//!         match *self {
//!             MySimpleError::Critical => "Critical error.",
//!             MySimpleError::Recoverable => "Recoverable error."
//!         }
//!     }
//! }
//!
//! impl fmt::Display for MySimpleError {
//!     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
//!         write!(f, "{} With human readable details", self.description())
//!     }
//! }
//!
//! fn main() {
//!     // Create a twig error, wrapping this error code + code location.
//!     let result: Result<(), Traced<MySimpleError>> = traced_err!(MySimpleError::Critical);
//!
//!     if let Err(error) = result {
//!         assert!(error.to_string().starts_with("Critical error. With human readable details at <anon>:"));
//!     }
//! }
//! ```

#[macro_use] mod macros;
use std::fmt::{self, Debug, Display};
use std::error::Error; // use std Error-trait to improve cross-crate compatibility
use std::ops::{Deref, DerefMut};

/// Extension trait for std::error::Error
///
/// adds conversion to `Traced` error
pub trait ErrorExt: Error {
    /// Returns generic twig error for this error code.
    /// You must provide the location, where the error occured.
    fn at(self, location: Location) -> Traced<Self>
        where Self: Sized
    {
        Traced::new(self, location)
    }
}

impl<T> ErrorExt for T where T: Error {}

/// Record current state of complex objects
///
/// The purpose of a dump is to be wrapped in error codes instead of complex objects.
/// This abstraction allows to
/// * keep error codes free of any lifetimes
/// * maintain the ability of receiver of error codes to decide about verbosity
///
/// The Dump::Data type may not contain lifetimes.
/// In practice this means cloning all referenced data into the dump.
///
/// For a type `X: Dump` you can reference the associated dump type via `<X as Dump>::Data`.
pub trait Dump {
    type Data: Debug + Display + 'static;

    fn dump(&self) -> Self::Data;
}

/// Generic error with backtrace.
///
/// Wrapper around some error type `T` implementing `std::error::Error`.
/// * Adds support for a backtrace.
pub struct Traced<T>
    where T: Error
{
    error: T,
    trace: Trace,
}

impl<T> Traced<T> where
    T: Error
{
    /// Create a new twig error out of some generic error code.
    ///
    /// # Examples
    ///
    /// ```
    /// # #[macro_use] extern crate twig;
    /// # fn main() {
    /// use twig::api::error;
    /// use std::env::VarError;
    ///
    /// error::Traced::new(VarError::NotPresent, loc!()); // shorthand: `trace!(VarError::NotPresent)`
    /// # }
    /// ```
    pub fn new(error: T, location: Location) -> Traced<T> {
        Traced {
            error: error,
            trace: Trace(vec![location]),
        }
    }

    /// Return the first location the error occured.
    pub fn location(&self) -> Option<&Location> {
        self.trace.first()
    }

    pub fn backtrace(&self) -> &[Location] {
        &self.trace
    }

    // should not name it `at`, because `Traced` derefs to `Error` which impl `ErrorExt::at()`
    pub fn trace<R>(self, loc: Location) -> Traced<R>
        where T: Into<R>,
              R: Error
    {
        let Traced { error, mut trace } = self;
        trace.push(loc);

        Traced {
            error: error.into(),
            trace: trace
        }
    }

    /// Creates an iterator to iterate along the error cause-chain.
    pub fn iter(&self) -> ErrorIter {
        ErrorIter {
            next: Some(&self.error),
        }
    }
}

impl<T> Error for Traced<T> where
    T: Error
{
    fn description(&self) -> &str {
        self.error.description()
    }

    fn cause(&self) -> Option<&Error> {
        self.error.cause()
    }
}

/// Iterator to iterate along the error cause-chain.
pub struct ErrorIter<'a> {
    next: Option<&'a Error>
}

impl<'a> Iterator for ErrorIter<'a> {
    type Item = &'a Error;

    fn next(&mut self) -> Option<Self::Item> {
        return match self.next {
            Some(err) => {
                self.next = err.cause();
                Some(err)
            }
            None => None,
        }
    }
}

impl<T> Debug for Traced<T> where
    T: Error
{
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        f.debug_struct("Traced")
            .field("error", &self.error)
            .field("at", &self.trace)
            .finish()
    }
}

impl<T> Display for Traced<T> where
    T: Error
{
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{error} at {trace}\n",
            error = self.error,
            trace = &self.trace)
    }
}

/// Trace wraps `Vec<Location>`  with specialized `Display` and `Debug` impls.
///
/// For everything else it just derefs to `Vec<Location>`.
pub struct Trace(Vec<Location>);

impl Deref for Trace {
    type Target = Vec<Location>;

    fn deref(&self) -> &Self::Target {
        let Trace(ref trace) = *self;

        trace
    }
}

impl DerefMut for Trace {
    fn deref_mut(&mut self) -> &mut Self::Target {
        let Trace(ref mut trace) = *self;

        trace
    }
}

impl Debug for Trace {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        self.iter().enumerate().fold(
            &mut f.debug_struct("Trace"),
            |s, i| s.field(&format!("#{}", i.0), i.1)
        ).finish()
    }
}

impl Display for Trace {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        match self.first() {
            Some(location) => write!(f, "{:?}", location),
            None =>  write!(f, "unknown (no backtrace)")
        }
    }
}

/// Location in rust source code
///
/// Debug::fmt() output is formatted in a compact way: `"{filename}:{line}:{column}"`.
#[derive(PartialEq)]
pub struct Location {
    pub filename : &'static str,    // e.g. /src/lexer/job/state/shared_traits.rs
    pub line : u32,
    pub column : u32,
}

impl Debug for Location {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{filename}:{line}:{column}",
            filename = self.filename,
            line     = self.line,
            column   = self.column)
    }
}
<|endoftext|>


<commit_before>use std::error::Error;

use serde::Serialize;
use serde::Serializer;
use serde::Deserialize;
use serde::Deserializer;
use serde::de::Visitor;
use serde::de::Error as SerdeError;
use chrono::naive::datetime::NaiveDateTime;

#[derive(Clone, Debug, Hash)]
pub struct Date(NaiveDateTime);

pub static TASKWARRIOR_DATETIME_TEMPLATE : &'static str = "%Y%m%dT%H%M%SZ";

impl Serialize for Date {

    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: Serializer
    {
        serializer.serialize_str(&format!("{}", self.0))
    }

}

impl Deserialize for Date {

    fn deserialize<D>(deserializer: &mut D) -> Result<Date, D::Error>
        where D: Deserializer
    {
        struct DateVisitor;

        impl Visitor for DateVisitor {
            type Value = Date;

            fn visit_str<E>(&mut self, value: &str) -> Result<Date, E>
                where E: SerdeError
            {
                NaiveDateTime::parse_from_str(value, TASKWARRIOR_DATETIME_TEMPLATE)
                    .map(|d| Date(d))
                    .map_err(|e| SerdeError::custom(e.description()))
            }
        }

        deserializer.deserialize(DateVisitor)
    }

}
<commit_msg>Add converter from for Date<commit_after>use std::error::Error;

use serde::Serialize;
use serde::Serializer;
use serde::Deserialize;
use serde::Deserializer;
use serde::de::Visitor;
use serde::de::Error as SerdeError;
use chrono::naive::datetime::NaiveDateTime;

#[derive(Clone, Debug, Hash, Eq, PartialEq)]
pub struct Date(NaiveDateTime);

impl From<NaiveDateTime> for Date {

    fn from(ndt: NaiveDateTime) -> Date {
        Date(ndt)
    }

}

pub static TASKWARRIOR_DATETIME_TEMPLATE : &'static str = "%Y%m%dT%H%M%SZ";

impl Serialize for Date {

    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: Serializer
    {
        serializer.serialize_str(&format!("{}", self.0))
    }

}

impl Deserialize for Date {

    fn deserialize<D>(deserializer: &mut D) -> Result<Date, D::Error>
        where D: Deserializer
    {
        struct DateVisitor;

        impl Visitor for DateVisitor {
            type Value = Date;

            fn visit_str<E>(&mut self, value: &str) -> Result<Date, E>
                where E: SerdeError
            {
                NaiveDateTime::parse_from_str(value, TASKWARRIOR_DATETIME_TEMPLATE)
                    .map(|d| Date(d))
                    .map_err(|e| SerdeError::custom(e.description()))
            }
        }

        deserializer.deserialize(DateVisitor)
    }

}
<|endoftext|>


<commit_before>#[macro_use]
extern crate itertools;

use itertools::Itertools;

fn main() {
    println!("24 days of Rust - itertools (day 7)");
    let mut words = "hello supercalifragilisticexpialidocious programmer".split(|c| c == ' ');
    words.foreach(|word| println!("{} is {} characters long", word, word.len()));
    let even = (1..10).map(|x| x * 2);
    let odd = (1..5).map(|x| x * 2 + 1);
    println!("{:?}", even.interleave(odd).collect::<Vec<_>>());
    println!("{:?}", (1..10).intersperse(15).collect::<Vec<_>>());
    let numbers = 1..4;
    let chars = vec!['a', 'b', 'c'];
    for (i, c) in iproduct!(numbers, chars.iter()) {
        println!("{}: {}", i, c);
    }
    let log = "GET / 4096\nGET /home/ 16301\nPOST /home/ 49\nGET / 4096\n";
    let lines = log.lines();
    let rows = icompr!(line.split(|c| c == ' ').collect::<Vec<_>>(), line, lines);
    let bytes = icompr!(row[2], row, rows, row[0] != "POST");
    let total = icompr!(b.parse::<u32>().unwrap(), b, bytes).fold(0, |acc, x| acc + x);
    println!("Total GET throughput: {} bytes", total);
}
<commit_msg>Remove `icompr!` macro from day 7.<commit_after>#[macro_use]
extern crate itertools;

use itertools::Itertools;

fn main() {
    println!("24 days of Rust - itertools (day 7)");
    let mut words = "hello supercalifragilisticexpialidocious programmer".split(|c| c == ' ');
    words.foreach(|word| println!("{} is {} characters long", word, word.len()));
    let even = (1..10).map(|x| x * 2);
    let odd = (1..5).map(|x| x * 2 + 1);
    println!("{:?}", even.interleave(odd).collect::<Vec<_>>());
    println!("{:?}", (1..10).intersperse(15).collect::<Vec<_>>());
    let numbers = 1..4;
    let chars = vec!['a', 'b', 'c'];
    for (i, c) in iproduct!(numbers, chars.iter()) {
        println!("{}: {}", i, c);
    }
}
<|endoftext|>


<commit_before>//! `las` file management.

use std::f64;
use std::fs;
use std::io::{BufReader, BufWriter, Seek, Read, Write};
use std::path::Path;

use byteorder::{LittleEndian, WriteBytesExt};

use Result;
use error::Error;
use header::Header;
use io::write_zeros;
use point::Point;
use scale::descale;
use stream::Stream;
use vlr::Vlr;

/// A las file.
#[derive(Debug, PartialEq)]
pub struct File {
    header: Header,
    vlrs: Vec<Vlr>,
    points: Vec<Point>,
}

impl File {
    /// Reads a las file from the filesystem.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::file::File;
    /// let file = File::from_path("data/1.0_0.las").unwrap();
    /// ```
    pub fn from_path<P: AsRef<Path>>(path: P) -> Result<File> {
        let reader = BufReader::new(try!(fs::File::open(path)));
        File::read_from(reader)
    }

    /// Reads a las file from a `Read`.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::fs;
    /// use las::file::File;
    /// let reader = fs::File::open("data/1.0_0.las").unwrap();
    /// let file = File::read_from(reader).unwrap();
    /// ```
    pub fn read_from<R: Read + Seek>(reader: R) -> Result<File> {
        let mut file = File::new();
        let mut stream = try!(Stream::new(reader));
        file.header = stream.header();
        file.vlrs = (*stream.vlrs()).clone();
        file.points.reserve(stream.npoints() as usize);
        loop {
            match try!(stream.next_point()) {
                Some(point) => file.points.push(point),
                None => break,
            }
        }
        Ok(file)
    }

    /// Creates a new, empty las file.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::file::File;
    /// let file = File::new();
    /// ```
    pub fn new() -> File {
        File {
            header: Header::new(),
            vlrs: Vec::new(),
            points: Vec::new(),
        }
    }

    /// Returns a reference to a vector of this file's points.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::file::File;
    /// let file = File::from_path("data/1.0_0.las").unwrap();
    /// let points = file.points();
    /// ```
    pub fn points(&self) -> &Vec<Point> {
        &self.points
    }

    /// Adds a point to this lasfile.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::file::File;
    /// use las::point::Point;
    /// let mut file = File::new();
    /// let point = Point::new();
    /// file.add_point(point);
    /// ```
    pub fn add_point(&mut self, point: Point) {
        self.points.push(point);
    }

    /// Writes out this las file to a `Path`.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::fs::remove_file;
    /// use las::file::File;
    /// let mut file = File::new();
    /// file.to_path("temp.las").unwrap();
    /// remove_file("temp.las");
    /// ```
    pub fn to_path<P: AsRef<Path>>(&mut self, path: P) -> Result<()> {
        let ref mut writer = BufWriter::new(try!(fs::File::create(path)));
        self.write_to(writer)
    }

    /// Writes this las file to a `Write`.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::io::Cursor;
    /// use las::file::File;
    /// let mut file = File::from_path("data/1.0_0.las").unwrap();
    /// let ref mut cursor = Cursor::new(Vec::new());
    /// file.write_to(cursor).unwrap();
    /// ```
    pub fn write_to<W: Write>(&mut self, writer: &mut W) -> Result<()> {
        self.header.calculate_size();
        self.header.number_of_point_records = self.points.len() as u32;
        self.header.offset_to_point_data = self.header.header_size as u32 +
                                           self.vlrs.iter().fold(0, |a, v| a + v.len());
        self.header.point_data_record_length = self.header.point_data_format.record_length();

        let mut number_of_points_by_return = [0u32; 5];
        let mut x_min = f64::MAX;
        let mut y_min = f64::MAX;
        let mut z_min = f64::MAX;
        let mut x_max = f64::MIN;
        let mut y_max = f64::MIN;
        let mut z_max = f64::MIN;
        for point in &self.points {
            let return_number = point.return_number.as_u8();
            if return_number > 0 {
                number_of_points_by_return[(return_number - 1) as usize] += 1;
            }
            if point.x < x_min {
                x_min = point.x;
            }
            if point.y < y_min {
                y_min = point.y;
            }
            if point.z < z_min {
                z_min = point.z;
            }
            if point.x > x_max {
                x_max = point.x;
            }
            if point.y > y_max {
                y_max = point.y;
            }
            if point.z > z_max {
                z_max = point.z;
            }
        }
        self.header.x_min = x_min;
        self.header.y_min = y_min;
        self.header.z_min = z_min;
        self.header.x_max = x_max;
        self.header.y_max = y_max;
        self.header.z_max = z_max;

        let mut bytes_written = try!(self.header.write_to(writer)) as usize;
        if bytes_written < self.header.header_size as usize {
            bytes_written += try!(write_zeros(writer,
                                              self.header.header_size as usize - bytes_written));
        }
        for vlr in &self.vlrs {
            bytes_written += try!(vlr.write_to(writer)) as usize;
        }
        if bytes_written < self.header.offset_to_point_data as usize {
            try!(write_zeros(writer,
                             self.header.offset_to_point_data as usize - bytes_written));
        }
        for point in &self.points {
            try!(self.write_point_to(writer, point));
        }
        Ok(())
    }

    fn write_point_to<W: Write>(&self, writer: &mut W, point: &Point) -> Result<()> {
        try!(writer.write_i32::<LittleEndian>(descale(point.x,
                                                      self.header.x_scale_factor,
                                                      self.header.x_offset)));
        try!(writer.write_i32::<LittleEndian>(descale(point.y,
                                                      self.header.y_scale_factor,
                                                      self.header.y_offset)));
        try!(writer.write_i32::<LittleEndian>(descale(point.z,
                                                      self.header.z_scale_factor,
                                                      self.header.z_offset)));
        try!(writer.write_u16::<LittleEndian>(point.intensity));
        let byte = point.return_number.as_u8() + (point.number_of_returns.as_u8() << 3) +
                   (point.scan_direction.as_u8() << 6) +
                   ((point.edge_of_flight_line as u8) << 7);
        try!(writer.write_u8(byte));
        let byte = point.classification.as_u8() + ((point.synthetic as u8) << 5) +
                   ((point.key_point as u8) << 6) +
                   ((point.withheld as u8) << 7);
        try!(writer.write_u8(byte));
        try!(writer.write_i8(point.scan_angle_rank));
        try!(writer.write_u8(point.user_data));
        try!(writer.write_u16::<LittleEndian>(point.point_source_id));
        if self.header.point_data_format.has_time() {
            match point.gps_time {
                Some(gps_time) => try!(writer.write_f64::<LittleEndian>(gps_time)),
                None => {
                    return Err(Error::PointFormat(self.header.point_data_format,
                                                     "gps_time".to_string()))
                }
            }
        }
        if self.header.point_data_format.has_color() {
            match point.red {
                Some(red) => try!(writer.write_u16::<LittleEndian>(red)),
                None => {
                    return Err(Error::PointFormat(self.header.point_data_format,
                                                     "red".to_string()))
                }
            }
            match point.green {
                Some(green) => try!(writer.write_u16::<LittleEndian>(green)),
                None => {
                    return Err(Error::PointFormat(self.header.point_data_format,
                                                     "green".to_string()))
                }
            }
            match point.blue {
                Some(blue) => try!(writer.write_u16::<LittleEndian>(blue)),
                None => {
                    return Err(Error::PointFormat(self.header.point_data_format,
                                                     "blue".to_string()))
                }
            }
        }
        match point.extra_bytes {
            Some(ref bytes) => try!(writer.write_all(&bytes[..])),
            None => {}
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::fs::remove_file;
    use std::io::Cursor;
    use std::path::Path;

    use point::Point;

    fn roundtrip<P: AsRef<Path>>(path: P) {
        let mut lasfile = File::from_path(path).unwrap();
        let ref mut cursor = Cursor::new(Vec::new());
        lasfile.write_to(cursor).unwrap();
        cursor.set_position(0);
        let lasfile2 = File::read_from(cursor).unwrap();
        assert_eq!(lasfile, lasfile2);
    }

    #[test]
    fn roundtrip_1_0_0() {
        roundtrip("data/1.0_0.las");
    }

    #[test]
    fn roundtrip_1_0_1() {
        roundtrip("data/1.0_1.las");
    }

    #[test]
    fn roundtrip_1_1_0() {
        roundtrip("data/1.1_0.las");
    }

    #[test]
    fn roundtrip_1_1_1() {
        roundtrip("data/1.1_1.las");
    }

    #[test]
    fn roundtrip_1_2_0() {
        roundtrip("data/1.2_0.las");
    }

    #[test]
    fn roundtrip_1_2_1() {
        roundtrip("data/1.2_1.las");
    }

    #[test]
    fn roundtrip_1_2_2() {
        roundtrip("data/1.2_2.las");
    }

    #[test]
    fn roundtrip_1_2_3() {
        roundtrip("data/1.2_3.las");
    }

    /// This file is good as it exercieses a weird use case, but the test fails at the moment. I'm
    /// not sure why, so I'm going to keep it around but ignore it.
    #[test]
    #[ignore]
    fn roundtrip_extrabytes() {
        roundtrip("data/extrabytes.las");
    }

    #[test]
    fn point_format_1_has_gps_time() {
        let lasfile = File::from_path("data/1.0_1.las").unwrap();
        let ref point = lasfile.points()[0];
        assert!(point.gps_time.is_some());
    }

    #[test]
    fn point_format_2_has_color() {
        let lasfile = File::from_path("data/1.2_2.las").unwrap();
        let ref point = lasfile.points()[0];
        assert!(point.red.is_some());
        assert!(point.green.is_some());
        assert!(point.blue.is_some());
    }

    #[test]
    fn write_one_point() {
        let mut point = Point::new();
        point.x = 1.0;
        point.y = 2.0;
        point.z = 3.0;
        let mut lasfile = File::new();
        lasfile.add_point(point);
        lasfile.to_path("temp.las").unwrap();

        let lasfile = File::from_path("temp.las").unwrap();
        let ref point = lasfile.points()[0];
        assert_eq!(1.0, point.x);
        assert_eq!(2.0, point.y);
        assert_eq!(3.0, point.z);

        remove_file("temp.las").unwrap();
    }
}
<commit_msg>Use the builder pattern to set scale factors<commit_after>//! `las` file management.

use std::f64;
use std::fs;
use std::io::{BufReader, BufWriter, Seek, Read, Write};
use std::path::Path;

use byteorder::{LittleEndian, WriteBytesExt};

use Result;
use error::Error;
use header::Header;
use io::write_zeros;
use point::Point;
use scale::descale;
use stream::Stream;
use vlr::Vlr;

/// A las file.
#[derive(Debug, PartialEq)]
pub struct File {
    header: Header,
    vlrs: Vec<Vlr>,
    points: Vec<Point>,
}

impl File {
    /// Reads a las file from the filesystem.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::file::File;
    /// let file = File::from_path("data/1.0_0.las").unwrap();
    /// ```
    pub fn from_path<P: AsRef<Path>>(path: P) -> Result<File> {
        let reader = BufReader::new(try!(fs::File::open(path)));
        File::read_from(reader)
    }

    /// Reads a las file from a `Read`.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::fs;
    /// use las::file::File;
    /// let reader = fs::File::open("data/1.0_0.las").unwrap();
    /// let file = File::read_from(reader).unwrap();
    /// ```
    pub fn read_from<R: Read + Seek>(reader: R) -> Result<File> {
        let mut file = File::new();
        let mut stream = try!(Stream::new(reader));
        file.header = stream.header();
        file.vlrs = (*stream.vlrs()).clone();
        file.points.reserve(stream.npoints() as usize);
        loop {
            match try!(stream.next_point()) {
                Some(point) => file.points.push(point),
                None => break,
            }
        }
        Ok(file)
    }

    /// Creates a new, empty las file.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::file::File;
    /// let file = File::new();
    /// ```
    pub fn new() -> File {
        File {
            header: Header::new(),
            vlrs: Vec::new(),
            points: Vec::new(),
        }
    }

    /// Set the scale factors on a file.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::file::File;
    /// let file = File::new().scale_factors(0.01, 0.01, 0.01);
    /// ```
    pub fn scale_factors(mut self,
                         x_scale_factor: f64,
                         y_scale_factor: f64,
                         z_scale_factor: f64)
                         -> File {
        self.header.x_scale_factor = x_scale_factor;
        self.header.y_scale_factor = y_scale_factor;
        self.header.z_scale_factor = z_scale_factor;
        self
    }

    /// Returns a reference to a vector of this file's points.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::file::File;
    /// let file = File::from_path("data/1.0_0.las").unwrap();
    /// let points = file.points();
    /// ```
    pub fn points(&self) -> &Vec<Point> {
        &self.points
    }

    /// Adds a point to this lasfile.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::file::File;
    /// use las::point::Point;
    /// let mut file = File::new();
    /// let point = Point::new();
    /// file.add_point(point);
    /// ```
    pub fn add_point(&mut self, point: Point) {
        self.points.push(point);
    }

    /// Writes out this las file to a `Path`.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::fs::remove_file;
    /// use las::file::File;
    /// let mut file = File::new();
    /// file.to_path("temp.las").unwrap();
    /// remove_file("temp.las");
    /// ```
    pub fn to_path<P: AsRef<Path>>(&mut self, path: P) -> Result<()> {
        let ref mut writer = BufWriter::new(try!(fs::File::create(path)));
        self.write_to(writer)
    }

    /// Writes this las file to a `Write`.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::io::Cursor;
    /// use las::file::File;
    /// let mut file = File::from_path("data/1.0_0.las").unwrap();
    /// let ref mut cursor = Cursor::new(Vec::new());
    /// file.write_to(cursor).unwrap();
    /// ```
    pub fn write_to<W: Write>(&mut self, writer: &mut W) -> Result<()> {
        self.header.calculate_size();
        self.header.number_of_point_records = self.points.len() as u32;
        self.header.offset_to_point_data = self.header.header_size as u32 +
                                           self.vlrs.iter().fold(0, |a, v| a + v.len());
        self.header.point_data_record_length = self.header.point_data_format.record_length();

        let mut number_of_points_by_return = [0u32; 5];
        let mut x_min = f64::MAX;
        let mut y_min = f64::MAX;
        let mut z_min = f64::MAX;
        let mut x_max = f64::MIN;
        let mut y_max = f64::MIN;
        let mut z_max = f64::MIN;
        for point in &self.points {
            let return_number = point.return_number.as_u8();
            if return_number > 0 {
                number_of_points_by_return[(return_number - 1) as usize] += 1;
            }
            if point.x < x_min {
                x_min = point.x;
            }
            if point.y < y_min {
                y_min = point.y;
            }
            if point.z < z_min {
                z_min = point.z;
            }
            if point.x > x_max {
                x_max = point.x;
            }
            if point.y > y_max {
                y_max = point.y;
            }
            if point.z > z_max {
                z_max = point.z;
            }
        }
        self.header.x_min = x_min;
        self.header.y_min = y_min;
        self.header.z_min = z_min;
        self.header.x_max = x_max;
        self.header.y_max = y_max;
        self.header.z_max = z_max;

        let mut bytes_written = try!(self.header.write_to(writer)) as usize;
        if bytes_written < self.header.header_size as usize {
            bytes_written += try!(write_zeros(writer,
                                              self.header.header_size as usize - bytes_written));
        }
        for vlr in &self.vlrs {
            bytes_written += try!(vlr.write_to(writer)) as usize;
        }
        if bytes_written < self.header.offset_to_point_data as usize {
            try!(write_zeros(writer,
                             self.header.offset_to_point_data as usize - bytes_written));
        }
        for point in &self.points {
            try!(self.write_point_to(writer, point));
        }
        Ok(())
    }

    fn write_point_to<W: Write>(&self, writer: &mut W, point: &Point) -> Result<()> {
        try!(writer.write_i32::<LittleEndian>(descale(point.x,
                                                      self.header.x_scale_factor,
                                                      self.header.x_offset)));
        try!(writer.write_i32::<LittleEndian>(descale(point.y,
                                                      self.header.y_scale_factor,
                                                      self.header.y_offset)));
        try!(writer.write_i32::<LittleEndian>(descale(point.z,
                                                      self.header.z_scale_factor,
                                                      self.header.z_offset)));
        try!(writer.write_u16::<LittleEndian>(point.intensity));
        let byte = point.return_number.as_u8() + (point.number_of_returns.as_u8() << 3) +
                   (point.scan_direction.as_u8() << 6) +
                   ((point.edge_of_flight_line as u8) << 7);
        try!(writer.write_u8(byte));
        let byte = point.classification.as_u8() + ((point.synthetic as u8) << 5) +
                   ((point.key_point as u8) << 6) +
                   ((point.withheld as u8) << 7);
        try!(writer.write_u8(byte));
        try!(writer.write_i8(point.scan_angle_rank));
        try!(writer.write_u8(point.user_data));
        try!(writer.write_u16::<LittleEndian>(point.point_source_id));
        if self.header.point_data_format.has_time() {
            match point.gps_time {
                Some(gps_time) => try!(writer.write_f64::<LittleEndian>(gps_time)),
                None => {
                    return Err(Error::PointFormat(self.header.point_data_format,
                                                     "gps_time".to_string()))
                }
            }
        }
        if self.header.point_data_format.has_color() {
            match point.red {
                Some(red) => try!(writer.write_u16::<LittleEndian>(red)),
                None => {
                    return Err(Error::PointFormat(self.header.point_data_format,
                                                     "red".to_string()))
                }
            }
            match point.green {
                Some(green) => try!(writer.write_u16::<LittleEndian>(green)),
                None => {
                    return Err(Error::PointFormat(self.header.point_data_format,
                                                     "green".to_string()))
                }
            }
            match point.blue {
                Some(blue) => try!(writer.write_u16::<LittleEndian>(blue)),
                None => {
                    return Err(Error::PointFormat(self.header.point_data_format,
                                                     "blue".to_string()))
                }
            }
        }
        match point.extra_bytes {
            Some(ref bytes) => try!(writer.write_all(&bytes[..])),
            None => {}
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::fs::remove_file;
    use std::io::Cursor;
    use std::path::Path;

    use point::Point;

    fn roundtrip<P: AsRef<Path>>(path: P) {
        let mut lasfile = File::from_path(path).unwrap();
        let ref mut cursor = Cursor::new(Vec::new());
        lasfile.write_to(cursor).unwrap();
        cursor.set_position(0);
        let lasfile2 = File::read_from(cursor).unwrap();
        assert_eq!(lasfile, lasfile2);
    }

    #[test]
    fn roundtrip_1_0_0() {
        roundtrip("data/1.0_0.las");
    }

    #[test]
    fn roundtrip_1_0_1() {
        roundtrip("data/1.0_1.las");
    }

    #[test]
    fn roundtrip_1_1_0() {
        roundtrip("data/1.1_0.las");
    }

    #[test]
    fn roundtrip_1_1_1() {
        roundtrip("data/1.1_1.las");
    }

    #[test]
    fn roundtrip_1_2_0() {
        roundtrip("data/1.2_0.las");
    }

    #[test]
    fn roundtrip_1_2_1() {
        roundtrip("data/1.2_1.las");
    }

    #[test]
    fn roundtrip_1_2_2() {
        roundtrip("data/1.2_2.las");
    }

    #[test]
    fn roundtrip_1_2_3() {
        roundtrip("data/1.2_3.las");
    }

    /// This file is good as it exercieses a weird use case, but the test fails at the moment. I'm
    /// not sure why, so I'm going to keep it around but ignore it.
    #[test]
    #[ignore]
    fn roundtrip_extrabytes() {
        roundtrip("data/extrabytes.las");
    }

    #[test]
    fn point_format_1_has_gps_time() {
        let lasfile = File::from_path("data/1.0_1.las").unwrap();
        let ref point = lasfile.points()[0];
        assert!(point.gps_time.is_some());
    }

    #[test]
    fn point_format_2_has_color() {
        let lasfile = File::from_path("data/1.2_2.las").unwrap();
        let ref point = lasfile.points()[0];
        assert!(point.red.is_some());
        assert!(point.green.is_some());
        assert!(point.blue.is_some());
    }

    #[test]
    fn write_one_point() {
        let mut point = Point::new();
        point.x = 1.0;
        point.y = 2.0;
        point.z = 3.0;
        let mut lasfile = File::new();
        lasfile.add_point(point);
        lasfile.to_path("temp.las").unwrap();

        let lasfile = File::from_path("temp.las").unwrap();
        let ref point = lasfile.points()[0];
        assert_eq!(1.0, point.x);
        assert_eq!(2.0, point.y);
        assert_eq!(3.0, point.z);

        remove_file("temp.las").unwrap();
    }
}
<|endoftext|>


<commit_before>extern crate unicode_segmentation;
use game::unicode_segmentation::UnicodeSegmentation;

use std::path::Path;
use std::collections::HashMap;
use std::io::{BufRead, BufReader};
use std::fs::File;

pub struct Results {
    white: u64,
    black: u64,
    draw: u64,
}

impl Clone for Results {
    fn clone(&self) -> Results {
        Results {
            white: self.white.clone(),
            black: self.black.clone(),
            draw: self.draw.clone(),
        }
    }
}

pub struct Game {
    moves: Vec<String>,
    tagpairs: HashMap<String, String>,
    results: Results,
}

impl Clone for Game {
    fn clone(&self) -> Game {
        Game {
            moves: self.moves.clone(),
            tagpairs: self.tagpairs.clone(),
            results: self.results.clone(),
        }
    }
}

fn remove_first_last(s: &String) -> String {
    s.graphemes(true).skip(1).take(s.len() - 1).collect::<String>()
}

impl Game {
    fn parse_moves(moves: &Vec<String>) -> Vec<String> {
        let mut v: Vec<String> = Vec::new();
        for line in moves {
            let parts: Vec<&str> = line.split_whitespace().collect();
            for part in parts {
                if !(String::from(part).ends_with(".")) {
                    v.push(String::from(part));
                }
            }
        }
        v.clone()
    }

    fn parse_tagpairs(tagpairs: &Vec<String>) -> HashMap<String, String> {
        let mut tp: HashMap<String, String> = HashMap::new();
        for s in tagpairs {
            let t = remove_first_last(s);
            let v: Vec<&str> = t.split_whitespace().collect();
            tp.insert(String::from(v[0]), remove_first_last(&String::from(v[1])));
        }
        tp.clone()
    }

    fn parse_results(tagpairs: &HashMap<String, String>) -> Results {
        match tagpairs.get("Result").map(String::as_ref) {
            Some("1-0") => Results { white: 1, black: 0, draw: 0 },
            Some("0-1") => Results { white: 0, black: 1, draw: 0 },
            Some("1/2-1/2") => Results { white: 0, black: 0, draw: 1 },
            _ => Results { white: 0, black: 0, draw: 0 },
        }
    }

    pub fn new(tagpairs: &Vec<String>, moves: &Vec<String>) -> Game {
        let tp = Game::parse_tagpairs(tagpairs);
        Game {
            moves: Game::parse_moves(moves),
            tagpairs: tp.clone(),
            results: Game::parse_results(&tp),
        }
    }

    pub fn from_pgn(path: &Path) -> Vec<Game> {
        let mut v = Vec::new();
        let display = path.display();
        let file = match File::open(path) {
            Ok(f) => f,
            _     => panic!("opening {}", display),
        };
        let reader = BufReader::new(file);
        let mut header = Vec::new();
        let mut moves = Vec::new();
        let mut reading_header = true;
        for l in reader.lines() {
            let line = match l {
                Ok(ln) => ln,
                _      => panic!("reading {}", display),
            };
            if line == "" {
                reading_header = !reading_header;
                if reading_header {
                    v.push(Game::new(&header, &moves));
                    header = Vec::new();
                    moves = Vec::new();
                }
                continue;
            }
            if reading_header {
                header.push(line);
            } else {
                moves.push(line);
            }
        }
        println!("{:?}", display); // remove in future
        v
    }
}

pub struct GameTree {
    game_vecs: HashMap<String, Vec<Game>>,
}

impl GameTree {
    pub fn new() -> GameTree {
        GameTree {
            game_vecs: HashMap::new(),
        }
    }

    pub fn add(&mut self, v: Vec<Game>) {
        for game in v {
            let first_move: &str = game.moves[0].as_ref();
            if !self.game_vecs.contains_key(first_move) {
                self.game_vecs.insert(String::from(first_move), Vec::new());
            }
            self.game_vecs.get_mut(first_move).unwrap().push(game.clone());
        }
    }
}
<commit_msg>New constructor for game tree<commit_after>extern crate unicode_segmentation;
use game::unicode_segmentation::UnicodeSegmentation;

use std::path::Path;
use std::collections::HashMap;
use std::io::{BufRead, BufReader};
use std::fs::File;

pub struct Results {
    white: u64,
    black: u64,
    draw: u64,
}

impl Clone for Results {
    fn clone(&self) -> Results {
        Results {
            white: self.white.clone(),
            black: self.black.clone(),
            draw: self.draw.clone(),
        }
    }
}

pub struct Game {
    moves: Vec<String>,
    tagpairs: HashMap<String, String>,
    results: Results,
}

impl Clone for Game {
    fn clone(&self) -> Game {
        Game {
            moves: self.moves.clone(),
            tagpairs: self.tagpairs.clone(),
            results: self.results.clone(),
        }
    }
}

fn remove_first_last(s: &String) -> String {
    s.graphemes(true).skip(1).take(s.len() - 1).collect::<String>()
}

impl Game {
    fn parse_moves(moves: &Vec<String>) -> Vec<String> {
        let mut v: Vec<String> = Vec::new();
        for line in moves {
            let parts: Vec<&str> = line.split_whitespace().collect();
            for part in parts {
                if !(String::from(part).ends_with(".")) {
                    v.push(String::from(part));
                }
            }
        }
        v.clone()
    }

    fn parse_tagpairs(tagpairs: &Vec<String>) -> HashMap<String, String> {
        let mut tp: HashMap<String, String> = HashMap::new();
        for s in tagpairs {
            let t = remove_first_last(s);
            let v: Vec<&str> = t.split_whitespace().collect();
            tp.insert(String::from(v[0]), remove_first_last(&String::from(v[1])));
        }
        tp.clone()
    }

    fn parse_results(tagpairs: &HashMap<String, String>) -> Results {
        match tagpairs.get("Result").map(String::as_ref) {
            Some("1-0") => Results { white: 1, black: 0, draw: 0 },
            Some("0-1") => Results { white: 0, black: 1, draw: 0 },
            Some("1/2-1/2") => Results { white: 0, black: 0, draw: 1 },
            _ => Results { white: 0, black: 0, draw: 0 },
        }
    }

    pub fn new(tagpairs: &Vec<String>, moves: &Vec<String>) -> Game {
        let tp = Game::parse_tagpairs(tagpairs);
        Game {
            moves: Game::parse_moves(moves),
            tagpairs: tp.clone(),
            results: Game::parse_results(&tp),
        }
    }

    pub fn from_pgn(path: &Path) -> Vec<Game> {
        let mut v = Vec::new();
        let display = path.display();
        let file = match File::open(path) {
            Ok(f) => f,
            _     => panic!("opening {}", display),
        };
        let reader = BufReader::new(file);
        let mut header = Vec::new();
        let mut moves = Vec::new();
        let mut reading_header = true;
        for l in reader.lines() {
            let line = match l {
                Ok(ln) => ln,
                _      => panic!("reading {}", display),
            };
            if line == "" {
                reading_header = !reading_header;
                if reading_header {
                    v.push(Game::new(&header, &moves));
                    header = Vec::new();
                    moves = Vec::new();
                }
                continue;
            }
            if reading_header {
                header.push(line);
            } else {
                moves.push(line);
            }
        }
        println!("{:?}", display); // remove in future
        v
    }
}

pub struct GameTree {
    moves: Vec<String>,
    game_vecs: HashMap<String, Vec<Game>>,
}

impl GameTree {
    pub fn new() -> GameTree {
        GameTree {
            moves: Vec::new(),
            game_vecs: HashMap::new(),
        }
    }
    pub fn from_moves(moves: &Vec<String>) -> GameTree {
        GameTree {
            moves: moves.clone(),
            game_vecs: HashMap::new(),
        }
    }

    pub fn add(&mut self, v: Vec<Game>) {
        for game in v {
            let n = self.moves.len();
            let next_move: &str = game.moves[n].as_ref();
            if !self.game_vecs.contains_key(next_move) {
                self.game_vecs.insert(String::from(next_move), Vec::new());
            }
            self.game_vecs.get_mut(next_move).unwrap().push(game.clone());
        }
    }
}
<|endoftext|>