<commit_before>use super::{AsStr, Element, XmlReader, XmlWriter};
use super::Event::*;
use super::error::ResultPos;
use std::str::from_utf8;
use std::io::Cursor;

macro_rules! next_eq {
    ($r: expr, $($t:path, $bytes:expr),*) => {
        $(
            match $r.next() {
                Some(Ok($t(ref e))) => {
                    assert!(e.name() == $bytes, "expecting {:?}, found {:?}",
                            from_utf8($bytes), e.content().as_str());
                },
                Some(Ok(e)) => {
                    assert!(false, "expecting {:?}, found {:?}",
                            $t(Element::from_buffer($bytes.to_vec(),
                                                    0,
                                                    $bytes.len(),
                                                    $bytes.len())),
                            e);
                },
                Some(Err((e, pos))) => {
                    assert!(false, "{:?} at buffer position {}", e, pos);
                },
                p => {
                    assert!(false, "expecting {:?}, found {:?}",
                            $t(Element::from_buffer($bytes.to_vec(),
                                                    0,
                                                    $bytes.len(),
                                                    $bytes.len())),
                            p);
                }
            }
        )*
    }
}

#[test]
fn test_start() {
    let mut r = XmlReader::from("").trim_text(true);
    next_eq!(r, Start, b"a");
}

#[test]
fn test_start_end() {
    let mut r = XmlReader::from("").trim_text(true);
    next_eq!(r, Start, b"a", End, b"a");
}

#[test]
fn test_start_end_attr() {
    let mut r = XmlReader::from("").trim_text(true);
    next_eq!(r, Start, b"a", End, b"a");
}

#[test]
fn test_start_end_comment() {
    let mut r = XmlReader::from(" ")
        .trim_text(true);
    next_eq!(r,
             Start,
             b"b",
             Start,
             b"a",
             End,
             b"a",
             Start,
             b"a",
             End,
             b"a",
             Comment,
             b"t",
             End,
             b"b");
}

#[test]
fn test_start_txt_end() {
    let mut r = XmlReader::from("test").trim_text(true);
    next_eq!(r, Start, b"a", Text, b"test", End, b"a");
}

#[test]
fn test_comment() {
    let mut r = XmlReader::from("").trim_text(true);
    next_eq!(r, Comment, b"test");
}

#[test]
fn test_xml_decl() {
    let mut r = XmlReader::from("<?xml version=\"1.0\" encoding='utf-8'?>")
        .trim_text(true);
    match r.next() {
        Some(Ok(Decl(ref e))) => {
            match e.version() {
                Ok(v) => {
                    assert!(v == b"1.0",
                            "expecting version '1.0', got '{:?}",
                            v.as_str())
                }
                Err(e) => assert!(false, "{:?}", e),
            }
            match e.encoding() {
                Some(Ok(v)) => {
                    assert!(v == b"utf-8",
                            "expecting encoding 'utf-8', got '{:?}",
                            v.as_str())
                }
                Some(Err(e)) => assert!(false, "{:?}", e),
                None => assert!(false, "cannot find encoding"),
            }
            match e.standalone() {
                None => assert!(true),
                e => assert!(false, "doesn't expect standalone, got {:?}", e),
            }
        }
        Some(Err((e, pos))) => {
            assert!(false, "{:?} at buffer position {}", e, pos);
        }
        _ => assert!(false, "unable to parse XmlDecl"),
    }
}

#[test]
fn test_trim_test() {
    let txt = "  ";
    let mut r = XmlReader::from(txt).trim_text(true);
    next_eq!(r, Start, b"a", Start, b"b", End, b"b", End, b"a");

    let mut r = XmlReader::from(txt).trim_text(false);
    next_eq!(r,
             Text,
             b"",
             Start,
             b"a",
             Text,
             b"",
             Start,
             b"b",
             Text,
             b"  ",
             End,
             b"b",
             Text,
             b"",
             End,
             b"a");
}

#[test]
fn test_cdata() {
    let mut r = XmlReader::from("<![CDATA[test]]>").trim_text(true);
    next_eq!(r, CData, b"test");
}

#[test]
fn test_cdata_open_close() {
    let mut r = XmlReader::from("<![CDATA[test <> test]]>").trim_text(true);
    next_eq!(r, CData, b"test <> test");
}

#[test]
fn test_start_attr() {
    let mut r = XmlReader::from("").trim_text(true);
    next_eq!(r, Start, b"a");
}

#[test]
fn test_nested() {
    let mut r = XmlReader::from("test<c/>").trim_text(true);
    next_eq!(r,
             Start,
             b"a",
             Start,
             b"b",
             Text,
             b"test",
             End,
             b"b",
             Start,
             b"c",
             End,
             b"c",
             End,
             b"a");
}

#[test]
fn test_writer() {
    let txt = r#"<?xml version="1.0" encoding="utf-8"?><manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.github.sample" android:versionName="Lollipop" android:versionCode="5.1"><application android:label="SampleApplication"></application></manifest>"#;
    let reader = XmlReader::from(txt).trim_text(true);
    let mut writer = XmlWriter::new(Cursor::new(Vec::new()));
    for event in reader {
        assert!(writer.write(event.unwrap()).is_ok());
    }

    let result = writer.into_inner().into_inner();
    assert_eq!(result, txt.as_bytes());
}

#[test]
fn test_write_attrs() {
    let str_from = r#"<source attr="val"></source>"#;
    let expected = r#"<copy attr="val" a="b" c="d" x="y"></copy>"#;
    let reader = XmlReader::from(str_from).trim_text(true);
    let mut writer = XmlWriter::new(Cursor::new(Vec::new()));
    for event in reader {
        let event = event.unwrap();
        let event = match event {
            Start(elem) => {
                let mut attrs = elem.attributes()
                    .collect::<ResultPos<Vec<_>>>().unwrap();
                attrs.extend_from_slice(&[(b"a", b"b"), (b"c", b"d")]);
                let mut elem = Element::new("copy").with_attributes(attrs);
                elem.push_attribute("x", "y");
                Start(elem)
            }
            End(_elem) => End(Element::new("copy")),
            _ => event,
        };
        assert!(writer.write(event).is_ok());
    }

    let result = writer.into_inner().into_inner();
    assert_eq!(result, expected.as_bytes());
}

#[test]
fn test_buf_position() {
    let mut r = XmlReader::from("")
        .trim_text(true)
        .with_check(true);

    match r.next() {
        Some(Err((_, 2))) => assert!(true), // error at char 2: no opening tag
        Some(Err((e, n))) => assert!(false,
                                     "expecting buf_pos = 2, found {}, err: {:?}", n, e),
        e => assert!(false, "expecting error, found {:?}", e),
    }

    r = XmlReader::from(" tag found
        Some(Err((e, n))) => assert!(false,
                                     "expecting buf_pos = 2, found {}, err: {:?}", n, e),
        e => assert!(false, "expecting error, found {:?}", e),
    }

}

#[test]
fn test_namespace() {
    let mut r = XmlReader::from("<myns:b>in namespace!</myns:b>")
        .trim_text(true)
        .namespaced();;

    if let Some(Ok((None, Start(_)))) = r.next() {
    } else {
        assert!(false, "expecting start element with no namespace");
    }

    if let Some(Ok((Some(a), Start(_)))) = r.next() {
        if &*a == b"www1" {
            assert!(true);
        } else {
            assert!(false, "expecting namespace to resolve to 'www1'");
        }
    } else {
        assert!(false, "expecting namespace resolution");
    }
}

#[test]
fn test_escaped_content() {
    let mut r = XmlReader::from("<test>").trim_text(true);
    next_eq!(r, Start, b"a");
    match r.next() {
        Some(Ok(Text(ref e))) => {
            if e.content() != b"<test>" {
                panic!("content unexpected: expecting '<test>', got '{:?}'",
                       e.content().as_str());
            }
            match e.unescaped_content() {
                Ok(ref c) => {
                    if &**c != b"<test>" {
                        panic!("unescaped content unexpected: expecting '<test<', got '{:?}'",
                               c.as_str())
                    }
                }
                Err((e, i)) => panic!("cannot escape content at position {}: {:?}", i, e),
            }
        }
        Some(Ok(ref e)) => panic!("Expecting text event, got {:?}", e),
        Some(Err((e, i))) => panic!("Cannot get next event at position {}: {:?}", i, e),
        None => panic!("Expecting text event, got None"),
    }
    next_eq!(r, End, b"a");
}
<commit_msg>Failing `write_empty_element_attrs` test<commit_after>use super::{AsStr, Element, XmlReader, XmlWriter};
use super::Event::*;
use super::error::ResultPos;
use std::str::from_utf8;
use std::io::Cursor;

macro_rules! next_eq {
    ($r: expr, $($t:path, $bytes:expr),*) => {
        $(
            match $r.next() {
                Some(Ok($t(ref e))) => {
                    assert!(e.name() == $bytes, "expecting {:?}, found {:?}",
                            from_utf8($bytes), e.content().as_str());
                },
                Some(Ok(e)) => {
                    assert!(false, "expecting {:?}, found {:?}",
                            $t(Element::from_buffer($bytes.to_vec(),
                                                    0,
                                                    $bytes.len(),
                                                    $bytes.len())),
                            e);
                },
                Some(Err((e, pos))) => {
                    assert!(false, "{:?} at buffer position {}", e, pos);
                },
                p => {
                    assert!(false, "expecting {:?}, found {:?}",
                            $t(Element::from_buffer($bytes.to_vec(),
                                                    0,
                                                    $bytes.len(),
                                                    $bytes.len())),
                            p);
                }
            }
        )*
    }
}

#[test]
fn test_start() {
    let mut r = XmlReader::from("").trim_text(true);
    next_eq!(r, Start, b"a");
}

#[test]
fn test_start_end() {
    let mut r = XmlReader::from("").trim_text(true);
    next_eq!(r, Start, b"a", End, b"a");
}

#[test]
fn test_start_end_attr() {
    let mut r = XmlReader::from("").trim_text(true);
    next_eq!(r, Start, b"a", End, b"a");
}

#[test]
fn test_start_end_comment() {
    let mut r = XmlReader::from(" ")
        .trim_text(true);
    next_eq!(r,
             Start,
             b"b",
             Start,
             b"a",
             End,
             b"a",
             Start,
             b"a",
             End,
             b"a",
             Comment,
             b"t",
             End,
             b"b");
}

#[test]
fn test_start_txt_end() {
    let mut r = XmlReader::from("test").trim_text(true);
    next_eq!(r, Start, b"a", Text, b"test", End, b"a");
}

#[test]
fn test_comment() {
    let mut r = XmlReader::from("").trim_text(true);
    next_eq!(r, Comment, b"test");
}

#[test]
fn test_xml_decl() {
    let mut r = XmlReader::from("<?xml version=\"1.0\" encoding='utf-8'?>")
        .trim_text(true);
    match r.next() {
        Some(Ok(Decl(ref e))) => {
            match e.version() {
                Ok(v) => {
                    assert!(v == b"1.0",
                            "expecting version '1.0', got '{:?}",
                            v.as_str())
                }
                Err(e) => assert!(false, "{:?}", e),
            }
            match e.encoding() {
                Some(Ok(v)) => {
                    assert!(v == b"utf-8",
                            "expecting encoding 'utf-8', got '{:?}",
                            v.as_str())
                }
                Some(Err(e)) => assert!(false, "{:?}", e),
                None => assert!(false, "cannot find encoding"),
            }
            match e.standalone() {
                None => assert!(true),
                e => assert!(false, "doesn't expect standalone, got {:?}", e),
            }
        }
        Some(Err((e, pos))) => {
            assert!(false, "{:?} at buffer position {}", e, pos);
        }
        _ => assert!(false, "unable to parse XmlDecl"),
    }
}

#[test]
fn test_trim_test() {
    let txt = "  ";
    let mut r = XmlReader::from(txt).trim_text(true);
    next_eq!(r, Start, b"a", Start, b"b", End, b"b", End, b"a");

    let mut r = XmlReader::from(txt).trim_text(false);
    next_eq!(r,
             Text,
             b"",
             Start,
             b"a",
             Text,
             b"",
             Start,
             b"b",
             Text,
             b"  ",
             End,
             b"b",
             Text,
             b"",
             End,
             b"a");
}

#[test]
fn test_cdata() {
    let mut r = XmlReader::from("<![CDATA[test]]>").trim_text(true);
    next_eq!(r, CData, b"test");
}

#[test]
fn test_cdata_open_close() {
    let mut r = XmlReader::from("<![CDATA[test <> test]]>").trim_text(true);
    next_eq!(r, CData, b"test <> test");
}

#[test]
fn test_start_attr() {
    let mut r = XmlReader::from("").trim_text(true);
    next_eq!(r, Start, b"a");
}

#[test]
fn test_nested() {
    let mut r = XmlReader::from("test<c/>").trim_text(true);
    next_eq!(r,
             Start,
             b"a",
             Start,
             b"b",
             Text,
             b"test",
             End,
             b"b",
             Start,
             b"c",
             End,
             b"c",
             End,
             b"a");
}

#[test]
fn test_writer() {
    let txt = r#"<?xml version="1.0" encoding="utf-8"?><manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.github.sample" android:versionName="Lollipop" android:versionCode="5.1"><application android:label="SampleApplication"></application></manifest>"#;
    let reader = XmlReader::from(txt).trim_text(true);
    let mut writer = XmlWriter::new(Cursor::new(Vec::new()));
    for event in reader {
        assert!(writer.write(event.unwrap()).is_ok());
    }

    let result = writer.into_inner().into_inner();
    assert_eq!(result, txt.as_bytes());
}

#[test]
fn test_write_empty_element_attrs() {
    let str_from = r#"<source attr="val"/>"#;
    let expected = r#"<source attr="val"></source>"#;
    let reader = XmlReader::from(str_from);
    let mut writer = XmlWriter::new(Cursor::new(Vec::new()));
    for event in reader {
        assert!(writer.write(event.unwrap()).is_ok());
    }

    let result = writer.into_inner().into_inner();
    assert_eq!(String::from_utf8(result).unwrap(), expected);
}

#[test]
fn test_write_attrs() {
    let str_from = r#"<source attr="val"></source>"#;
    let expected = r#"<copy attr="val" a="b" c="d" x="y"></copy>"#;
    let reader = XmlReader::from(str_from).trim_text(true);
    let mut writer = XmlWriter::new(Cursor::new(Vec::new()));
    for event in reader {
        let event = event.unwrap();
        let event = match event {
            Start(elem) => {
                let mut attrs = elem.attributes()
                    .collect::<ResultPos<Vec<_>>>().unwrap();
                attrs.extend_from_slice(&[(b"a", b"b"), (b"c", b"d")]);
                let mut elem = Element::new("copy").with_attributes(attrs);
                elem.push_attribute("x", "y");
                Start(elem)
            }
            End(_elem) => End(Element::new("copy")),
            _ => event,
        };
        assert!(writer.write(event).is_ok());
    }

    let result = writer.into_inner().into_inner();
    assert_eq!(result, expected.as_bytes());
}

#[test]
fn test_buf_position() {
    let mut r = XmlReader::from("")
        .trim_text(true)
        .with_check(true);

    match r.next() {
        Some(Err((_, 2))) => assert!(true), // error at char 2: no opening tag
        Some(Err((e, n))) => assert!(false,
                                     "expecting buf_pos = 2, found {}, err: {:?}", n, e),
        e => assert!(false, "expecting error, found {:?}", e),
    }

    r = XmlReader::from(" tag found
        Some(Err((e, n))) => assert!(false,
                                     "expecting buf_pos = 2, found {}, err: {:?}", n, e),
        e => assert!(false, "expecting error, found {:?}", e),
    }

}

#[test]
fn test_namespace() {
    let mut r = XmlReader::from("<myns:b>in namespace!</myns:b>")
        .trim_text(true)
        .namespaced();;

    if let Some(Ok((None, Start(_)))) = r.next() {
    } else {
        assert!(false, "expecting start element with no namespace");
    }

    if let Some(Ok((Some(a), Start(_)))) = r.next() {
        if &*a == b"www1" {
            assert!(true);
        } else {
            assert!(false, "expecting namespace to resolve to 'www1'");
        }
    } else {
        assert!(false, "expecting namespace resolution");
    }
}

#[test]
fn test_escaped_content() {
    let mut r = XmlReader::from("<test>").trim_text(true);
    next_eq!(r, Start, b"a");
    match r.next() {
        Some(Ok(Text(ref e))) => {
            if e.content() != b"<test>" {
                panic!("content unexpected: expecting '<test>', got '{:?}'",
                       e.content().as_str());
            }
            match e.unescaped_content() {
                Ok(ref c) => {
                    if &**c != b"<test>" {
                        panic!("unescaped content unexpected: expecting '<test<', got '{:?}'",
                               c.as_str())
                    }
                }
                Err((e, i)) => panic!("cannot escape content at position {}: {:?}", i, e),
            }
        }
        Some(Ok(ref e)) => panic!("Expecting text event, got {:?}", e),
        Some(Err((e, i))) => panic!("Cannot get next event at position {}: {:?}", i, e),
        None => panic!("Expecting text event, got None"),
    }
    next_eq!(r, End, b"a");
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use crate::exec;
use crate::exit_codes::{merge_exitcodes, ExitCode};
use crate::fshelper;
use crate::internal::{opts::FdOptions, osstr_to_bytes, MAX_BUFFER_LENGTH};
use crate::output;

use std::borrow::Cow;
use std::convert::TryFrom;
use std::error::Error;
use std::ffi::OsStr;
use std::fs;
use std::io;
use std::path::PathBuf;
use std::process;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{channel, Receiver, Sender};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time;

use ignore::overrides::OverrideBuilder;
use ignore::{self, WalkBuilder};
use regex::bytes::Regex;

/// The receiver thread can either be buffering results or directly streaming to the console.
enum ReceiverMode {
    /// Receiver is still buffering in order to sort the results, if the search finishes fast
    /// enough.
    Buffering,

    /// Receiver is directly printing results to the output.
    Streaming,
}

/// The Worker threads can result in a valid entry having PathBuf or an error.
pub enum WorkerResult {
    Entry(PathBuf),
    Error(ignore::Error),
}

/// Converts a WorkerResult to a PathBuf. This is required as in some cases,
/// the underlying `ignore` libary will return an error when we actually
/// still want to show the path.
///
/// Currently, the cases supported are:
///   - a broken symlink should still be found and the path returned
impl TryFrom<WorkerResult> for PathBuf {
    type Error = ignore::Error;

    fn try_from(worker_result: WorkerResult) -> Result<PathBuf, Self::Error> {
        match worker_result {
            WorkerResult::Entry(value) => return Ok(value),
            WorkerResult::Error(ignore_error) => {
                if let ignore::Error::WithPath { path, err } = &ignore_error {
                    if let ignore::Error::Io(ref io_err) = **err {
                        if io_err.kind() == io::ErrorKind::NotFound
                            && fs::symlink_metadata(&path)
                                .map_or(false, |m| m.file_type().is_symlink())
                        {
                            // Broken symlink
                            return Ok(path.to_path_buf());
                        }
                    }
                }
                return Err(ignore_error);
            }
        }
    }
}

/// Recursively scan the given search path for files / pathnames matching the pattern.
///
/// If the `--exec` argument was supplied, this will create a thread pool for executing
/// jobs in parallel from a given command line and the discovered paths. Otherwise, each
/// path will simply be written to standard output.
pub fn scan(path_vec: &[PathBuf], pattern: Arc<Regex>, config: Arc<FdOptions>) -> ExitCode {
    let mut path_iter = path_vec.iter();
    let first_path_buf = path_iter
        .next()
        .expect("Error: Path vector can not be empty");
    let (tx, rx) = channel();

    let mut override_builder = OverrideBuilder::new(first_path_buf.as_path());

    for pattern in &config.exclude_patterns {
        let res = override_builder.add(pattern);
        if res.is_err() {
            print_error_and_exit!("Malformed exclude pattern '{}'", pattern);
        }
    }
    let overrides = override_builder.build().unwrap_or_else(|_| {
        print_error_and_exit!("Mismatch in exclude patterns");
    });

    let mut walker = WalkBuilder::new(first_path_buf.as_path());
    walker
        .hidden(config.ignore_hidden)
        .ignore(config.read_fdignore)
        .parents(config.read_fdignore || config.read_vcsignore)
        .git_ignore(config.read_vcsignore)
        .git_global(config.read_vcsignore)
        .git_exclude(config.read_vcsignore)
        .overrides(overrides)
        .follow_links(config.follow_links)
        // No need to check for supported platforms, option is unavailable on unsupported ones
        .same_file_system(config.one_file_system)
        .max_depth(config.max_depth);

    if config.read_fdignore {
        walker.add_custom_ignore_filename(".fdignore");
    }

    for ignore_file in &config.ignore_files {
        let result = walker.add_ignore(ignore_file);
        match result {
            Some(ignore::Error::Partial(_)) => (),
            Some(err) => {
                print_error!(
                    "{}",
                    format!(
                        "Malformed pattern in custom ignore file '{}': {}.",
                        ignore_file.to_string_lossy(),
                        err.description()
                    )
                );
            }
            None => (),
        }
    }

    for path_entry in path_iter {
        walker.add(path_entry.as_path());
    }

    let parallel_walker = walker.threads(config.threads).build_parallel();

    let wants_to_quit = Arc::new(AtomicBool::new(false));
    if config.ls_colors.is_some() && config.command.is_none() {
        let wq = Arc::clone(&wants_to_quit);
        ctrlc::set_handler(move || {
            if wq.load(Ordering::Relaxed) {
                // Ctrl-C has been pressed twice, exit NOW
                process::exit(ExitCode::KilledBySigint.into());
            } else {
                wq.store(true, Ordering::Relaxed);
            }
        })
        .unwrap();
    }

    // Spawn the thread that receives all results through the channel.
    let receiver_thread = spawn_receiver(&config, &wants_to_quit, rx);

    // Spawn the sender threads.
    spawn_senders(&config, &wants_to_quit, pattern, parallel_walker, tx);

    // Wait for the receiver thread to print out all results.
    let exit_code = receiver_thread.join().unwrap();

    if wants_to_quit.load(Ordering::Relaxed) {
        process::exit(ExitCode::KilledBySigint.into());
    }

    exit_code
}

fn spawn_receiver(
    config: &Arc<FdOptions>,
    wants_to_quit: &Arc<AtomicBool>,
    rx: Receiver<WorkerResult>,
) -> thread::JoinHandle<ExitCode> {
    let config = Arc::clone(config);
    let wants_to_quit = Arc::clone(wants_to_quit);

    let show_filesystem_errors = config.show_filesystem_errors;
    let threads = config.threads;

    thread::spawn(move || {
        // This will be set to `Some` if the `--exec` argument was supplied.
        if let Some(ref cmd) = config.command {
            if cmd.in_batch_mode() {
                exec::batch(rx, cmd, show_filesystem_errors)
            } else {
                let shared_rx = Arc::new(Mutex::new(rx));

                let out_perm = Arc::new(Mutex::new(()));

                // Each spawned job will store it's thread handle in here.
                let mut handles = Vec::with_capacity(threads);
                for _ in 0..threads {
                    let rx = Arc::clone(&shared_rx);
                    let cmd = Arc::clone(cmd);
                    let out_perm = Arc::clone(&out_perm);

                    // Spawn a job thread that will listen for and execute inputs.
                    let handle =
                        thread::spawn(move || exec::job(rx, cmd, out_perm, show_filesystem_errors));

                    // Push the handle of the spawned thread into the vector for later joining.
                    handles.push(handle);
                }

                // Wait for all threads to exit before exiting the program.
                let mut results: Vec<ExitCode> = Vec::new();
                for h in handles {
                    results.push(h.join().unwrap());
                }

                merge_exitcodes(results)
            }
        } else {
            let start = time::Instant::now();

            let mut buffer = vec![];

            // Start in buffering mode
            let mut mode = ReceiverMode::Buffering;

            // Maximum time to wait before we start streaming to the console.
            let max_buffer_time = config
                .max_buffer_time
                .unwrap_or_else(|| time::Duration::from_millis(100));

            let stdout = io::stdout();
            let mut stdout = stdout.lock();

            for worker_result in rx {
                match PathBuf::try_from(worker_result) {
                    Ok(value) => {
                        match mode {
                            ReceiverMode::Buffering => {
                                buffer.push(value);

                                // Have we reached the maximum buffer size or maximum buffering time?
                                if buffer.len() > MAX_BUFFER_LENGTH
                                    || time::Instant::now() - start > max_buffer_time
                                {
                                    // Flush the buffer
                                    for v in &buffer {
                                        output::print_entry(
                                            &mut stdout,
                                            v,
                                            &config,
                                            &wants_to_quit,
                                        );
                                    }
                                    buffer.clear();

                                    // Start streaming
                                    mode = ReceiverMode::Streaming;
                                }
                            }
                            ReceiverMode::Streaming => {
                                output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                            }
                        }
                    }
                    Err(err) => {
                        if show_filesystem_errors {
                            print_error!("{}", err);
                        }
                    }
                }
            }

            // If we have finished fast enough (faster than max_buffer_time), we haven't streamed
            // anything to the console, yet. In this case, sort the results and print them:
            if !buffer.is_empty() {
                buffer.sort();
                for value in buffer {
                    output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                }
            }

            ExitCode::Success
        }
    })
}

fn spawn_senders(
    config: &Arc<FdOptions>,
    wants_to_quit: &Arc<AtomicBool>,
    pattern: Arc<Regex>,
    parallel_walker: ignore::WalkParallel,
    tx: Sender<WorkerResult>,
) {
    parallel_walker.run(|| {
        let config = Arc::clone(config);
        let pattern = Arc::clone(&pattern);
        let tx_thread = tx.clone();
        let wants_to_quit = Arc::clone(wants_to_quit);

        Box::new(move |entry_o| {
            if wants_to_quit.load(Ordering::Relaxed) {
                return ignore::WalkState::Quit;
            }

            let entry = match entry_o {
                Ok(e) => e,
                Err(err) => {
                    tx_thread.send(WorkerResult::Error(err)).unwrap();
                    return ignore::WalkState::Continue;
                }
            };

            if entry.depth() == 0 {
                return ignore::WalkState::Continue;
            }

            // Check the name first, since it doesn't require metadata
            let entry_path = entry.path();

            let search_str: Cow<OsStr> = if config.search_full_path {
                match fshelper::path_absolute_form(entry_path) {
                    Ok(path_abs_buf) => Cow::Owned(path_abs_buf.as_os_str().to_os_string()),
                    Err(_) => {
                        print_error_and_exit!("Unable to retrieve absolute path.");
                    }
                }
            } else {
                match entry_path.file_name() {
                    Some(filename) => Cow::Borrowed(filename),
                    None => unreachable!(
                        "Encountered file system entry without a file name. This should only \
                         happen for paths like 'foo/bar/..' or '/' which are not supposed to \
                         appear in a file system traversal."
                    ),
                }
            };

            if !pattern.is_match(&osstr_to_bytes(search_str.as_ref())) {
                return ignore::WalkState::Continue;
            }

            // Filter out unwanted extensions.
            if let Some(ref exts_regex) = config.extensions {
                if let Some(path_str) = entry_path.file_name() {
                    if !exts_regex.is_match(&osstr_to_bytes(path_str)) {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted file types.
            if let Some(ref file_types) = config.file_types {
                if let Some(ref entry_type) = entry.file_type() {
                    if (!file_types.files && entry_type.is_file())
                        || (!file_types.directories && entry_type.is_dir())
                        || (!file_types.symlinks && entry_type.is_symlink())
                        || (file_types.executables_only
                            && !entry
                                .metadata()
                                .map(|m| fshelper::is_executable(&m))
                                .unwrap_or(false))
                        || (file_types.empty_only && !fshelper::is_empty(&entry))
                        || !(entry_type.is_file() || entry_type.is_dir() || entry_type.is_symlink())
                    {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted sizes if it is a file and we have been given size constraints.
            if !config.size_constraints.is_empty() {
                if entry_path.is_file() {
                    if let Ok(metadata) = entry_path.metadata() {
                        let file_size = metadata.len();
                        if config
                            .size_constraints
                            .iter()
                            .any(|sc| !sc.is_within(file_size))
                        {
                            return ignore::WalkState::Continue;
                        }
                    } else {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted modification times
            if !config.time_constraints.is_empty() {
                let mut matched = false;
                if let Ok(metadata) = entry_path.metadata() {
                    if let Ok(modified) = metadata.modified() {
                        matched = config
                            .time_constraints
                            .iter()
                            .all(|tf| tf.applies_to(&modified));
                    }
                }
                if !matched {
                    return ignore::WalkState::Continue;
                }
            }

            let send_result = tx_thread.send(WorkerResult::Entry(entry_path.to_owned()));

            if !send_result.is_ok() {
                return ignore::WalkState::Quit;
            }

            ignore::WalkState::Continue
        })
    });
}
<commit_msg>Revert back to master state<commit_after>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use crate::exec;
use crate::exit_codes::{merge_exitcodes, ExitCode};
use crate::fshelper;
use crate::internal::{opts::FdOptions, osstr_to_bytes, MAX_BUFFER_LENGTH};
use crate::output;

use std::borrow::Cow;
use std::error::Error;
use std::ffi::OsStr;
use std::io;
use std::path::PathBuf;
use std::process;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{channel, Receiver, Sender};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time;

use ignore::overrides::OverrideBuilder;
use ignore::{self, WalkBuilder};
use regex::bytes::Regex;

/// The receiver thread can either be buffering results or directly streaming to the console.
enum ReceiverMode {
    /// Receiver is still buffering in order to sort the results, if the search finishes fast
    /// enough.
    Buffering,

    /// Receiver is directly printing results to the output.
    Streaming,
}

/// The Worker threads can result in a valid entry having PathBuf or an error.
pub enum WorkerResult {
    Entry(PathBuf),
    Error(ignore::Error),
}

/// Recursively scan the given search path for files / pathnames matching the pattern.
///
/// If the `--exec` argument was supplied, this will create a thread pool for executing
/// jobs in parallel from a given command line and the discovered paths. Otherwise, each
/// path will simply be written to standard output.
pub fn scan(path_vec: &[PathBuf], pattern: Arc<Regex>, config: Arc<FdOptions>) -> ExitCode {
    let mut path_iter = path_vec.iter();
    let first_path_buf = path_iter
        .next()
        .expect("Error: Path vector can not be empty");
    let (tx, rx) = channel();

    let mut override_builder = OverrideBuilder::new(first_path_buf.as_path());

    for pattern in &config.exclude_patterns {
        let res = override_builder.add(pattern);
        if res.is_err() {
            print_error_and_exit!("Malformed exclude pattern '{}'", pattern);
        }
    }
    let overrides = override_builder.build().unwrap_or_else(|_| {
        print_error_and_exit!("Mismatch in exclude patterns");
    });

    let mut walker = WalkBuilder::new(first_path_buf.as_path());
    walker
        .hidden(config.ignore_hidden)
        .ignore(config.read_fdignore)
        .parents(config.read_fdignore || config.read_vcsignore)
        .git_ignore(config.read_vcsignore)
        .git_global(config.read_vcsignore)
        .git_exclude(config.read_vcsignore)
        .overrides(overrides)
        .follow_links(config.follow_links)
        // No need to check for supported platforms, option is unavailable on unsupported ones
        .same_file_system(config.one_file_system)
        .max_depth(config.max_depth);

    if config.read_fdignore {
        walker.add_custom_ignore_filename(".fdignore");
    }

    for ignore_file in &config.ignore_files {
        let result = walker.add_ignore(ignore_file);
        match result {
            Some(ignore::Error::Partial(_)) => (),
            Some(err) => {
                print_error!(
                    "{}",
                    format!(
                        "Malformed pattern in custom ignore file '{}': {}.",
                        ignore_file.to_string_lossy(),
                        err.description()
                    )
                );
            }
            None => (),
        }
    }

    for path_entry in path_iter {
        walker.add(path_entry.as_path());
    }

    let parallel_walker = walker.threads(config.threads).build_parallel();

    let wants_to_quit = Arc::new(AtomicBool::new(false));
    if config.ls_colors.is_some() && config.command.is_none() {
        let wq = Arc::clone(&wants_to_quit);
        ctrlc::set_handler(move || {
            if wq.load(Ordering::Relaxed) {
                // Ctrl-C has been pressed twice, exit NOW
                process::exit(ExitCode::KilledBySigint.into());
            } else {
                wq.store(true, Ordering::Relaxed);
            }
        })
        .unwrap();
    }

    // Spawn the thread that receives all results through the channel.
    let receiver_thread = spawn_receiver(&config, &wants_to_quit, rx);

    // Spawn the sender threads.
    spawn_senders(&config, &wants_to_quit, pattern, parallel_walker, tx);

    // Wait for the receiver thread to print out all results.
    let exit_code = receiver_thread.join().unwrap();

    if wants_to_quit.load(Ordering::Relaxed) {
        process::exit(ExitCode::KilledBySigint.into());
    }

    exit_code
}

fn spawn_receiver(
    config: &Arc<FdOptions>,
    wants_to_quit: &Arc<AtomicBool>,
    rx: Receiver<WorkerResult>,
) -> thread::JoinHandle<ExitCode> {
    let config = Arc::clone(config);
    let wants_to_quit = Arc::clone(wants_to_quit);

    let show_filesystem_errors = config.show_filesystem_errors;
    let threads = config.threads;

    thread::spawn(move || {
        // This will be set to `Some` if the `--exec` argument was supplied.
        if let Some(ref cmd) = config.command {
            if cmd.in_batch_mode() {
                exec::batch(rx, cmd, show_filesystem_errors)
            } else {
                let shared_rx = Arc::new(Mutex::new(rx));

                let out_perm = Arc::new(Mutex::new(()));

                // Each spawned job will store it's thread handle in here.
                let mut handles = Vec::with_capacity(threads);
                for _ in 0..threads {
                    let rx = Arc::clone(&shared_rx);
                    let cmd = Arc::clone(cmd);
                    let out_perm = Arc::clone(&out_perm);

                    // Spawn a job thread that will listen for and execute inputs.
                    let handle =
                        thread::spawn(move || exec::job(rx, cmd, out_perm, show_filesystem_errors));

                    // Push the handle of the spawned thread into the vector for later joining.
                    handles.push(handle);
                }

                // Wait for all threads to exit before exiting the program.
                let mut results: Vec<ExitCode> = Vec::new();
                for h in handles {
                    results.push(h.join().unwrap());
                }

                merge_exitcodes(results)
            }
        } else {
            let start = time::Instant::now();

            let mut buffer = vec![];

            // Start in buffering mode
            let mut mode = ReceiverMode::Buffering;

            // Maximum time to wait before we start streaming to the console.
            let max_buffer_time = config
                .max_buffer_time
                .unwrap_or_else(|| time::Duration::from_millis(100));

            let stdout = io::stdout();
            let mut stdout = stdout.lock();

            for worker_result in rx {
                match worker_result {
                    WorkerResult::Entry(value) => {
                        match mode {
                            ReceiverMode::Buffering => {
                                buffer.push(value);

                                // Have we reached the maximum buffer size or maximum buffering time?
                                if buffer.len() > MAX_BUFFER_LENGTH
                                    || time::Instant::now() - start > max_buffer_time
                                {
                                    // Flush the buffer
                                    for v in &buffer {
                                        output::print_entry(
                                            &mut stdout,
                                            v,
                                            &config,
                                            &wants_to_quit,
                                        );
                                    }
                                    buffer.clear();

                                    // Start streaming
                                    mode = ReceiverMode::Streaming;
                                }
                            }
                            ReceiverMode::Streaming => {
                                output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                            }
                        }
                    }
                    WorkerResult::Error(err) => {
                        if show_filesystem_errors {
                            print_error!("{}", err);
                        }
                    }
                }
            }

            // If we have finished fast enough (faster than max_buffer_time), we haven't streamed
            // anything to the console, yet. In this case, sort the results and print them:
            if !buffer.is_empty() {
                buffer.sort();
                for value in buffer {
                    output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                }
            }

            ExitCode::Success
        }
    })
}

fn spawn_senders(
    config: &Arc<FdOptions>,
    wants_to_quit: &Arc<AtomicBool>,
    pattern: Arc<Regex>,
    parallel_walker: ignore::WalkParallel,
    tx: Sender<WorkerResult>,
) {
    parallel_walker.run(|| {
        let config = Arc::clone(config);
        let pattern = Arc::clone(&pattern);
        let tx_thread = tx.clone();
        let wants_to_quit = Arc::clone(wants_to_quit);

        Box::new(move |entry_o| {
            if wants_to_quit.load(Ordering::Relaxed) {
                return ignore::WalkState::Quit;
            }

            let entry = match entry_o {
                Ok(e) => e,
                Err(err) => {
                    tx_thread.send(WorkerResult::Error(err)).unwrap();
                    return ignore::WalkState::Continue;
                }
            };

            if entry.depth() == 0 {
                return ignore::WalkState::Continue;
            }

            // Check the name first, since it doesn't require metadata
            let entry_path = entry.path();

            let search_str: Cow<OsStr> = if config.search_full_path {
                match fshelper::path_absolute_form(entry_path) {
                    Ok(path_abs_buf) => Cow::Owned(path_abs_buf.as_os_str().to_os_string()),
                    Err(_) => {
                        print_error_and_exit!("Unable to retrieve absolute path.");
                    }
                }
            } else {
                match entry_path.file_name() {
                    Some(filename) => Cow::Borrowed(filename),
                    None => unreachable!(
                        "Encountered file system entry without a file name. This should only \
                         happen for paths like 'foo/bar/..' or '/' which are not supposed to \
                         appear in a file system traversal."
                    ),
                }
            };

            if !pattern.is_match(&osstr_to_bytes(search_str.as_ref())) {
                return ignore::WalkState::Continue;
            }

            // Filter out unwanted extensions.
            if let Some(ref exts_regex) = config.extensions {
                if let Some(path_str) = entry_path.file_name() {
                    if !exts_regex.is_match(&osstr_to_bytes(path_str)) {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted file types.
            if let Some(ref file_types) = config.file_types {
                if let Some(ref entry_type) = entry.file_type() {
                    if (!file_types.files && entry_type.is_file())
                        || (!file_types.directories && entry_type.is_dir())
                        || (!file_types.symlinks && entry_type.is_symlink())
                        || (file_types.executables_only
                            && !entry
                                .metadata()
                                .map(|m| fshelper::is_executable(&m))
                                .unwrap_or(false))
                        || (file_types.empty_only && !fshelper::is_empty(&entry))
                        || !(entry_type.is_file() || entry_type.is_dir() || entry_type.is_symlink())
                    {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted sizes if it is a file and we have been given size constraints.
            if !config.size_constraints.is_empty() {
                if entry_path.is_file() {
                    if let Ok(metadata) = entry_path.metadata() {
                        let file_size = metadata.len();
                        if config
                            .size_constraints
                            .iter()
                            .any(|sc| !sc.is_within(file_size))
                        {
                            return ignore::WalkState::Continue;
                        }
                    } else {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted modification times
            if !config.time_constraints.is_empty() {
                let mut matched = false;
                if let Ok(metadata) = entry_path.metadata() {
                    if let Ok(modified) = metadata.modified() {
                        matched = config
                            .time_constraints
                            .iter()
                            .all(|tf| tf.applies_to(&modified));
                    }
                }
                if !matched {
                    return ignore::WalkState::Continue;
                }
            }

            let send_result = tx_thread.send(WorkerResult::Entry(entry_path.to_owned()));

            if !send_result.is_ok() {
                return ignore::WalkState::Quit;
            }

            ignore::WalkState::Continue
        })
    });
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use exec::{self, TokenizedCommand};
use fshelper;
use internal::{error, FdOptions};
use output;

use std::path::Path;
use std::sync::{Arc, Mutex};
use std::sync::mpsc::channel;
use std::thread;
use std::time;

use ignore::{self, WalkBuilder};
use ignore::overrides::OverrideBuilder;
use regex::Regex;

/// The receiver thread can either be buffering results or directly streaming to the console.
enum ReceiverMode {
    /// Receiver is still buffering in order to sort the results, if the search finishes fast
    /// enough.
    Buffering,

    /// Receiver is directly printing results to the output.
    Streaming,
}

/// The type of file to search for.
#[derive(Copy, Clone)]
pub enum FileType {
    Any,
    RegularFile,
    Directory,
    SymLink,
}

/// Recursively scan the given search path for files / pathnames matching the pattern.
///
/// If the `--exec` argument was supplied, this will create a thread pool for executing
/// jobs in parallel from a given command line and the discovered paths. Otherwise, each
/// path will simply be written to standard output.
pub fn scan(root: &Path, pattern: Arc<Regex>, config: Arc<FdOptions>) {
    let (tx, rx) = channel();
    let threads = config.threads;

    let mut override_builder = OverrideBuilder::new(root);

    for pattern in config.exclude_patterns.iter() {
        let res = override_builder.add(pattern);
        if res.is_err() {
            error(&format!("Error: malformed exclude pattern '{}'", pattern));
        }
    }
    let overrides = override_builder.build().unwrap_or_else(|_| {
        error("Mismatch in exclude patterns");
    });

    let walker = WalkBuilder::new(root)
        .hidden(config.ignore_hidden)
        .ignore(config.read_ignore)
        .git_ignore(config.read_ignore)
        .parents(config.read_ignore)
        .git_global(config.read_ignore)
        .git_exclude(config.read_ignore)
        .overrides(overrides)
        .follow_links(config.follow_links)
        .max_depth(config.max_depth)
        .threads(threads)
        .build_parallel();

    // Spawn the thread that receives all results through the channel.
    let rx_config = Arc::clone(&config);
    let receiver_thread = thread::spawn(move || {
        // This will be set to `Some` if the `--exec` argument was supplied.
        if let Some(ref cmd) = rx_config.command {
            let shared_rx = Arc::new(Mutex::new(rx));

            let out_perm = Arc::new(Mutex::new(()));

            // This is safe because `cmd` will exist beyond the end of this scope.
            // It's required to tell Rust that it's safe to share across threads.
            let cmd = unsafe { Arc::from_raw(cmd as *const TokenizedCommand) };

            // Each spawned job will store it's thread handle in here.
            let mut handles = Vec::with_capacity(threads);
            for _ in 0..threads {
                let rx = Arc::clone(&shared_rx);
                let cmd = Arc::clone(&cmd);
                let out_perm = Arc::clone(&out_perm);

                // Spawn a job thread that will listen for and execute inputs.
                let handle = thread::spawn(move || exec::job(rx, cmd, out_perm));

                // Push the handle of the spawned thread into the vector for later joining.
                handles.push(handle);
            }

            // Wait for all threads to exit before exiting the program.
            for h in handles {
                h.join().unwrap();
            }
        } else {
            let start = time::Instant::now();

            let mut buffer = vec![];

            // Start in buffering mode
            let mut mode = ReceiverMode::Buffering;

            // Maximum time to wait before we start streaming to the console.
            let max_buffer_time = rx_config.max_buffer_time.unwrap_or_else(
                || time::Duration::from_millis(100),
            );

            for value in rx {
                match mode {
                    ReceiverMode::Buffering => {
                        buffer.push(value);

                        // Have we reached the maximum time?
                        if time::Instant::now() - start > max_buffer_time {
                            // Flush the buffer
                            for v in &buffer {
                                output::print_entry(v, &rx_config);
                            }
                            buffer.clear();

                            // Start streaming
                            mode = ReceiverMode::Streaming;
                        }
                    }
                    ReceiverMode::Streaming => {
                        output::print_entry(&value, &rx_config);
                    }
                }
            }

            // If we have finished fast enough (faster than max_buffer_time), we haven't streamed
            // anything to the console, yet. In this case, sort the results and print them:
            if !buffer.is_empty() {
                buffer.sort();
                for value in buffer {
                    output::print_entry(&value, &rx_config);
                }
            }
        }
    });

    // Spawn the sender threads.
    walker.run(|| {
        let config = Arc::clone(&config);
        let pattern = Arc::clone(&pattern);
        let tx_thread = tx.clone();
        let root = root.to_owned();

        Box::new(move |entry_o| {
            let entry = match entry_o {
                Ok(e) => e,
                Err(_) => return ignore::WalkState::Continue,
            };

            let entry_path = entry.path();

            if entry_path == root {
                return ignore::WalkState::Continue;
            }

            // Filter out unwanted file types.
            match config.file_type {
                FileType::Any => (),
                FileType::RegularFile => {
                    if entry.file_type().map_or(true, |ft| !ft.is_file()) {
                        return ignore::WalkState::Continue;
                    }
                }
                FileType::Directory => {
                    if entry.file_type().map_or(true, |ft| !ft.is_dir()) {
                        return ignore::WalkState::Continue;
                    }
                }
                FileType::SymLink => {
                    if entry.file_type().map_or(true, |ft| !ft.is_symlink()) {
                        return ignore::WalkState::Continue;
                    }
                }
            }

            // Filter out unwanted extensions.
            if let Some(ref filter_ext) = config.extension {
                let entry_ext = entry_path.extension().map(
                    |e| e.to_string_lossy().to_lowercase(),
                );
                if entry_ext.map_or(true, |ext| ext != *filter_ext) {
                    return ignore::WalkState::Continue;
                }
            }

            let search_str_o = if config.search_full_path {
                match fshelper::path_absolute_form(entry_path) {
                    Ok(path_abs_buf) => Some(path_abs_buf.to_string_lossy().into_owned().into()),
                    Err(_) => error("Error: unable to get full path."),
                }
            } else {
                entry_path.file_name().map(|f| f.to_string_lossy())
            };

            if let Some(search_str) = search_str_o {
                if pattern.is_match(&*search_str) {
                    // TODO: take care of the unwrap call
                    tx_thread.send(entry_path.to_owned()).unwrap()
                }
            }

            ignore::WalkState::Continue
        })
    });

    // Drop the initial sender. If we don't do this, the receiver will block even
    // if all threads have finished, since there is still one sender around.
    drop(tx);

    // Wait for the receiver thread to print out all results.
    receiver_thread.join().unwrap();
}
<commit_msg>Workaround for the `unsafe` block.<commit_after>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use exec;
use fshelper;
use internal::{error, FdOptions};
use output;

use std::path::Path;
use std::sync::{Arc, Mutex};
use std::sync::mpsc::channel;
use std::thread;
use std::time;

use ignore::{self, WalkBuilder};
use ignore::overrides::OverrideBuilder;
use regex::Regex;

/// The receiver thread can either be buffering results or directly streaming to the console.
enum ReceiverMode {
    /// Receiver is still buffering in order to sort the results, if the search finishes fast
    /// enough.
    Buffering,

    /// Receiver is directly printing results to the output.
    Streaming,
}

/// The type of file to search for.
#[derive(Copy, Clone)]
pub enum FileType {
    Any,
    RegularFile,
    Directory,
    SymLink,
}

/// Recursively scan the given search path for files / pathnames matching the pattern.
///
/// If the `--exec` argument was supplied, this will create a thread pool for executing
/// jobs in parallel from a given command line and the discovered paths. Otherwise, each
/// path will simply be written to standard output.
pub fn scan(root: &Path, pattern: Arc<Regex>, config: Arc<FdOptions>) {
    let (tx, rx) = channel();
    let threads = config.threads;

    let mut override_builder = OverrideBuilder::new(root);

    for pattern in config.exclude_patterns.iter() {
        let res = override_builder.add(pattern);
        if res.is_err() {
            error(&format!("Error: malformed exclude pattern '{}'", pattern));
        }
    }
    let overrides = override_builder.build().unwrap_or_else(|_| {
        error("Mismatch in exclude patterns");
    });

    let walker = WalkBuilder::new(root)
        .hidden(config.ignore_hidden)
        .ignore(config.read_ignore)
        .git_ignore(config.read_ignore)
        .parents(config.read_ignore)
        .git_global(config.read_ignore)
        .git_exclude(config.read_ignore)
        .overrides(overrides)
        .follow_links(config.follow_links)
        .max_depth(config.max_depth)
        .threads(threads)
        .build_parallel();

    // Spawn the thread that receives all results through the channel.
    let rx_config = Arc::clone(&config);
    let receiver_thread = thread::spawn(move || {
        // This will be set to `Some` if the `--exec` argument was supplied.
        if let Some(ref cmd) = rx_config.command {
            let shared_rx = Arc::new(Mutex::new(rx));

            let out_perm = Arc::new(Mutex::new(()));

            // TODO: the following line is a workaround to replace the `unsafe` block that was
            // previously used here to avoid the (unnecessary?) cloning of the command. The
            // `unsafe` block caused problems on some platforms (SIGILL instructions on Linux) and
            // therefore had to be removed.
            let cmd = Arc::new(cmd.clone());

            // Each spawned job will store it's thread handle in here.
            let mut handles = Vec::with_capacity(threads);
            for _ in 0..threads {
                let rx = Arc::clone(&shared_rx);
                let cmd = Arc::clone(&cmd);
                let out_perm = Arc::clone(&out_perm);

                // Spawn a job thread that will listen for and execute inputs.
                let handle = thread::spawn(move || exec::job(rx, cmd, out_perm));

                // Push the handle of the spawned thread into the vector for later joining.
                handles.push(handle);
            }

            // Wait for all threads to exit before exiting the program.
            for h in handles {
                h.join().unwrap();
            }
        } else {
            let start = time::Instant::now();

            let mut buffer = vec![];

            // Start in buffering mode
            let mut mode = ReceiverMode::Buffering;

            // Maximum time to wait before we start streaming to the console.
            let max_buffer_time = rx_config.max_buffer_time.unwrap_or_else(
                || time::Duration::from_millis(100),
            );

            for value in rx {
                match mode {
                    ReceiverMode::Buffering => {
                        buffer.push(value);

                        // Have we reached the maximum time?
                        if time::Instant::now() - start > max_buffer_time {
                            // Flush the buffer
                            for v in &buffer {
                                output::print_entry(v, &rx_config);
                            }
                            buffer.clear();

                            // Start streaming
                            mode = ReceiverMode::Streaming;
                        }
                    }
                    ReceiverMode::Streaming => {
                        output::print_entry(&value, &rx_config);
                    }
                }
            }

            // If we have finished fast enough (faster than max_buffer_time), we haven't streamed
            // anything to the console, yet. In this case, sort the results and print them:
            if !buffer.is_empty() {
                buffer.sort();
                for value in buffer {
                    output::print_entry(&value, &rx_config);
                }
            }
        }
    });

    // Spawn the sender threads.
    walker.run(|| {
        let config = Arc::clone(&config);
        let pattern = Arc::clone(&pattern);
        let tx_thread = tx.clone();
        let root = root.to_owned();

        Box::new(move |entry_o| {
            let entry = match entry_o {
                Ok(e) => e,
                Err(_) => return ignore::WalkState::Continue,
            };

            let entry_path = entry.path();

            if entry_path == root {
                return ignore::WalkState::Continue;
            }

            // Filter out unwanted file types.
            match config.file_type {
                FileType::Any => (),
                FileType::RegularFile => {
                    if entry.file_type().map_or(true, |ft| !ft.is_file()) {
                        return ignore::WalkState::Continue;
                    }
                }
                FileType::Directory => {
                    if entry.file_type().map_or(true, |ft| !ft.is_dir()) {
                        return ignore::WalkState::Continue;
                    }
                }
                FileType::SymLink => {
                    if entry.file_type().map_or(true, |ft| !ft.is_symlink()) {
                        return ignore::WalkState::Continue;
                    }
                }
            }

            // Filter out unwanted extensions.
            if let Some(ref filter_ext) = config.extension {
                let entry_ext = entry_path.extension().map(
                    |e| e.to_string_lossy().to_lowercase(),
                );
                if entry_ext.map_or(true, |ext| ext != *filter_ext) {
                    return ignore::WalkState::Continue;
                }
            }

            let search_str_o = if config.search_full_path {
                match fshelper::path_absolute_form(entry_path) {
                    Ok(path_abs_buf) => Some(path_abs_buf.to_string_lossy().into_owned().into()),
                    Err(_) => error("Error: unable to get full path."),
                }
            } else {
                entry_path.file_name().map(|f| f.to_string_lossy())
            };

            if let Some(search_str) = search_str_o {
                if pattern.is_match(&*search_str) {
                    // TODO: take care of the unwrap call
                    tx_thread.send(entry_path.to_owned()).unwrap()
                }
            }

            ignore::WalkState::Continue
        })
    });

    // Drop the initial sender. If we don't do this, the receiver will block even
    // if all threads have finished, since there is still one sender around.
    drop(tx);

    // Wait for the receiver thread to print out all results.
    receiver_thread.join().unwrap();
}
<|endoftext|>


<commit_before>use std::io::Write;
use std::io::Read;
use std::io::Cursor;
use std::io::SeekFrom;
use std::io::Seek;
use std::marker::PhantomData;
use core::schema::DocId;
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use core::serialize::*;
use std::io;

struct LayerBuilder<T: BinarySerializable> {
    period: usize,
    buffer: Vec<u8>,
    remaining: usize,
    len: usize,
    _phantom_: PhantomData<T>,
}

impl<T: BinarySerializable> LayerBuilder<T> {

    fn written_size(&self,) -> usize {
        self.buffer.len()
    }

    fn write(&self, output: &mut Write) -> Result<(), io::Error> {
        try!(output.write_all(&self.buffer));
        Ok(())
    }

    fn with_period(period: usize) -> LayerBuilder<T> {
        LayerBuilder {
            period: period,
            buffer: Vec::new(),
            remaining: period,
            len: 0,
            _phantom_: PhantomData,
        }
    }

    fn insert(&mut self, doc_id: DocId, value: &T) -> Option<(DocId, u32)> {
        self.remaining -= 1;
        self.len += 1;
        let offset = self.written_size() as u32; // TODO not sure if we want after or here
        let res;
        if self.remaining == 0 {
            self.remaining = self.period;
            res = Some((doc_id, offset));
        }
        else {
            res = None;
        }
        self.buffer.write_u32::<BigEndian>(doc_id);
        value.serialize(&mut self.buffer);
        res
    }
}

pub struct SkipListBuilder<T: BinarySerializable> {
    period: usize,
    data_layer: LayerBuilder<T>,
    skip_layers: Vec<LayerBuilder<u32>>,
}

impl<T: BinarySerializable> SkipListBuilder<T> {

    pub fn new(period: usize) -> SkipListBuilder<T> {
        SkipListBuilder {
            period: period,
            data_layer: LayerBuilder::with_period(period),
            skip_layers: Vec::new(),
        }
    }

    fn get_skip_layer<'a>(&'a mut self, layer_id: usize) -> &mut LayerBuilder<u32> {
        if layer_id == self.skip_layers.len() {
            let layer_builder = LayerBuilder::with_period(self.period);
            self.skip_layers.push(layer_builder);
        }
        &mut self.skip_layers[layer_id]
    }

    pub fn insert(&mut self, doc_id: DocId, dest: &T) {
        let mut layer_id = 0;
        let mut skip_pointer = self.data_layer.insert(doc_id, dest);
        loop {
            skip_pointer = match skip_pointer {
                Some((skip_doc_id, skip_offset)) =>
                    self
                        .get_skip_layer(layer_id)
                        .insert(skip_doc_id, &skip_offset),
                None => { return; }
            };
            layer_id += 1;
        }
    }

    pub fn write<W: Write>(self, output: &mut Write) -> io::Result<()> {
        let mut size: u32 = 0;
        let mut layer_sizes: Vec<u32> = Vec::new();
        size += self.data_layer.buffer.len() as u32;
        layer_sizes.push(size);
        for layer in self.skip_layers.iter() {
            size += layer.buffer.len() as u32;
            layer_sizes.push(size);
        }
        try!(layer_sizes.serialize(output));
        try!(self.data_layer.write(output));
        for layer in self.skip_layers.iter() {
            try!(layer.write(output));
        }
        Ok(())
    }
}

struct Layer<'a, T> {
    cursor: Cursor<&'a [u8]>,
    next_id: DocId,
    _phantom_: PhantomData<T>,
}

impl<'a, T: BinarySerializable> Iterator for Layer<'a, T> {

    type Item = (DocId, T);

    fn next(&mut self,)-> Option<(DocId, T)> {
        if self.next_id == u32::max_value() {
            None
        }
        else {
            let cur_val = T::deserialize(&mut self.cursor).unwrap();
            let cur_id = self.next_id;
            self.next_id =
                match u32::deserialize(&mut self.cursor) {
                    Ok(val) => val,
                    Err(_) => u32::max_value()
                };
            Some((cur_id, cur_val))
        }
    }
}

static EMPTY: [u8; 0] = [];

impl<'a, T: BinarySerializable> Layer<'a, T> {

    fn read(mut cursor: Cursor<&'a [u8]>) -> Layer<'a, T> {
        // TODO error handling?
        let next_id = match cursor.read_u32::<BigEndian>() {
            Ok(val) => val,
            Err(_) => u32::max_value(),
        };
        Layer {
            cursor: cursor,
            next_id: next_id,
            _phantom_: PhantomData,
        }
    }

    fn empty() -> Layer<'a, T> {
        Layer {
            cursor: Cursor::new(&EMPTY),
            next_id: DocId::max_value(),
            _phantom_: PhantomData,
        }
    }

    fn seek_offset(&mut self, offset: usize)  {
        self.cursor.seek(SeekFrom::Start(offset as u64)).unwrap();
        self.next_id = match self.cursor.read_u32::<BigEndian>() {
            Ok(val) => val,
            Err(_) => u32::max_value(),
        };
    }

    // Returns the last element (key, val)
    // such that (key < doc_id)
    //
    // If there is no such element anymore,
    // returns None.
    fn seek(&mut self, doc_id: DocId) -> Option<(DocId, T)> {
        let mut val = None;
        while self.next_id < doc_id {
            match self.next() {
                None => { break; },
                v => { val = v; }
            }
        }
        val
    }
}

pub struct SkipList<'a, T: BinarySerializable> {
    data_layer: Layer<'a, T>,
    skip_layers: Vec<Layer<'a, u32>>,
}

impl<'a, T: BinarySerializable> Iterator for SkipList<'a, T> {

    type Item = (DocId, T);

    fn next(&mut self,)-> Option<(DocId, T)> {
        self.data_layer.next()
    }
}

impl<'a, T: BinarySerializable> SkipList<'a, T> {

    pub fn seek(&mut self, doc_id: DocId) -> Option<(DocId, T)> {
        let mut next_layer_skip: Option<(DocId, u32)> = None;
        for skip_layer_id in 0..self.skip_layers.len() {
            let mut skip_layer: &mut Layer<'a, u32> = &mut self.skip_layers[skip_layer_id];
            match next_layer_skip {
                 Some((_, offset)) => { skip_layer.seek_offset(offset as usize); },
                 None => {}
             };
             next_layer_skip = skip_layer.seek(doc_id);
         }
         match next_layer_skip {
             Some((_, offset)) => { self.data_layer.seek_offset(offset as usize); },
             None => {}
         };
         self.data_layer.seek(doc_id)
    }

    pub fn read(data: &'a [u8]) -> SkipList<'a, T> {
        let mut cursor = Cursor::new(data);
        let offsets: Vec<u32> = Vec::deserialize(&mut cursor).unwrap();
        let num_layers = offsets.len();
        let start_position = cursor.position() as usize;
        let layers_data: &[u8] = &data[start_position..data.len()];
        let data_layer: Layer<'a, T> =
            if num_layers == 0 { Layer::empty() }
            else {
                let first_layer_data: &[u8] = &layers_data[..offsets[0] as usize];
                let first_layer_cursor = Cursor::new(first_layer_data);
                Layer::read(first_layer_cursor)
            };
        let mut skip_layers =
            if num_layers > 0 {
                offsets.iter()
                    .zip(&offsets[1..])
                    .map(|(start, stop)| {
                        let layer_data: &[u8] = &layers_data[*start as usize..*stop as usize];
                        let cursor = Cursor::new(layer_data);
                        Layer::read(cursor)
                    })
                    .collect()
            }
            else {
                Vec::new()
            };
        skip_layers.reverse();
        SkipList {
            skip_layers: skip_layers,
            data_layer: data_layer,
        }
    }
}

#[test]
fn test_skip_list_builder() {
    {
        let mut output: Vec<u8> = Vec::new();
        let mut skip_list_builder: SkipListBuilder<u32> = SkipListBuilder::new(10);
        skip_list_builder.insert(2, &3);
        skip_list_builder.write::<Vec<u8>>(&mut output);
        assert_eq!(output.len(), 16);
        assert_eq!(output[0], 0);
    }
    {
        let mut output: Vec<u8> = Vec::new();
        let mut skip_list_builder: SkipListBuilder<u32> = SkipListBuilder::new(3);
        for i in 0..9 {
            skip_list_builder.insert(i, &i);
        }
        skip_list_builder.write::<Vec<u8>>(&mut output);
        assert_eq!(output.len(), 120);
        assert_eq!(output[0], 0);
    }
    {
        // checking that void gets serialized to nothing.
        let mut output: Vec<u8> = Vec::new();
        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(3);
        for i in 0..9 {
            skip_list_builder.insert(i, &());
        }
        skip_list_builder.write::<Vec<u8>>(&mut output);
        assert_eq!(output.len(), 84);
        assert_eq!(output[0], 0);
    }
}

#[test]
fn test_skip_list_reader() {
    {
        let mut output: Vec<u8> = Vec::new();
        let mut skip_list_builder: SkipListBuilder<u32> = SkipListBuilder::new(10);
        skip_list_builder.insert(2, &3);
        skip_list_builder.write::<Vec<u8>>(&mut output);
        let mut skip_list: SkipList<u32> = SkipList::read(&mut output);
        assert_eq!(skip_list.next(), Some((2, 3)));
    }
    {
        let mut output: Vec<u8> = Vec::new();
        let skip_list_builder: SkipListBuilder<u32> = SkipListBuilder::new(10);
        skip_list_builder.write::<Vec<u8>>(&mut output);
        let mut skip_list: SkipList<u32> = SkipList::read(&mut output);
        assert_eq!(skip_list.next(), None);
    }
    {
        let mut output: Vec<u8> = Vec::new();
        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(2);
        skip_list_builder.insert(2, &());
        skip_list_builder.insert(3, &());
        skip_list_builder.insert(5, &());
        skip_list_builder.insert(7, &());
        skip_list_builder.insert(9, &());
        skip_list_builder.write::<Vec<u8>>(&mut output);
        let mut skip_list: SkipList<()> = SkipList::read(&mut output);
        assert_eq!(skip_list.next().unwrap(), (2, ()));
        assert_eq!(skip_list.next().unwrap(), (3, ()));
        assert_eq!(skip_list.next().unwrap(), (5, ()));
        assert_eq!(skip_list.next().unwrap(), (7, ()));
        assert_eq!(skip_list.next().unwrap(), (9, ()));
        assert_eq!(skip_list.next(), None);
    }
    {
        let mut output: Vec<u8> = Vec::new();
        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(2);
        skip_list_builder.insert(2, &());
        skip_list_builder.insert(3, &());
        skip_list_builder.insert(5, &());
        skip_list_builder.insert(7, &());
        skip_list_builder.insert(9, &());
        skip_list_builder.write::<Vec<u8>>(&mut output);
        let mut skip_list: SkipList<()> = SkipList::read(&mut output);
        assert_eq!(skip_list.next().unwrap(), (2, ()));
        skip_list.seek(5);
        assert_eq!(skip_list.next().unwrap(), (5, ()));
        assert_eq!(skip_list.next().unwrap(), (7, ()));
        assert_eq!(skip_list.next().unwrap(), (9, ()));
        assert_eq!(skip_list.next(), None);
    }
    {
        let mut output: Vec<u8> = Vec::new();
        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(3);
        skip_list_builder.insert(2, &());
        skip_list_builder.insert(3, &());
        skip_list_builder.insert(5, &());
        skip_list_builder.insert(6, &());
        skip_list_builder.write::<Vec<u8>>(&mut output);
        let mut skip_list: SkipList<()> = SkipList::read(&mut output);
        assert_eq!(skip_list.next().unwrap(), (2, ()));
        skip_list.seek(6);
        assert_eq!(skip_list.next().unwrap(), (6, ()));
        assert_eq!(skip_list.next(), None);

    }
    {
        let mut output: Vec<u8> = Vec::new();
        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(2);
        skip_list_builder.insert(2, &());
        skip_list_builder.insert(3, &());
        skip_list_builder.insert(5, &());
        skip_list_builder.insert(7, &());
        skip_list_builder.insert(9, &());
        skip_list_builder.write::<Vec<u8>>(&mut output);
        let mut skip_list: SkipList<()> = SkipList::read(&mut output);
        assert_eq!(skip_list.next().unwrap(), (2, ()));
        skip_list.seek(10);
        assert_eq!(skip_list.next(), None);
    }
    {
        let mut output: Vec<u8> = Vec::new();
        let mut skip_list_builder: SkipListBuilder<()> = SkipListBuilder::new(3);
        for i in 0..1000 {
            skip_list_builder.insert(i, &());
        }
        skip_list_builder.insert(1004, &());
        skip_list_builder.write::<Vec<u8>>(&mut output);
        let mut skip_list: SkipList<()> = SkipList::read(&mut output);
        assert_eq!(skip_list.next().unwrap(), (0, ()));
        skip_list.seek(431);
        assert_eq!(skip_list.next().unwrap(), (431,()) );
        skip_list.seek(1003);
        assert_eq!(skip_list.next().unwrap(), (1004,()) );
        assert_eq!(skip_list.next(), None);
    }
}
<commit_msg>remove skip list temporarily<commit_after><|endoftext|>