<commit_before>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use crate::exec;
use crate::exit_codes::{merge_exitcodes, ExitCode};
use crate::fshelper;
use crate::internal::{opts::FdOptions, osstr_to_bytes, MAX_BUFFER_LENGTH};
use crate::output;

use std::borrow::Cow;
use std::error::Error;
use std::ffi::OsStr;
use std::io;
use std::path::PathBuf;
use std::process;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{channel, Receiver, Sender};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time;

use ignore::overrides::OverrideBuilder;
use ignore::{self, WalkBuilder};
use regex::bytes::Regex;

/// The receiver thread can either be buffering results or directly streaming to the console.
enum ReceiverMode {
    /// Receiver is still buffering in order to sort the results, if the search finishes fast
    /// enough.
    Buffering,

    /// Receiver is directly printing results to the output.
    Streaming,
}

/// The Worker threads can result in a valid entry having PathBuf or an error.
pub enum WorkerResult {
    Entry(PathBuf),
    Error(ignore::Error),
}

/// Recursively scan the given search path for files / pathnames matching the pattern.
///
/// If the `--exec` argument was supplied, this will create a thread pool for executing
/// jobs in parallel from a given command line and the discovered paths. Otherwise, each
/// path will simply be written to standard output.
pub fn scan(path_vec: &[PathBuf], pattern: Arc<Regex>, config: Arc<FdOptions>) -> ExitCode {
    let mut path_iter = path_vec.iter();
    let first_path_buf = path_iter
        .next()
        .expect("Error: Path vector can not be empty");
    let (tx, rx) = channel();

    let mut override_builder = OverrideBuilder::new(first_path_buf.as_path());

    for pattern in &config.exclude_patterns {
        let res = override_builder.add(pattern);
        if res.is_err() {
            print_error_and_exit!("Malformed exclude pattern '{}'", pattern);
        }
    }
    let overrides = override_builder.build().unwrap_or_else(|_| {
        print_error_and_exit!("Mismatch in exclude patterns");
    });

    let mut walker = WalkBuilder::new(first_path_buf.as_path());
    walker
        .hidden(config.ignore_hidden)
        .ignore(config.read_fdignore)
        .parents(config.read_fdignore || config.read_vcsignore)
        .git_ignore(config.read_vcsignore)
        .git_global(config.read_vcsignore)
        .git_exclude(config.read_vcsignore)
        .overrides(overrides)
        .follow_links(config.follow_links)
        // No need to check for supported platforms, option is unavailable on unsupported ones
        .same_file_system(config.one_file_system)
        .max_depth(config.max_depth);

    if config.read_fdignore {
        walker.add_custom_ignore_filename(".fdignore");
    }

    for ignore_file in &config.ignore_files {
        let result = walker.add_ignore(ignore_file);
        match result {
            Some(ignore::Error::Partial(_)) => (),
            Some(err) => {
                print_error!(
                    "{}",
                    format!(
                        "Malformed pattern in custom ignore file '{}': {}.",
                        ignore_file.to_string_lossy(),
                        err.description()
                    )
                );
            }
            None => (),
        }
    }

    for path_entry in path_iter {
        walker.add(path_entry.as_path());
    }

    let parallel_walker = walker.threads(config.threads).build_parallel();

    let wants_to_quit = Arc::new(AtomicBool::new(false));
    if config.ls_colors.is_some() && config.command.is_none() {
        let wq = Arc::clone(&wants_to_quit);
        ctrlc::set_handler(move || {
            if wq.load(Ordering::Relaxed) {
                // Ctrl-C has been pressed twice, exit NOW
                process::exit(ExitCode::KilledBySigint.into());
            } else {
                wq.store(true, Ordering::Relaxed);
            }
        })
        .unwrap();
    }

    // Spawn the thread that receives all results through the channel.
    let receiver_thread = spawn_receiver(&config, &wants_to_quit, rx);

    // Spawn the sender threads.
    spawn_senders(&config, &wants_to_quit, pattern, parallel_walker, tx);

    // Wait for the receiver thread to print out all results.
    let exit_code = receiver_thread.join().unwrap();

    if wants_to_quit.load(Ordering::Relaxed) {
        process::exit(ExitCode::KilledBySigint.into());
    }

    exit_code
}

fn spawn_receiver(
    config: &Arc<FdOptions>,
    wants_to_quit: &Arc<AtomicBool>,
    rx: Receiver<WorkerResult>,
) -> thread::JoinHandle<ExitCode> {
    let config = Arc::clone(config);
    let wants_to_quit = Arc::clone(wants_to_quit);

    let show_filesystem_errors = config.show_filesystem_errors;
    let threads = config.threads;

    thread::spawn(move || {
        // This will be set to `Some` if the `--exec` argument was supplied.
        if let Some(ref cmd) = config.command {
            if cmd.in_batch_mode() {
                exec::batch(rx, cmd, show_filesystem_errors)
            } else {
                let shared_rx = Arc::new(Mutex::new(rx));

                let out_perm = Arc::new(Mutex::new(()));

                // Each spawned job will store it's thread handle in here.
                let mut handles = Vec::with_capacity(threads);
                for _ in 0..threads {
                    let rx = Arc::clone(&shared_rx);
                    let cmd = Arc::clone(cmd);
                    let out_perm = Arc::clone(&out_perm);

                    // Spawn a job thread that will listen for and execute inputs.
                    let handle =
                        thread::spawn(move || exec::job(rx, cmd, out_perm, show_filesystem_errors));

                    // Push the handle of the spawned thread into the vector for later joining.
                    handles.push(handle);
                }

                // Wait for all threads to exit before exiting the program.
                let mut results: Vec<ExitCode> = Vec::new();
                for h in handles {
                    results.push(h.join().unwrap());
                }

                merge_exitcodes(results)
            }
        } else {
            let start = time::Instant::now();

            let mut buffer = vec![];

            // Start in buffering mode
            let mut mode = ReceiverMode::Buffering;

            // Maximum time to wait before we start streaming to the console.
            let max_buffer_time = config
                .max_buffer_time
                .unwrap_or_else(|| time::Duration::from_millis(100));

            let stdout = io::stdout();
            let mut stdout = stdout.lock();

            for worker_result in rx {
                match worker_result {
                    WorkerResult::Entry(value) => {
                        match mode {
                            ReceiverMode::Buffering => {
                                buffer.push(value);

                                // Have we reached the maximum buffer size or maximum buffering time?
                                if buffer.len() > MAX_BUFFER_LENGTH
                                    || time::Instant::now() - start > max_buffer_time
                                {
                                    // Flush the buffer
                                    for v in &buffer {
                                        output::print_entry(
                                            &mut stdout,
                                            v,
                                            &config,
                                            &wants_to_quit,
                                        );
                                    }
                                    buffer.clear();

                                    // Start streaming
                                    mode = ReceiverMode::Streaming;
                                }
                            }
                            ReceiverMode::Streaming => {
                                output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                            }
                        }
                    }
                    WorkerResult::Error(err) => {
                        if show_filesystem_errors {
                            print_error!("{}", err);
                        }
                    }
                }
            }

            // If we have finished fast enough (faster than max_buffer_time), we haven't streamed
            // anything to the console, yet. In this case, sort the results and print them:
            if !buffer.is_empty() {
                buffer.sort();
                for value in buffer {
                    output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                }
            }

            ExitCode::Success
        }
    })
}

fn spawn_senders(
    config: &Arc<FdOptions>,
    wants_to_quit: &Arc<AtomicBool>,
    pattern: Arc<Regex>,
    parallel_walker: ignore::WalkParallel,
    tx: Sender<WorkerResult>,
) {
    parallel_walker.run(|| {
        let config = Arc::clone(config);
        let pattern = Arc::clone(&pattern);
        let tx_thread = tx.clone();
        let wants_to_quit = Arc::clone(wants_to_quit);

        Box::new(move |entry_o| {
            if wants_to_quit.load(Ordering::Relaxed) {
                return ignore::WalkState::Quit;
            }

            let entry = match entry_o {
                Ok(e) => e,
                Err(err) => {
                    tx_thread.send(WorkerResult::Error(err)).unwrap();
                    return ignore::WalkState::Continue;
                }
            };

            if entry.depth() == 0 {
                return ignore::WalkState::Continue;
            }

            // Check the name first, since it doesn't require metadata
            let entry_path = entry.path();

            let search_str: Cow<OsStr> = if config.search_full_path {
                match fshelper::path_absolute_form(entry_path) {
                    Ok(path_abs_buf) => Cow::Owned(path_abs_buf.as_os_str().to_os_string()),
                    Err(_) => {
                        print_error_and_exit!("Unable to retrieve absolute path.");
                    }
                }
            } else {
                match entry_path.file_name() {
                    Some(filename) => Cow::Borrowed(filename),
                    None => unreachable!(
                        "Encountered file system entry without a file name. This should only \
                         happen for paths like 'foo/bar/..' or '/' which are not supposed to \
                         appear in a file system traversal."
                    ),
                }
            };

            if !pattern.is_match(&osstr_to_bytes(search_str.as_ref())) {
                return ignore::WalkState::Continue;
            }

            // Filter out unwanted extensions.
            if let Some(ref exts_regex) = config.extensions {
                if let Some(path_str) = entry_path.file_name() {
                    if !exts_regex.is_match(&osstr_to_bytes(path_str)) {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted file types.
            if let Some(ref file_types) = config.file_types {
                if let Some(ref entry_type) = entry.file_type() {
                    if (!file_types.files && entry_type.is_file())
                        || (!file_types.directories && entry_type.is_dir())
                        || (!file_types.symlinks && entry_type.is_symlink())
                        || (file_types.executables_only
                            && !entry
                                .metadata()
                                .map(|m| fshelper::is_executable(&m))
                                .unwrap_or(false))
                        || (file_types.empty_only && !fshelper::is_empty(&entry))
                        || !(entry_type.is_file() || entry_type.is_dir() || entry_type.is_symlink())
                    {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted sizes if it is a file and we have been given size constraints.
            if !config.size_constraints.is_empty() {
                if entry_path.is_file() {
                    if let Ok(metadata) = entry_path.metadata() {
                        let file_size = metadata.len();
                        if config
                            .size_constraints
                            .iter()
                            .any(|sc| !sc.is_within(file_size))
                        {
                            return ignore::WalkState::Continue;
                        }
                    } else {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted modification times
            if !config.time_constraints.is_empty() {
                let mut matched = false;
                if let Ok(metadata) = entry_path.metadata() {
                    if let Ok(modified) = metadata.modified() {
                        matched = config
                            .time_constraints
                            .iter()
                            .all(|tf| tf.applies_to(&modified));
                    }
                }
                if !matched {
                    return ignore::WalkState::Continue;
                }
            }

            let send_result = tx_thread.send(WorkerResult::Entry(entry_path.to_owned()));

            if !send_result.is_ok() {
                return ignore::WalkState::Quit;
            }

            ignore::WalkState::Continue
        })
    });
}
<commit_msg>walk: catch ignore NotFound error in the case of a broken symlink<commit_after>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

use crate::exec;
use crate::exit_codes::{merge_exitcodes, ExitCode};
use crate::fshelper;
use crate::internal::{opts::FdOptions, osstr_to_bytes, MAX_BUFFER_LENGTH};
use crate::output;

use std::borrow::Cow;
use std::convert::TryFrom;
use std::error::Error;
use std::ffi::OsStr;
use std::fs;
use std::io;
use std::path::PathBuf;
use std::process;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{channel, Receiver, Sender};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time;

use ignore::overrides::OverrideBuilder;
use ignore::{self, WalkBuilder};
use regex::bytes::Regex;

/// The receiver thread can either be buffering results or directly streaming to the console.
enum ReceiverMode {
    /// Receiver is still buffering in order to sort the results, if the search finishes fast
    /// enough.
    Buffering,

    /// Receiver is directly printing results to the output.
    Streaming,
}

/// The Worker threads can result in a valid entry having PathBuf or an error.
pub enum WorkerResult {
    Entry(PathBuf),
    Error(ignore::Error),
}

/// Converts a WorkerResult to a PathBuf. This is required as in some cases,
/// the underlying `ignore` libary will return an error when we actually
/// still want to show the path.
///
/// Currently, the cases supported are:
///   - a broken symlink should still be found and the path returned
impl TryFrom<WorkerResult> for PathBuf {
    type Error = ignore::Error;

    fn try_from(worker_result: WorkerResult) -> Result<PathBuf, Self::Error> {
        match worker_result {
            WorkerResult::Entry(value) => return Ok(value),
            WorkerResult::Error(err) => {
                if let ignore::Error::WithPath {
                    path,
                    err: path_err,
                } = &err
                {
                    if let ignore::Error::Io(io_err) = &**path_err {
                        if io_err.kind() == io::ErrorKind::NotFound {
                            if let Ok(metadata) = fs::symlink_metadata(&path) {
                                if metadata.file_type().is_symlink() {
                                    return Ok(path.to_path_buf());
                                }
                            }
                        }
                    }
                }
                return Err(err);
            }
        }
    }
}

/// Recursively scan the given search path for files / pathnames matching the pattern.
///
/// If the `--exec` argument was supplied, this will create a thread pool for executing
/// jobs in parallel from a given command line and the discovered paths. Otherwise, each
/// path will simply be written to standard output.
pub fn scan(path_vec: &[PathBuf], pattern: Arc<Regex>, config: Arc<FdOptions>) -> ExitCode {
    let mut path_iter = path_vec.iter();
    let first_path_buf = path_iter
        .next()
        .expect("Error: Path vector can not be empty");
    let (tx, rx) = channel();

    let mut override_builder = OverrideBuilder::new(first_path_buf.as_path());

    for pattern in &config.exclude_patterns {
        let res = override_builder.add(pattern);
        if res.is_err() {
            print_error_and_exit!("Malformed exclude pattern '{}'", pattern);
        }
    }
    let overrides = override_builder.build().unwrap_or_else(|_| {
        print_error_and_exit!("Mismatch in exclude patterns");
    });

    let mut walker = WalkBuilder::new(first_path_buf.as_path());
    walker
        .hidden(config.ignore_hidden)
        .ignore(config.read_fdignore)
        .parents(config.read_fdignore || config.read_vcsignore)
        .git_ignore(config.read_vcsignore)
        .git_global(config.read_vcsignore)
        .git_exclude(config.read_vcsignore)
        .overrides(overrides)
        .follow_links(config.follow_links)
        // No need to check for supported platforms, option is unavailable on unsupported ones
        .same_file_system(config.one_file_system)
        .max_depth(config.max_depth);

    if config.read_fdignore {
        walker.add_custom_ignore_filename(".fdignore");
    }

    for ignore_file in &config.ignore_files {
        let result = walker.add_ignore(ignore_file);
        match result {
            Some(ignore::Error::Partial(_)) => (),
            Some(err) => {
                print_error!(
                    "{}",
                    format!(
                        "Malformed pattern in custom ignore file '{}': {}.",
                        ignore_file.to_string_lossy(),
                        err.description()
                    )
                );
            }
            None => (),
        }
    }

    for path_entry in path_iter {
        walker.add(path_entry.as_path());
    }

    let parallel_walker = walker.threads(config.threads).build_parallel();

    let wants_to_quit = Arc::new(AtomicBool::new(false));
    if config.ls_colors.is_some() && config.command.is_none() {
        let wq = Arc::clone(&wants_to_quit);
        ctrlc::set_handler(move || {
            if wq.load(Ordering::Relaxed) {
                // Ctrl-C has been pressed twice, exit NOW
                process::exit(ExitCode::KilledBySigint.into());
            } else {
                wq.store(true, Ordering::Relaxed);
            }
        })
        .unwrap();
    }

    // Spawn the thread that receives all results through the channel.
    let receiver_thread = spawn_receiver(&config, &wants_to_quit, rx);

    // Spawn the sender threads.
    spawn_senders(&config, &wants_to_quit, pattern, parallel_walker, tx);

    // Wait for the receiver thread to print out all results.
    let exit_code = receiver_thread.join().unwrap();

    if wants_to_quit.load(Ordering::Relaxed) {
        process::exit(ExitCode::KilledBySigint.into());
    }

    exit_code
}

fn spawn_receiver(
    config: &Arc<FdOptions>,
    wants_to_quit: &Arc<AtomicBool>,
    rx: Receiver<WorkerResult>,
) -> thread::JoinHandle<ExitCode> {
    let config = Arc::clone(config);
    let wants_to_quit = Arc::clone(wants_to_quit);

    let show_filesystem_errors = config.show_filesystem_errors;
    let threads = config.threads;

    thread::spawn(move || {
        // This will be set to `Some` if the `--exec` argument was supplied.
        if let Some(ref cmd) = config.command {
            if cmd.in_batch_mode() {
                exec::batch(rx, cmd, show_filesystem_errors)
            } else {
                let shared_rx = Arc::new(Mutex::new(rx));

                let out_perm = Arc::new(Mutex::new(()));

                // Each spawned job will store it's thread handle in here.
                let mut handles = Vec::with_capacity(threads);
                for _ in 0..threads {
                    let rx = Arc::clone(&shared_rx);
                    let cmd = Arc::clone(cmd);
                    let out_perm = Arc::clone(&out_perm);

                    // Spawn a job thread that will listen for and execute inputs.
                    let handle =
                        thread::spawn(move || exec::job(rx, cmd, out_perm, show_filesystem_errors));

                    // Push the handle of the spawned thread into the vector for later joining.
                    handles.push(handle);
                }

                // Wait for all threads to exit before exiting the program.
                let mut results: Vec<ExitCode> = Vec::new();
                for h in handles {
                    results.push(h.join().unwrap());
                }

                merge_exitcodes(results)
            }
        } else {
            let start = time::Instant::now();

            let mut buffer = vec![];

            // Start in buffering mode
            let mut mode = ReceiverMode::Buffering;

            // Maximum time to wait before we start streaming to the console.
            let max_buffer_time = config
                .max_buffer_time
                .unwrap_or_else(|| time::Duration::from_millis(100));

            let stdout = io::stdout();
            let mut stdout = stdout.lock();

            for worker_result in rx {
                match PathBuf::try_from(worker_result) {
                    Ok(value) => {
                        match mode {
                            ReceiverMode::Buffering => {
                                buffer.push(value);

                                // Have we reached the maximum buffer size or maximum buffering time?
                                if buffer.len() > MAX_BUFFER_LENGTH
                                    || time::Instant::now() - start > max_buffer_time
                                {
                                    // Flush the buffer
                                    for v in &buffer {
                                        output::print_entry(
                                            &mut stdout,
                                            v,
                                            &config,
                                            &wants_to_quit,
                                        );
                                    }
                                    buffer.clear();

                                    // Start streaming
                                    mode = ReceiverMode::Streaming;
                                }
                            }
                            ReceiverMode::Streaming => {
                                output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                            }
                        }
                    }
                    Err(err) => {
                        if show_filesystem_errors {
                            print_error!("{}", err);
                        }
                    }
                }
            }

            // If we have finished fast enough (faster than max_buffer_time), we haven't streamed
            // anything to the console, yet. In this case, sort the results and print them:
            if !buffer.is_empty() {
                buffer.sort();
                for value in buffer {
                    output::print_entry(&mut stdout, &value, &config, &wants_to_quit);
                }
            }

            ExitCode::Success
        }
    })
}

fn spawn_senders(
    config: &Arc<FdOptions>,
    wants_to_quit: &Arc<AtomicBool>,
    pattern: Arc<Regex>,
    parallel_walker: ignore::WalkParallel,
    tx: Sender<WorkerResult>,
) {
    parallel_walker.run(|| {
        let config = Arc::clone(config);
        let pattern = Arc::clone(&pattern);
        let tx_thread = tx.clone();
        let wants_to_quit = Arc::clone(wants_to_quit);

        Box::new(move |entry_o| {
            if wants_to_quit.load(Ordering::Relaxed) {
                return ignore::WalkState::Quit;
            }

            let entry = match entry_o {
                Ok(e) => e,
                Err(err) => {
                    tx_thread.send(WorkerResult::Error(err)).unwrap();
                    return ignore::WalkState::Continue;
                }
            };

            if entry.depth() == 0 {
                return ignore::WalkState::Continue;
            }

            // Check the name first, since it doesn't require metadata
            let entry_path = entry.path();

            let search_str: Cow<OsStr> = if config.search_full_path {
                match fshelper::path_absolute_form(entry_path) {
                    Ok(path_abs_buf) => Cow::Owned(path_abs_buf.as_os_str().to_os_string()),
                    Err(_) => {
                        print_error_and_exit!("Unable to retrieve absolute path.");
                    }
                }
            } else {
                match entry_path.file_name() {
                    Some(filename) => Cow::Borrowed(filename),
                    None => unreachable!(
                        "Encountered file system entry without a file name. This should only \
                         happen for paths like 'foo/bar/..' or '/' which are not supposed to \
                         appear in a file system traversal."
                    ),
                }
            };

            if !pattern.is_match(&osstr_to_bytes(search_str.as_ref())) {
                return ignore::WalkState::Continue;
            }

            // Filter out unwanted extensions.
            if let Some(ref exts_regex) = config.extensions {
                if let Some(path_str) = entry_path.file_name() {
                    if !exts_regex.is_match(&osstr_to_bytes(path_str)) {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted file types.
            if let Some(ref file_types) = config.file_types {
                if let Some(ref entry_type) = entry.file_type() {
                    if (!file_types.files && entry_type.is_file())
                        || (!file_types.directories && entry_type.is_dir())
                        || (!file_types.symlinks && entry_type.is_symlink())
                        || (file_types.executables_only
                            && !entry
                                .metadata()
                                .map(|m| fshelper::is_executable(&m))
                                .unwrap_or(false))
                        || (file_types.empty_only && !fshelper::is_empty(&entry))
                        || !(entry_type.is_file() || entry_type.is_dir() || entry_type.is_symlink())
                    {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted sizes if it is a file and we have been given size constraints.
            if !config.size_constraints.is_empty() {
                if entry_path.is_file() {
                    if let Ok(metadata) = entry_path.metadata() {
                        let file_size = metadata.len();
                        if config
                            .size_constraints
                            .iter()
                            .any(|sc| !sc.is_within(file_size))
                        {
                            return ignore::WalkState::Continue;
                        }
                    } else {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            // Filter out unwanted modification times
            if !config.time_constraints.is_empty() {
                let mut matched = false;
                if let Ok(metadata) = entry_path.metadata() {
                    if let Ok(modified) = metadata.modified() {
                        matched = config
                            .time_constraints
                            .iter()
                            .all(|tf| tf.applies_to(&modified));
                    }
                }
                if !matched {
                    return ignore::WalkState::Continue;
                }
            }

            let send_result = tx_thread.send(WorkerResult::Entry(entry_path.to_owned()));

            if !send_result.is_ok() {
                return ignore::WalkState::Quit;
            }

            ignore::WalkState::Continue
        })
    });
}
<|endoftext|>


<commit_before>use yaml_rust::yaml::Yaml;
use yaml_rust::{YamlLoader, YamlEmitter};
use environment::{IEnvironment, Environment};
use ast::AST;
use evaluator::Evaluator;
use parser::Parser;
use helpers::is_keyword;
use lexer;

#[derive(Debug, PartialEq)]
pub enum YamlType {
    Value(Yaml),
    Return(Yaml),
}

pub fn evaluate(yaml: &Yaml, env: &mut IEnvironment) -> YamlType {
    match yaml {
        &Yaml::String(ref s) => {
            if s.as_str().contains("~>") {
                let split_vec = s.as_str().split("~>").collect::<Vec<_>>();
                let mut evaluator = Evaluator::new(env);
                let mut parser = Parser::new();

                let (mut var_deque, mut op_deque) = lexer::parse_string(split_vec[1]).unwrap();
                let ast = parser.parse_to_ast(&mut var_deque, &mut op_deque).unwrap_or(AST::None);
                let result = evaluator.evaluate(ast).unwrap_or(AST::None);

                YamlType::Value(match result {
                    AST::Decimal(d) => Yaml::Real(d.to_string()),
                    AST::Number(n) => Yaml::Integer(n as i64),
                    AST::String(s) => Yaml::String(s),
                    _ => Yaml::String(split_vec[1].to_string()),
                })
            } else {
                YamlType::Value(Yaml::String(s.clone()))
            }
        }
        &Yaml::Array(ref arr) => {
            let mut last_value: Option<Yaml> = None;
            for v in arr {
                let result = evaluate(v, env);
                if let YamlType::Return(val) = result {
                    return YamlType::Return(val);
                } else if let YamlType::Value(val) = result {
                    last_value = Some(val);
                }
            }

            if let Some(val) = last_value {
                YamlType::Return(val)
            } else {
                YamlType::Value(Yaml::Array(arr.clone()))
            }
        }
        &Yaml::Hash(ref h) => {
            for (k, v) in h {
                if let &Yaml::String(ref s) = k {
                    match s.as_str() {
                        "return" => {
                            let result = evaluate(&v, env);
                            if let YamlType::Value(val) = result {
                                return YamlType::Return(val);
                            }
                            return result;
                        }
                        _ => {}
                    }
                }
            }
            YamlType::Value(Yaml::Hash(h.clone()))
        }
        &ref val @ _ => YamlType::Value(val.clone()),
    }
}

#[test]
fn test_yaml_eval() {
    // Test if evaluating "foo" returns 10

    let s = "
    foo:
      - '~> a := 2'
      - if:
        - '~> a == 2'
        - do:
          - '~> a = 3'
      - return: '~> a * (2 + 3)'
    ";
    let mut env = Environment::new();
    env.set("a".to_string(), AST::Number(1));
    env.set("b".to_string(), AST::Number(2));

    let docs = YamlLoader::load_from_str(s).unwrap();

    for doc in &docs {
        assert_eq!(evaluate(&doc["foo"], &mut env),
                   YamlType::Return(Yaml::Integer(10)));
    }
}
<commit_msg>Added ugly if implementation<commit_after>use yaml_rust::yaml::Yaml;
use yaml_rust::{YamlLoader, YamlEmitter};
use environment::{IEnvironment, Environment};
use ast::AST;
use evaluator::Evaluator;
use parser::Parser;
use helpers::is_keyword;
use lexer;

#[derive(Debug, PartialEq)]
pub enum YamlType {
    Value(Yaml),
    Return(Yaml),
}

// TODO: clean up super ugly code
fn apply_keyword(s: &str, k: &Yaml, v: &Yaml, env: &mut IEnvironment) -> YamlType {
    match s {
         "if" => {
            if let &Yaml::Array(ref arr) = v {
                let mut prop_str = String::new();
                for val in arr {
                    if let &Yaml::String(ref prop) = val {
                        let str_len = prop_str.len();
                        if str_len == 0 {
                            prop_str = format!("({})", prop.clone());
                        } else {
                            prop_str = format!("{} && ({})", prop_str, prop.clone());
                        }
                    } else if let &Yaml::Hash(ref h) = val {
                        for (key, val) in h {
                            if let &Yaml::String(ref keyword) = key {
                                let result = evaluate(&Yaml::String(prop_str.clone()), env);

                                match keyword.as_str() {
                                    "do" => {
                                        if let YamlType::Value(Yaml::Integer(i)) = result {
                                            if i > 0 {
                                                return evaluate(val, env);
                                            }
                                        }
                                    }
                                    "else" => {
                                        let result = evaluate(&Yaml::String(prop_str.clone()), env);
                                        if let YamlType::Value(Yaml::Integer(i)) = result {
                                            if i == 0 {
                                                return evaluate(val, env);
                                            }
                                        }
                                    }
                                    _ => {},
                                }
                            }
                        }
                    }
                }
            }
         }
        "return" => {
            let result = evaluate(&v, env);
            if let YamlType::Value(val) = result {
                return YamlType::Return(val);
            }
            return result;
        }
        _ => {}
    }

    YamlType::Value(v.clone())
}

pub fn evaluate(yaml: &Yaml, env: &mut IEnvironment) -> YamlType {
    match yaml {
        &Yaml::String(ref s) => {
            if s.as_str().contains("~>") {
                let split_vec = s.as_str().split("~>").collect::<Vec<_>>();
                let mut evaluator = Evaluator::new(env);
                let mut parser = Parser::new();

                let (mut var_deque, mut op_deque) = lexer::parse_string(split_vec[1]).unwrap();
                let ast = parser.parse_to_ast(&mut var_deque, &mut op_deque).unwrap_or(AST::None);
                let result = evaluator.evaluate(ast).unwrap_or(AST::None);

                YamlType::Value(match result {
                    AST::Decimal(d) => Yaml::Real(d.to_string()),
                    AST::Number(n) => Yaml::Integer(n as i64),
                    AST::String(s) => Yaml::String(s),
                    _ => Yaml::String(split_vec[1].to_string()),
                })
            } else {
                YamlType::Value(Yaml::String(s.clone()))
            }
        }
        &Yaml::Array(ref arr) => {
            let mut last_value: Option<Yaml> = None;
            for v in arr {
                let result = evaluate(v, env);
                if let YamlType::Return(val) = result {
                    return YamlType::Return(val);
                } else if let YamlType::Value(val) = result {
                    last_value = Some(val);
                }
            }

            if let Some(val) = last_value {
                YamlType::Value(val)
            } else {
                YamlType::Value(Yaml::Array(arr.clone()))
            }
        }
        &Yaml::Hash(ref h) => {
            for (k, v) in h {
                if let &Yaml::String(ref s) = k {
                    return apply_keyword(s.as_str(), k, v, env);
                }
            }
            YamlType::Value(Yaml::Hash(h.clone()))
        }
        &ref val @ _ => YamlType::Value(val.clone()),
    }
}

#[test]
fn test_yaml_eval() {
    // Test if evaluating "foo" returns 15

    let s = "
    foo:
      - '~> a := 2'
      - if:
        - '~> a == 2'
        - do:
          \
             - '~> a = 3'
      - return: '~> a * (2 + 3)'
    ";
    let mut env = Environment::new();
    env.set("a".to_string(), AST::Number(1));
    env.set("b".to_string(), AST::Number(2));

    let docs = YamlLoader::load_from_str(s).unwrap();

    assert_eq!(evaluate(&docs[0]["foo"], &mut env),
               YamlType::Return(Yaml::Integer(15)));
}

#[test]
fn test_yaml_else() {
    // Test if evaluating "foo" returns 20

    let s = "
    foo:
      - '~> a := 2'
      - if:
        - '~> a == 3'
        - do:
          \
             - '~> a = 3'
          else:
          \
             - '~> a = 4'
      - return: '~> a * (2 + 3)'
    ";
    let mut env = Environment::new();
    env.set("a".to_string(), AST::Number(1));
    env.set("b".to_string(), AST::Number(2));

    let docs = YamlLoader::load_from_str(s).unwrap();

    assert_eq!(evaluate(&docs[0]["foo"], &mut env),
               YamlType::Return(Yaml::Integer(20)));
}

#[test]
fn test_return() {
    // Test that return doesn't execute statements after it

    let s = "
    foo:
      - return: '~> 2 * (2 + 3)'
      - '~> a := 2'
      - if:
        \
             - '~> a == 2'
        - do:
          - '~> a = 3'
    ";
    let mut env = Environment::new();
    env.set("a".to_string(), AST::Number(1));
    env.set("b".to_string(), AST::Number(2));

    let docs = YamlLoader::load_from_str(s).unwrap();

    assert_eq!(evaluate(&docs[0]["foo"], &mut env),
               YamlType::Return(Yaml::Integer(10)));

    assert_eq!(env.get("a".to_string()), Some(&AST::Number(1)));
}

#[test]
fn test_return_last_val() {
    // Test that the last value is returned as value instead of return

    let s = "
    foo:
      - '~> a := 2'
      - if:
        \
             - '~> a == 2'
        - do:
          - '~> a = 3'
      - '~> 2 * (2 + 3)'
    ";
    let mut env = Environment::new();
    env.set("a".to_string(), AST::Number(1));
    env.set("b".to_string(), AST::Number(2));

    let docs = YamlLoader::load_from_str(s).unwrap();

    assert_eq!(evaluate(&docs[0]["foo"], &mut env),
               YamlType::Value(Yaml::Integer(10)));
}
<|endoftext|>


<commit_before>//! Library service providers implementation.

extern crate hyper;

use hyper::client::{Client, Response};

/// Used to specify which provider to use to generate a short URL.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum Provider {
    /// https://bn.gy provider
    BnGy,
    /// https://hec.su provider
    ///
    /// * Limited to 3000 API requests per day
    HecSu,
    /// https://is.gd provider
    IsGd,
    /// http://psbe.co provider
    PsbeCo,
    /// http://readbility.com provider
    Rdd,
    /// http://rlu.ru provider
    ///
    /// * Attention! If you send a lot of requests from one IP, it can be
    /// blocked. If you plan to add more then 100 URLs in one hour, please let
    /// the technical support know. Otherwise your IP can be blocked
    /// unexpectedly. Prior added URLs can be deleted.
    Rlu,
    /// http://tinyurl.com provider
    /// * Note: this service does not provide any API.
    /// The implementation result depends on the service result web page.
    TinyUrl,
    /// https://v.gd provider
    VGd,
}

impl Provider {
    /// Converts the Provider variant into its domain name equivilant
    pub fn to_name(&self) -> &str {
        match *self {
            Provider::BnGy => "bn.gy",
            Provider::HecSu => "hec.su",
            Provider::IsGd => "is.gd",
            Provider::PsbeCo => "psbe.co",
            Provider::Rdd => "readability.com",
            Provider::Rlu => "rlu.ru",
            Provider::TinyUrl => "tinyurl.com",
            Provider::VGd => "v.gd",
        }
    }
}

/// Returns a vector of all `Provider` variants.
///
/// The providers which are discouraged from use due to limitations such as
/// rate limitations are at the end of the resultant vector.
pub fn providers() -> Vec<Provider> {
    vec![
        Provider::IsGd,
        Provider::BnGy,
        Provider::VGd,
        Provider::Rdd,

        // The following list are items that are discouraged from use.
        // Reason: rate limit (3000 requests per day)
        Provider::HecSu,
        // Reason: does not provide an api
        Provider::TinyUrl,
        // Reason: unstable work
        Provider::PsbeCo,
        // Reason: rate limit (100 requests per hour)
        Provider::Rlu,
    ]
}

fn bngy_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let iter = string.split("<ShortenedUrl>").skip(1).next();
    if iter.is_none() {
        return None
    }
    if let Some(string) = iter.unwrap().split("</ShortenedUrl>").next() {
        Some(string.to_owned())
    } else {
        None
    }
}

fn bngy_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("https://bn.gy/API.asmx/CreateUrl?real_url={}", url))
        .send()
        .ok()
}

fn hecsu_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let iter = string.split("<short>").skip(1).next();
    if iter.is_none() {
        return None
    }
    if let Some(string) = iter.unwrap().split("</short>").next() {
        Some(string.to_owned())
    } else {
        None
    }
}

fn hecsu_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("https://hec.su/api?url={}&method=xml", url))
        .send()
        .ok()
}

fn isgd_parse(res: &str) -> Option<String> {
    Some(res.to_owned())
}

fn isgd_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("https://is.gd/create.php?format=simple&url={}", url))
        .send()
        .ok()
}

fn psbeco_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let iter = string.split("<ShortUrl>").skip(1).next();
    if iter.is_none() {
        return None
    }
    if let Some(string) = iter.unwrap().split("</ShortUrl>").next() {
        Some(string.to_owned())
    } else {
        None
    }
}

fn psbeco_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("http://psbe.co/API.asmx/CreateUrl?real_url={}", url))
        .send()
        .ok()
}

fn rdd_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let value = string.split("\"rdd_url\"")
                      .nth(1).unwrap_or("")
                      .split(",").next().unwrap_or("")
                      .split("\"").nth(1);
    if let Some(string) = value {
        let mut short_url = string.to_owned();
        let _ = short_url.pop();
        Some(short_url)
    } else {
        None
    }
}

fn rdd_request(url: &str, client: &Client) -> Option<Response> {
    client.post("https://readability.com/api/shortener/v1/urls")
        .body(&format!("url={}", url))
        .send()
        .ok()
}

fn rlu_parse(res: &str) -> Option<String> {
    Some(res.to_owned())
}

fn rlu_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("http://rlu.ru/index.sema?a=api&link={}", url))
        .send()
        .ok()
}

fn tinyurl_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let value = string.split("data-clipboard-text=\"")
                      .nth(1).unwrap_or("")
                      .split("\">").next();
    if let Some(string) = value {
        Some(string.to_owned())
    } else {
        None
    }
}

fn tinyurl_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("http://tinyurl.com/create.php?url={}", url))
        .send()
        .ok()
}

fn vgd_parse(res: &str) -> Option<String> {
    Some(res.to_owned())
}

fn vgd_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("http://v.gd/create.php?format=simple&url={}", url))
        .send()
        .ok()
}

/// Parses the response from a successful request to a provider into the
/// URL-shortened string.
pub fn parse(res: &str, provider: Provider) -> Option<String> {
    match provider {
        Provider::BnGy => bngy_parse(res),
        Provider::HecSu => hecsu_parse(res),
        Provider::IsGd => isgd_parse(res),
        Provider::PsbeCo => psbeco_parse(res),
        Provider::Rdd => rdd_parse(res),
        Provider::Rlu => rlu_parse(res),
        Provider::TinyUrl => tinyurl_parse(res),
        Provider::VGd => vgd_parse(res),
    }
}

/// Performs a request to the short link provider.
/// Response to be parsed or `None` on a error.
pub fn request(url: &str, client: &Client, provider: Provider) -> Option<Response> {
    match provider {
        Provider::BnGy => bngy_request(url, client),
        Provider::HecSu => hecsu_request(url, client),
        Provider::IsGd => isgd_request(url, client),
        Provider::PsbeCo => psbeco_request(url, client),
        Provider::Rdd => rdd_request(url, client),
        Provider::Rlu => rlu_request(url, client),
        Provider::TinyUrl => tinyurl_request(url, client),
        Provider::VGd => vgd_request(url, client),
    }
}
<commit_msg>fifo.cc provider implemented<commit_after>//! Library service providers implementation.

extern crate hyper;

use hyper::client::{Client, Response};

/// Used to specify which provider to use to generate a short URL.
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum Provider {
    /// https://bn.gy provider
    BnGy,
    /// http://fifo.cc provider
    FifoCc,
    /// https://hec.su provider
    ///
    /// * Limited to 3000 API requests per day
    HecSu,
    /// https://is.gd provider
    IsGd,
    /// http://psbe.co provider
    PsbeCo,
    /// http://readbility.com provider
    Rdd,
    /// http://rlu.ru provider
    ///
    /// * Attention! If you send a lot of requests from one IP, it can be
    /// blocked. If you plan to add more then 100 URLs in one hour, please let
    /// the technical support know. Otherwise your IP can be blocked
    /// unexpectedly. Prior added URLs can be deleted.
    Rlu,
    /// http://tinyurl.com provider
    /// * Note: this service does not provide any API.
    /// The implementation result depends on the service result web page.
    TinyUrl,
    /// https://v.gd provider
    VGd,
}

impl Provider {
    /// Converts the Provider variant into its domain name equivilant
    pub fn to_name(&self) -> &str {
        match *self {
            Provider::BnGy => "bn.gy",
            Provider::FifoCc => "fifo.cc",
            Provider::HecSu => "hec.su",
            Provider::IsGd => "is.gd",
            Provider::PsbeCo => "psbe.co",
            Provider::Rdd => "readability.com",
            Provider::Rlu => "rlu.ru",
            Provider::TinyUrl => "tinyurl.com",
            Provider::VGd => "v.gd",
        }
    }
}

/// Returns a vector of all `Provider` variants.
///
/// The providers which are discouraged from use due to limitations such as
/// rate limitations are at the end of the resultant vector.
pub fn providers() -> Vec<Provider> {
    vec![
        Provider::IsGd,
        Provider::BnGy,
        Provider::VGd,
        Provider::Rdd,
        Provider::FifoCc,

        // The following list are items that are discouraged from use.
        // Reason: rate limit (3000 requests per day)
        Provider::HecSu,
        // Reason: does not provide an api
        Provider::TinyUrl,
        // Reason: unstable work
        Provider::PsbeCo,
        // Reason: rate limit (100 requests per hour)
        Provider::Rlu,
    ]
}

fn bngy_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let iter = string.split("<ShortenedUrl>").skip(1).next();
    if iter.is_none() {
        return None
    }
    if let Some(string) = iter.unwrap().split("</ShortenedUrl>").next() {
        Some(string.to_owned())
    } else {
        None
    }
}

fn bngy_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("https://bn.gy/API.asmx/CreateUrl?real_url={}", url))
        .send()
        .ok()
}

fn fifocc_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let value = string.split("\"shortner\"")
                      .nth(1).unwrap_or("")
                      .split(",").next().unwrap_or("")
                      .split("\"").nth(1);
    if let Some(string) = value {
        let mut short_url = string.to_owned();
        short_url = format!("http://fifo.cc/{}", short_url);
        Some(short_url)
    } else {
        None
    }
}

fn fifocc_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("https://fifo.cc/api/v2?url={}", url))
        .send()
        .ok()
}

fn hecsu_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let iter = string.split("<short>").skip(1).next();
    if iter.is_none() {
        return None
    }
    if let Some(string) = iter.unwrap().split("</short>").next() {
        Some(string.to_owned())
    } else {
        None
    }
}

fn hecsu_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("https://hec.su/api?url={}&method=xml", url))
        .send()
        .ok()
}

fn isgd_parse(res: &str) -> Option<String> {
    Some(res.to_owned())
}

fn isgd_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("https://is.gd/create.php?format=simple&url={}", url))
        .send()
        .ok()
}

fn psbeco_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let iter = string.split("<ShortUrl>").skip(1).next();
    if iter.is_none() {
        return None
    }
    if let Some(string) = iter.unwrap().split("</ShortUrl>").next() {
        Some(string.to_owned())
    } else {
        None
    }
}

fn psbeco_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("http://psbe.co/API.asmx/CreateUrl?real_url={}", url))
        .send()
        .ok()
}

fn rdd_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let value = string.split("\"rdd_url\"")
                      .nth(1).unwrap_or("")
                      .split(",").next().unwrap_or("")
                      .split("\"").nth(1);
    if let Some(string) = value {
        let mut short_url = string.to_owned();
        let _ = short_url.pop();
        Some(short_url)
    } else {
        None
    }
}

fn rdd_request(url: &str, client: &Client) -> Option<Response> {
    client.post("https://readability.com/api/shortener/v1/urls")
        .body(&format!("url={}", url))
        .send()
        .ok()
}

fn rlu_parse(res: &str) -> Option<String> {
    Some(res.to_owned())
}

fn rlu_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("http://rlu.ru/index.sema?a=api&link={}", url))
        .send()
        .ok()
}

fn tinyurl_parse(res: &str) -> Option<String> {
    if res.is_empty() {
        return None
    }
    let string = res.to_owned();
    let value = string.split("data-clipboard-text=\"")
                      .nth(1).unwrap_or("")
                      .split("\">").next();
    if let Some(string) = value {
        Some(string.to_owned())
    } else {
        None
    }
}

fn tinyurl_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("http://tinyurl.com/create.php?url={}", url))
        .send()
        .ok()
}

fn vgd_parse(res: &str) -> Option<String> {
    Some(res.to_owned())
}

fn vgd_request(url: &str, client: &Client) -> Option<Response> {
    client.get(&format!("http://v.gd/create.php?format=simple&url={}", url))
        .send()
        .ok()
}

/// Parses the response from a successful request to a provider into the
/// URL-shortened string.
pub fn parse(res: &str, provider: Provider) -> Option<String> {
    match provider {
        Provider::BnGy => bngy_parse(res),
        Provider::FifoCc => fifocc_parse(res),
        Provider::HecSu => hecsu_parse(res),
        Provider::IsGd => isgd_parse(res),
        Provider::PsbeCo => psbeco_parse(res),
        Provider::Rdd => rdd_parse(res),
        Provider::Rlu => rlu_parse(res),
        Provider::TinyUrl => tinyurl_parse(res),
        Provider::VGd => vgd_parse(res),
    }
}

/// Performs a request to the short link provider.
/// Response to be parsed or `None` on a error.
pub fn request(url: &str, client: &Client, provider: Provider) -> Option<Response> {
    match provider {
        Provider::BnGy => bngy_request(url, client),
        Provider::FifoCc => fifocc_request(url, client),
        Provider::HecSu => hecsu_request(url, client),
        Provider::IsGd => isgd_request(url, client),
        Provider::PsbeCo => psbeco_request(url, client),
        Provider::Rdd => rdd_request(url, client),
        Provider::Rlu => rlu_request(url, client),
        Provider::TinyUrl => tinyurl_request(url, client),
        Provider::VGd => vgd_request(url, client),
    }
}
<|endoftext|>


<commit_before>pub mod sentiment {
    use std::collections::HashMap;
    use std::fs::File;
    use std::io::prelude::*;
    use std::io::BufReader;

    pub struct Analizer {
        wordlist: HashMap<String, i8>,
    }

    impl Analizer {
        pub fn analyze(&self, s: &str) -> f32 {
            let tokens = s.split(" ");
            let word_count = tokens.clone().count();

            let score = tokens.map(|word| word.to_string().to_lowercase())
                .map(|word| match self.wordlist.get(&word) {
                    Some(word_score) => *word_score,
                    None => 0 as i8,
                })
                .fold(0, |sum, curr| sum + curr);

            return score as f32 / word_count as f32;
        }
    }

    pub fn new(path: String) -> Analizer {
        let mut wordlist = HashMap::new();

        let f = match File::open(&path) {
            Err(why) => panic!("couldn't open {}: {}", path, why),
            Ok(f) => f,
        };

        let f = BufReader::new(f);

        for line in f.lines() {
            let l = line.unwrap();
            let parts = l.split("\t").collect::<Vec<&str>>();

            let word = parts[0].to_string();
            let score = parts[1].parse::<i8>().unwrap();

            wordlist.insert(word, score);
        }

        return Analizer { wordlist: wordlist };
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn sentiment_test() {
        let analizer = sentiment::new("/Users/coornail/rust/ircbot/src/wordlist.txt".to_string());
        assert!(analizer.analyze("Hey you worthless scumbag") == -1.5);
        assert!(analizer.analyze("I am happy") == 1 as f32);
        assert!(analizer.analyze("I am so happy") == 0.75);
        assert!(analizer.analyze("I am extremely happy") == 0.75 as f32);
        assert!(analizer.analyze("I am really sad") == -0.5 as f32);
    }
}
<commit_msg>Fix test wordlist path<commit_after>
pub mod sentiment {
    use std::collections::HashMap;
    use std::fs::File;
    use std::io::prelude::*;
    use std::io::BufReader;

    pub struct Analizer {
        wordlist: HashMap<String, i8>,
    }

    impl Analizer {
        pub fn analyze(&self, s: &str) -> f32 {
            let tokens = s.split(" ");
            let word_count = tokens.clone().count();

            let score = tokens.map(|word| word.to_string().to_lowercase())
                .map(|word| match self.wordlist.get(&word) {
                    Some(word_score) => *word_score,
                    None => 0 as i8,
                })
                .fold(0, |sum, curr| sum + curr);

            return score as f32 / word_count as f32;
        }
    }

    pub fn new(path: String) -> Analizer {
        let mut wordlist = HashMap::new();

        let f = match File::open(&path) {
            Err(why) => panic!("couldn't open {}: {}", path, why),
            Ok(f) => f,
        };

        let f = BufReader::new(f);

        for line in f.lines() {
            let l = line.unwrap();
            let parts = l.split("\t").collect::<Vec<&str>>();

            let word = parts[0].to_string();
            let score = parts[1].parse::<i8>().unwrap();

            wordlist.insert(word, score);
        }

        return Analizer { wordlist: wordlist };
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn sentiment_test() {
        let analizer = sentiment::new("./src/wordlist.txt".to_string());
        assert!(analizer.analyze("Hey you worthless scumbag") == -1.5);
        assert!(analizer.analyze("I am happy") == 1 as f32);
        assert!(analizer.analyze("I am so happy") == 0.75);
        assert!(analizer.analyze("I am extremely happy") == 0.75 as f32);
        assert!(analizer.analyze("I am really sad") == -0.5 as f32);
    }
}
<|endoftext|>


<commit_before>use std::io;

use byteorder::{ByteOrder, BigEndian, WriteBytesExt};

use error::NbtError;

pub trait NbtFmt {
    fn write_nbt_fmt<W>(&self, dst: &mut W) -> Result<(), NbtError>
       where W: io::Write;

    #[inline]
    fn write_nbt_fmt_with_name<W, S>(&self, dst: &mut W, name: S) -> Result<(), NbtError>
       where W: io::Write,
             S: AsRef<str> {

        try!(dst.write_u8(Self::tag()));
        try!(write_bare_string(dst, name.as_ref()));
        self.write_nbt_fmt(dst)
    }

    #[inline] fn tag() -> u8 { 0x0a }
    #[inline] fn is_bare() -> bool { false }
}

pub fn close_nbt<W>(dst: &mut W) -> Result<(), NbtError>
    where W: io::Write {

    dst.write_u8(0x00).map_err(From::from)
}

pub fn to_writer<W, T>(dst: &mut W, obj: T) -> Result<(), NbtError>
    where W: io::Write,
          T: NbtFmt
{
    obj.write_nbt_fmt_with_name(dst, "")
}

macro_rules! nbtfmt_value {
  ($T:ty, $method:ident, $tag:expr, $bare:expr) => (
    impl NbtFmt for $T {
      fn write_nbt_fmt<W>(&self, dst: &mut W) -> Result<(), NbtError>
           where W: io::Write {
            $method(dst, *self)
      }
        #[inline] fn tag() -> u8 { $tag }
        #[inline] fn is_bare() -> bool { $bare }
    }
  )
}

macro_rules! nbtfmt_ptr {
  ($T:ty, $method:ident, $tag:expr, $bare:expr) => (
    impl NbtFmt for $T {
      fn write_nbt_fmt<W>(&self, dst: &mut W) -> Result<(), NbtError>
           where W: io::Write {
            $method(dst, self)
      }
        #[inline] fn tag() -> u8 { $tag }
        #[inline] fn is_bare() -> bool { $bare }
    }
  )
}

macro_rules! nbtfmt_slice {
  ($T:ty, $method:ident, $tag:expr, $bare:expr) => (
    impl NbtFmt for $T {
      fn write_nbt_fmt<W>(&self, dst: &mut W) -> Result<(), NbtError>
           where W: io::Write {
            $method(dst, &self[..])
      }
        #[inline] fn tag() -> u8 { $tag }
        #[inline] fn is_bare() -> bool { $bare }
    }
  )
}

nbtfmt_value!(i8, write_bare_byte, 0x01, true);
nbtfmt_value!(i16, write_bare_short, 0x02, true);
nbtfmt_value!(i32, write_bare_int, 0x03, true);
nbtfmt_value!(i64, write_bare_long, 0x04, true);
nbtfmt_value!(f32, write_bare_float, 0x05, true);
nbtfmt_value!(f64, write_bare_double, 0x06, true);
nbtfmt_ptr!([i8], write_bare_byte_array, 0x07, true);
nbtfmt_slice!(Vec<i8>, write_bare_byte_array, 0x07, true);
nbtfmt_ptr!(str, write_bare_string, 0x08, true);
nbtfmt_slice!(String, write_bare_string, 0x08, true);
nbtfmt_ptr!([i32], write_bare_int_array, 0x0b, true);
nbtfmt_slice!(Vec<i32>, write_bare_int_array, 0x0b, true);

// impl<T> NbtFmt for [T] where T: NbtFmt {
//  fn write_nbt_fmt<W>(&self, dst: &mut W) -> Result<(), NbtError>
//        where W: io::Write {

//          write_bare_list(dst, self.iter())
//  }
//     #[inline] fn tag() -> u8 { 0x09 }
//     #[inline] fn is_bare() -> bool { true }
// }

// impl<T> NbtFmt for Vec<T> where T: NbtFmt {
//  fn write_nbt_fmt<W>(&self, dst: &mut W) -> Result<(), NbtError>
//        where W: io::Write {

//          write_bare_list(dst, self.iter())
//  }
//     #[inline] fn tag() -> u8 { 0x09 }
//     #[inline] fn is_bare() -> bool { true }
// }

#[inline]
fn write_bare_byte<W>(dst: &mut W, value: i8) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_i8(value).map_err(From::from)
}

#[inline]
fn write_bare_short<W>(dst: &mut W, value: i16) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_i16::<BigEndian>(value).map_err(From::from)
}

#[inline]
fn write_bare_int<W>(dst: &mut W, value: i32) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_i32::<BigEndian>(value).map_err(From::from)
}

#[inline]
fn write_bare_long<W>(dst: &mut W, value: i64) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_i64::<BigEndian>(value).map_err(From::from)
}

#[inline]
fn write_bare_float<W>(dst: &mut W, value: f32) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_f32::<BigEndian>(value).map_err(From::from)
}

#[inline]
fn write_bare_double<W>(dst: &mut W, value: f64) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_f64::<BigEndian>(value).map_err(From::from)
}

#[inline]
fn write_bare_byte_array<W>(dst: &mut W, value: &[i8]) -> Result<(), NbtError>
   where W: io::Write {

    try!(dst.write_i32::<BigEndian>(value.len() as i32));
    for &v in value {
        try!(dst.write_i8(v));
    }
    Ok(())
}

#[inline]
fn write_bare_int_array<W>(dst: &mut W, value: &[i32]) -> Result<(), NbtError>
   where W: io::Write {

    try!(dst.write_i32::<BigEndian>(value.len() as i32));
    for &v in value {
        try!(dst.write_i32::<BigEndian>(v));
    }
    Ok(())
}

#[inline]
fn write_bare_string<W>(dst: &mut W, value: &str) -> Result<(), NbtError>
   where W: io::Write {

    try!(dst.write_u16::<BigEndian>(value.len() as u16));
    dst.write_all(value.as_bytes()).map_err(From::from)
}

#[inline]
fn write_bare_list<'a, W, I, T>(dst: &mut W, values: I) -> Result<(), NbtError>
   where W: io::Write,
         I: Iterator<Item=&'a T> + ExactSizeIterator,
         T: 'a + NbtFmt {

    // The list contents are prefixed by a byte tag for the type and the
    // length of the list (a big-endian i32).
    try!(dst.write_u8(T::tag()));
    try!(dst.write_i32::<BigEndian>(values.len() as i32));

    for ref value in values {
       try!(value.write_nbt_fmt(dst));
    }

    Ok(())
}

#[inline]
fn write_bare_compound<'a, W, I, T, S>(dst: &mut W, values: I) -> Result<(), NbtError>
   where W: io::Write,
         I: Iterator<Item=(&'a S, &'a T)>,
         S: 'a + AsRef<str>,
         T: 'a + NbtFmt {

    for (key, ref value) in values {
        try!(value.write_nbt_fmt_with_name(dst, key.as_ref()));
    }

    // Write the marker for the end of the Compound.
    dst.write_u8(0x00).map_err(From::from)
}

#[test]
fn nbt_test_struct_serialize() {
  struct TestStruct {
    name: String,
    health: i8,
    food: f32,
    emeralds: i16,
    timestamp: i32
  }

  impl NbtFmt for TestStruct {
    fn write_nbt_fmt<W>(&self, dst: &mut W) -> Result<(), NbtError>
           where W: io::Write {

            self.name.write_nbt_fmt_with_name(dst, "name");
            self.health.write_nbt_fmt_with_name(dst, "health");
            self.food.write_nbt_fmt_with_name(dst, "food");
            self.emeralds.write_nbt_fmt_with_name(dst, "emeralds");
            self.timestamp.write_nbt_fmt_with_name(dst, "timestamp");

            dst.write_u8(0x00).map_err(From::from)
        }
  }

  let test = TestStruct {
    name: "Herobrine".to_string(),
    health: 100, food: 20.0, emeralds: 12345, timestamp: 1424778774
  };

  let mut test_encoded = Vec::new();
  test.write_nbt_fmt_with_name(&mut test_encoded, "");

  let bytes = [
        0x0a,
            0x00, 0x00,
            0x08,
                0x00, 0x04,
                0x6e, 0x61, 0x6d, 0x65,
                0x00, 0x09,
                0x48, 0x65, 0x72, 0x6f, 0x62, 0x72, 0x69, 0x6e, 0x65,
            0x01,
                0x00, 0x06,
                0x68, 0x65, 0x61, 0x6c, 0x74, 0x68,
                0x64,
            0x05,
                0x00, 0x04,
                0x66, 0x6f, 0x6f, 0x64,
                0x41, 0xa0, 0x00, 0x00,
            0x02,
                0x00, 0x08,
                0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x64, 0x73,
                0x30, 0x39,
            0x03,
                0x00, 0x09,
                0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
                0x54, 0xec, 0x66, 0x16,
        0x00
    ];

    assert_eq!(&bytes[..], &test_encoded[..]);
}
<commit_msg>Better method names for `NbtFmt` and some other `serialize` clean-up.<commit_after>use std::io;

use byteorder::{ByteOrder, BigEndian, WriteBytesExt};

use error::NbtError;

pub trait NbtFmt {
    fn to_bare_nbt<W>(&self, dst: &mut W) -> Result<(), NbtError>
       where W: io::Write;

    #[inline]
    fn to_nbt<W, S>(&self, dst: &mut W, name: S) -> Result<(), NbtError>
       where W: io::Write,
             S: AsRef<str>
    {
        try!(dst.write_u8(Self::tag()));
        try!(write_bare_string(dst, name.as_ref()));
        self.to_bare_nbt(dst)
    }

    #[inline] fn tag() -> u8 { 0x0a }
    #[inline] fn is_bare() -> bool { false }
}

pub fn close_nbt<W>(dst: &mut W) -> Result<(), NbtError>
    where W: io::Write {

    dst.write_u8(0x00).map_err(From::from)
}

pub fn to_writer<W, T>(dst: &mut W, obj: T) -> Result<(), NbtError>
    where W: io::Write,
          T: NbtFmt
{
    match T::is_bare() {
        // Refuse to blindly serialize types not wrapped in an NBT Compound.
        true  => { return Err(NbtError::NoRootCompound); },
        false => obj.to_nbt(dst, ""),
    }
}

macro_rules! nbtfmt_value {
  ($T:ty, $method:ident, $tag:expr, $bare:expr) => (
    impl NbtFmt for $T {
      fn to_bare_nbt<W>(&self, dst: &mut W) -> Result<(), NbtError>
           where W: io::Write {
            $method(dst, *self)
      }
        #[inline] fn tag() -> u8 { $tag }
        #[inline] fn is_bare() -> bool { $bare }
    }
  )
}

macro_rules! nbtfmt_ptr {
  ($T:ty, $method:ident, $tag:expr, $bare:expr) => (
    impl NbtFmt for $T {
      fn to_bare_nbt<W>(&self, dst: &mut W) -> Result<(), NbtError>
           where W: io::Write {
            $method(dst, self)
      }
        #[inline] fn tag() -> u8 { $tag }
        #[inline] fn is_bare() -> bool { $bare }
    }
  )
}

macro_rules! nbtfmt_slice {
  ($T:ty, $method:ident, $tag:expr, $bare:expr) => (
    impl NbtFmt for $T {
      fn to_bare_nbt<W>(&self, dst: &mut W) -> Result<(), NbtError>
           where W: io::Write {
            $method(dst, &self[..])
      }
        #[inline] fn tag() -> u8 { $tag }
        #[inline] fn is_bare() -> bool { $bare }
    }
  )
}

nbtfmt_value!(i8, write_bare_byte, 0x01, true);
nbtfmt_value!(i16, write_bare_short, 0x02, true);
nbtfmt_value!(i32, write_bare_int, 0x03, true);
nbtfmt_value!(i64, write_bare_long, 0x04, true);
nbtfmt_value!(f32, write_bare_float, 0x05, true);
nbtfmt_value!(f64, write_bare_double, 0x06, true);
nbtfmt_ptr!([i8], write_bare_byte_array, 0x07, true);
nbtfmt_slice!(Vec<i8>, write_bare_byte_array, 0x07, true);
nbtfmt_ptr!(str, write_bare_string, 0x08, true);
nbtfmt_slice!(String, write_bare_string, 0x08, true);
nbtfmt_ptr!([i32], write_bare_int_array, 0x0b, true);
nbtfmt_slice!(Vec<i32>, write_bare_int_array, 0x0b, true);

// impl<T> NbtFmt for [T] where T: NbtFmt {
//  fn to_bare_nbt<W>(&self, dst: &mut W) -> Result<(), NbtError>
//        where W: io::Write {

//          write_bare_list(dst, self.iter())
//  }
//     #[inline] fn tag() -> u8 { 0x09 }
//     #[inline] fn is_bare() -> bool { true }
// }

// impl<T> NbtFmt for Vec<T> where T: NbtFmt {
//  fn to_bare_nbt<W>(&self, dst: &mut W) -> Result<(), NbtError>
//        where W: io::Write {

//          write_bare_list(dst, self.iter())
//  }
//     #[inline] fn tag() -> u8 { 0x09 }
//     #[inline] fn is_bare() -> bool { true }
// }

#[inline]
fn write_bare_byte<W>(dst: &mut W, value: i8) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_i8(value).map_err(From::from)
}

#[inline]
fn write_bare_short<W>(dst: &mut W, value: i16) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_i16::<BigEndian>(value).map_err(From::from)
}

#[inline]
fn write_bare_int<W>(dst: &mut W, value: i32) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_i32::<BigEndian>(value).map_err(From::from)
}

#[inline]
fn write_bare_long<W>(dst: &mut W, value: i64) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_i64::<BigEndian>(value).map_err(From::from)
}

#[inline]
fn write_bare_float<W>(dst: &mut W, value: f32) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_f32::<BigEndian>(value).map_err(From::from)
}

#[inline]
fn write_bare_double<W>(dst: &mut W, value: f64) -> Result<(), NbtError>
   where W: io::Write {

    dst.write_f64::<BigEndian>(value).map_err(From::from)
}

#[inline]
fn write_bare_byte_array<W>(dst: &mut W, value: &[i8]) -> Result<(), NbtError>
   where W: io::Write {

    try!(dst.write_i32::<BigEndian>(value.len() as i32));
    for &v in value {
        try!(dst.write_i8(v));
    }
    Ok(())
}

#[inline]
fn write_bare_int_array<W>(dst: &mut W, value: &[i32]) -> Result<(), NbtError>
   where W: io::Write {

    try!(dst.write_i32::<BigEndian>(value.len() as i32));
    for &v in value {
        try!(dst.write_i32::<BigEndian>(v));
    }
    Ok(())
}

#[inline]
fn write_bare_string<W>(dst: &mut W, value: &str) -> Result<(), NbtError>
   where W: io::Write {

    try!(dst.write_u16::<BigEndian>(value.len() as u16));
    dst.write_all(value.as_bytes()).map_err(From::from)
}

#[inline]
fn write_bare_list<'a, W, I, T>(dst: &mut W, values: I) -> Result<(), NbtError>
   where W: io::Write,
         I: Iterator<Item=&'a T> + ExactSizeIterator,
         T: 'a + NbtFmt {

    // The list contents are prefixed by a byte tag for the type and the
    // length of the list (a big-endian i32).
    try!(dst.write_u8(T::tag()));
    try!(dst.write_i32::<BigEndian>(values.len() as i32));

    for ref value in values {
        // Note the use of bare values.
        try!(value.to_bare_nbt(dst));
    }

    Ok(())
}

#[inline]
fn write_bare_compound<'a, W, I, T, S>(dst: &mut W, values: I) -> Result<(), NbtError>
   where W: io::Write,
         I: Iterator<Item=(&'a S, &'a T)>,
         S: 'a + AsRef<str>,
         T: 'a + NbtFmt {

    for (key, ref value) in values {
        try!(value.to_nbt(dst, key.as_ref()));
    }

    // Write the marker for the end of the Compound.
    close_nbt(dst)
}

#[test]
fn nbt_test_struct_serialize() {
  struct TestStruct {
    name: String,
    health: i8,
    food: f32,
    emeralds: i16,
    timestamp: i32
  }

  impl NbtFmt for TestStruct {
    fn to_bare_nbt<W>(&self, dst: &mut W) -> Result<(), NbtError>
           where W: io::Write {

            try!(self.name.to_nbt(dst, "name"));
            try!(self.health.to_nbt(dst, "health"));
            try!(self.food.to_nbt(dst, "food"));
            try!(self.emeralds.to_nbt(dst, "emeralds"));
            try!(self.timestamp.to_nbt(dst, "timestamp"));

            close_nbt(dst)
        }
  }

  let test = TestStruct {
    name: "Herobrine".to_string(),
    health: 100, food: 20.0, emeralds: 12345, timestamp: 1424778774
  };

  let mut test_encoded = Vec::new();
  to_writer(&mut dst, test);

  let bytes = [
        0x0a,
            0x00, 0x00,
            0x08,
                0x00, 0x04,
                0x6e, 0x61, 0x6d, 0x65,
                0x00, 0x09,
                0x48, 0x65, 0x72, 0x6f, 0x62, 0x72, 0x69, 0x6e, 0x65,
            0x01,
                0x00, 0x06,
                0x68, 0x65, 0x61, 0x6c, 0x74, 0x68,
                0x64,
            0x05,
                0x00, 0x04,
                0x66, 0x6f, 0x6f, 0x64,
                0x41, 0xa0, 0x00, 0x00,
            0x02,
                0x00, 0x08,
                0x65, 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x64, 0x73,
                0x30, 0x39,
            0x03,
                0x00, 0x09,
                0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
                0x54, 0xec, 0x66, 0x16,
        0x00
    ];

    assert_eq!(&bytes[..], &test_encoded[..]);
}
<|endoftext|>