<commit_before>#[macro_use]
extern crate log;
extern crate env_logger;
extern crate irc;
#[macro_use]
extern crate lazy_static;
extern crate regex;
extern crate hyper;
extern crate hubcaps;
extern crate hyper_native_tls;
extern crate serde_json;

use std::env;
use std::fmt;
use std::thread;
use std::collections::HashMap;
use std::ascii::AsciiExt;
use regex::Regex;

use irc::client::prelude::*;
use irc::client::data::command::Command;

use hyper::Client;
use hyper::net::HttpsConnector;
use hyper_native_tls::NativeTlsClient;
use hubcaps::{Credentials, Github};
use hubcaps::comments::CommentOptions;

fn main() {
    env_logger::init().unwrap();

    let config_file =
        {
            let mut args = env::args_os().skip(1); // skip program name
            let config_file = args.next().expect("Expected a single command-line argument, the JSON configuration file.");
            if args.next().is_some() {
                panic!("Expected only a single command-line argument, the JSON configuration file.");
            }
            config_file
        };

    let server = IrcServer::new(config_file).expect("Couldn't initialize server with given configuration file");
    server.identify().unwrap();

    let options = server
        .config()
        .options
        .as_ref()
        .expect("No options property within configuration?");

    // FIXME: Add a way to ask the bot to reboot itself?
    let mut channel_data: HashMap<String, ChannelData> = HashMap::new();

    for message in server.iter() {
        let message = message.unwrap(); // panic if there's an error

        match message.command {
            Command::PRIVMSG(ref target, ref msg) => {
                match message.source_nickname() {
                    None => {
                        warn!("PRIVMSG without a source! {}",
                              format!("{}", message).trim());
                    }
                    Some(ref source) => {
                        let source_ = String::from(*source);
                        let line = if msg.starts_with("\x01ACTION ") && msg.ends_with("\x01") {
                            ChannelLine {
                                source: source_,
                                is_action: true,
                                message: String::from(&msg[8..msg.len() - 1]),
                            }
                        } else {
                            ChannelLine {
                                source: source_,
                                is_action: false,
                                message: msg.clone(),
                            }
                        };
                        let mynick = server.current_nickname();
                        if target == mynick {
                            // An actual private message.
                            info!("[{}] {}", source, line);
                            handle_bot_command(&server,
                                               options,
                                               &mut channel_data,
                                               &line.message,
                                               source,
                                               false,
                                               None)
                        } else if target.starts_with('#') {
                            // A message in a channel.
                            info!("[{}] {}", target, line);
                            match check_command_in_channel(mynick, &line.message) {
                                Some(ref command) => {
                                    handle_bot_command(&server,
                                                       options,
                                                       &mut channel_data,
                                                       command,
                                                       target,
                                                       line.is_action,
                                                       Some(source))
                                }
                                None => {
                                    // FIXME: code duplication
                                    let this_channel_data =
                                        channel_data
                                            .entry(target.clone())
                                            .or_insert_with(|| ChannelData::new(options));
                                    if let Some(response) = this_channel_data.add_line(line) {
                                        server.send_privmsg(target, &*response).unwrap();
                                    }
                                }
                            }
                        } else {
                            warn!("UNEXPECTED TARGET {} in message {}",
                                  target,
                                  format!("{}", message).trim());
                        }
                    }
                }
            }
            Command::INVITE(ref target, ref channel) => {
                if target == server.current_nickname() {
                    // Join channels when invited.
                    server.send_join(channel).unwrap();
                }
            }
            _ => (),
        }
    }
}

// Take a message in the channel, and see if it was a message sent to
// this bot.
fn check_command_in_channel(mynick: &str, msg: &String) -> Option<String> {
    if !msg.starts_with(mynick) {
        return None;
    }
    let after_nick = &msg[mynick.len()..];
    if !after_nick.starts_with(":") && !after_nick.starts_with(",") {
        return None;
    }
    let after_punct = &after_nick[1..];
    Some(String::from(after_punct.trim_left()))
}

fn handle_bot_command<'opts>(server: &IrcServer,
                             options: &'opts HashMap<String, String>,
                             all_channel_data: &mut HashMap<String, ChannelData<'opts>>,
                             command: &str,
                             response_target: &str,
                             response_is_action: bool,
                             response_username: Option<&str>) {

    let send_line = |response_username: Option<&str>, line: &str| {
        let line_with_nick = match response_username {
            None => String::from(line),
            Some(username) => String::from(username) + ", " + line,
        };
        let adjusted_line = if response_is_action {
            format!("\x01ACTION {}\x01", line_with_nick)
        } else {
            line_with_nick
        };
        server
            .send_privmsg(response_target, &adjusted_line)
            .unwrap();
    };

    if command == "help" {
        send_line(response_username, "The commands I understand are:");
        send_line(None, "  help     Send this message.");
        send_line(None, "  intro    Send a message describing what I do.");
        send_line(None, "  status   Send a message with current bot status.");
        send_line(None,
                  "  bye      Leave the channel.  (You can /invite me back.)");
        return;
    }

    if command == "intro" {
        let config = server.config();
        send_line(None,
                  "My job is to leave comments in github when the group discusses github issues and takes minutes in IRC.");
        send_line(None,
                  "I separate discussions by the \"Topic:\" lines, and I know what github issues to use only by lines of the form \"GitHub topic: <url> | none\", or if the \"Topic:\" line is a github URL.");
        send_line(None,
                  &*format!("I'm only allowed to comment on issues in the repositories: {}.",
                            options["github_repos_allowed"]));
        let owners = if let Some(v) = config.owners.as_ref() {
            v.join(" ")
        } else {
            String::from("")
        };
        send_line(None,
                  &*format!("My source code is at {} and I'm run by {}.",
                            options["source"],
                            owners));
        return;
    }

    if command == "status" {
        // FIXME: Give the changeset we were compiled from.
        send_line(response_username,
                  "I currently have data for the following channels:");
        let mut sorted_channels: Vec<&String> = all_channel_data.keys().collect();
        sorted_channels.sort();
        for channel in sorted_channels {
            let ref channel_data = all_channel_data[channel];
            if let Some(ref topic) = channel_data.current_topic {
                send_line(None,
                          &*format!("  {} ({} lines buffered on \"{}\")",
                                    channel,
                                    topic.lines.len(),
                                    topic.topic));
                match topic.github_url {
                    None => send_line(None, "    no GitHub URL to comment on"),
                    Some(ref github_url) => {
                        send_line(None, &*format!("    will comment on {}", github_url))
                    }
                };
            } else {
                send_line(None, &*format!("  {} (no topic data buffered)", channel));
            }
        }
        return;
    }

    if command == "bye" {
        if response_target.starts_with('#') {
            // FIXME: code duplication
            let this_channel_data = all_channel_data
                .entry(String::from(response_target))
                .or_insert_with(|| ChannelData::new(options));
            this_channel_data.end_topic();
            server.send(Command::PART(String::from(response_target),
                        Some(format!("Leaving at request of {}.  Feel free to /invite me back.",
                                     response_username.unwrap())))).unwrap();
        } else {
            send_line(response_username, "'bye' only works in a channel");
        }
        return;
    }

    send_line(response_username,
              "Sorry, I don't understand that command.  Try 'help'.");
}

struct ChannelLine {
    source: String,
    is_action: bool,
    message: String,
}

struct TopicData {
    topic: String,
    github_url: Option<String>,
    lines: Vec<ChannelLine>,
    resolutions: Vec<String>,
}

struct ChannelData<'opts> {
    current_topic: Option<TopicData>,
    options: &'opts HashMap<String, String>,
}

impl fmt::Display for ChannelLine {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        if self.is_action {
            write!(f, "* {} {}", self.source, self.message)
        } else {
            write!(f, "<{}> {}", self.source, self.message)
        }
    }
}

impl TopicData {
    fn new(topic: &str) -> TopicData {
        let topic_ = String::from(topic);
        TopicData {
            topic: topic_,
            github_url: None,
            lines: vec![],
            resolutions: vec![],
        }
    }
}

impl fmt::Display for TopicData {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        if self.resolutions.len() == 0 {
            try!(write!(f,
                        "The CSS Working Group just discussed {}.\n\n",
                        // FIXME: escape self.topic
                        self.topic));
        } else {
            try!(write!(f,
                        "The CSS Working Group just discussed {}, and agreed to the following resolutions:\n\n",
                        self.topic));
            for resolution in &self.resolutions {
                try!(write!(f, "```\n{}\n```\n\n", resolution));
            }
        }

        try!(write!(f,
                    "The full IRC log of that discussion\n"));
        try!(write!(f, "\n```\n"));
        for line in &self.lines {
            try!(write!(f, "{}\n", line));
        }
        try!(write!(f, "```\n\n"));
        Ok(())
    }
}

/// A case-insensitive version of starts_with.
fn ci_starts_with(s: &str, prefix: &str) -> bool {
    assert!(prefix.to_lowercase() == prefix);
    assert!(prefix.len() == prefix.chars().count());

    s.len() >= prefix.len() &&
    prefix
        .as_bytes()
        .eq_ignore_ascii_case(&s.as_bytes()[..prefix.len()])
}

/// Remove a case-insensitive start of the line, and if that prefix is
/// present return the rest of the line.
fn strip_ci_prefix(s: &str, prefix: &str) -> Option<String> {
    if ci_starts_with(s, prefix) {
        Some(String::from(s[prefix.len()..].trim_left()))
    } else {
        None
    }
}

impl<'opts> ChannelData<'opts> {
    fn new(options_: &'opts HashMap<String, String>) -> ChannelData {
        ChannelData {
            current_topic: None,
            options: options_,
        }
    }

    // Returns the response that should be sent to the message over IRC.
    fn add_line(&mut self, line: ChannelLine) -> Option<String> {
        if let Some(ref topic) = strip_ci_prefix(&line.message, "topic:") {
            self.start_topic(topic);
        }
        if line.source == "trackbot" && line.is_action == true &&
           line.message == "is ending a teleconference." {
            self.end_topic();
        }
        match self.current_topic {
            None => None,
            Some(ref mut data) => {
                let new_url_option = extract_github_url(&line.message, self.options);
                let response = match (new_url_option.as_ref(), &data.github_url) {
                    (None, _) => None,
                    (Some(&None), &None) => None,
                    (Some(&None), _) => Some(String::from("OK, I won't post this discussion to GitHub")),
                    (Some(&Some(ref new_url)), &None) => {
                        Some(format!("OK, I'll post this discussion to {}", new_url))
                    }
                    (Some(new_url), old_url) if *old_url == *new_url => None,
                    (Some(&Some(ref new_url)), &Some(ref old_url)) => {
                        Some(format!("OK, I'll post this discussion to {} instead of {} like you said before",
                                     new_url,
                                     old_url))
                    }
                };

                if let Some(new_url) = new_url_option {
                    data.github_url = new_url;
                }

                if line.message.starts_with("RESOLUTION") || line.message.starts_with("RESOLVED") {
                    data.resolutions.push(line.message.clone());
                }

                data.lines.push(line);

                response
            }
        }
    }

    fn start_topic(&mut self, topic: &str) {
        self.end_topic();
        self.current_topic = Some(TopicData::new(topic));
    }

    fn end_topic(&mut self) {
        // TODO: Test the topic boundary code.
        if let Some(topic) = self.current_topic.take() {
            if topic.github_url.is_some() {
                let task = GithubCommentTask::new(topic, self.options);
                task.run();
            }
        }
    }
}

fn extract_github_url(message: &str, options: &HashMap<String, String>) -> Option<Option<String>> {
    lazy_static! {
        static ref GITHUB_URL_RE: Regex =
            Regex::new(r"^https://github.com/(?P<repo>[^/]*/[^/]*)/issues/(?P<number>[0-9]+)$")
            .unwrap();
    }
    let ref allowed_repos = options["github_repos_allowed"];
    for prefix in ["topic:", "github topic:"].into_iter() {
        if let Some(ref maybe_url) = strip_ci_prefix(&message, prefix) {
            if maybe_url.to_lowercase() == "none" {
                return Some(None);
            } else if let Some(ref caps) = GITHUB_URL_RE.captures(maybe_url) {
                for repo in allowed_repos.split_whitespace() {
                    if caps["repo"] == *repo {
                        return Some(Some(maybe_url.clone()));
                    }
                }
            }
        }
    }
    None
}

struct GithubCommentTask {
    data: TopicData,
    github: Github,
}

impl GithubCommentTask {
    fn new(data_: TopicData, options: &HashMap<String, String>) -> GithubCommentTask {
        let github_ =
            Github::new(&*options["github_uastring"],
                        Client::with_connector(HttpsConnector::new(NativeTlsClient::new()
                                                                       .unwrap())),
                        Credentials::Token(options["github_access_token"].clone()));
        GithubCommentTask {
            data: data_,
            github: github_,
        }
    }
    fn run(self) {
        thread::spawn(move || { self.main(); });
    }
    fn main(&self) {
        lazy_static! {
            static ref GITHUB_URL_RE: Regex =
                Regex::new(r"^https://github.com/(?P<owner>[^/]*)/(?P<repo>[^/]*)/issues/(?P<number>[0-9]+)$")
                .unwrap();
        }

        if let Some(ref github_url) = self.data.github_url {
            if let Some(ref caps) = GITHUB_URL_RE.captures(github_url) {
                let repo = self.github
                    .repo(String::from(&caps["owner"]), String::from(&caps["repo"]));
                let issue = repo.issue(caps["number"].parse::<u64>().unwrap());
                let comments = issue.comments();

                let comment_text = format!("{}", self.data);
                comments
                    .create(&CommentOptions { body: comment_text })
                    .unwrap();
            } else {
                warn!("How does {} fail to match now when it matched before?",
                      github_url)
            }
        }
    }
}
<commit_msg>Only post lines to the issue (including resolutions) if they're not /me commands.<commit_after>#[macro_use]
extern crate log;
extern crate env_logger;
extern crate irc;
#[macro_use]
extern crate lazy_static;
extern crate regex;
extern crate hyper;
extern crate hubcaps;
extern crate hyper_native_tls;
extern crate serde_json;

use std::env;
use std::fmt;
use std::thread;
use std::collections::HashMap;
use std::ascii::AsciiExt;
use regex::Regex;

use irc::client::prelude::*;
use irc::client::data::command::Command;

use hyper::Client;
use hyper::net::HttpsConnector;
use hyper_native_tls::NativeTlsClient;
use hubcaps::{Credentials, Github};
use hubcaps::comments::CommentOptions;

fn main() {
    env_logger::init().unwrap();

    let config_file =
        {
            let mut args = env::args_os().skip(1); // skip program name
            let config_file = args.next().expect("Expected a single command-line argument, the JSON configuration file.");
            if args.next().is_some() {
                panic!("Expected only a single command-line argument, the JSON configuration file.");
            }
            config_file
        };

    let server = IrcServer::new(config_file).expect("Couldn't initialize server with given configuration file");
    server.identify().unwrap();

    let options = server
        .config()
        .options
        .as_ref()
        .expect("No options property within configuration?");

    // FIXME: Add a way to ask the bot to reboot itself?
    let mut channel_data: HashMap<String, ChannelData> = HashMap::new();

    for message in server.iter() {
        let message = message.unwrap(); // panic if there's an error

        match message.command {
            Command::PRIVMSG(ref target, ref msg) => {
                match message.source_nickname() {
                    None => {
                        warn!("PRIVMSG without a source! {}",
                              format!("{}", message).trim());
                    }
                    Some(ref source) => {
                        let source_ = String::from(*source);
                        let line = if msg.starts_with("\x01ACTION ") && msg.ends_with("\x01") {
                            ChannelLine {
                                source: source_,
                                is_action: true,
                                message: String::from(&msg[8..msg.len() - 1]),
                            }
                        } else {
                            ChannelLine {
                                source: source_,
                                is_action: false,
                                message: msg.clone(),
                            }
                        };
                        let mynick = server.current_nickname();
                        if target == mynick {
                            // An actual private message.
                            info!("[{}] {}", source, line);
                            handle_bot_command(&server,
                                               options,
                                               &mut channel_data,
                                               &line.message,
                                               source,
                                               false,
                                               None)
                        } else if target.starts_with('#') {
                            // A message in a channel.
                            info!("[{}] {}", target, line);
                            match check_command_in_channel(mynick, &line.message) {
                                Some(ref command) => {
                                    handle_bot_command(&server,
                                                       options,
                                                       &mut channel_data,
                                                       command,
                                                       target,
                                                       line.is_action,
                                                       Some(source))
                                }
                                None => {
                                    // FIXME: code duplication
                                    let this_channel_data =
                                        channel_data
                                            .entry(target.clone())
                                            .or_insert_with(|| ChannelData::new(options));
                                    if let Some(response) = this_channel_data.add_line(line) {
                                        server.send_privmsg(target, &*response).unwrap();
                                    }
                                }
                            }
                        } else {
                            warn!("UNEXPECTED TARGET {} in message {}",
                                  target,
                                  format!("{}", message).trim());
                        }
                    }
                }
            }
            Command::INVITE(ref target, ref channel) => {
                if target == server.current_nickname() {
                    // Join channels when invited.
                    server.send_join(channel).unwrap();
                }
            }
            _ => (),
        }
    }
}

// Take a message in the channel, and see if it was a message sent to
// this bot.
fn check_command_in_channel(mynick: &str, msg: &String) -> Option<String> {
    if !msg.starts_with(mynick) {
        return None;
    }
    let after_nick = &msg[mynick.len()..];
    if !after_nick.starts_with(":") && !after_nick.starts_with(",") {
        return None;
    }
    let after_punct = &after_nick[1..];
    Some(String::from(after_punct.trim_left()))
}

fn handle_bot_command<'opts>(server: &IrcServer,
                             options: &'opts HashMap<String, String>,
                             all_channel_data: &mut HashMap<String, ChannelData<'opts>>,
                             command: &str,
                             response_target: &str,
                             response_is_action: bool,
                             response_username: Option<&str>) {

    let send_line = |response_username: Option<&str>, line: &str| {
        let line_with_nick = match response_username {
            None => String::from(line),
            Some(username) => String::from(username) + ", " + line,
        };
        let adjusted_line = if response_is_action {
            format!("\x01ACTION {}\x01", line_with_nick)
        } else {
            line_with_nick
        };
        server
            .send_privmsg(response_target, &adjusted_line)
            .unwrap();
    };

    if command == "help" {
        send_line(response_username, "The commands I understand are:");
        send_line(None, "  help     Send this message.");
        send_line(None, "  intro    Send a message describing what I do.");
        send_line(None, "  status   Send a message with current bot status.");
        send_line(None,
                  "  bye      Leave the channel.  (You can /invite me back.)");
        return;
    }

    if command == "intro" {
        let config = server.config();
        send_line(None,
                  "My job is to leave comments in github when the group discusses github issues and takes minutes in IRC.");
        send_line(None,
                  "I separate discussions by the \"Topic:\" lines, and I know what github issues to use only by lines of the form \"GitHub topic: <url> | none\", or if the \"Topic:\" line is a github URL.");
        send_line(None,
                  &*format!("I'm only allowed to comment on issues in the repositories: {}.",
                            options["github_repos_allowed"]));
        let owners = if let Some(v) = config.owners.as_ref() {
            v.join(" ")
        } else {
            String::from("")
        };
        send_line(None,
                  &*format!("My source code is at {} and I'm run by {}.",
                            options["source"],
                            owners));
        return;
    }

    if command == "status" {
        // FIXME: Give the changeset we were compiled from.
        send_line(response_username,
                  "I currently have data for the following channels:");
        let mut sorted_channels: Vec<&String> = all_channel_data.keys().collect();
        sorted_channels.sort();
        for channel in sorted_channels {
            let ref channel_data = all_channel_data[channel];
            if let Some(ref topic) = channel_data.current_topic {
                send_line(None,
                          &*format!("  {} ({} lines buffered on \"{}\")",
                                    channel,
                                    topic.lines.len(),
                                    topic.topic));
                match topic.github_url {
                    None => send_line(None, "    no GitHub URL to comment on"),
                    Some(ref github_url) => {
                        send_line(None, &*format!("    will comment on {}", github_url))
                    }
                };
            } else {
                send_line(None, &*format!("  {} (no topic data buffered)", channel));
            }
        }
        return;
    }

    if command == "bye" {
        if response_target.starts_with('#') {
            // FIXME: code duplication
            let this_channel_data = all_channel_data
                .entry(String::from(response_target))
                .or_insert_with(|| ChannelData::new(options));
            this_channel_data.end_topic();
            server.send(Command::PART(String::from(response_target),
                        Some(format!("Leaving at request of {}.  Feel free to /invite me back.",
                                     response_username.unwrap())))).unwrap();
        } else {
            send_line(response_username, "'bye' only works in a channel");
        }
        return;
    }

    send_line(response_username,
              "Sorry, I don't understand that command.  Try 'help'.");
}

struct ChannelLine {
    source: String,
    is_action: bool,
    message: String,
}

struct TopicData {
    topic: String,
    github_url: Option<String>,
    lines: Vec<ChannelLine>,
    resolutions: Vec<String>,
}

struct ChannelData<'opts> {
    current_topic: Option<TopicData>,
    options: &'opts HashMap<String, String>,
}

impl fmt::Display for ChannelLine {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        if self.is_action {
            write!(f, "* {} {}", self.source, self.message)
        } else {
            write!(f, "<{}> {}", self.source, self.message)
        }
    }
}

impl TopicData {
    fn new(topic: &str) -> TopicData {
        let topic_ = String::from(topic);
        TopicData {
            topic: topic_,
            github_url: None,
            lines: vec![],
            resolutions: vec![],
        }
    }
}

impl fmt::Display for TopicData {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        if self.resolutions.len() == 0 {
            try!(write!(f,
                        "The CSS Working Group just discussed {}.\n\n",
                        // FIXME: escape self.topic
                        self.topic));
        } else {
            try!(write!(f,
                        "The CSS Working Group just discussed {}, and agreed to the following resolutions:\n\n",
                        self.topic));
            for resolution in &self.resolutions {
                try!(write!(f, "```\n{}\n```\n\n", resolution));
            }
        }

        try!(write!(f,
                    "The full IRC log of that discussion\n"));
        try!(write!(f, "\n```\n"));
        for line in &self.lines {
            try!(write!(f, "{}\n", line));
        }
        try!(write!(f, "```\n\n"));
        Ok(())
    }
}

/// A case-insensitive version of starts_with.
fn ci_starts_with(s: &str, prefix: &str) -> bool {
    assert!(prefix.to_lowercase() == prefix);
    assert!(prefix.len() == prefix.chars().count());

    s.len() >= prefix.len() &&
    prefix
        .as_bytes()
        .eq_ignore_ascii_case(&s.as_bytes()[..prefix.len()])
}

/// Remove a case-insensitive start of the line, and if that prefix is
/// present return the rest of the line.
fn strip_ci_prefix(s: &str, prefix: &str) -> Option<String> {
    if ci_starts_with(s, prefix) {
        Some(String::from(s[prefix.len()..].trim_left()))
    } else {
        None
    }
}

impl<'opts> ChannelData<'opts> {
    fn new(options_: &'opts HashMap<String, String>) -> ChannelData {
        ChannelData {
            current_topic: None,
            options: options_,
        }
    }

    // Returns the response that should be sent to the message over IRC.
    fn add_line(&mut self, line: ChannelLine) -> Option<String> {
        if let Some(ref topic) = strip_ci_prefix(&line.message, "topic:") {
            self.start_topic(topic);
        }
        if line.source == "trackbot" && line.is_action == true &&
           line.message == "is ending a teleconference." {
            self.end_topic();
        }
        match self.current_topic {
            None => None,
            Some(ref mut data) => {
                let new_url_option = extract_github_url(&line.message, self.options);
                let response = match (new_url_option.as_ref(), &data.github_url) {
                    (None, _) => None,
                    (Some(&None), &None) => None,
                    (Some(&None), _) => Some(String::from("OK, I won't post this discussion to GitHub")),
                    (Some(&Some(ref new_url)), &None) => {
                        Some(format!("OK, I'll post this discussion to {}", new_url))
                    }
                    (Some(new_url), old_url) if *old_url == *new_url => None,
                    (Some(&Some(ref new_url)), &Some(ref old_url)) => {
                        Some(format!("OK, I'll post this discussion to {} instead of {} like you said before",
                                     new_url,
                                     old_url))
                    }
                };

                if let Some(new_url) = new_url_option {
                    data.github_url = new_url;
                }

                if !line.is_action {
                    if line.message.starts_with("RESOLUTION") ||
                       line.message.starts_with("RESOLVED") {
                        data.resolutions.push(line.message.clone());
                    }

                    data.lines.push(line);
                }

                response
            }
        }
    }

    fn start_topic(&mut self, topic: &str) {
        self.end_topic();
        self.current_topic = Some(TopicData::new(topic));
    }

    fn end_topic(&mut self) {
        // TODO: Test the topic boundary code.
        if let Some(topic) = self.current_topic.take() {
            if topic.github_url.is_some() {
                let task = GithubCommentTask::new(topic, self.options);
                task.run();
            }
        }
    }
}

fn extract_github_url(message: &str, options: &HashMap<String, String>) -> Option<Option<String>> {
    lazy_static! {
        static ref GITHUB_URL_RE: Regex =
            Regex::new(r"^https://github.com/(?P<repo>[^/]*/[^/]*)/issues/(?P<number>[0-9]+)$")
            .unwrap();
    }
    let ref allowed_repos = options["github_repos_allowed"];
    for prefix in ["topic:", "github topic:"].into_iter() {
        if let Some(ref maybe_url) = strip_ci_prefix(&message, prefix) {
            if maybe_url.to_lowercase() == "none" {
                return Some(None);
            } else if let Some(ref caps) = GITHUB_URL_RE.captures(maybe_url) {
                for repo in allowed_repos.split_whitespace() {
                    if caps["repo"] == *repo {
                        return Some(Some(maybe_url.clone()));
                    }
                }
            }
        }
    }
    None
}

struct GithubCommentTask {
    data: TopicData,
    github: Github,
}

impl GithubCommentTask {
    fn new(data_: TopicData, options: &HashMap<String, String>) -> GithubCommentTask {
        let github_ =
            Github::new(&*options["github_uastring"],
                        Client::with_connector(HttpsConnector::new(NativeTlsClient::new()
                                                                       .unwrap())),
                        Credentials::Token(options["github_access_token"].clone()));
        GithubCommentTask {
            data: data_,
            github: github_,
        }
    }
    fn run(self) {
        thread::spawn(move || { self.main(); });
    }
    fn main(&self) {
        lazy_static! {
            static ref GITHUB_URL_RE: Regex =
                Regex::new(r"^https://github.com/(?P<owner>[^/]*)/(?P<repo>[^/]*)/issues/(?P<number>[0-9]+)$")
                .unwrap();
        }

        if let Some(ref github_url) = self.data.github_url {
            if let Some(ref caps) = GITHUB_URL_RE.captures(github_url) {
                let repo = self.github
                    .repo(String::from(&caps["owner"]), String::from(&caps["repo"]));
                let issue = repo.issue(caps["number"].parse::<u64>().unwrap());
                let comments = issue.comments();

                let comment_text = format!("{}", self.data);
                comments
                    .create(&CommentOptions { body: comment_text })
                    .unwrap();
            } else {
                warn!("How does {} fail to match now when it matched before?",
                      github_url)
            }
        }
    }
}
<|endoftext|>


<commit_before>extern crate discord;

use std::env;
use discord::{Discord, State};
use discord::model::Event;

pub fn main() {
	// Log in to Discord using a bot token from the environment
	let discord = Discord::from_bot_token(
		&env::var("DISCORD_TOKEN").expect("Expected token"),
	).expect("login failed");

	// establish websocket and voice connection
	let (mut connection, ready) = discord.connect().expect("connect failed");
	println!("[Ready] {} is serving {} servers", ready.user.username, ready.servers.len());
	let mut state = State::new(ready);
	connection.sync_calls(&state.all_private_channels());

	// receive events forever
	loop {
		let event = match connection.recv_event() {
			Ok(event) => event,
			Err(err) => {
				println!("[Warning] Receive error: {:?}", err);
				if let discord::Error::WebSocket(..) = err {
					// Handle the websocket connection being dropped
					let (new_connection, ready) = discord.connect().expect("connect failed");
					connection = new_connection;
					state = State::new(ready);
					println!("[Ready] Reconnected successfully.");
				}
				if let discord::Error::Closed(..) = err {
					break
				}
				continue
			},
		};
		state.update(&event);

		match event {
			Event::MessageCreate(message) => {
				use std::ascii::AsciiExt;
				// safeguard: stop if the message is from us
				if message.author.id == state.user().id {
					continue
				}

				// reply to a command if there was one
				let mut split = message.content.split(' ');
				let first_word = split.next().unwrap_or("");
				let argument = split.next().unwrap_or("");

                if first_word.eq_ignore_ascii_case("!help") {
                    if argument.eq_ignore_ascii_case("commands") {
                        warn(discord.send_message(message.channel_id, "help, about, uber, distrowiki, and sourcecode are the avaliable commands. (!dj is buggy)", "", false))
                    }
                    else {
                    warn(discord.send_message(message.channel_id, "Umm, you gotta use it like this. !help [args]", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!about") {
                    warn(discord.send_message(message.channel_id, "I have no clue. All I know is that I was written by Arroz in a language called Rust, and my code is still being developed.", "", false))
                }

                if first_word.eq_ignore_ascii_case("!uber"){
			if argument.eq_ignore_ascii_case("ddog75") {
                        warn(discord.send_message(message.channel_id, "Alright, 10 minutes.", "", false))
                    } else {
                        warn(discord.send_message(message.channel_id, "Fuck off!", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!sourcecode") {
                    warn(discord.send_message(message.channel_id, "Okay, here you go. https://github.com/aarroz/rice_bot2", "", false))
                }
				//Section that prints wikis for linux distributions.
				if first_word.eq_ignore_ascii_case("!distrowiki") {
					if argument.eq_ignore_ascii_case("arch") {
						warn(discord.send_message(message.channel_id, "Here's the Arch Wiki! https://wiki.archlinux.org/", "", false))
					}
					if argument.eq_ignore_ascii_case("ubuntu") {
						warn(discord.send_message(message.channel_id, "Here's the Ubuntu Wiki! https://wiki.ubuntu.com/", "", false))
					}
					if argument.eq_ignore_ascii_case("debian") {
						warn(discord.send_message(message.channel_id, "Here's the Debian Wiki! https://wiki.debian.org/", "", false))
					}
					if argument.eq_ignore_ascii_case("antergos") {
						warn(discord.send_message(message.channel_id, "Here's the Antergos Wiki! https://antergos.com/wiki/", "", false))
					}
					if argument.eq_ignore_ascii_case("centos") {
						warn(discord.send_message(message.channel_id, "Here's the CentOS Wiki! https://wiki.centos.org/", "", false))
					}
					//Sorry OpenBSD users.
					if argument.eq_ignore_ascii_case("openbsd") {
						warn(discord.send_message(message.channel_id, "No, just no.", "", false))
					}
				}

                // Command for using dj.
				if first_word.eq_ignore_ascii_case("!dj") {
					let vchan = state.find_voice_user(message.author.id);
					if argument.eq_ignore_ascii_case("stop") {
						vchan.map(|(sid, _)| connection.voice(sid).stop());
					} else if argument.eq_ignore_ascii_case("quit") {
						vchan.map(|(sid, _)| connection.drop_voice(sid));
					} else {
						let output = if let Some((server_id, channel_id)) = vchan {
							match discord::voice::open_ytdl_stream(argument) {
								Ok(stream) => {
									let voice = connection.voice(server_id);
									voice.set_deaf(true);
									voice.connect(channel_id);
									voice.play(stream);
									String::new()
								},
								Err(error) => format!("Error: {}", error),
							}
						} else {
							"You must be in a voice channel to DJ".to_owned()
						};
						if !output.is_empty() {
							warn(discord.send_message(message.channel_id, &output, "", false));
						}
					}
				}
			}
			Event::VoiceStateUpdate(server_id, _) => {
				// If someone moves/hangs up, and we are in a voice channel,
				if let Some(cur_channel) = connection.voice(server_id).current_channel() {
					// and our current voice channel is empty, disconnect from voice
					match server_id {
						Some(server_id) => if let Some(srv) = state.servers().iter().find(|srv| srv.id == server_id) {
							if srv.voice_states.iter().filter(|vs| vs.channel_id == Some(cur_channel)).count() <= 1 {
								connection.voice(Some(server_id)).disconnect();
							}
						},
						None => if let Some(call) = state.calls().get(&cur_channel) {
							if call.voice_states.len() <= 1 {
								connection.voice(server_id).disconnect();
							}
						}
					}
				}
			}
			_ => {}, // discard other events
		}
	}
}

fn warn<T, E: ::std::fmt::Debug>(result: Result<T, E>) {
	match result {
		Ok(_) => {},
		Err(err) => println!("[Warning] {:?}", err)
	}
}
<commit_msg>Fixed syntax. Dammit Gordo<commit_after>extern crate discord;

use std::env;
use discord::{Discord, State};
use discord::model::Event;

pub fn main() {
	// Log in to Discord using a bot token from the environment
	let discord = Discord::from_bot_token(
		&env::var("DISCORD_TOKEN").expect("Expected token"),
	).expect("login failed");

	// establish websocket and voice connection
	let (mut connection, ready) = discord.connect().expect("connect failed");
	println!("[Ready] {} is serving {} servers", ready.user.username, ready.servers.len());
	let mut state = State::new(ready);
	connection.sync_calls(&state.all_private_channels());

	// receive events forever
	loop {
		let event = match connection.recv_event() {
			Ok(event) => event,
			Err(err) => {
				println!("[Warning] Receive error: {:?}", err);
				if let discord::Error::WebSocket(..) = err {
					// Handle the websocket connection being dropped
					let (new_connection, ready) = discord.connect().expect("connect failed");
					connection = new_connection;
					state = State::new(ready);
					println!("[Ready] Reconnected successfully.");
				}
				if let discord::Error::Closed(..) = err {
					break
				}
				continue
			},
		};
		state.update(&event);

		match event {
			Event::MessageCreate(message) => {
				use std::ascii::AsciiExt;
				// safeguard: stop if the message is from us
				if message.author.id == state.user().id {
					continue
				}

				// reply to a command if there was one
				let mut split = message.content.split(' ');
				let first_word = split.next().unwrap_or("");
				let argument = split.next().unwrap_or("");

                if first_word.eq_ignore_ascii_case("!help") {
                    if argument.eq_ignore_ascii_case("commands") {
                        warn(discord.send_message(message.channel_id, "help, about, uber, distrowiki, and sourcecode are the avaliable commands. (!dj is buggy)", "", false))
                    }
                    else {
                    warn(discord.send_message(message.channel_id, "Umm, you gotta use it like this. !help [args]", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!about") {
                    warn(discord.send_message(message.channel_id, "I have no clue. All I know is that I was written by Arroz in a language called Rust, and my code is still being developed.", "", false))
                }

                if first_word.eq_ignore_ascii_case("!uber"){
					if argument.eq_ignore_ascii_case("ddog75") {
                        warn(discord.send_message(message.channel_id, "Alright, 10 minutes.", "", false))
                    } else {
                        warn(discord.send_message(message.channel_id, "Fuck off!", "", false))
                    }
                }

                if first_word.eq_ignore_ascii_case("!sourcecode") {
                    warn(discord.send_message(message.channel_id, "Okay, here you go. https://github.com/aarroz/rice_bot2", "", false))
                }
				//Section that prints wikis for linux distributions.
				if first_word.eq_ignore_ascii_case("!distrowiki") {
					if argument.eq_ignore_ascii_case("arch") {
						warn(discord.send_message(message.channel_id, "Here's the Arch Wiki! https://wiki.archlinux.org/", "", false))
					}
					if argument.eq_ignore_ascii_case("ubuntu") {
						warn(discord.send_message(message.channel_id, "Here's the Ubuntu Wiki! https://wiki.ubuntu.com/", "", false))
					}
					if argument.eq_ignore_ascii_case("debian") {
						warn(discord.send_message(message.channel_id, "Here's the Debian Wiki! https://wiki.debian.org/", "", false))
					}
					if argument.eq_ignore_ascii_case("antergos") {
						warn(discord.send_message(message.channel_id, "Here's the Antergos Wiki! https://antergos.com/wiki/", "", false))
					}
					if argument.eq_ignore_ascii_case("centos") {
						warn(discord.send_message(message.channel_id, "Here's the CentOS Wiki! https://wiki.centos.org/", "", false))
					}
					//Sorry OpenBSD users.
					if argument.eq_ignore_ascii_case("openbsd") {
						warn(discord.send_message(message.channel_id, "No, just no.", "", false))
					}
				}

                // Command for using dj.
				if first_word.eq_ignore_ascii_case("!dj") {
					let vchan = state.find_voice_user(message.author.id);
					if argument.eq_ignore_ascii_case("stop") {
						vchan.map(|(sid, _)| connection.voice(sid).stop());
					} else if argument.eq_ignore_ascii_case("quit") {
						vchan.map(|(sid, _)| connection.drop_voice(sid));
					} else {
						let output = if let Some((server_id, channel_id)) = vchan {
							match discord::voice::open_ytdl_stream(argument) {
								Ok(stream) => {
									let voice = connection.voice(server_id);
									voice.set_deaf(true);
									voice.connect(channel_id);
									voice.play(stream);
									String::new()
								},
								Err(error) => format!("Error: {}", error),
							}
						} else {
							"You must be in a voice channel to DJ".to_owned()
						};
						if !output.is_empty() {
							warn(discord.send_message(message.channel_id, &output, "", false));
						}
					}
				}
			}
			Event::VoiceStateUpdate(server_id, _) => {
				// If someone moves/hangs up, and we are in a voice channel,
				if let Some(cur_channel) = connection.voice(server_id).current_channel() {
					// and our current voice channel is empty, disconnect from voice
					match server_id {
						Some(server_id) => if let Some(srv) = state.servers().iter().find(|srv| srv.id == server_id) {
							if srv.voice_states.iter().filter(|vs| vs.channel_id == Some(cur_channel)).count() <= 1 {
								connection.voice(Some(server_id)).disconnect();
							}
						},
						None => if let Some(call) = state.calls().get(&cur_channel) {
							if call.voice_states.len() <= 1 {
								connection.voice(server_id).disconnect();
							}
						}
					}
				}
			}
			_ => {}, // discard other events
		}
	}
}

fn warn<T, E: ::std::fmt::Debug>(result: Result<T, E>) {
	match result {
		Ok(_) => {},
		Err(err) => println!("[Warning] {:?}", err)
	}
}
<|endoftext|>


<commit_before>mod app;
mod error;
mod exec;
mod exit_codes;
mod filesystem;
mod filetypes;
mod filter;
mod options;
mod output;
mod regex_helper;
mod walk;

use std::env;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::Arc;
use std::time;

use anyhow::{anyhow, Context, Result};
use atty::Stream;
use globset::GlobBuilder;
use lscolors::LsColors;
use regex::bytes::{RegexBuilder, RegexSetBuilder};

use crate::exec::CommandTemplate;
use crate::exit_codes::ExitCode;
use crate::filetypes::FileTypes;
#[cfg(unix)]
use crate::filter::OwnerFilter;
use crate::filter::{SizeFilter, TimeFilter};
use crate::options::Options;
use crate::regex_helper::pattern_has_uppercase_char;

// We use jemalloc for performance reasons, see https://github.com/sharkdp/fd/pull/481
#[cfg(all(not(windows), not(target_env = "musl")))]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

fn run() -> Result<ExitCode> {
    let matches = app::build_app().get_matches_from(env::args_os());

    // Set the current working directory of the process
    if let Some(base_directory) = matches.value_of_os("base-directory") {
        let base_directory = Path::new(base_directory);
        if !filesystem::is_dir(base_directory) {
            return Err(anyhow!(
                "The '--base-directory' path '{}' is not a directory.",
                base_directory.to_string_lossy()
            ));
        }
        env::set_current_dir(base_directory).with_context(|| {
            format!(
                "Could not set '{}' as the current working directory.",
                base_directory.to_string_lossy()
            )
        })?;
    }

    let current_directory = Path::new(".");
    if !filesystem::is_dir(current_directory) {
        return Err(anyhow!(
            "Could not retrieve current directory (has it been deleted?)."
        ));
    }

    // Get the search pattern
    let pattern = matches
        .value_of_os("pattern")
        .map(|p| {
            p.to_str()
                .ok_or_else(|| anyhow!("The search pattern includes invalid UTF-8 sequences."))
        })
        .transpose()?
        .unwrap_or("");

    // Get one or more root directories to search.
    let mut dir_vec: Vec<_> = match matches
        .values_of_os("path")
        .or_else(|| matches.values_of_os("search-path"))
    {
        Some(paths) => paths
            .map(|path| {
                let path_buffer = PathBuf::from(path);
                if filesystem::is_dir(&path_buffer) {
                    Ok(path_buffer)
                } else {
                    Err(anyhow!(
                        "Search path '{}' is not a directory.",
                        path_buffer.to_string_lossy()
                    ))
                }
            })
            .collect::<Result<Vec<_>>>()?,
        None => vec![current_directory.to_path_buf()],
    };

    if matches.is_present("absolute-path") {
        dir_vec = dir_vec
            .iter()
            .map(|path_buffer| {
                path_buffer
                    .canonicalize()
                    .and_then(|pb| filesystem::absolute_path(pb.as_path()))
                    .unwrap()
            })
            .collect();
    }

    // Detect if the user accidentally supplied a path instead of a search pattern
    if !matches.is_present("full-path")
        && pattern.contains(std::path::MAIN_SEPARATOR)
        && filesystem::is_dir(Path::new(pattern))
    {
        return Err(anyhow!(
            "The search pattern '{pattern}' contains a path-separation character ('{sep}') \
             and will not lead to any search results.\n\n\
             If you want to search for all files inside the '{pattern}' directory, use a match-all pattern:\n\n  \
             fd . '{pattern}'\n\n\
             Instead, if you want your pattern to match the full file path, use:\n\n  \
             fd --full-path '{pattern}'",
            pattern = pattern,
            sep = std::path::MAIN_SEPARATOR,
        ));
    }

    let pattern_regex = if matches.is_present("glob") && !pattern.is_empty() {
        let glob = GlobBuilder::new(pattern).literal_separator(true).build()?;
        glob.regex().to_owned()
    } else if matches.is_present("fixed-strings") {
        // Treat pattern as literal string if '--fixed-strings' is used
        regex::escape(pattern)
    } else {
        String::from(pattern)
    };

    // The search will be case-sensitive if the command line flag is set or
    // if the pattern has an uppercase character (smart case).
    let case_sensitive = !matches.is_present("ignore-case")
        && (matches.is_present("case-sensitive") || pattern_has_uppercase_char(&pattern_regex));

    let interactive_terminal = atty::is(Stream::Stdout);
    let colored_output = match matches.value_of("color") {
        Some("always") => true,
        Some("never") => false,
        _ => env::var_os("NO_COLOR").is_none() && interactive_terminal,
    };

    let path_separator = matches.value_of("path-separator").map(|str| str.to_owned());

    #[cfg(windows)]
    ansi_term::enable_ansi_support().ok();

    let ls_colors = if colored_output {
        Some(LsColors::from_env().unwrap_or_default())
    } else {
        None
    };

    let command = if let Some(args) = matches.values_of("exec") {
        Some(CommandTemplate::new(args))
    } else if let Some(args) = matches.values_of("exec-batch") {
        Some(CommandTemplate::new_batch(args)?)
    } else if matches.is_present("list-details") {
        let color = matches.value_of("color").unwrap_or("auto");
        let color_arg = ["--color=", color].concat();

        #[allow(unused)]
        let gnu_ls = |command_name| {
            vec![
                command_name,
                "-l",               // long listing format
                "--human-readable", // human readable file sizes
                "--directory",      // list directories themselves, not their contents
                &color_arg,
            ]
        };

        let cmd: Vec<&str> = if cfg!(unix) {
            if !cfg!(any(
                target_os = "macos",
                target_os = "dragonfly",
                target_os = "freebsd",
                target_os = "netbsd",
                target_os = "openbsd"
            )) {
                // Assume ls is GNU ls
                gnu_ls("ls")
            } else {
                // MacOS, DragonFlyBSD, FreeBSD
                use std::process::{Command, Stdio};

                // Use GNU ls, if available (support for --color=auto, better LS_COLORS support)
                let gnu_ls_exists = Command::new("gls")
                    .arg("--version")
                    .stdout(Stdio::null())
                    .stderr(Stdio::null())
                    .status()
                    .is_ok();

                if gnu_ls_exists {
                    gnu_ls("gls")
                } else {
                    let mut cmd = vec![
                        "ls", // BSD version of ls
                        "-l", // long listing format
                        "-h", // '--human-readable' is not available, '-h' is
                        "-d", // '--directory' is not available, but '-d' is
                    ];

                    if !cfg!(any(target_os = "netbsd", target_os = "openbsd")) && colored_output {
                        // -G is not available in NetBSD's and OpenBSD's ls
                        cmd.push("-G");
                    }

                    cmd
                }
            }
        } else if cfg!(windows) {
            use std::process::{Command, Stdio};

            // Use GNU ls, if available
            let gnu_ls_exists = Command::new("ls")
                .arg("--version")
                .stdout(Stdio::null())
                .stderr(Stdio::null())
                .status()
                .is_ok();

            if gnu_ls_exists {
                gnu_ls("ls")
            } else {
                return Err(anyhow!(
                    "'fd --list-details' is not supported on Windows unless GNU 'ls' is installed."
                ));
            }
        } else {
            return Err(anyhow!(
                "'fd --list-details' is not supported on this platform."
            ));
        };

        Some(CommandTemplate::new_batch(&cmd).unwrap())
    } else {
        None
    };

    let size_limits = if let Some(vs) = matches.values_of("size") {
        vs.map(|sf| {
            SizeFilter::from_string(sf)
                .ok_or_else(|| anyhow!("'{}' is not a valid size constraint. See 'fd --help'.", sf))
        })
        .collect::<Result<Vec<_>>>()?
    } else {
        vec![]
    };

    let now = time::SystemTime::now();
    let mut time_constraints: Vec<TimeFilter> = Vec::new();
    if let Some(t) = matches.value_of("changed-within") {
        if let Some(f) = TimeFilter::after(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }
    if let Some(t) = matches.value_of("changed-before") {
        if let Some(f) = TimeFilter::before(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }

    #[cfg(unix)]
    let owner_constraint = if let Some(s) = matches.value_of("owner") {
        OwnerFilter::from_string(s)?
    } else {
        None
    };

    let config = Options {
        case_sensitive,
        search_full_path: matches.is_present("full-path"),
        ignore_hidden: !(matches.is_present("hidden")
            || matches.occurrences_of("rg-alias-hidden-ignore") >= 2),
        read_fdignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")),
        read_vcsignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")
            || matches.is_present("no-ignore-vcs")),
        read_global_ignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")
            || matches.is_present("no-global-ignore-file")),
        follow_links: matches.is_present("follow"),
        one_file_system: matches.is_present("one-file-system"),
        null_separator: matches.is_present("null_separator"),
        max_depth: matches
            .value_of("max-depth")
            .or_else(|| matches.value_of("rg-depth"))
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        min_depth: matches
            .value_of("min-depth")
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        threads: std::cmp::max(
            matches
                .value_of("threads")
                .and_then(|n| usize::from_str_radix(n, 10).ok())
                .unwrap_or_else(num_cpus::get),
            1,
        ),
        max_buffer_time: matches
            .value_of("max-buffer-time")
            .and_then(|n| u64::from_str_radix(n, 10).ok())
            .map(time::Duration::from_millis),
        ls_colors,
        interactive_terminal,
        file_types: matches.values_of("file-type").map(|values| {
            let mut file_types = FileTypes::default();
            for value in values {
                match value {
                    "f" | "file" => file_types.files = true,
                    "d" | "directory" => file_types.directories = true,
                    "l" | "symlink" => file_types.symlinks = true,
                    "x" | "executable" => {
                        file_types.executables_only = true;
                        file_types.files = true;
                    }
                    "e" | "empty" => file_types.empty_only = true,
                    "s" | "socket" => file_types.sockets = true,
                    "p" | "pipe" => file_types.pipes = true,
                    _ => unreachable!(),
                }
            }

            // If only 'empty' was specified, search for both files and directories:
            if file_types.empty_only && !(file_types.files || file_types.directories) {
                file_types.files = true;
                file_types.directories = true;
            }

            file_types
        }),
        extensions: matches
            .values_of("extension")
            .map(|exts| {
                let patterns = exts
                    .map(|e| e.trim_start_matches('.'))
                    .map(|e| format!(r".\.{}$", regex::escape(e)));
                RegexSetBuilder::new(patterns)
                    .case_insensitive(true)
                    .build()
            })
            .transpose()?,
        command: command.map(Arc::new),
        exclude_patterns: matches
            .values_of("exclude")
            .map(|v| v.map(|p| String::from("!") + p).collect())
            .unwrap_or_else(|| vec![]),
        ignore_files: matches
            .values_of("ignore-file")
            .map(|vs| vs.map(PathBuf::from).collect())
            .unwrap_or_else(|| vec![]),
        size_constraints: size_limits,
        time_constraints,
        #[cfg(unix)]
        owner_constraint,
        show_filesystem_errors: matches.is_present("show-errors"),
        path_separator,
        max_results: matches
            .value_of("max-results")
            .and_then(|n| usize::from_str_radix(n, 10).ok())
            .filter(|&n| n != 0)
            .or_else(|| {
                if matches.is_present("max-one-result") {
                    Some(1)
                } else {
                    None
                }
            }),
    };

    let re = RegexBuilder::new(&pattern_regex)
        .case_insensitive(!config.case_sensitive)
        .dot_matches_new_line(true)
        .build()
        .map_err(|e| {
            anyhow!(
                "{}\n\nNote: You can use the '--fixed-strings' option to search for a \
                 literal string instead of a regular expression. Alternatively, you can \
                 also use the '--glob' option to match on a glob pattern.",
                e.to_string()
            )
        })?;

    walk::scan(&dir_vec, Arc::new(re), Arc::new(config))
}

fn main() {
    let result = run();
    match result {
        Ok(exit_code) => {
            process::exit(exit_code.into());
        }
        Err(err) => {
            eprintln!("[fd error]: {}", err);
            process::exit(ExitCode::GeneralError.into());
        }
    }
}
<commit_msg>Allow for missing directories<commit_after>mod app;
mod error;
mod exec;
mod exit_codes;
mod filesystem;
mod filetypes;
mod filter;
mod options;
mod output;
mod regex_helper;
mod walk;

use std::env;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::Arc;
use std::time;

use anyhow::{anyhow, Context, Result};
use atty::Stream;
use globset::GlobBuilder;
use lscolors::LsColors;
use regex::bytes::{RegexBuilder, RegexSetBuilder};

use crate::error::print_error;
use crate::exec::CommandTemplate;
use crate::exit_codes::ExitCode;
use crate::filetypes::FileTypes;
#[cfg(unix)]
use crate::filter::OwnerFilter;
use crate::filter::{SizeFilter, TimeFilter};
use crate::options::Options;
use crate::regex_helper::pattern_has_uppercase_char;

// We use jemalloc for performance reasons, see https://github.com/sharkdp/fd/pull/481
#[cfg(all(not(windows), not(target_env = "musl")))]
#[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;

fn run() -> Result<ExitCode> {
    let matches = app::build_app().get_matches_from(env::args_os());

    // Set the current working directory of the process
    if let Some(base_directory) = matches.value_of_os("base-directory") {
        let base_directory = Path::new(base_directory);
        if !filesystem::is_dir(base_directory) {
            return Err(anyhow!(
                "The '--base-directory' path '{}' is not a directory.",
                base_directory.to_string_lossy()
            ));
        }
        env::set_current_dir(base_directory).with_context(|| {
            format!(
                "Could not set '{}' as the current working directory.",
                base_directory.to_string_lossy()
            )
        })?;
    }

    let current_directory = Path::new(".");
    if !filesystem::is_dir(current_directory) {
        return Err(anyhow!(
            "Could not retrieve current directory (has it been deleted?)."
        ));
    }

    // Get the search pattern
    let pattern = matches
        .value_of_os("pattern")
        .map(|p| {
            p.to_str()
                .ok_or_else(|| anyhow!("The search pattern includes invalid UTF-8 sequences."))
        })
        .transpose()?
        .unwrap_or("");

    // Get one or more root directories to search.
    let mut dir_vec: Vec<_> = match matches
        .values_of_os("path")
        .or_else(|| matches.values_of_os("search-path"))
    {
        Some(paths) => paths
            .map(|path| {
                let path_buffer = PathBuf::from(path);
                if filesystem::is_dir(&path_buffer) {
                    Ok(path_buffer)
                } else {
                    Err(anyhow!(
                        "Search path '{}' is not a directory.",
                        path_buffer.to_string_lossy()
                    ))
                }
            })
            .inspect(|res| {
                if let Err(e) = res {
                    print_error(format!("{}", e))
                }
            })
            .filter_map(Result::ok)
            .collect::<Vec<_>>(),
        None => vec![current_directory.to_path_buf()],
    };

    if matches.is_present("absolute-path") {
        dir_vec = dir_vec
            .iter()
            .map(|path_buffer| {
                path_buffer
                    .canonicalize()
                    .and_then(|pb| filesystem::absolute_path(pb.as_path()))
                    .unwrap()
            })
            .collect();
    }

    // Detect if the user accidentally supplied a path instead of a search pattern
    if !matches.is_present("full-path")
        && pattern.contains(std::path::MAIN_SEPARATOR)
        && filesystem::is_dir(Path::new(pattern))
    {
        return Err(anyhow!(
            "The search pattern '{pattern}' contains a path-separation character ('{sep}') \
             and will not lead to any search results.\n\n\
             If you want to search for all files inside the '{pattern}' directory, use a match-all pattern:\n\n  \
             fd . '{pattern}'\n\n\
             Instead, if you want your pattern to match the full file path, use:\n\n  \
             fd --full-path '{pattern}'",
            pattern = pattern,
            sep = std::path::MAIN_SEPARATOR,
        ));
    }

    let pattern_regex = if matches.is_present("glob") && !pattern.is_empty() {
        let glob = GlobBuilder::new(pattern).literal_separator(true).build()?;
        glob.regex().to_owned()
    } else if matches.is_present("fixed-strings") {
        // Treat pattern as literal string if '--fixed-strings' is used
        regex::escape(pattern)
    } else {
        String::from(pattern)
    };

    // The search will be case-sensitive if the command line flag is set or
    // if the pattern has an uppercase character (smart case).
    let case_sensitive = !matches.is_present("ignore-case")
        && (matches.is_present("case-sensitive") || pattern_has_uppercase_char(&pattern_regex));

    let interactive_terminal = atty::is(Stream::Stdout);
    let colored_output = match matches.value_of("color") {
        Some("always") => true,
        Some("never") => false,
        _ => env::var_os("NO_COLOR").is_none() && interactive_terminal,
    };

    let path_separator = matches.value_of("path-separator").map(|str| str.to_owned());

    #[cfg(windows)]
    ansi_term::enable_ansi_support().ok();

    let ls_colors = if colored_output {
        Some(LsColors::from_env().unwrap_or_default())
    } else {
        None
    };

    let command = if let Some(args) = matches.values_of("exec") {
        Some(CommandTemplate::new(args))
    } else if let Some(args) = matches.values_of("exec-batch") {
        Some(CommandTemplate::new_batch(args)?)
    } else if matches.is_present("list-details") {
        let color = matches.value_of("color").unwrap_or("auto");
        let color_arg = ["--color=", color].concat();

        #[allow(unused)]
        let gnu_ls = |command_name| {
            vec![
                command_name,
                "-l",               // long listing format
                "--human-readable", // human readable file sizes
                "--directory",      // list directories themselves, not their contents
                &color_arg,
            ]
        };

        let cmd: Vec<&str> = if cfg!(unix) {
            if !cfg!(any(
                target_os = "macos",
                target_os = "dragonfly",
                target_os = "freebsd",
                target_os = "netbsd",
                target_os = "openbsd"
            )) {
                // Assume ls is GNU ls
                gnu_ls("ls")
            } else {
                // MacOS, DragonFlyBSD, FreeBSD
                use std::process::{Command, Stdio};

                // Use GNU ls, if available (support for --color=auto, better LS_COLORS support)
                let gnu_ls_exists = Command::new("gls")
                    .arg("--version")
                    .stdout(Stdio::null())
                    .stderr(Stdio::null())
                    .status()
                    .is_ok();

                if gnu_ls_exists {
                    gnu_ls("gls")
                } else {
                    let mut cmd = vec![
                        "ls", // BSD version of ls
                        "-l", // long listing format
                        "-h", // '--human-readable' is not available, '-h' is
                        "-d", // '--directory' is not available, but '-d' is
                    ];

                    if !cfg!(any(target_os = "netbsd", target_os = "openbsd")) && colored_output {
                        // -G is not available in NetBSD's and OpenBSD's ls
                        cmd.push("-G");
                    }

                    cmd
                }
            }
        } else if cfg!(windows) {
            use std::process::{Command, Stdio};

            // Use GNU ls, if available
            let gnu_ls_exists = Command::new("ls")
                .arg("--version")
                .stdout(Stdio::null())
                .stderr(Stdio::null())
                .status()
                .is_ok();

            if gnu_ls_exists {
                gnu_ls("ls")
            } else {
                return Err(anyhow!(
                    "'fd --list-details' is not supported on Windows unless GNU 'ls' is installed."
                ));
            }
        } else {
            return Err(anyhow!(
                "'fd --list-details' is not supported on this platform."
            ));
        };

        Some(CommandTemplate::new_batch(&cmd).unwrap())
    } else {
        None
    };

    let size_limits = if let Some(vs) = matches.values_of("size") {
        vs.map(|sf| {
            SizeFilter::from_string(sf)
                .ok_or_else(|| anyhow!("'{}' is not a valid size constraint. See 'fd --help'.", sf))
        })
        .collect::<Result<Vec<_>>>()?
    } else {
        vec![]
    };

    let now = time::SystemTime::now();
    let mut time_constraints: Vec<TimeFilter> = Vec::new();
    if let Some(t) = matches.value_of("changed-within") {
        if let Some(f) = TimeFilter::after(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }
    if let Some(t) = matches.value_of("changed-before") {
        if let Some(f) = TimeFilter::before(&now, t) {
            time_constraints.push(f);
        } else {
            return Err(anyhow!(
                "'{}' is not a valid date or duration. See 'fd --help'.",
                t
            ));
        }
    }

    #[cfg(unix)]
    let owner_constraint = if let Some(s) = matches.value_of("owner") {
        OwnerFilter::from_string(s)?
    } else {
        None
    };

    let config = Options {
        case_sensitive,
        search_full_path: matches.is_present("full-path"),
        ignore_hidden: !(matches.is_present("hidden")
            || matches.occurrences_of("rg-alias-hidden-ignore") >= 2),
        read_fdignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")),
        read_vcsignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")
            || matches.is_present("no-ignore-vcs")),
        read_global_ignore: !(matches.is_present("no-ignore")
            || matches.is_present("rg-alias-hidden-ignore")
            || matches.is_present("no-global-ignore-file")),
        follow_links: matches.is_present("follow"),
        one_file_system: matches.is_present("one-file-system"),
        null_separator: matches.is_present("null_separator"),
        max_depth: matches
            .value_of("max-depth")
            .or_else(|| matches.value_of("rg-depth"))
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        min_depth: matches
            .value_of("min-depth")
            .or_else(|| matches.value_of("exact-depth"))
            .and_then(|n| usize::from_str_radix(n, 10).ok()),
        threads: std::cmp::max(
            matches
                .value_of("threads")
                .and_then(|n| usize::from_str_radix(n, 10).ok())
                .unwrap_or_else(num_cpus::get),
            1,
        ),
        max_buffer_time: matches
            .value_of("max-buffer-time")
            .and_then(|n| u64::from_str_radix(n, 10).ok())
            .map(time::Duration::from_millis),
        ls_colors,
        interactive_terminal,
        file_types: matches.values_of("file-type").map(|values| {
            let mut file_types = FileTypes::default();
            for value in values {
                match value {
                    "f" | "file" => file_types.files = true,
                    "d" | "directory" => file_types.directories = true,
                    "l" | "symlink" => file_types.symlinks = true,
                    "x" | "executable" => {
                        file_types.executables_only = true;
                        file_types.files = true;
                    }
                    "e" | "empty" => file_types.empty_only = true,
                    "s" | "socket" => file_types.sockets = true,
                    "p" | "pipe" => file_types.pipes = true,
                    _ => unreachable!(),
                }
            }

            // If only 'empty' was specified, search for both files and directories:
            if file_types.empty_only && !(file_types.files || file_types.directories) {
                file_types.files = true;
                file_types.directories = true;
            }

            file_types
        }),
        extensions: matches
            .values_of("extension")
            .map(|exts| {
                let patterns = exts
                    .map(|e| e.trim_start_matches('.'))
                    .map(|e| format!(r".\.{}$", regex::escape(e)));
                RegexSetBuilder::new(patterns)
                    .case_insensitive(true)
                    .build()
            })
            .transpose()?,
        command: command.map(Arc::new),
        exclude_patterns: matches
            .values_of("exclude")
            .map(|v| v.map(|p| String::from("!") + p).collect())
            .unwrap_or_else(|| vec![]),
        ignore_files: matches
            .values_of("ignore-file")
            .map(|vs| vs.map(PathBuf::from).collect())
            .unwrap_or_else(|| vec![]),
        size_constraints: size_limits,
        time_constraints,
        #[cfg(unix)]
        owner_constraint,
        show_filesystem_errors: matches.is_present("show-errors"),
        path_separator,
        max_results: matches
            .value_of("max-results")
            .and_then(|n| usize::from_str_radix(n, 10).ok())
            .filter(|&n| n != 0)
            .or_else(|| {
                if matches.is_present("max-one-result") {
                    Some(1)
                } else {
                    None
                }
            }),
    };

    let re = RegexBuilder::new(&pattern_regex)
        .case_insensitive(!config.case_sensitive)
        .dot_matches_new_line(true)
        .build()
        .map_err(|e| {
            anyhow!(
                "{}\n\nNote: You can use the '--fixed-strings' option to search for a \
                 literal string instead of a regular expression. Alternatively, you can \
                 also use the '--glob' option to match on a glob pattern.",
                e.to_string()
            )
        })?;

    walk::scan(&dir_vec, Arc::new(re), Arc::new(config))
}

fn main() {
    let result = run();
    match result {
        Ok(exit_code) => {
            process::exit(exit_code.into());
        }
        Err(err) => {
            eprintln!("[fd error]: {}", err);
            process::exit(ExitCode::GeneralError.into());
        }
    }
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

use failure::{format_err, Error};
use slog_scope::info;
use std::path::PathBuf;
use structopt::StructOpt;

use common::config::load_es_config_for;
use mimir2::{
    adapters::secondary::elasticsearch::{self, ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ},
    domain::ports::secondary::remote::Remote,
};

#[derive(Debug, StructOpt)]
struct Args {
    /// NTFS directory.
    #[structopt(short = "i", long = "input", parse(from_os_str), default_value = ".")]
    input: PathBuf,
    /// Name of the dataset.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,
    /// Elasticsearch parameters.
    #[structopt(
        short = "c",
        long = "connection-string",
        default_value = "http://localhost:9200/munin"
    )]
    connection_string: String,
    #[structopt(parse(from_os_str), long)]
    mappings: Option<PathBuf>,
    #[structopt(parse(from_os_str), long)]
    settings: Option<PathBuf>,
    /// Override value of settings using `key.subkey=value` syntax
    override_settings: Vec<String>,
}

#[tokio::main]
async fn main() {
    mimirsbrunn::utils::launch_async(index_ntfs).await;
}

/// Uses the commandline arguments to index an ntfs directory into Elasticsearch.
async fn index_ntfs(args: Args) -> Result<(), Error> {
    info!("Launching ntfs2mimir...");

    let pool = elasticsearch::remote::connection_pool_url(&args.connection_string)
        .await
        .map_err(|err| {
            format_err!(
                "could not create elasticsearch connection pool: {}",
                err.to_string()
            )
        })?;

    let client = pool
        .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
        .await
        .map_err(|err| format_err!("could not connect elasticsearch pool: {}", err.to_string()))?;

    let config = load_es_config_for::<places::stop::Stop>(
        args.mappings,
        args.settings,
        args.override_settings,
        args.dataset,
    )
    .map_err(|err| format_err!("could not load configuration: {}", err))?;

    mimirsbrunn::stops::index_ntfs(args.input, config, &client)
        .await
        .map_err(|err| format_err!("could not index ntfs: {}", err.to_string()))
}

#[cfg(test)]
mod tests {
    use futures::stream::TryStreamExt;

    use super::*;
    // use common::document::ContainerDocument;
    // use mimir2::domain::model::query::Query;
    use mimir2::domain::ports::primary::list_documents::ListDocuments;
    // use mimir2::domain::ports::primary::search_documents::SearchDocuments;
    use mimir2::{adapters::secondary::elasticsearch::remote, utils::docker};
    use places::stop::Stop;
    // use places::Place;

    fn elasticsearch_test_url() -> String {
        std::env::var(elasticsearch::remote::ES_TEST_KEY).expect("env var")
    }

    #[tokio::test]
    async fn should_return_an_error_when_given_an_invalid_es_url() {
        let url = String::from("http://example.com:demo");
        let args = Args {
            input: PathBuf::from("NA"),
            connection_string: url,
            dataset: String::from("test"),
            mappings: Some("./config/elasticsearch/stop/mappings.json".into()),
            settings: Some("./config/elasticsearch/stop/settings.json".into()),
            override_settings: vec![],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;
        assert!(res
            .unwrap_err()
            .to_string()
            .contains("could not create elasticsearch connection pool: Invalid Elasticsearch URL"));
    }

    #[tokio::test]
    async fn should_return_an_error_when_given_an_url_not_es() {
        let url = String::from("http://no-es.test");
        let args = Args {
            input: PathBuf::from("NA"),
            connection_string: url,
            dataset: String::from("test"),
            mappings: None,
            settings: None,
            override_settings: vec![],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;
        assert!(res
            .unwrap_err()
            .to_string()
            .contains("could not connect elasticsearch pool: Connection Error: Elasticsearch Connection Error"));
    }

    #[tokio::test]
    async fn should_return_an_error_when_given_an_invalid_path_for_config() {
        let guard = docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("NA"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: None,
            settings: None,
            override_settings: vec![],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        drop(guard);

        assert!(res
            .unwrap_err()
            .to_string()
            .contains("is neither a file nor a directory, cannot read a ntfs from it"));
    }

    #[tokio::test]
    async fn should_return_an_error_when_given_an_invalid_mappings() {
        let guard = docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("foo"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: Some("./tests/fixtures/config/invalid/mappings.json".into()), // exists, but not json
            settings: None,
            override_settings: vec![],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        drop(guard);

        assert!(dbg!(res.unwrap_err().to_string()).contains("expected value at line 1 column 1"));
    }

    #[tokio::test]
    async fn should_return_an_error_when_given_an_invalid_path_for_input() {
        let guard = docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("NA"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: None,
            settings: None,
            override_settings: vec![],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        drop(guard);

        assert!(res
            .unwrap_err()
            .to_string()
            .contains("is neither a file nor a directory, cannot read a ntfs from it"));
    }

    #[tokio::test]
    async fn should_return_an_error_when_given_an_invalid_setting_override() {
        let guard = docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("NA"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: None,
            settings: None,
            override_settings: vec!["no-value".to_string()],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        drop(guard);

        assert!(res
            .unwrap_err()
            .to_string()
            .contains("couldn't override settings"));
    }

    #[tokio::test]
    async fn should_correctly_index_a_small_ntfs_() {
        let guard = docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("./tests/fixtures/ntfs"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: None,
            settings: None,
            override_settings: vec![],
        };

        let _res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        // Now we query the index we just created. Since it's a small NTFS dataset with few entries,
        // we'll just list all the documents in the index, and check them.
        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");

        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");

        let stops: Vec<Stop> = client
            .list_documents()
            .await
            .unwrap()
            .try_collect()
            .await
            .unwrap();

        drop(guard);

        assert!(stops.iter().all(|stop| stop.id.starts_with("stop_area")));
        assert!(stops.iter().all(|stop| stop.weight != 0f64));
    }

    #[tokio::test]
    async fn should_index_ntfs_with_correct_values() {
        let guard = docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("./tests/fixtures/ntfs"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: PathBuf::from("./config/elasticsearch/stop/mappings.json").into(),
            settings: PathBuf::from("./config/elasticsearch/stop/settings.json").into(),
            override_settings: vec![],
        };

        let _res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");

        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");

        let stops: Vec<Stop> = client
            .list_documents()
            .await
            .unwrap()
            .try_collect()
            .await
            .unwrap();

        drop(guard);

        assert_eq!(stops.len(), 6);

        for stop in &stops {
            println!("stop: {}", stop.name);
        }
        let stop1 = stops.iter().find(|&stop| stop.name == "Châtelet").unwrap();

        assert_eq!(stop1.id, "stop_area:CHA");
        assert_eq!(stop1.lines.len(), 1);
    }
}
<commit_msg>Use serial_test for ntfs2mimir<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

use failure::{format_err, Error};
use slog_scope::info;
use std::path::PathBuf;
use structopt::StructOpt;

use common::config::load_es_config_for;
use mimir2::{
    adapters::secondary::elasticsearch::{self, ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ},
    domain::ports::secondary::remote::Remote,
};

#[derive(Debug, StructOpt)]
struct Args {
    /// NTFS directory.
    #[structopt(short = "i", long = "input", parse(from_os_str), default_value = ".")]
    input: PathBuf,
    /// Name of the dataset.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,
    /// Elasticsearch parameters.
    #[structopt(
        short = "c",
        long = "connection-string",
        default_value = "http://localhost:9200/munin"
    )]
    connection_string: String,
    #[structopt(parse(from_os_str), long)]
    mappings: Option<PathBuf>,
    #[structopt(parse(from_os_str), long)]
    settings: Option<PathBuf>,
    /// Override value of settings using `key.subkey=value` syntax
    override_settings: Vec<String>,
}

#[tokio::main]
async fn main() {
    mimirsbrunn::utils::launch_async(index_ntfs).await;
}

/// Uses the commandline arguments to index an ntfs directory into Elasticsearch.
async fn index_ntfs(args: Args) -> Result<(), Error> {
    info!("Launching ntfs2mimir...");

    let pool = elasticsearch::remote::connection_pool_url(&args.connection_string)
        .await
        .map_err(|err| {
            format_err!(
                "could not create elasticsearch connection pool: {}",
                err.to_string()
            )
        })?;

    let client = pool
        .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
        .await
        .map_err(|err| format_err!("could not connect elasticsearch pool: {}", err.to_string()))?;

    let config = load_es_config_for::<places::stop::Stop>(
        args.mappings,
        args.settings,
        args.override_settings,
        args.dataset,
    )
    .map_err(|err| format_err!("could not load configuration: {}", err))?;

    mimirsbrunn::stops::index_ntfs(args.input, config, &client)
        .await
        .map_err(|err| format_err!("could not index ntfs: {}", err.to_string()))
}

#[cfg(test)]
mod tests {
    use futures::stream::TryStreamExt;
    use serial_test::serial;

    use super::*;
    use mimir2::domain::ports::primary::list_documents::ListDocuments;
    use mimir2::{adapters::secondary::elasticsearch::remote, utils::docker};
    use places::stop::Stop;

    fn elasticsearch_test_url() -> String {
        std::env::var(elasticsearch::remote::ES_TEST_KEY).expect("env var")
    }

    #[tokio::test]
    async fn should_return_an_error_when_given_an_invalid_es_url() {
        let url = String::from("http://example.com:demo");
        let args = Args {
            input: PathBuf::from("NA"),
            connection_string: url,
            dataset: String::from("test"),
            mappings: Some("./config/elasticsearch/stop/mappings.json".into()),
            settings: Some("./config/elasticsearch/stop/settings.json".into()),
            override_settings: vec![],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;
        assert!(res
            .unwrap_err()
            .to_string()
            .contains("could not create elasticsearch connection pool: Invalid Elasticsearch URL"));
    }

    #[tokio::test]
    async fn should_return_an_error_when_given_an_url_not_es() {
        let url = String::from("http://no-es.test");
        let args = Args {
            input: PathBuf::from("NA"),
            connection_string: url,
            dataset: String::from("test"),
            mappings: None,
            settings: None,
            override_settings: vec![],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;
        assert!(res
            .unwrap_err()
            .to_string()
            .contains("could not connect elasticsearch pool: Connection Error: Elasticsearch Connection Error"));
    }

    #[tokio::test]
    #[serial]
    async fn should_return_an_error_when_given_an_invalid_path_for_config() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("NA"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: None,
            settings: None,
            override_settings: vec![],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        assert!(res
            .unwrap_err()
            .to_string()
            .contains("is neither a file nor a directory, cannot read a ntfs from it"));
    }

    #[tokio::test]
    #[serial]
    async fn should_return_an_error_when_given_an_invalid_mappings() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("foo"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: Some("./tests/fixtures/config/invalid/mappings.json".into()), // exists, but not json
            settings: None,
            override_settings: vec![],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        assert!(dbg!(res.unwrap_err().to_string()).contains("expected value at line 1 column 1"));
    }

    #[tokio::test]
    #[serial]
    async fn should_return_an_error_when_given_an_invalid_path_for_input() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("NA"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: None,
            settings: None,
            override_settings: vec![],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        assert!(res
            .unwrap_err()
            .to_string()
            .contains("is neither a file nor a directory, cannot read a ntfs from it"));
    }

    #[tokio::test]
    #[serial]
    async fn should_return_an_error_when_given_an_invalid_setting_override() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("NA"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: None,
            settings: None,
            override_settings: vec!["no-value".to_string()],
        };

        let res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        assert!(res
            .unwrap_err()
            .to_string()
            .contains("couldn't override settings"));
    }

    #[tokio::test]
    #[serial]
    async fn should_correctly_index_a_small_ntfs_() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("./tests/fixtures/ntfs"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: None,
            settings: None,
            override_settings: vec![],
        };

        let _res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        // Now we query the index we just created. Since it's a small NTFS dataset with few entries,
        // we'll just list all the documents in the index, and check them.
        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");

        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");

        let stops: Vec<Stop> = client
            .list_documents()
            .await
            .unwrap()
            .try_collect()
            .await
            .unwrap();

        assert!(stops.iter().all(|stop| stop.id.starts_with("stop_area")));
        assert!(stops.iter().all(|stop| stop.weight != 0f64));
    }

    #[tokio::test]
    #[serial]
    async fn should_index_ntfs_with_correct_values() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let args = Args {
            input: PathBuf::from("./tests/fixtures/ntfs"),
            connection_string: elasticsearch_test_url(),
            dataset: String::from("test"),
            mappings: PathBuf::from("./config/elasticsearch/stop/mappings.json").into(),
            settings: PathBuf::from("./config/elasticsearch/stop/settings.json").into(),
            override_settings: vec![],
        };

        let _res = mimirsbrunn::utils::launch_async_args(index_ntfs, args).await;

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");

        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");

        let stops: Vec<Stop> = client
            .list_documents()
            .await
            .unwrap()
            .try_collect()
            .await
            .unwrap();

        assert_eq!(stops.len(), 6);

        let stop1 = stops.iter().find(|&stop| stop.name == "Châtelet").unwrap();

        assert_eq!(stop1.id, "stop_area:CHA");
        assert_eq!(stop1.lines.len(), 1);
    }
}
<|endoftext|>