<commit_before>/// Contains all functionality pertaining to parsing, tokenizing, and generating input arguments.
pub mod errors;
mod jobs;
mod man;
mod quote;

use std::env;
use std::ffi::OsStr;
use std::fs;
use std::io::{self, BufRead, BufReader};
use std::path::Path;
use std::process::exit;
use arrayvec::ArrayVec;
use permutate::Permutator;
use num_cpus;

use super::disk_buffer::{self, DiskBufferTrait};
use super::input_iterator::InputIterator;
use super::tokenizer::Token;
use self::errors::ParseErr;

// Re-export key items from internal modules.
pub use self::errors::{FileErr, InputIteratorErr};

#[derive(PartialEq)]
enum Mode { Arguments, Command, Inputs, Files }

pub const INPUTS_ARE_COMMANDS: u8 = 1;
pub const PIPE_IS_ENABLED:     u8 = 2;
pub const SHELL_ENABLED:       u8 = 4;
pub const QUIET_MODE:          u8 = 8;
pub const VERBOSE_MODE:        u8 = 16;
pub const DASH_EXISTS:         u8 = 32;

/// Defines what quoting mode to use when expanding the command.
enum Quoting { None, Basic, Shell }

/// `Args` is a collection of critical options and arguments that were collected at
/// startup of the application.
pub struct Args<'a> {
    pub flags:        u8,
    pub ncores:       usize,
    pub arguments:    ArrayVec<[Token<'a>; 128]>,
    pub ninputs:      usize,
}

impl<'a> Args<'a> {
    pub fn new() -> Args<'a> {
        Args {
            ncores:       num_cpus::get(),
            flags:        SHELL_ENABLED,
            arguments:    ArrayVec::new(),
            ninputs:      0,
        }
    }

    pub fn parse(&mut self, comm: &mut String, unprocessed_path: &Path) -> Result<InputIterator, ParseErr> {
        let mut quote = Quoting::None;

        // Create a write buffer that automatically writes data to the disk when the buffer is full.
        let mut disk_buffer = disk_buffer::DiskBuffer::new(unprocessed_path).write()
            .map_err(|why| ParseErr::File(FileErr::Open(unprocessed_path.to_owned(), why)))?;

        // Temporary stores for input arguments.
        let mut raw_args                    = env::args().skip(1).peekable();
        let mut lists: Vec<Vec<String>>     = Vec::new();
        let mut current_inputs: Vec<String> = Vec::with_capacity(1024);
        let mut number_of_arguments = 0;

        if env::args().len() > 1 {
            // The purpose of this is to set the initial parsing mode.
            let mut mode = match raw_args.peek().unwrap().as_ref() {
                ":::"  => Mode::Inputs,
                "::::" => Mode::Files,
                _      => Mode::Arguments
            };

            // If there are no arguments to be parsed, then the inputs are commands.
            if mode == Mode::Inputs || mode == Mode::Files {
                self.flags |= INPUTS_ARE_COMMANDS;
            } else {
                self.flags &= 255 ^ INPUTS_ARE_COMMANDS;
            }

            // Parse each and every input argument supplied to the program.
            while let Some(argument) = raw_args.next() {
                let argument = argument.as_str();
                match mode {
                    Mode::Arguments => {
                        let mut char_iter = argument.chars().peekable();

                        // If the first character is a '-' then it will be processed as an argument.
                        // We can guarantee that there will always be at least one character.
                        if char_iter.next().unwrap() == '-' {
                            // If the second character exists, everything's OK.
                            if let Some(character) = char_iter.next() {
                                // This scope of code allows users to utilize the GNU style
                                // command line arguments, to allow for laziness.
                                if character == 'j' {
                                    // The short-hand job argument needs to be handled specially.
                                    self.ncores = if char_iter.peek().is_some() {
                                        // Each character that follows after `j` will be considered an
                                        // input value.
                                        jobs::parse(&argument[2..])?
                                    } else {
                                        // If there was no character after `j`, the following argument
                                        // must be the job value.
                                        let val = &raw_args.next().ok_or(ParseErr::JobsNoValue)?;
                                        jobs::parse(val)?
                                    }
                                } else if character != '-' {
                                    // NOTE: Short mode versions of arguments
                                    for character in argument[1..].chars() {
                                        match character {
                                            'h' => {
                                                println!("{}", man::MAN_PAGE);
                                                exit(0);
                                            },
                                            'n' => self.flags &= 255 ^ SHELL_ENABLED,
                                            'p' => self.flags |= PIPE_IS_ENABLED,
                                            'q' => quote = Quoting::Basic,
                                            's' => self.flags |= QUIET_MODE,
                                            'v' => self.flags |= VERBOSE_MODE,
                                            _ => {
                                                return Err(ParseErr::InvalidArgument(argument.to_owned()))
                                            }
                                        }
                                    }
                                } else {
                                    // NOTE: Long mode versions of arguments
                                    match &argument[2..] {
                                        "help" => {
                                            println!("{}", man::MAN_PAGE);
                                            exit(0);
                                        },
                                        "jobs" => {
                                            let val = &raw_args.next().ok_or(ParseErr::JobsNoValue)?;
                                            self.ncores = jobs::parse(val)?
                                        },
                                        "no-shell" => self.flags &= 255 ^ SHELL_ENABLED,
                                        "num-cpu-cores" => {
                                            println!("{}", num_cpus::get());
                                            exit(0);
                                        },
                                        "pipe" => self.flags |= PIPE_IS_ENABLED,
                                        "quiet" | "silent" => self.flags |= QUIET_MODE,
                                        "quote" => quote = Quoting::Basic,
                                        "shellquote" => quote = Quoting::Shell,
                                        "verbose" => self.flags |= VERBOSE_MODE,
                                        "version" => {
                                            println!("parallel 0.6.2\n\nCrate Dependencies:");
                                            println!("    libc      0.2.15");
                                            println!("    num_cpus  1.0.0");
                                            println!("    permutate 0.1.3");
                                            exit(0);
                                        }
                                        _ => {
                                            return Err(ParseErr::InvalidArgument(argument.to_owned()));
                                        }
                                    }
                                }
                            } else {
                                // `-` will never be a valid argument
                                return Err(ParseErr::InvalidArgument("-".to_owned()));
                            }
                        } else {
                            match argument {
                                ":::" => {
                                    mode = Mode::Inputs;
                                    self.flags |= INPUTS_ARE_COMMANDS;
                                },
                                "::::" => {
                                    mode = Mode::Files;
                                    self.flags |= INPUTS_ARE_COMMANDS;
                                }
                                _ => {
                                    // The command has been supplied, and argument parsing is over.
                                    comm.push_str(argument);
                                    mode = Mode::Command;
                                }
                            }
                        }
                    },
                    Mode::Command => match argument {
                        // Arguments after `:::` are input values.
                        ":::" | ":::+" => mode = Mode::Inputs,
                        // Arguments after `::::` are files with inputs.
                        "::::" | "::::+" => mode = Mode::Files,
                        // All other arguments are command arguments.
                        _ => {
                            comm.push(' ');
                            comm.push_str(argument);
                        }
                    },
                    _ => match argument {
                        // `:::` denotes that the next set of inputs will be added to a new list.
                        ":::"  => {
                            mode = Mode::Inputs;
                            if !current_inputs.is_empty() {
                                lists.push(current_inputs.clone());
                                current_inputs.clear();
                            }
                        },
                        // `:::+` denotes that the next set of inputs will be added to the current list.
                        ":::+" => mode = Mode::Inputs,
                        // `::::` denotes that the next set of inputs will be added to a new list.
                        "::::"  => {
                            mode = Mode::Files;
                            if !current_inputs.is_empty() {
                                lists.push(current_inputs.clone());
                                current_inputs.clear();
                            }
                        },
                        // `:::+` denotes that the next set of inputs will be added to the current list.
                        "::::+" => mode = Mode::Files,
                        // All other arguments will be added to the current list.
                        _ => match mode {
                            Mode::Inputs => current_inputs.push(argument.to_owned()),
                            Mode::Files => file_parse(&mut current_inputs, argument)?,
                            _ => unreachable!()
                        }
                    }
                }
            }

            if !current_inputs.is_empty() {
                lists.push(current_inputs.clone());
            }

            if lists.len() > 1 {
                // Convert the Vec<Vec<String>> into a Vec<Vec<&str>>
                let tmp: Vec<Vec<&str>> = lists.iter()
                    .map(|list| list.iter().map(AsRef::as_ref).collect::<Vec<&str>>())
                    .collect();

                // Convert the Vec<Vec<&str>> into a Vec<&[&str]>
                let list_array: Vec<&[&str]> = tmp.iter().map(AsRef::as_ref).collect();

                // Create a `Permutator` with the &[&[&str]] as the input.
                let permutator = Permutator::new(&list_array[..]);

                for permutation in permutator {
                    let mut iter = permutation.iter();
                    disk_buffer.write(iter.next().unwrap().as_bytes())
                        .map_err(|why| ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    for element in iter {
                        disk_buffer.write_byte(b' ')
                            .map_err(|why| ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                        disk_buffer.write(element.as_bytes())
                            .map_err(|why| ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    }
                    disk_buffer.write_byte(b'\n')
                        .map_err(|why| ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    number_of_arguments += 1;
                }
            } else {
                for input in current_inputs {
                    disk_buffer.write(input.as_bytes()).map_err(|why|
                        ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    disk_buffer.write_byte(b'\n').map_err(|why|
                        ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    number_of_arguments += 1;
                }
            }
        }

        // If no inputs are provided, read from stdin instead.
        if disk_buffer.is_empty() {
            let stdin = io::stdin();
            for line in stdin.lock().lines() {
                if let Ok(line) = line {
                    disk_buffer.write(line.as_bytes()).map_err(|why|
                        ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    disk_buffer.write_byte(b'\n').map_err(|why|
                        ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    number_of_arguments += 1;
                }
            }
        }

        if number_of_arguments == 0 { return Err(ParseErr::NoArguments); }

        // Flush the contents of the buffer to the disk before tokenizing the command argument.
        disk_buffer.flush().map_err(|why| ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;

        // Expand the command if quoting is enabled
        match quote {
            Quoting::None  => (),
            Quoting::Basic => quote::basic(comm),
            Quoting::Shell => quote::shell(comm),
        }

        if dash_exists() {
            self.flags |= DASH_EXISTS;
        }

        if !shell_required(&self.arguments) {
            self.flags &= 255 ^ SHELL_ENABLED;
            if self.flags & SHELL_ENABLED == 1 {
                println!("Shell enabled");
            } else {
                println!("Shell disabled");
            }
        }

        // Return an `InputIterator` of the arguments contained within the unprocessed file.
        let inputs = InputIterator::new(unprocessed_path, number_of_arguments).map_err(ParseErr::File)?;
        Ok(inputs)
    }
}

fn shell_required(arguments: &[Token]) -> bool {
    for token in arguments {
        if let &Token::Argument(ref arg) = token {
            if arg.contains(';') || arg.contains('&') {
                return true
            }
        }
    }
    false
}

fn dash_exists() -> bool {
    if let Ok(path) = env::var("PATH") {
        for path in path.split(':') {
            if let Ok(directory) = fs::read_dir(path) {
                for entry in directory {
                    if let Ok(entry) = entry {
                        let path = entry.path();
                        if path.is_file() && path.file_name() == Some(OsStr::new("dash")) { return true; }
                    }
                }
            }
        }
    }
    false
}

/// Attempts to open an input argument and adds each line to the `inputs` list.
fn file_parse<P: AsRef<Path>>(inputs: &mut Vec<String>, path: P) -> Result<(), ParseErr> {
    let path = path.as_ref();
    let file = fs::File::open(path).map_err(|err| ParseErr::File(FileErr::Open(path.to_owned(), err)))?;
    for line in BufReader::new(file).lines() {
        if let Ok(line) = line { inputs.push(line); }
    }
    Ok(())
}
<commit_msg>Enable shell when '|' is found This is a real slap in the face that I didn't think about it earlier.<commit_after>/// Contains all functionality pertaining to parsing, tokenizing, and generating input arguments.
pub mod errors;
mod jobs;
mod man;
mod quote;

use std::env;
use std::ffi::OsStr;
use std::fs;
use std::io::{self, BufRead, BufReader};
use std::path::Path;
use std::process::exit;
use arrayvec::ArrayVec;
use permutate::Permutator;
use num_cpus;

use super::disk_buffer::{self, DiskBufferTrait};
use super::input_iterator::InputIterator;
use super::tokenizer::Token;
use self::errors::ParseErr;

// Re-export key items from internal modules.
pub use self::errors::{FileErr, InputIteratorErr};

#[derive(PartialEq)]
enum Mode { Arguments, Command, Inputs, Files }

pub const INPUTS_ARE_COMMANDS: u8 = 1;
pub const PIPE_IS_ENABLED:     u8 = 2;
pub const SHELL_ENABLED:       u8 = 4;
pub const QUIET_MODE:          u8 = 8;
pub const VERBOSE_MODE:        u8 = 16;
pub const DASH_EXISTS:         u8 = 32;

/// Defines what quoting mode to use when expanding the command.
enum Quoting { None, Basic, Shell }

/// `Args` is a collection of critical options and arguments that were collected at
/// startup of the application.
pub struct Args<'a> {
    pub flags:        u8,
    pub ncores:       usize,
    pub arguments:    ArrayVec<[Token<'a>; 128]>,
    pub ninputs:      usize,
}

impl<'a> Args<'a> {
    pub fn new() -> Args<'a> {
        Args {
            ncores:       num_cpus::get(),
            flags:        SHELL_ENABLED,
            arguments:    ArrayVec::new(),
            ninputs:      0,
        }
    }

    pub fn parse(&mut self, comm: &mut String, unprocessed_path: &Path) -> Result<InputIterator, ParseErr> {
        let mut quote = Quoting::None;

        // Create a write buffer that automatically writes data to the disk when the buffer is full.
        let mut disk_buffer = disk_buffer::DiskBuffer::new(unprocessed_path).write()
            .map_err(|why| ParseErr::File(FileErr::Open(unprocessed_path.to_owned(), why)))?;

        // Temporary stores for input arguments.
        let mut raw_args                    = env::args().skip(1).peekable();
        let mut lists: Vec<Vec<String>>     = Vec::new();
        let mut current_inputs: Vec<String> = Vec::with_capacity(1024);
        let mut number_of_arguments = 0;

        if env::args().len() > 1 {
            // The purpose of this is to set the initial parsing mode.
            let mut mode = match raw_args.peek().unwrap().as_ref() {
                ":::"  => Mode::Inputs,
                "::::" => Mode::Files,
                _      => Mode::Arguments
            };

            // If there are no arguments to be parsed, then the inputs are commands.
            if mode == Mode::Inputs || mode == Mode::Files {
                self.flags |= INPUTS_ARE_COMMANDS;
            } else {
                self.flags &= 255 ^ INPUTS_ARE_COMMANDS;
            }

            // Parse each and every input argument supplied to the program.
            while let Some(argument) = raw_args.next() {
                let argument = argument.as_str();
                match mode {
                    Mode::Arguments => {
                        let mut char_iter = argument.chars().peekable();

                        // If the first character is a '-' then it will be processed as an argument.
                        // We can guarantee that there will always be at least one character.
                        if char_iter.next().unwrap() == '-' {
                            // If the second character exists, everything's OK.
                            if let Some(character) = char_iter.next() {
                                // This scope of code allows users to utilize the GNU style
                                // command line arguments, to allow for laziness.
                                if character == 'j' {
                                    // The short-hand job argument needs to be handled specially.
                                    self.ncores = if char_iter.peek().is_some() {
                                        // Each character that follows after `j` will be considered an
                                        // input value.
                                        jobs::parse(&argument[2..])?
                                    } else {
                                        // If there was no character after `j`, the following argument
                                        // must be the job value.
                                        let val = &raw_args.next().ok_or(ParseErr::JobsNoValue)?;
                                        jobs::parse(val)?
                                    }
                                } else if character != '-' {
                                    // NOTE: Short mode versions of arguments
                                    for character in argument[1..].chars() {
                                        match character {
                                            'h' => {
                                                println!("{}", man::MAN_PAGE);
                                                exit(0);
                                            },
                                            'n' => self.flags &= 255 ^ SHELL_ENABLED,
                                            'p' => self.flags |= PIPE_IS_ENABLED,
                                            'q' => quote = Quoting::Basic,
                                            's' => self.flags |= QUIET_MODE,
                                            'v' => self.flags |= VERBOSE_MODE,
                                            _ => {
                                                return Err(ParseErr::InvalidArgument(argument.to_owned()))
                                            }
                                        }
                                    }
                                } else {
                                    // NOTE: Long mode versions of arguments
                                    match &argument[2..] {
                                        "help" => {
                                            println!("{}", man::MAN_PAGE);
                                            exit(0);
                                        },
                                        "jobs" => {
                                            let val = &raw_args.next().ok_or(ParseErr::JobsNoValue)?;
                                            self.ncores = jobs::parse(val)?
                                        },
                                        "no-shell" => self.flags &= 255 ^ SHELL_ENABLED,
                                        "num-cpu-cores" => {
                                            println!("{}", num_cpus::get());
                                            exit(0);
                                        },
                                        "pipe" => self.flags |= PIPE_IS_ENABLED,
                                        "quiet" | "silent" => self.flags |= QUIET_MODE,
                                        "quote" => quote = Quoting::Basic,
                                        "shellquote" => quote = Quoting::Shell,
                                        "verbose" => self.flags |= VERBOSE_MODE,
                                        "version" => {
                                            println!("parallel 0.6.2\n\nCrate Dependencies:");
                                            println!("    libc      0.2.15");
                                            println!("    num_cpus  1.0.0");
                                            println!("    permutate 0.1.3");
                                            exit(0);
                                        }
                                        _ => {
                                            return Err(ParseErr::InvalidArgument(argument.to_owned()));
                                        }
                                    }
                                }
                            } else {
                                // `-` will never be a valid argument
                                return Err(ParseErr::InvalidArgument("-".to_owned()));
                            }
                        } else {
                            match argument {
                                ":::" => {
                                    mode = Mode::Inputs;
                                    self.flags |= INPUTS_ARE_COMMANDS;
                                },
                                "::::" => {
                                    mode = Mode::Files;
                                    self.flags |= INPUTS_ARE_COMMANDS;
                                }
                                _ => {
                                    // The command has been supplied, and argument parsing is over.
                                    comm.push_str(argument);
                                    mode = Mode::Command;
                                }
                            }
                        }
                    },
                    Mode::Command => match argument {
                        // Arguments after `:::` are input values.
                        ":::" | ":::+" => mode = Mode::Inputs,
                        // Arguments after `::::` are files with inputs.
                        "::::" | "::::+" => mode = Mode::Files,
                        // All other arguments are command arguments.
                        _ => {
                            comm.push(' ');
                            comm.push_str(argument);
                        }
                    },
                    _ => match argument {
                        // `:::` denotes that the next set of inputs will be added to a new list.
                        ":::"  => {
                            mode = Mode::Inputs;
                            if !current_inputs.is_empty() {
                                lists.push(current_inputs.clone());
                                current_inputs.clear();
                            }
                        },
                        // `:::+` denotes that the next set of inputs will be added to the current list.
                        ":::+" => mode = Mode::Inputs,
                        // `::::` denotes that the next set of inputs will be added to a new list.
                        "::::"  => {
                            mode = Mode::Files;
                            if !current_inputs.is_empty() {
                                lists.push(current_inputs.clone());
                                current_inputs.clear();
                            }
                        },
                        // `:::+` denotes that the next set of inputs will be added to the current list.
                        "::::+" => mode = Mode::Files,
                        // All other arguments will be added to the current list.
                        _ => match mode {
                            Mode::Inputs => current_inputs.push(argument.to_owned()),
                            Mode::Files => file_parse(&mut current_inputs, argument)?,
                            _ => unreachable!()
                        }
                    }
                }
            }

            if !current_inputs.is_empty() {
                lists.push(current_inputs.clone());
            }

            if lists.len() > 1 {
                // Convert the Vec<Vec<String>> into a Vec<Vec<&str>>
                let tmp: Vec<Vec<&str>> = lists.iter()
                    .map(|list| list.iter().map(AsRef::as_ref).collect::<Vec<&str>>())
                    .collect();

                // Convert the Vec<Vec<&str>> into a Vec<&[&str]>
                let list_array: Vec<&[&str]> = tmp.iter().map(AsRef::as_ref).collect();

                // Create a `Permutator` with the &[&[&str]] as the input.
                let permutator = Permutator::new(&list_array[..]);

                for permutation in permutator {
                    let mut iter = permutation.iter();
                    disk_buffer.write(iter.next().unwrap().as_bytes())
                        .map_err(|why| ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    for element in iter {
                        disk_buffer.write_byte(b' ')
                            .map_err(|why| ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                        disk_buffer.write(element.as_bytes())
                            .map_err(|why| ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    }
                    disk_buffer.write_byte(b'\n')
                        .map_err(|why| ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    number_of_arguments += 1;
                }
            } else {
                for input in current_inputs {
                    disk_buffer.write(input.as_bytes()).map_err(|why|
                        ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    disk_buffer.write_byte(b'\n').map_err(|why|
                        ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    number_of_arguments += 1;
                }
            }
        }

        // If no inputs are provided, read from stdin instead.
        if disk_buffer.is_empty() {
            let stdin = io::stdin();
            for line in stdin.lock().lines() {
                if let Ok(line) = line {
                    disk_buffer.write(line.as_bytes()).map_err(|why|
                        ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    disk_buffer.write_byte(b'\n').map_err(|why|
                        ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;
                    number_of_arguments += 1;
                }
            }
        }

        if number_of_arguments == 0 { return Err(ParseErr::NoArguments); }

        // Flush the contents of the buffer to the disk before tokenizing the command argument.
        disk_buffer.flush().map_err(|why| ParseErr::File(FileErr::Write(disk_buffer.path.clone(), why)))?;

        // Expand the command if quoting is enabled
        match quote {
            Quoting::None  => (),
            Quoting::Basic => quote::basic(comm),
            Quoting::Shell => quote::shell(comm),
        }

        if dash_exists() {
            self.flags |= DASH_EXISTS;
        }

        if !shell_required(&self.arguments) {
            self.flags &= 255 ^ SHELL_ENABLED;
            if self.flags & SHELL_ENABLED == 1 {
                println!("Shell enabled");
            } else {
                println!("Shell disabled");
            }
        }

        // Return an `InputIterator` of the arguments contained within the unprocessed file.
        let inputs = InputIterator::new(unprocessed_path, number_of_arguments).map_err(ParseErr::File)?;
        Ok(inputs)
    }
}

fn shell_required(arguments: &[Token]) -> bool {
    for token in arguments {
        if let &Token::Argument(ref arg) = token {
            if arg.contains(';') || arg.contains('&') || arg.contains('|') {
                return true
            }
        }
    }
    false
}

fn dash_exists() -> bool {
    if let Ok(path) = env::var("PATH") {
        for path in path.split(':') {
            if let Ok(directory) = fs::read_dir(path) {
                for entry in directory {
                    if let Ok(entry) = entry {
                        let path = entry.path();
                        if path.is_file() && path.file_name() == Some(OsStr::new("dash")) { return true; }
                    }
                }
            }
        }
    }
    false
}

/// Attempts to open an input argument and adds each line to the `inputs` list.
fn file_parse<P: AsRef<Path>>(inputs: &mut Vec<String>, path: P) -> Result<(), ParseErr> {
    let path = path.as_ref();
    let file = fs::File::open(path).map_err(|err| ParseErr::File(FileErr::Open(path.to_owned(), err)))?;
    for line in BufReader::new(file).lines() {
        if let Ok(line) = line { inputs.push(line); }
    }
    Ok(())
}
<|endoftext|>


<commit_before>// Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// Inspired by Paul Woolcock's cargo-fmt (https://github.com/pwoolcoc/cargo-fmt/)

#![cfg(not(test))]
#![deny(warnings)]

extern crate getopts;
extern crate rustc_serialize;

use std::env;
use std::io::Write;
use std::path::PathBuf;
use std::process::{Command, ExitStatus};
use std::str;

use getopts::Options;
use rustc_serialize::json::Json;

fn main() {
    let exit_status = execute();
    std::io::stdout().flush().unwrap();
    std::process::exit(exit_status);
}

fn execute() -> i32 {
    let success = 0;
    let failure = 1;

    let mut opts = getopts::Options::new();
    opts.optflag("h", "help", "show this message");
    opts.optflag("q", "quiet", "no output printed to stdout");
    opts.optflag("v", "verbose", "use verbose output");

    let matches = match opts.parse(env::args().skip(1).take_while(|a| a != "--")) {
        Ok(m) => m,
        Err(e) => {
            print_usage(&opts, &e.to_string());
            return failure;
        }
    };

    let verbosity = match (matches.opt_present("v"), matches.opt_present("q")) {
        (false, false) => Verbosity::Normal,
        (false, true) => Verbosity::Quiet,
        (true, false) => Verbosity::Verbose,
        (true, true) => {
            print_usage(&opts, "quiet mode and verbose mode are not compatible");
            return failure;
        }
    };

    if matches.opt_present("h") {
        print_usage(&opts, "");
        return success;
    }

    match format_crate(verbosity) {
        Err(e) => {
            print_usage(&opts, &e.to_string());
            failure
        }
        Ok(status) => {
            if status.success() {
                success
            } else {
                status.code().unwrap_or(failure)
            }
        }
    }
}

fn print_usage(opts: &Options, reason: &str) {
    let msg = format!("{}\nusage: cargo fmt [options]", reason);
    println!("{}\nThis utility formats all bin and lib files of the current crate using rustfmt. \
              Arguments after `--` are passed to rustfmt.",
             opts.usage(&msg));
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Verbosity {
    Verbose,
    Normal,
    Quiet,
}

fn format_crate(verbosity: Verbosity) -> Result<ExitStatus, std::io::Error> {
    let targets = try!(get_targets());

    // Currently only bin and lib files get formatted
    let files: Vec<_> = targets.into_iter()
        .filter(|t| t.kind.is_lib() | t.kind.is_bin())
        .inspect(|t| {
            if verbosity == Verbosity::Verbose {
                println!("[{:?}] {:?}", t.kind, t.path)
            }
        })
        .map(|t| t.path)
        .collect();

    format_files(&files, &get_fmt_args(), verbosity)
}

fn get_fmt_args() -> Vec<String> {
    // All arguments after -- are passed to rustfmt
    env::args().skip_while(|a| a != "--").skip(1).collect()
}

#[derive(Debug)]
enum TargetKind {
    Lib, // dylib, staticlib, lib
    Bin, // bin
    Other, // test, plugin,...
}

impl TargetKind {
    fn is_lib(&self) -> bool {
        match *self {
            TargetKind::Lib => true,
            _ => false,
        }
    }

    fn is_bin(&self) -> bool {
        match *self {
            TargetKind::Bin => true,
            _ => false,
        }
    }
}

#[derive(Debug)]
pub struct Target {
    path: PathBuf,
    kind: TargetKind,
}

// Returns a vector of all compile targets of a crate
fn get_targets() -> Result<Vec<Target>, std::io::Error> {
    let mut targets: Vec<Target> = vec![];
    let output = try!(Command::new("cargo").arg("read-manifest").output());
    if output.status.success() {
        // None of the unwraps should fail if output of `cargo read-manifest` is correct
        let data = &String::from_utf8(output.stdout).unwrap();
        let json = Json::from_str(data).unwrap();
        let jtargets = json.find("targets").unwrap().as_array().unwrap();
        for jtarget in jtargets {
            targets.push(target_from_json(jtarget));
        }

        Ok(targets)
    } else {
        // This happens when cargo-fmt is not used inside a crate
        Err(std::io::Error::new(std::io::ErrorKind::NotFound,
                                str::from_utf8(&output.stderr).unwrap()))
    }
}

fn target_from_json(jtarget: &Json) -> Target {
    let jtarget = jtarget.as_object().unwrap();
    let path = PathBuf::from(jtarget.get("src_path").unwrap().as_string().unwrap());
    let kinds = jtarget.get("kind").unwrap().as_array().unwrap();
    let kind = match kinds[0].as_string().unwrap() {
        "bin" => TargetKind::Bin,
        "lib" | "dylib" | "staticlib" => TargetKind::Lib,
        _ => TargetKind::Other,
    };

    Target {
        path: path,
        kind: kind,
    }
}

fn format_files(files: &[PathBuf],
                fmt_args: &[String],
                verbosity: Verbosity)
                -> Result<ExitStatus, std::io::Error> {
    let stdout = if verbosity == Verbosity::Quiet {
        std::process::Stdio::null()
    } else {
        std::process::Stdio::inherit()
    };
    if verbosity == Verbosity::Verbose {
        print!("rustfmt");
        for a in fmt_args.iter() {
            print!(" {}", a);
        }
        for f in files.iter() {
            print!(" {}", f.display());
        }
        println!("");
    }
    let mut command = try!(Command::new("rustfmt")
        .stdout(stdout)
        .args(files)
        .args(fmt_args)
        .spawn()
        .map_err(|e| {
            match e.kind() {
                std::io::ErrorKind::NotFound => {
                    std::io::Error::new(std::io::ErrorKind::Other,
                                        "Could not run rustfmt, please make sure it is in your \
                                         PATH.")
                }
                _ => e,
            }
        }));
    command.wait()
}
<commit_msg>cargo-fmt processes bench, test, example files<commit_after>// Copyright 2015-2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// Inspired by Paul Woolcock's cargo-fmt (https://github.com/pwoolcoc/cargo-fmt/)

#![cfg(not(test))]
#![deny(warnings)]

extern crate getopts;
extern crate rustc_serialize;

use std::env;
use std::io::Write;
use std::path::PathBuf;
use std::process::{Command, ExitStatus};
use std::str;

use getopts::Options;
use rustc_serialize::json::Json;

fn main() {
    let exit_status = execute();
    std::io::stdout().flush().unwrap();
    std::process::exit(exit_status);
}

fn execute() -> i32 {
    let success = 0;
    let failure = 1;

    let mut opts = getopts::Options::new();
    opts.optflag("h", "help", "show this message");
    opts.optflag("q", "quiet", "no output printed to stdout");
    opts.optflag("v", "verbose", "use verbose output");

    let matches = match opts.parse(env::args().skip(1).take_while(|a| a != "--")) {
        Ok(m) => m,
        Err(e) => {
            print_usage(&opts, &e.to_string());
            return failure;
        }
    };

    let verbosity = match (matches.opt_present("v"), matches.opt_present("q")) {
        (false, false) => Verbosity::Normal,
        (false, true) => Verbosity::Quiet,
        (true, false) => Verbosity::Verbose,
        (true, true) => {
            print_usage(&opts, "quiet mode and verbose mode are not compatible");
            return failure;
        }
    };

    if matches.opt_present("h") {
        print_usage(&opts, "");
        return success;
    }

    match format_crate(verbosity) {
        Err(e) => {
            print_usage(&opts, &e.to_string());
            failure
        }
        Ok(status) => {
            if status.success() {
                success
            } else {
                status.code().unwrap_or(failure)
            }
        }
    }
}

fn print_usage(opts: &Options, reason: &str) {
    let msg = format!("{}\nusage: cargo fmt [options]", reason);
    println!("{}\nThis utility formats all bin and lib files of the current crate using rustfmt. \
              Arguments after `--` are passed to rustfmt.",
             opts.usage(&msg));
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Verbosity {
    Verbose,
    Normal,
    Quiet,
}

fn format_crate(verbosity: Verbosity) -> Result<ExitStatus, std::io::Error> {
    let targets = try!(get_targets());

    // Currently only bin and lib files get formatted
    let files: Vec<_> = targets.into_iter()
        .filter(|t| t.kind.should_format())
        .inspect(|t| {
            if verbosity == Verbosity::Verbose {
                println!("[{:?}] {:?}", t.kind, t.path)
            }
        })
        .map(|t| t.path)
        .collect();

    format_files(&files, &get_fmt_args(), verbosity)
}

fn get_fmt_args() -> Vec<String> {
    // All arguments after -- are passed to rustfmt
    env::args().skip_while(|a| a != "--").skip(1).collect()
}

#[derive(Debug)]
enum TargetKind {
    Lib, // dylib, staticlib, lib
    Bin, // bin
    Example, // example file
    Test, // test file
    Bench, // bench file
    Other, // plugin,...
}

impl TargetKind {
    fn should_format(&self) -> bool {
        match *self {
            TargetKind::Lib | TargetKind::Bin | TargetKind::Example | TargetKind::Test |
            TargetKind::Bench => true,
            _ => false,
        }
    }
}

#[derive(Debug)]
pub struct Target {
    path: PathBuf,
    kind: TargetKind,
}

// Returns a vector of all compile targets of a crate
fn get_targets() -> Result<Vec<Target>, std::io::Error> {
    let mut targets: Vec<Target> = vec![];
    let output = try!(Command::new("cargo").arg("read-manifest").output());
    if output.status.success() {
        // None of the unwraps should fail if output of `cargo read-manifest` is correct
        let data = &String::from_utf8(output.stdout).unwrap();
        let json = Json::from_str(data).unwrap();
        let jtargets = json.find("targets").unwrap().as_array().unwrap();
        for jtarget in jtargets {
            targets.push(target_from_json(jtarget));
        }

        Ok(targets)
    } else {
        // This happens when cargo-fmt is not used inside a crate
        Err(std::io::Error::new(std::io::ErrorKind::NotFound,
                                str::from_utf8(&output.stderr).unwrap()))
    }
}

fn target_from_json(jtarget: &Json) -> Target {
    let jtarget = jtarget.as_object().unwrap();
    let path = PathBuf::from(jtarget.get("src_path").unwrap().as_string().unwrap());
    let kinds = jtarget.get("kind").unwrap().as_array().unwrap();
    let kind = match kinds[0].as_string().unwrap() {
        "bin" => TargetKind::Bin,
        "lib" | "dylib" | "staticlib" => TargetKind::Lib,
        "test" => TargetKind::Test,
        "example" => TargetKind::Example,
        "bench" => TargetKind::Bench,
        _ => TargetKind::Other,
    };

    Target {
        path: path,
        kind: kind,
    }
}

fn format_files(files: &[PathBuf],
                fmt_args: &[String],
                verbosity: Verbosity)
                -> Result<ExitStatus, std::io::Error> {
    let stdout = if verbosity == Verbosity::Quiet {
        std::process::Stdio::null()
    } else {
        std::process::Stdio::inherit()
    };
    if verbosity == Verbosity::Verbose {
        print!("rustfmt");
        for a in fmt_args.iter() {
            print!(" {}", a);
        }
        for f in files.iter() {
            print!(" {}", f.display());
        }
        println!("");
    }
    let mut command = try!(Command::new("rustfmt")
        .stdout(stdout)
        .args(files)
        .args(fmt_args)
        .spawn()
        .map_err(|e| {
            match e.kind() {
                std::io::ErrorKind::NotFound => {
                    std::io::Error::new(std::io::ErrorKind::Other,
                                        "Could not run rustfmt, please make sure it is in your \
                                         PATH.")
                }
                _ => e,
            }
        }));
    command.wait()
}
<|endoftext|>


<commit_before>// This file is part of the uutils coreutils package.
//
// (c) Michael Gehring <mg@ebfe.org>
//
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.

use clap::App;
use clap::Shell;
use std::cmp;
use std::collections::hash_map::HashMap;
use std::ffi::OsString;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process;

const VERSION: &str = env!("CARGO_PKG_VERSION");

include!(concat!(env!("OUT_DIR"), "/uutils_map.rs"));

fn usage<T>(utils: &UtilityMap<T>, name: &str) {
    println!("{} {} (multi-call binary)\n", name, VERSION);
    println!("Usage: {} [function [arguments...]]\n", name);
    println!("Currently defined functions:\n");
    #[allow(clippy::map_clone)]
    let mut utils: Vec<&str> = utils.keys().map(|&s| s).collect();
    utils.sort_unstable();
    let display_list = utils.join(", ");
    let width = cmp::min(textwrap::termwidth(), 100) - 4 * 2; // (opinion/heuristic) max 100 chars wide with 4 character side indentions
    println!(
        "{}",
        textwrap::indent(&textwrap::fill(&display_list, width), "    ")
    );
}

fn binary_path(args: &mut impl Iterator<Item = OsString>) -> PathBuf {
    match args.next() {
        Some(ref s) if !s.is_empty() => PathBuf::from(s),
        _ => std::env::current_exe().unwrap(),
    }
}

fn name(binary_path: &Path) -> &str {
    binary_path.file_stem().unwrap().to_str().unwrap()
}

fn main() {
    uucore::panic::mute_sigpipe_panic();

    let utils = util_map();
    let mut args = uucore::args_os();

    let binary = binary_path(&mut args);
    let binary_as_util = name(&binary);

    // binary name equals util name?
    if let Some(&(uumain, _)) = utils.get(binary_as_util) {
        process::exit(uumain((vec![binary.into()].into_iter()).chain(args)));
    }

    // binary name equals prefixed util name?
    // * prefix/stem may be any string ending in a non-alphanumeric character
    let util_name = if let Some(util) = utils.keys().find(|util| {
        binary_as_util.ends_with(*util)
            && !(&binary_as_util[..binary_as_util.len() - (*util).len()])
                .ends_with(char::is_alphanumeric)
    }) {
        // prefixed util => replace 0th (aka, executable name) argument
        Some(OsString::from(*util))
    } else {
        // unmatched binary name => regard as multi-binary container and advance argument list
        args.next()
    };

    // 0th argument equals util name?
    if let Some(util_os) = util_name {
        let util = util_os.as_os_str().to_string_lossy();

        if util == "completion" {
            gen_completions(args, utils);
        }

        match utils.get(&util[..]) {
            Some(&(uumain, _)) => {
                process::exit(uumain((vec![util_os].into_iter()).chain(args)));
            }
            None => {
                if util == "--help" || util == "-h" {
                    // see if they want help on a specific util
                    if let Some(util_os) = args.next() {
                        let util = util_os.as_os_str().to_string_lossy();

                        match utils.get(&util[..]) {
                            Some(&(uumain, _)) => {
                                let code = uumain(
                                    (vec![util_os, OsString::from("--help")].into_iter())
                                        .chain(args),
                                );
                                io::stdout().flush().expect("could not flush stdout");
                                process::exit(code);
                            }
                            None => {
                                println!("{}: function/utility not found", util);
                                process::exit(1);
                            }
                        }
                    }
                    usage(&utils, binary_as_util);
                    process::exit(0);
                } else {
                    println!("{}: function/utility not found", util);
                    process::exit(1);
                }
            }
        }
    } else {
        // no arguments provided
        usage(&utils, binary_as_util);
        process::exit(0);
    }
}

/// Prints completions for the utility in the first parameter for the shell in the second parameter to stdout
fn gen_completions<T: uucore::Args>(
    mut args: impl Iterator<Item = OsString>,
    util_map: UtilityMap<T>,
) -> ! {
    let utility = args
        .next()
        .expect("expected utility as the first parameter")
        .to_str()
        .expect("utility name was not valid utf-8")
        .to_owned();
    let shell = args
        .next()
        .expect("expected shell as the second parameter")
        .to_str()
        .expect("shell name was not valid utf-8")
        .to_owned();
    let mut app = if utility == "coreutils" {
        gen_coreutils_app(util_map)
    } else if let Some((_, app)) = util_map.get(utility.as_str()) {
        app()
    } else {
        eprintln!("{} is not a valid utility", utility);
        process::exit(1)
    };
    let shell: Shell = shell.parse().unwrap();
    let bin_name = std::env::var("PROG_PREFIX").unwrap_or_default() + &utility;
    app.gen_completions_to(bin_name, shell, &mut io::stdout());
    io::stdout().flush().unwrap();
    process::exit(0);
}

fn gen_coreutils_app<T: uucore::Args>(util_map: UtilityMap<T>) -> App<'static, 'static> {
    let mut app = App::new("coreutils");
    for (_, (_, sub_app)) in util_map {
        app = app.subcommand(sub_app());
    }
    app
}
<commit_msg>coreutils: better errors for invalid args for completions<commit_after>// This file is part of the uutils coreutils package.
//
// (c) Michael Gehring <mg@ebfe.org>
//
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.

use clap::App;
use clap::Arg;
use clap::Shell;
use std::cmp;
use std::collections::hash_map::HashMap;
use std::ffi::OsString;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process;

const VERSION: &str = env!("CARGO_PKG_VERSION");

include!(concat!(env!("OUT_DIR"), "/uutils_map.rs"));

fn usage<T>(utils: &UtilityMap<T>, name: &str) {
    println!("{} {} (multi-call binary)\n", name, VERSION);
    println!("Usage: {} [function [arguments...]]\n", name);
    println!("Currently defined functions:\n");
    #[allow(clippy::map_clone)]
    let mut utils: Vec<&str> = utils.keys().map(|&s| s).collect();
    utils.sort_unstable();
    let display_list = utils.join(", ");
    let width = cmp::min(textwrap::termwidth(), 100) - 4 * 2; // (opinion/heuristic) max 100 chars wide with 4 character side indentions
    println!(
        "{}",
        textwrap::indent(&textwrap::fill(&display_list, width), "    ")
    );
}

fn binary_path(args: &mut impl Iterator<Item = OsString>) -> PathBuf {
    match args.next() {
        Some(ref s) if !s.is_empty() => PathBuf::from(s),
        _ => std::env::current_exe().unwrap(),
    }
}

fn name(binary_path: &Path) -> &str {
    binary_path.file_stem().unwrap().to_str().unwrap()
}

fn main() {
    uucore::panic::mute_sigpipe_panic();

    let utils = util_map();
    let mut args = uucore::args_os();

    let binary = binary_path(&mut args);
    let binary_as_util = name(&binary);

    // binary name equals util name?
    if let Some(&(uumain, _)) = utils.get(binary_as_util) {
        process::exit(uumain((vec![binary.into()].into_iter()).chain(args)));
    }

    // binary name equals prefixed util name?
    // * prefix/stem may be any string ending in a non-alphanumeric character
    let util_name = if let Some(util) = utils.keys().find(|util| {
        binary_as_util.ends_with(*util)
            && !(&binary_as_util[..binary_as_util.len() - (*util).len()])
                .ends_with(char::is_alphanumeric)
    }) {
        // prefixed util => replace 0th (aka, executable name) argument
        Some(OsString::from(*util))
    } else {
        // unmatched binary name => regard as multi-binary container and advance argument list
        args.next()
    };

    // 0th argument equals util name?
    if let Some(util_os) = util_name {
        let util = util_os.as_os_str().to_string_lossy();

        if util == "completion" {
            gen_completions(args, utils);
        }

        match utils.get(&util[..]) {
            Some(&(uumain, _)) => {
                process::exit(uumain((vec![util_os].into_iter()).chain(args)));
            }
            None => {
                if util == "--help" || util == "-h" {
                    // see if they want help on a specific util
                    if let Some(util_os) = args.next() {
                        let util = util_os.as_os_str().to_string_lossy();

                        match utils.get(&util[..]) {
                            Some(&(uumain, _)) => {
                                let code = uumain(
                                    (vec![util_os, OsString::from("--help")].into_iter())
                                        .chain(args),
                                );
                                io::stdout().flush().expect("could not flush stdout");
                                process::exit(code);
                            }
                            None => {
                                println!("{}: function/utility not found", util);
                                process::exit(1);
                            }
                        }
                    }
                    usage(&utils, binary_as_util);
                    process::exit(0);
                } else {
                    println!("{}: function/utility not found", util);
                    process::exit(1);
                }
            }
        }
    } else {
        // no arguments provided
        usage(&utils, binary_as_util);
        process::exit(0);
    }
}

/// Prints completions for the utility in the first parameter for the shell in the second parameter to stdout
fn gen_completions<T: uucore::Args>(
    args: impl Iterator<Item = OsString>,
    util_map: UtilityMap<T>,
) -> ! {
    let all_utilities: Vec<_> = std::iter::once("coreutils")
        .chain(util_map.keys().copied())
        .collect();

    let matches = App::new("completion")
        .about("Prints completions to stdout")
        .arg(
            Arg::with_name("utility")
                .possible_values(&all_utilities)
                .required(true),
        )
        .arg(
            Arg::with_name("shell")
                .possible_values(&Shell::variants())
                .required(true),
        )
        .get_matches_from(std::iter::once(OsString::from("completion")).chain(args));

    let utility = matches.value_of("utility").unwrap();
    let shell = matches.value_of("shell").unwrap();

    let mut app = if utility == "coreutils" {
        gen_coreutils_app(util_map)
    } else {
        util_map.get(utility).unwrap().1()
    };
    let shell: Shell = shell.parse().unwrap();
    let bin_name = std::env::var("PROG_PREFIX").unwrap_or_default() + utility;

    app.gen_completions_to(bin_name, shell, &mut io::stdout());
    io::stdout().flush().unwrap();
    process::exit(0);
}

fn gen_coreutils_app<T: uucore::Args>(util_map: UtilityMap<T>) -> App<'static, 'static> {
    let mut app = App::new("coreutils");
    for (_, (_, sub_app)) in util_map {
        app = app.subcommand(sub_app());
    }
    app
}
<|endoftext|>


<commit_before>// Copyright © 2019, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

use slog_scope::info;

use failure::format_err;
use lazy_static::lazy_static;
use mimir::objects::{Coord, I18nProperties, Poi, PoiType, Property};
use mimir::rubber::{IndexSettings, IndexVisibility, Rubber, TypedIndex};
use mimirsbrunn::{labels, utils};
use navitia_poi_model::{Model as NavitiaModel, Poi as NavitiaPoi, PoiType as NavitiaPoiType};
use std::collections::HashMap;
use std::ops::Deref;
use std::path::PathBuf;
use structopt::StructOpt;

lazy_static! {
    static ref DEFAULT_NB_THREADS: String = num_cpus::get().to_string();
}

// This function takes a Poi from the navitia model, ie from the CSV deserialization, and returns
// a Poi from the mimir model, with all the contextual information added.
fn into_mimir_poi(
    poi: NavitiaPoi,
    poi_types: &HashMap<String, NavitiaPoiType>,
    rubber: &mut Rubber,
) -> Result<Poi, mimirsbrunn::Error> {
    let poi_type = poi_types
        .get(&poi.poi_type_id)
        .ok_or_else(|| format_err!("could not find Poi Type '{}'", poi.poi_type_id))
        .map(|poi_type| PoiType::from(poi_type))?;

    let coord = Coord::from(&poi.coord);

    let place = rubber
        .get_address(&coord) // No timeout
        .ok()
        .and_then(|addrs| addrs.into_iter().next()); // Take the first place

    let addr = place.as_ref().and_then(|addr| addr.address());

    let admins = place.map_or_else(|| Vec::new(), |addr| addr.admins());

    if admins.is_empty() {
        return Err(format_err!("Could not find admins for POI {}", &poi.id));
    }

    // The weight is that of the city, or 0.0 if there is no such admin.
    let weight: f64 = admins
        .iter()
        .filter(|adm| adm.is_city())
        .map(|adm| adm.weight)
        .next()
        .unwrap_or(0.0);

    let country_codes = utils::find_country_codes(admins.iter().map(|a| a.deref()));

    let label =
        labels::format_poi_label(&poi.name, admins.iter().map(|a| a.deref()), &country_codes);

    let poi = Poi {
        id: poi.id,
        label: label,
        name: poi.name,
        coord: coord.clone(),
        approx_coord: Some(coord.into()),
        administrative_regions: admins,
        weight: weight,
        zip_codes: vec![],
        poi_type: poi_type,
        properties: poi
            .properties
            .into_iter()
            .map(|p| Property::from(p))
            .collect(),
        address: addr,
        country_codes: country_codes,
        names: I18nProperties::default(),
        labels: I18nProperties::default(),
        distance: None,
        context: None,
    };

    Ok(poi)
}

fn import_pois(
    rubber: &mut Rubber,
    index: &TypedIndex<Poi>,
    file: &PathBuf,
) -> Result<(), mimirsbrunn::Error>
where
{
    info!("Add data in elasticsearch db.");

    let model = NavitiaModel::try_from_path(file)?;
    let poi_types = model.poi_types;

    // Note: We're ignoring those POIs that fail to be enriched.
    let pois: Vec<_> = model
        .pois
        .into_iter()
        .filter_map(|(id, poi)| {
            into_mimir_poi(poi, &poi_types, rubber)
                .map_err(|err| info!("Could not extract information for POI '{}': {}", id, err))
                .ok()
        })
        .collect(); // TODO Can we get rid of collect, and chain with the following rubber...?

    let count = rubber
        .bulk_index(&index, pois.into_iter())
        .map_err(|err| format_err!("Failed bulk insertion {}", err))?;

    info!("importing POIs: {} POIs added.", count);

    Ok(())
}

/// This function initializes the ES context: It creates an index for this dataset,
/// and then import the POIs in it.
fn index_poi(
    cnx_string: &str,
    dataset: &str,
    file: &PathBuf,
    visibility: IndexVisibility,
    nb_shards: usize,
    nb_replicas: usize,
) -> Result<(), mimirsbrunn::Error>
where
{
    let mut rubber = Rubber::new(cnx_string);
    rubber.initialize_templates()?;

    let settings = IndexSettings {
        nb_shards: nb_shards,
        nb_replicas: nb_replicas,
    };

    let index = rubber.make_index(dataset, &settings)?;

    import_pois(&mut rubber, &index, file)?;

    rubber
        .publish_index(dataset, index, visibility)
        .map_err(|err| format_err!("Failed to publish index {}.", err))
}

#[derive(StructOpt, Debug)]
struct Args {
    /// POI file
    #[structopt(short = "i", long = "input", parse(from_os_str))]
    input: PathBuf,

    /// Elasticsearch parameters.
    #[structopt(
        short = "c",
        long = "connection-string",
        default_value = "http://localhost:9200/munin"
    )]
    connection_string: String,

    /// Name of the dataset.
    /// A dataset is a label, that can be used for filtering the data.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,

    /// Indicate if the POI dataset is private
    #[structopt(short = "p", long = "private")]
    private: bool,

    /// Number of threads to use
    #[structopt(
        short = "t",
        long = "nb-threads",
        default_value = &DEFAULT_NB_THREADS
    )]
    nb_threads: usize,

    /// Number of shards for the es index
    #[structopt(short = "s", long = "nb-shards", default_value = "1")]
    nb_shards: usize,

    /// Number of replicas for the es index
    #[structopt(short = "r", long = "nb-replicas", default_value = "1")]
    nb_replicas: usize,
}

fn run(args: Args) -> Result<(), mimirsbrunn::Error> {
    let visibility = if args.private {
        IndexVisibility::Private
    } else {
        IndexVisibility::Public
    };

    index_poi(
        &args.connection_string,
        &args.dataset,
        &args.input,
        visibility,
        args.nb_shards,
        args.nb_replicas,
    )
}
fn main() {
    mimirsbrunn::utils::launch_run(run);
}
<commit_msg>Use the AdminGeoFinder with poi2mimir<commit_after>// Copyright © 2019, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

use slog_scope::{error, info};

use failure::format_err;
use lazy_static::lazy_static;
use mimir::objects::{Coord, I18nProperties, Poi, PoiType, Property};
use mimir::rubber::{IndexSettings, IndexVisibility, Rubber, TypedIndex};
use mimirsbrunn::{admin_geofinder::AdminGeoFinder, labels, utils};
use navitia_poi_model::{Model as NavitiaModel, Poi as NavitiaPoi, PoiType as NavitiaPoiType};
use std::collections::HashMap;
use std::ops::Deref;
use std::path::PathBuf;
use structopt::StructOpt;

lazy_static! {
    static ref DEFAULT_NB_THREADS: String = num_cpus::get().to_string();
}

// This function takes a Poi from the navitia model, ie from the CSV deserialization, and returns
// a Poi from the mimir model, with all the contextual information added.
fn into_mimir_poi(
    poi: NavitiaPoi,
    poi_types: &HashMap<String, NavitiaPoiType>,
    rubber: &mut Rubber,
    admins_geofinder: &AdminGeoFinder,
) -> Result<Poi, mimirsbrunn::Error> {
    let poi_type = poi_types
        .get(&poi.poi_type_id)
        .ok_or_else(|| format_err!("could not find Poi Type '{}'", poi.poi_type_id))
        .map(|poi_type| PoiType::from(poi_type))?;

    let coord = Coord::from(&poi.coord);

    let place = rubber
        .get_address(&coord) // No timeout
        .ok()
        .and_then(|addrs| addrs.into_iter().next()); // Take the first place

    let addr = place.as_ref().and_then(|addr| addr.address());

    let admins = admins_geofinder.get(&poi.coord);

    if admins.is_empty() {
        return Err(format_err!("Could not find admins for POI {}", &poi.id));
    }

    // The weight is that of the city, or 0.0 if there is no such admin.
    let weight: f64 = admins
        .iter()
        .filter(|adm| adm.is_city())
        .map(|adm| adm.weight)
        .next()
        .unwrap_or(0.0);

    let country_codes = utils::find_country_codes(admins.iter().map(|a| a.deref()));

    let label =
        labels::format_poi_label(&poi.name, admins.iter().map(|a| a.deref()), &country_codes);

    let poi = Poi {
        id: poi.id,
        label: label,
        name: poi.name,
        coord: coord.clone(),
        approx_coord: Some(coord.into()),
        administrative_regions: admins,
        weight: weight,
        zip_codes: vec![],
        poi_type: poi_type,
        properties: poi
            .properties
            .into_iter()
            .map(|p| Property::from(p))
            .collect(),
        address: addr,
        country_codes: country_codes,
        names: I18nProperties::default(),
        labels: I18nProperties::default(),
        distance: None,
        context: None,
    };

    Ok(poi)
}

fn import_pois(
    rubber: &mut Rubber,
    index: &TypedIndex<Poi>,
    admins_geofinder: AdminGeoFinder,
    file: &PathBuf,
) -> Result<(), mimirsbrunn::Error>
where
{
    info!("Add data in elasticsearch db.");

    let model = NavitiaModel::try_from_path(file)?;
    let poi_types = model.poi_types;

    // Note: We're ignoring those POIs that fail to be enriched.
    let pois: Vec<_> = model
        .pois
        .into_iter()
        .filter_map(|(id, poi)| {
            into_mimir_poi(poi, &poi_types, rubber, &admins_geofinder)
                .map_err(|err| info!("Could not extract information for POI '{}': {}", id, err))
                .ok()
        })
        .collect(); // TODO Can we get rid of collect, and chain with the following rubber...?

    let count = rubber
        .bulk_index(&index, pois.into_iter())
        .map_err(|err| format_err!("Failed bulk insertion {}", err))?;

    info!("importing POIs: {} POIs added.", count);

    Ok(())
}

/// This function initializes the ES context: It creates an index for this dataset,
/// and then import the POIs in it.
fn index_poi(
    cnx_string: &str,
    dataset: &str,
    file: &PathBuf,
    visibility: IndexVisibility,
    nb_shards: usize,
    nb_replicas: usize,
) -> Result<(), mimirsbrunn::Error>
where
{
    let mut rubber = Rubber::new(cnx_string);
    rubber.initialize_templates()?;

    let settings = IndexSettings {
        nb_shards: nb_shards,
        nb_replicas: nb_replicas,
    };

    let index = rubber.make_index(dataset, &settings)?;

    let admins = rubber.get_all_admins().map_err(|err| {
        error!("Administratives regions not found in es db");
        err
    })?;
    let admins_geofinder = admins.into_iter().collect();

    import_pois(&mut rubber, &index, admins_geofinder, file)?;

    rubber
        .publish_index(dataset, index, visibility)
        .map_err(|err| format_err!("Failed to publish index {}.", err))
}

#[derive(StructOpt, Debug)]
struct Args {
    /// POI file
    #[structopt(short = "i", long = "input", parse(from_os_str))]
    input: PathBuf,

    /// Elasticsearch parameters.
    #[structopt(
        short = "c",
        long = "connection-string",
        default_value = "http://localhost:9200/munin"
    )]
    connection_string: String,

    /// Name of the dataset.
    /// A dataset is a label, that can be used for filtering the data.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,

    /// Indicate if the POI dataset is private
    #[structopt(short = "p", long = "private")]
    private: bool,

    /// Number of threads to use
    #[structopt(
        short = "t",
        long = "nb-threads",
        default_value = &DEFAULT_NB_THREADS
    )]
    nb_threads: usize,

    /// Number of shards for the es index
    #[structopt(short = "s", long = "nb-shards", default_value = "1")]
    nb_shards: usize,

    /// Number of replicas for the es index
    #[structopt(short = "r", long = "nb-replicas", default_value = "1")]
    nb_replicas: usize,
}

fn run(args: Args) -> Result<(), mimirsbrunn::Error> {
    let visibility = if args.private {
        IndexVisibility::Private
    } else {
        IndexVisibility::Public
    };

    index_poi(
        &args.connection_string,
        &args.dataset,
        &args.input,
        visibility,
        args.nb_shards,
        args.nb_replicas,
    )
}
fn main() {
    mimirsbrunn::utils::launch_run(run);
}
<|endoftext|>


<commit_before>//! Character specific parsers and combinators
//!
//! Functions recognizing specific characters

#[macro_use]
mod macros;

pub mod complete;
pub mod streaming;

/// Tests if byte is ASCII alphabetic: A-Z, a-z
///
/// # Example
///
/// ```
/// # use nom::character::is_alphabetic;
/// assert_eq!(is_alphabetic(b'9'), false);
/// assert_eq!(is_alphabetic(b'a'), true);
/// ```
#[inline]
pub fn is_alphabetic(chr: u8) -> bool {
  (chr >= 0x41 && chr <= 0x5A) || (chr >= 0x61 && chr <= 0x7A)
}

/// Tests if byte is ASCII digit: 0-9
///
/// # Example
///
/// ```
/// # use nom::character::is_digit;
/// assert_eq!(is_digit(b'a'), false);
/// assert_eq!(is_digit(b'9'), true);
/// ```
#[inline]
pub fn is_digit(chr: u8) -> bool {
  chr >= 0x30 && chr <= 0x39
}

/// Tests if byte is ASCII hex digit: 0-9, A-F, a-f
///
/// # Example
///
/// ```
/// # use nom::character::is_hex_digit;
/// assert_eq!(is_hex_digit(b'a'), true);
/// assert_eq!(is_hex_digit(b'9'), true);
/// assert_eq!(is_hex_digit(b'A'), true);
/// assert_eq!(is_hex_digit(b'x'), false);
/// ```
#[inline]
pub fn is_hex_digit(chr: u8) -> bool {
  (chr >= 0x30 && chr <= 0x39) || (chr >= 0x41 && chr <= 0x46) || (chr >= 0x61 && chr <= 0x66)
}

/// Tests if byte is ASCII octal digit: 0-7
///
/// # Example
///
/// ```
/// # use nom::character::is_oct_digit;
/// assert_eq!(is_oct_digit(b'a'), false);
/// assert_eq!(is_oct_digit(b'9'), false);
/// assert_eq!(is_oct_digit(b'6'), true);
/// ```
#[inline]
pub fn is_oct_digit(chr: u8) -> bool {
  chr >= 0x30 && chr <= 0x37
}

/// Tests if byte is ASCII alphanumeric: A-Z, a-z, 0-9
///
/// # Example
///
/// ```
/// # use nom::character::is_alphanumeric;
/// assert_eq!(is_alphanumeric(b'-'), false);
/// assert_eq!(is_alphanumeric(b'a'), true);
/// assert_eq!(is_alphanumeric(b'9'), true);
/// assert_eq!(is_alphanumeric(b'A'), true);
/// ```
#[inline]
pub fn is_alphanumeric(chr: u8) -> bool {
  is_alphabetic(chr) || is_digit(chr)
}

/// Tests if byte is ASCII space or tab
///
/// # Example
///
/// ```
/// # use nom::character::is_space;
/// assert_eq!(is_space(b'\n'), false);
/// assert_eq!(is_space(b'\r'), false);
/// assert_eq!(is_space(b' '), true);
/// assert_eq!(is_space(b'\t'), true);
/// ```
#[inline]
pub fn is_space(chr: u8) -> bool {
  chr == b' ' || chr == b'\t'
}
<commit_msg>Add a is_newline function<commit_after>//! Character specific parsers and combinators
//!
//! Functions recognizing specific characters

#[macro_use]
mod macros;

pub mod complete;
pub mod streaming;

/// Tests if byte is ASCII alphabetic: A-Z, a-z
///
/// # Example
///
/// ```
/// # use nom::character::is_alphabetic;
/// assert_eq!(is_alphabetic(b'9'), false);
/// assert_eq!(is_alphabetic(b'a'), true);
/// ```
#[inline]
pub fn is_alphabetic(chr: u8) -> bool {
  (chr >= 0x41 && chr <= 0x5A) || (chr >= 0x61 && chr <= 0x7A)
}

/// Tests if byte is ASCII digit: 0-9
///
/// # Example
///
/// ```
/// # use nom::character::is_digit;
/// assert_eq!(is_digit(b'a'), false);
/// assert_eq!(is_digit(b'9'), true);
/// ```
#[inline]
pub fn is_digit(chr: u8) -> bool {
  chr >= 0x30 && chr <= 0x39
}

/// Tests if byte is ASCII hex digit: 0-9, A-F, a-f
///
/// # Example
///
/// ```
/// # use nom::character::is_hex_digit;
/// assert_eq!(is_hex_digit(b'a'), true);
/// assert_eq!(is_hex_digit(b'9'), true);
/// assert_eq!(is_hex_digit(b'A'), true);
/// assert_eq!(is_hex_digit(b'x'), false);
/// ```
#[inline]
pub fn is_hex_digit(chr: u8) -> bool {
  (chr >= 0x30 && chr <= 0x39) || (chr >= 0x41 && chr <= 0x46) || (chr >= 0x61 && chr <= 0x66)
}

/// Tests if byte is ASCII octal digit: 0-7
///
/// # Example
///
/// ```
/// # use nom::character::is_oct_digit;
/// assert_eq!(is_oct_digit(b'a'), false);
/// assert_eq!(is_oct_digit(b'9'), false);
/// assert_eq!(is_oct_digit(b'6'), true);
/// ```
#[inline]
pub fn is_oct_digit(chr: u8) -> bool {
  chr >= 0x30 && chr <= 0x37
}

/// Tests if byte is ASCII alphanumeric: A-Z, a-z, 0-9
///
/// # Example
///
/// ```
/// # use nom::character::is_alphanumeric;
/// assert_eq!(is_alphanumeric(b'-'), false);
/// assert_eq!(is_alphanumeric(b'a'), true);
/// assert_eq!(is_alphanumeric(b'9'), true);
/// assert_eq!(is_alphanumeric(b'A'), true);
/// ```
#[inline]
pub fn is_alphanumeric(chr: u8) -> bool {
  is_alphabetic(chr) || is_digit(chr)
}

/// Tests if byte is ASCII space or tab
///
/// # Example
///
/// ```
/// # use nom::character::is_space;
/// assert_eq!(is_space(b'\n'), false);
/// assert_eq!(is_space(b'\r'), false);
/// assert_eq!(is_space(b' '), true);
/// assert_eq!(is_space(b'\t'), true);
/// ```
#[inline]
pub fn is_space(chr: u8) -> bool {
  chr == b' ' || chr == b'\t'
}

/// Tests if byte is ASCII newline: \n
///
/// # Example
///
/// ```
/// # use nom::character::is_newline;
/// assert_eq!(is_newline(b'\n'), true);
/// assert_eq!(is_newline(b'\r'), false);
/// assert_eq!(is_newline(b' '), false);
/// assert_eq!(is_newline(b'\t'), false);
/// ```
#[inline]
pub fn is_newline(chr: u8) -> bool {
  chr == b'\n'
}<|endoftext|>