<commit_before>mod command;
mod input;
mod job;
mod token;

use std::borrow::Cow;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::{Arc, Mutex};

use anyhow::{anyhow, Result};
use lazy_static::lazy_static;
use regex::Regex;

use crate::exit_codes::ExitCode;
use crate::filesystem::strip_current_dir;

use self::command::execute_command;
use self::input::{basename, dirname, remove_extension};
pub use self::job::{batch, job};
use self::token::Token;

/// Execution mode of the command
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum ExecutionMode {
    /// Command is executed for each search result
    OneByOne,
    /// Command is run for a batch of results at once
    Batch,
}

/// Represents a template that is utilized to generate command strings.
///
/// The template is meant to be coupled with an input in order to generate a command. The
/// `generate_and_execute()` method will be used to generate a command and execute it.
#[derive(Debug, Clone, PartialEq)]
pub struct CommandTemplate {
    args: Vec<ArgumentTemplate>,
    mode: ExecutionMode,
}

impl CommandTemplate {
    pub fn new<I, S>(input: I) -> CommandTemplate
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        Self::build(input, ExecutionMode::OneByOne)
    }

    pub fn new_batch<I, S>(input: I) -> Result<CommandTemplate>
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        let cmd = Self::build(input, ExecutionMode::Batch);
        if cmd.number_of_tokens() > 1 {
            return Err(anyhow!("Only one placeholder allowed for batch commands"));
        }
        if cmd.args[0].has_tokens() {
            return Err(anyhow!(
                "First argument of exec-batch is expected to be a fixed executable"
            ));
        }
        Ok(cmd)
    }

    fn build<I, S>(input: I, mode: ExecutionMode) -> CommandTemplate
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        lazy_static! {
            static ref PLACEHOLDER_PATTERN: Regex = Regex::new(r"\{(/?\.?|//)\}").unwrap();
        }

        let mut args = Vec::new();
        let mut has_placeholder = false;

        for arg in input {
            let arg = arg.as_ref();

            let mut tokens = Vec::new();
            let mut start = 0;

            for placeholder in PLACEHOLDER_PATTERN.find_iter(arg) {
                // Leading text before the placeholder.
                if placeholder.start() > start {
                    tokens.push(Token::Text(arg[start..placeholder.start()].to_owned()));
                }

                start = placeholder.end();

                match placeholder.as_str() {
                    "{}" => tokens.push(Token::Placeholder),
                    "{.}" => tokens.push(Token::NoExt),
                    "{/}" => tokens.push(Token::Basename),
                    "{//}" => tokens.push(Token::Parent),
                    "{/.}" => tokens.push(Token::BasenameNoExt),
                    _ => panic!("Unhandled placeholder"),
                }

                has_placeholder = true;
            }

            // Without a placeholder, the argument is just fixed text.
            if tokens.is_empty() {
                args.push(ArgumentTemplate::Text(arg.to_owned()));
                continue;
            }

            if start < arg.len() {
                // Trailing text after last placeholder.
                tokens.push(Token::Text(arg[start..].to_owned()));
            }

            args.push(ArgumentTemplate::Tokens(tokens));
        }

        // If a placeholder token was not supplied, append one at the end of the command.
        if !has_placeholder {
            args.push(ArgumentTemplate::Tokens(vec![Token::Placeholder]));
        }

        CommandTemplate { args, mode }
    }

    fn number_of_tokens(&self) -> usize {
        self.args.iter().filter(|arg| arg.has_tokens()).count()
    }

    /// Generates and executes a command.
    ///
    /// Using the internal `args` field, and a supplied `input` variable, a `Command` will be
    /// build. Once all arguments have been processed, the command is executed.
    pub fn generate_and_execute(&self, input: &Path, out_perm: Arc<Mutex<()>>) -> ExitCode {
        let input = strip_current_dir(input);

        let mut cmd = Command::new(self.args[0].generate(&input).as_ref());
        for arg in &self.args[1..] {
            cmd.arg(arg.generate(&input).as_ref());
        }

        execute_command(cmd, &out_perm)
    }

    pub fn in_batch_mode(&self) -> bool {
        self.mode == ExecutionMode::Batch
    }

    pub fn generate_and_execute_batch<I>(&self, paths: I) -> ExitCode
    where
        I: Iterator<Item = PathBuf>,
    {
        let mut cmd = Command::new(self.args[0].generate("").as_ref());
        cmd.stdin(Stdio::inherit());
        cmd.stdout(Stdio::inherit());
        cmd.stderr(Stdio::inherit());

        let mut paths: Vec<_> = paths.collect();
        let mut has_path = false;

        for arg in &self.args[1..] {
            if arg.has_tokens() {
                paths.sort();

                // A single `Tokens` is expected
                // So we can directly consume the iterator once and for all
                for path in &mut paths {
                    cmd.arg(arg.generate(strip_current_dir(path)).as_ref());
                    has_path = true;
                }
            } else {
                cmd.arg(arg.generate("").as_ref());
            }
        }

        if has_path {
            execute_command(cmd, &Mutex::new(()))
        } else {
            ExitCode::Success
        }
    }
}

/// Represents a template for a single command argument.
///
/// The argument is either a collection of `Token`s including at least one placeholder variant, or
/// a fixed text.
#[derive(Clone, Debug, PartialEq)]
enum ArgumentTemplate {
    Tokens(Vec<Token>),
    Text(String),
}

impl ArgumentTemplate {
    pub fn has_tokens(&self) -> bool {
        match self {
            ArgumentTemplate::Tokens(_) => true,
            _ => false,
        }
    }

    pub fn generate<'a>(&'a self, path: impl AsRef<Path>) -> Cow<'a, str> {
        use self::Token::*;

        let path = path.as_ref().to_string_lossy(); // TODO

        match *self {
            ArgumentTemplate::Tokens(ref tokens) => {
                let mut s = String::new();
                for token in tokens {
                    match *token {
                        Basename => s += basename(&path),
                        BasenameNoExt => s += remove_extension(basename(&path)),
                        NoExt => s += remove_extension(&path),
                        Parent => s += dirname(&path),
                        Placeholder => s += &path,
                        Text(ref string) => s += string,
                    }
                }
                Cow::Owned(s)
            }
            ArgumentTemplate::Text(ref text) => Cow::Borrowed(text),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn tokens_with_placeholder() {
        assert_eq!(
            CommandTemplate::new(&[&"echo", &"${SHELL}:"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Text("${SHELL}:".into()),
                    ArgumentTemplate::Tokens(vec![Token::Placeholder]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_with_no_extension() {
        assert_eq!(
            CommandTemplate::new(&["echo", "{.}"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Tokens(vec![Token::NoExt]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_with_basename() {
        assert_eq!(
            CommandTemplate::new(&["echo", "{/}"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Tokens(vec![Token::Basename]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_with_parent() {
        assert_eq!(
            CommandTemplate::new(&["echo", "{//}"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Tokens(vec![Token::Parent]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_with_basename_no_extension() {
        assert_eq!(
            CommandTemplate::new(&["echo", "{/.}"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Tokens(vec![Token::BasenameNoExt]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_multiple() {
        assert_eq!(
            CommandTemplate::new(&["cp", "{}", "{/.}.ext"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("cp".into()),
                    ArgumentTemplate::Tokens(vec![Token::Placeholder]),
                    ArgumentTemplate::Tokens(vec![
                        Token::BasenameNoExt,
                        Token::Text(".ext".into())
                    ]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_single_batch() {
        assert_eq!(
            CommandTemplate::new_batch(&["echo", "{.}"]).unwrap(),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Tokens(vec![Token::NoExt]),
                ],
                mode: ExecutionMode::Batch,
            }
        );
    }

    #[test]
    fn tokens_multiple_batch() {
        assert!(CommandTemplate::new_batch(&["echo", "{.}", "{}"]).is_err());
    }
}
<commit_msg>panic => unreachable<commit_after>mod command;
mod input;
mod job;
mod token;

use std::borrow::Cow;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::{Arc, Mutex};

use anyhow::{anyhow, Result};
use lazy_static::lazy_static;
use regex::Regex;

use crate::exit_codes::ExitCode;
use crate::filesystem::strip_current_dir;

use self::command::execute_command;
use self::input::{basename, dirname, remove_extension};
pub use self::job::{batch, job};
use self::token::Token;

/// Execution mode of the command
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum ExecutionMode {
    /// Command is executed for each search result
    OneByOne,
    /// Command is run for a batch of results at once
    Batch,
}

/// Represents a template that is utilized to generate command strings.
///
/// The template is meant to be coupled with an input in order to generate a command. The
/// `generate_and_execute()` method will be used to generate a command and execute it.
#[derive(Debug, Clone, PartialEq)]
pub struct CommandTemplate {
    args: Vec<ArgumentTemplate>,
    mode: ExecutionMode,
}

impl CommandTemplate {
    pub fn new<I, S>(input: I) -> CommandTemplate
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        Self::build(input, ExecutionMode::OneByOne)
    }

    pub fn new_batch<I, S>(input: I) -> Result<CommandTemplate>
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        let cmd = Self::build(input, ExecutionMode::Batch);
        if cmd.number_of_tokens() > 1 {
            return Err(anyhow!("Only one placeholder allowed for batch commands"));
        }
        if cmd.args[0].has_tokens() {
            return Err(anyhow!(
                "First argument of exec-batch is expected to be a fixed executable"
            ));
        }
        Ok(cmd)
    }

    fn build<I, S>(input: I, mode: ExecutionMode) -> CommandTemplate
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        lazy_static! {
            static ref PLACEHOLDER_PATTERN: Regex = Regex::new(r"\{(/?\.?|//)\}").unwrap();
        }

        let mut args = Vec::new();
        let mut has_placeholder = false;

        for arg in input {
            let arg = arg.as_ref();

            let mut tokens = Vec::new();
            let mut start = 0;

            for placeholder in PLACEHOLDER_PATTERN.find_iter(arg) {
                // Leading text before the placeholder.
                if placeholder.start() > start {
                    tokens.push(Token::Text(arg[start..placeholder.start()].to_owned()));
                }

                start = placeholder.end();

                match placeholder.as_str() {
                    "{}" => tokens.push(Token::Placeholder),
                    "{.}" => tokens.push(Token::NoExt),
                    "{/}" => tokens.push(Token::Basename),
                    "{//}" => tokens.push(Token::Parent),
                    "{/.}" => tokens.push(Token::BasenameNoExt),
                    _ => unreachable!("Unhandled placeholder"),
                }

                has_placeholder = true;
            }

            // Without a placeholder, the argument is just fixed text.
            if tokens.is_empty() {
                args.push(ArgumentTemplate::Text(arg.to_owned()));
                continue;
            }

            if start < arg.len() {
                // Trailing text after last placeholder.
                tokens.push(Token::Text(arg[start..].to_owned()));
            }

            args.push(ArgumentTemplate::Tokens(tokens));
        }

        // If a placeholder token was not supplied, append one at the end of the command.
        if !has_placeholder {
            args.push(ArgumentTemplate::Tokens(vec![Token::Placeholder]));
        }

        CommandTemplate { args, mode }
    }

    fn number_of_tokens(&self) -> usize {
        self.args.iter().filter(|arg| arg.has_tokens()).count()
    }

    /// Generates and executes a command.
    ///
    /// Using the internal `args` field, and a supplied `input` variable, a `Command` will be
    /// build. Once all arguments have been processed, the command is executed.
    pub fn generate_and_execute(&self, input: &Path, out_perm: Arc<Mutex<()>>) -> ExitCode {
        let input = strip_current_dir(input);

        let mut cmd = Command::new(self.args[0].generate(&input).as_ref());
        for arg in &self.args[1..] {
            cmd.arg(arg.generate(&input).as_ref());
        }

        execute_command(cmd, &out_perm)
    }

    pub fn in_batch_mode(&self) -> bool {
        self.mode == ExecutionMode::Batch
    }

    pub fn generate_and_execute_batch<I>(&self, paths: I) -> ExitCode
    where
        I: Iterator<Item = PathBuf>,
    {
        let mut cmd = Command::new(self.args[0].generate("").as_ref());
        cmd.stdin(Stdio::inherit());
        cmd.stdout(Stdio::inherit());
        cmd.stderr(Stdio::inherit());

        let mut paths: Vec<_> = paths.collect();
        let mut has_path = false;

        for arg in &self.args[1..] {
            if arg.has_tokens() {
                paths.sort();

                // A single `Tokens` is expected
                // So we can directly consume the iterator once and for all
                for path in &mut paths {
                    cmd.arg(arg.generate(strip_current_dir(path)).as_ref());
                    has_path = true;
                }
            } else {
                cmd.arg(arg.generate("").as_ref());
            }
        }

        if has_path {
            execute_command(cmd, &Mutex::new(()))
        } else {
            ExitCode::Success
        }
    }
}

/// Represents a template for a single command argument.
///
/// The argument is either a collection of `Token`s including at least one placeholder variant, or
/// a fixed text.
#[derive(Clone, Debug, PartialEq)]
enum ArgumentTemplate {
    Tokens(Vec<Token>),
    Text(String),
}

impl ArgumentTemplate {
    pub fn has_tokens(&self) -> bool {
        match self {
            ArgumentTemplate::Tokens(_) => true,
            _ => false,
        }
    }

    pub fn generate<'a>(&'a self, path: impl AsRef<Path>) -> Cow<'a, str> {
        use self::Token::*;

        let path = path.as_ref().to_string_lossy(); // TODO

        match *self {
            ArgumentTemplate::Tokens(ref tokens) => {
                let mut s = String::new();
                for token in tokens {
                    match *token {
                        Basename => s += basename(&path),
                        BasenameNoExt => s += remove_extension(basename(&path)),
                        NoExt => s += remove_extension(&path),
                        Parent => s += dirname(&path),
                        Placeholder => s += &path,
                        Text(ref string) => s += string,
                    }
                }
                Cow::Owned(s)
            }
            ArgumentTemplate::Text(ref text) => Cow::Borrowed(text),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn tokens_with_placeholder() {
        assert_eq!(
            CommandTemplate::new(&[&"echo", &"${SHELL}:"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Text("${SHELL}:".into()),
                    ArgumentTemplate::Tokens(vec![Token::Placeholder]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_with_no_extension() {
        assert_eq!(
            CommandTemplate::new(&["echo", "{.}"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Tokens(vec![Token::NoExt]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_with_basename() {
        assert_eq!(
            CommandTemplate::new(&["echo", "{/}"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Tokens(vec![Token::Basename]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_with_parent() {
        assert_eq!(
            CommandTemplate::new(&["echo", "{//}"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Tokens(vec![Token::Parent]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_with_basename_no_extension() {
        assert_eq!(
            CommandTemplate::new(&["echo", "{/.}"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Tokens(vec![Token::BasenameNoExt]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_multiple() {
        assert_eq!(
            CommandTemplate::new(&["cp", "{}", "{/.}.ext"]),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("cp".into()),
                    ArgumentTemplate::Tokens(vec![Token::Placeholder]),
                    ArgumentTemplate::Tokens(vec![
                        Token::BasenameNoExt,
                        Token::Text(".ext".into())
                    ]),
                ],
                mode: ExecutionMode::OneByOne,
            }
        );
    }

    #[test]
    fn tokens_single_batch() {
        assert_eq!(
            CommandTemplate::new_batch(&["echo", "{.}"]).unwrap(),
            CommandTemplate {
                args: vec![
                    ArgumentTemplate::Text("echo".into()),
                    ArgumentTemplate::Tokens(vec![Token::NoExt]),
                ],
                mode: ExecutionMode::Batch,
            }
        );
    }

    #[test]
    fn tokens_multiple_batch() {
        assert!(CommandTemplate::new_batch(&["echo", "{.}", "{}"]).is_err());
    }
}
<|endoftext|>


<commit_before>use glob::glob;
use reqwest;
use reqwest::header::Authorization;
use serde_json;
use std;
use std::collections::HashMap;
use std::fs::File;
use std::io::prelude::*;
use structs::{CliError, Config, Member, PublicKey, Repo, Sector, SectorGroup, Team};

pub struct GithubClient {
    client: reqwest::Client,
    conf: Config,
}

impl GithubClient {
    pub fn new(config: &Config) -> GithubClient {
        if std::env::var("SSL_CERT_FILE").is_err() {
            std::env::set_var("SSL_CERT_FILE", &config.cert_path);
        }
        GithubClient { client: match config.proxy_url {
                           Some(ref proxy_url) => {
                               let p = reqwest::Proxy::all(proxy_url).unwrap();
                               reqwest::Client::builder().proxy(p).build().unwrap()
                           }
                           None => reqwest::Client::new(),
                       },
                       conf: config.clone(), }
    }

    fn get_cache_path(url: &str) -> std::path::PathBuf {
        let mut path = std::env::temp_dir();
        path.push("sectora-cache");
        path.push(url);
        path
    }

    fn load_content_from_cache(&self, url: &str) -> Result<(std::fs::Metadata, String), CliError> {
        let path = Self::get_cache_path(url);
        let metadata = std::fs::metadata(path.to_str().unwrap())?;
        let mut f = File::open(path.to_str().unwrap())?;
        let mut content = String::new();
        f.read_to_string(&mut content)?;
        Ok((metadata, content))
    }

    fn store_content_to_cache(&self, url: &str, content: &str) -> Result<(), CliError> {
        let path = Self::get_cache_path(url);
        std::fs::create_dir_all(path.parent().unwrap())?;
        let mut f = File::create(path.to_str().unwrap())?;
        f.write(content.as_bytes())?;
        Ok(())
    }

    fn get_content_from_url(&self, url: &str) -> Result<String, CliError> {
        let token = String::from("token ") + &self.conf.token;
        let res = self.client.get(url).header(Authorization(token)).send();
        let mut content = String::new();
        res?.read_to_string(&mut content)?;
        self.store_content_to_cache(url, &content)?;
        Ok(content)
    }

    fn get_content(&self, url: &str) -> Result<String, CliError> {
        match self.load_content_from_cache(url) {
            Ok((metadata, cache_content)) => match std::time::SystemTime::now().duration_since(metadata.modified()?) {
                Ok(caching_duration) => {
                    if caching_duration.as_secs() > self.conf.cache_duration {
                        match self.get_content_from_url(url) {
                            Ok(content_from_url) => Ok(content_from_url),
                            Err(_) => Ok(cache_content),
                        }
                    } else {
                        Ok(cache_content)
                    }
                }
                Err(_) => Ok(cache_content),
            },
            Err(_) => self.get_content_from_url(url),
        }
    }

    #[allow(dead_code)]
    pub fn print_user_public_key(&self, user: &str) -> Result<(), CliError> {
        let keys = self.get_user_public_key(user)?;
        println!("{}", keys);
        Ok(())
    }

    fn get_user_public_key(&self, user: &str) -> Result<String, CliError> {
        let url = format!("{}/users/{}/keys", self.conf.endpoint, user);
        let content = self.get_content(&url)?;
        let keys = serde_json::from_str::<Vec<PublicKey>>(&content)?;
        Ok(keys.iter().map(|k| k.key.clone())
               .collect::<Vec<String>>()
               .join("\n"))
    }

    #[allow(dead_code)]
    pub fn check_pam(&self, user: &str) -> Result<bool, CliError> {
        let sectors = self.get_sectors();
        Ok(sectors.iter().any(|team| team.members.contains_key(user)))
    }

    pub fn get_sectors(&self) -> Vec<SectorGroup> {
        let mut sectors: Vec<SectorGroup> = self.get_teams_result().unwrap_or(Vec::new());
        sectors.append(&mut self.get_repos_result().unwrap_or(Vec::new()));
        sectors
    }

    fn get_teams_result(&self) -> Result<Vec<SectorGroup>, CliError> {
        let gh_teams = self.get_team_map()?;
        let mut teams = Vec::new();
        for team_conf in &self.conf.team {
            if let &Some(gh_team) = &gh_teams.get(&team_conf.name) {
                teams.push(SectorGroup { sector: Sector::from(gh_team.clone()),
                                         gid: team_conf.gid.clone(),
                                         group: team_conf.group.clone(),
                                         members: self.get_team_members(gh_team.id)?, });
            }
        }
        Ok(teams)
    }

    fn get_team_map(&self) -> Result<HashMap<String, Team>, CliError> {
        let url = format!("{}/orgs/{}/teams", self.conf.endpoint, self.conf.org);
        let content = self.get_content(&url)?;
        let teams = serde_json::from_str::<Vec<Team>>(&content)?;
        Ok(teams.iter().map(|t| (t.name.clone(), t.clone())).collect())
    }

    fn get_team_members(&self, mid: u64) -> Result<HashMap<String, Member>, CliError> {
        let url = format!("{}/teams/{}/members", self.conf.endpoint, mid);
        let content = self.get_content(&url)?;
        let members = serde_json::from_str::<Vec<Member>>(&content)?;
        Ok(members.iter()
                  .map(|m| (m.login.clone(), m.clone()))
                  .collect())
    }

    fn get_repos_result(&self) -> Result<Vec<SectorGroup>, CliError> {
        let gh_repos = self.get_repo_map()?;
        let mut repos = Vec::new();
        for repo_conf in &self.conf.repo {
            if let &Some(gh_repo) = &gh_repos.get(&repo_conf.name) {
                repos.push(SectorGroup { sector: Sector::from(gh_repo.clone()),
                                         gid: repo_conf.gid.clone(),
                                         group: repo_conf.group.clone(),
                                         members: self.get_repo_collaborators(&gh_repo.name)?, });
            }
        }
        Ok(repos)
    }

    fn get_repo_map(&self) -> Result<HashMap<String, Repo>, CliError> {
        let url = format!("{}/orgs/{}/repos", self.conf.endpoint, self.conf.org);
        let content = self.get_content(&url)?;
        let repos = serde_json::from_str::<Vec<Repo>>(&content)?;
        Ok(repos.iter().map(|t| (t.name.clone(), t.clone())).collect())
    }

    fn get_repo_collaborators(&self, repo_name: &str) -> Result<HashMap<String, Member>, CliError> {
        let url = format!("{}/repos/{}/{}/collaborators?affiliation=outside",
                          self.conf.endpoint, self.conf.org, repo_name);
        let content = self.get_content(&url)?;
        let members = serde_json::from_str::<Vec<Member>>(&content)?;
        Ok(members.iter()
                  .map(|m| (m.login.clone(), m.clone()))
                  .collect())
    }

    #[allow(dead_code)]
    pub fn clear_all_caches(&self) -> Result<(), CliError> {
        let mut path = std::env::temp_dir();
        path.push("sectora-cache/**/*");
        for entry in glob(&path.to_str().unwrap()).unwrap() {
            match entry {
                Ok(path) => {
                    if path.is_file() {
                        std::fs::remove_file(path)?
                    }
                }
                Err(e) => println!("{:?}", e),
            }
        }
        Ok(())
    }
}
<commit_msg>Increase page size of repo list<commit_after>use glob::glob;
use reqwest;
use reqwest::header::Authorization;
use serde_json;
use std;
use std::collections::HashMap;
use std::fs::File;
use std::io::prelude::*;
use structs::{CliError, Config, Member, PublicKey, Repo, Sector, SectorGroup, Team};

pub struct GithubClient {
    client: reqwest::Client,
    conf: Config,
}

impl GithubClient {
    pub fn new(config: &Config) -> GithubClient {
        if std::env::var("SSL_CERT_FILE").is_err() {
            std::env::set_var("SSL_CERT_FILE", &config.cert_path);
        }
        GithubClient { client: match config.proxy_url {
                           Some(ref proxy_url) => {
                               let p = reqwest::Proxy::all(proxy_url).unwrap();
                               reqwest::Client::builder().proxy(p).build().unwrap()
                           }
                           None => reqwest::Client::new(),
                       },
                       conf: config.clone(), }
    }

    fn get_cache_path(url: &str) -> std::path::PathBuf {
        let mut path = std::env::temp_dir();
        path.push("sectora-cache");
        path.push(url);
        path
    }

    fn load_content_from_cache(&self, url: &str) -> Result<(std::fs::Metadata, String), CliError> {
        let path = Self::get_cache_path(url);
        let metadata = std::fs::metadata(path.to_str().unwrap())?;
        let mut f = File::open(path.to_str().unwrap())?;
        let mut content = String::new();
        f.read_to_string(&mut content)?;
        Ok((metadata, content))
    }

    fn store_content_to_cache(&self, url: &str, content: &str) -> Result<(), CliError> {
        let path = Self::get_cache_path(url);
        std::fs::create_dir_all(path.parent().unwrap())?;
        let mut f = File::create(path.to_str().unwrap())?;
        f.write(content.as_bytes())?;
        Ok(())
    }

    fn get_content_from_url(&self, url: &str) -> Result<String, CliError> {
        let token = String::from("token ") + &self.conf.token;
        let res = self.client.get(url).header(Authorization(token)).send();
        let mut content = String::new();
        res?.read_to_string(&mut content)?;
        self.store_content_to_cache(url, &content)?;
        Ok(content)
    }

    fn get_content(&self, url: &str) -> Result<String, CliError> {
        match self.load_content_from_cache(url) {
            Ok((metadata, cache_content)) => match std::time::SystemTime::now().duration_since(metadata.modified()?) {
                Ok(caching_duration) => {
                    if caching_duration.as_secs() > self.conf.cache_duration {
                        match self.get_content_from_url(url) {
                            Ok(content_from_url) => Ok(content_from_url),
                            Err(_) => Ok(cache_content),
                        }
                    } else {
                        Ok(cache_content)
                    }
                }
                Err(_) => Ok(cache_content),
            },
            Err(_) => self.get_content_from_url(url),
        }
    }

    #[allow(dead_code)]
    pub fn print_user_public_key(&self, user: &str) -> Result<(), CliError> {
        let keys = self.get_user_public_key(user)?;
        println!("{}", keys);
        Ok(())
    }

    fn get_user_public_key(&self, user: &str) -> Result<String, CliError> {
        let url = format!("{}/users/{}/keys", self.conf.endpoint, user);
        let content = self.get_content(&url)?;
        let keys = serde_json::from_str::<Vec<PublicKey>>(&content)?;
        Ok(keys.iter().map(|k| k.key.clone())
               .collect::<Vec<String>>()
               .join("\n"))
    }

    #[allow(dead_code)]
    pub fn check_pam(&self, user: &str) -> Result<bool, CliError> {
        let sectors = self.get_sectors();
        Ok(sectors.iter().any(|team| team.members.contains_key(user)))
    }

    pub fn get_sectors(&self) -> Vec<SectorGroup> {
        let mut sectors: Vec<SectorGroup> = self.get_teams_result().unwrap_or(Vec::new());
        sectors.append(&mut self.get_repos_result().unwrap_or(Vec::new()));
        sectors
    }

    fn get_teams_result(&self) -> Result<Vec<SectorGroup>, CliError> {
        let gh_teams = self.get_team_map()?;
        let mut teams = Vec::new();
        for team_conf in &self.conf.team {
            if let &Some(gh_team) = &gh_teams.get(&team_conf.name) {
                teams.push(SectorGroup { sector: Sector::from(gh_team.clone()),
                                         gid: team_conf.gid.clone(),
                                         group: team_conf.group.clone(),
                                         members: self.get_team_members(gh_team.id)?, });
            }
        }
        Ok(teams)
    }

    fn get_team_map(&self) -> Result<HashMap<String, Team>, CliError> {
        let url = format!("{}/orgs/{}/teams", self.conf.endpoint, self.conf.org);
        let content = self.get_content(&url)?;
        let teams = serde_json::from_str::<Vec<Team>>(&content)?;
        Ok(teams.iter().map(|t| (t.name.clone(), t.clone())).collect())
    }

    fn get_team_members(&self, mid: u64) -> Result<HashMap<String, Member>, CliError> {
        let url = format!("{}/teams/{}/members", self.conf.endpoint, mid);
        let content = self.get_content(&url)?;
        let members = serde_json::from_str::<Vec<Member>>(&content)?;
        Ok(members.iter()
                  .map(|m| (m.login.clone(), m.clone()))
                  .collect())
    }

    fn get_repos_result(&self) -> Result<Vec<SectorGroup>, CliError> {
        let gh_repos = self.get_repo_map()?;
        let mut repos = Vec::new();
        for repo_conf in &self.conf.repo {
            if let &Some(gh_repo) = &gh_repos.get(&repo_conf.name) {
                repos.push(SectorGroup { sector: Sector::from(gh_repo.clone()),
                                         gid: repo_conf.gid.clone(),
                                         group: repo_conf.group.clone(),
                                         members: self.get_repo_collaborators(&gh_repo.name)?, });
            }
        }
        Ok(repos)
    }

    fn get_repo_map(&self) -> Result<HashMap<String, Repo>, CliError> {
        let url = format!("{}/orgs/{}/repos?per_page=100",
                          self.conf.endpoint, self.conf.org);
        let content = self.get_content(&url)?;
        let repos = serde_json::from_str::<Vec<Repo>>(&content)?;
        Ok(repos.iter().map(|t| (t.name.clone(), t.clone())).collect())
    }

    fn get_repo_collaborators(&self, repo_name: &str) -> Result<HashMap<String, Member>, CliError> {
        let url = format!("{}/repos/{}/{}/collaborators?affiliation=outside",
                          self.conf.endpoint, self.conf.org, repo_name);
        let content = self.get_content(&url)?;
        let members = serde_json::from_str::<Vec<Member>>(&content)?;
        Ok(members.iter()
                  .map(|m| (m.login.clone(), m.clone()))
                  .collect())
    }

    #[allow(dead_code)]
    pub fn clear_all_caches(&self) -> Result<(), CliError> {
        let mut path = std::env::temp_dir();
        path.push("sectora-cache/**/*");
        for entry in glob(&path.to_str().unwrap()).unwrap() {
            match entry {
                Ok(path) => {
                    if path.is_file() {
                        std::fs::remove_file(path)?
                    }
                }
                Err(e) => println!("{:?}", e),
            }
        }
        Ok(())
    }
}
<|endoftext|>


<commit_before>use std::cell::RefCell;

use proc_macro2::{Delimiter, Span};

use buffer::Cursor;
use error::{self, Error};
use span::IntoSpans;
use token::Token;

/// Support for checking the next token in a stream to decide how to parse.
///
/// An important advantage over [`ParseStream::peek`] is that here we
/// automatically construct an appropriate error message based on the token
/// alternatives that get peeked. If you are producing your own error message,
/// go ahead and use `ParseStream::peek` instead.
///
/// Use [`ParseStream::lookahead1`] to construct this object.
///
/// [`ParseStream::peek`]: struct.ParseBuffer.html#method.peek
/// [`ParseStream::lookahead1`]: struct.ParseBuffer.html#method.lookahead1
///
/// # Example
///
/// ```
/// # extern crate syn;
/// #
/// use syn::{ConstParam, Ident, Lifetime, LifetimeDef, Token, TypeParam};
/// use syn::parse::{Parse, ParseStream, Result};
///
/// // A generic parameter, a single one of the comma-separated elements inside
/// // angle brackets in:
/// //
/// //     fn f<T: Clone, 'a, 'b: 'a, const N: usize>() { ... }
/// //
/// // On invalid input, lookahead gives us a reasonable error message.
/// //
/// //     error: expected one of: identifier, lifetime, `const`
/// //       |
/// //     5 |     fn f<!Sized>() {}
/// //       |          ^
/// enum GenericParam {
///     Type(TypeParam),
///     Lifetime(LifetimeDef),
///     Const(ConstParam),
/// }
///
/// impl Parse for GenericParam {
///     fn parse(input: ParseStream) -> Result<Self> {
///         let lookahead = input.lookahead1();
///         if lookahead.peek(Ident) {
///             input.parse().map(GenericParam::Type)
///         } else if lookahead.peek(Lifetime) {
///             input.parse().map(GenericParam::Lifetime)
///         } else if lookahead.peek(Token![const]) {
///             input.parse().map(GenericParam::Const)
///         } else {
///             Err(lookahead.error())
///         }
///     }
/// }
/// #
/// # fn main() {}
/// ```
pub struct Lookahead1<'a> {
    scope: Span,
    cursor: Cursor<'a>,
    comparisons: RefCell<Vec<&'static str>>,
}

pub fn new(scope: Span, cursor: Cursor) -> Lookahead1 {
    Lookahead1 {
        scope: scope,
        cursor: cursor,
        comparisons: RefCell::new(Vec::new()),
    }
}

fn peek_impl(
    lookahead: &Lookahead1,
    peek: fn(Cursor) -> bool,
    display: fn() -> &'static str,
) -> bool {
    if peek(lookahead.cursor) {
        return true;
    }
    lookahead.comparisons.borrow_mut().push(display());
    false
}

impl<'a> Lookahead1<'a> {
    pub fn peek<T: Peek>(&self, token: T) -> bool {
        let _ = token;
        peek_impl(self, T::Token::peek, T::Token::display)
    }

    pub fn error(self) -> Error {
        let comparisons = self.comparisons.borrow();
        match comparisons.len() {
            0 => if self.cursor.eof() {
                Error::new(self.scope, "unexpected end of input")
            } else {
                Error::new(self.cursor.span(), "unexpected token")
            },
            1 => {
                let message = format!("expected {}", comparisons[0]);
                error::new_at(self.scope, self.cursor, message)
            }
            _ => {
                let join = comparisons.join(", ");
                let message = format!("expected one of: {}", join);
                error::new_at(self.scope, self.cursor, message)
            }
        }
    }
}

/// Types that can be parsed by looking at just one token.
///
/// Use [`ParseStream::peek`] to peek one of these types in a parse stream
/// without consuming it from the stream.
///
/// This trait is sealed and cannot be implemented for types outside of Syn.
///
/// [`ParseStream::peek`]: struct.ParseBuffer.html#method.peek
pub trait Peek: private::Sealed {
    // Not public API.
    #[doc(hidden)]
    type Token: Token;
}

impl<F: FnOnce(TokenMarker) -> T, T: Token> Peek for F {
    type Token = T;
}

pub enum TokenMarker {}

impl<S> IntoSpans<S> for TokenMarker {
    fn into_spans(self) -> S {
        match self {}
    }
}

pub fn is_delimiter(cursor: Cursor, delimiter: Delimiter) -> bool {
    cursor.group(delimiter).is_some()
}

mod private {
    use super::{Token, TokenMarker};
    pub trait Sealed {}
    impl<F: FnOnce(TokenMarker) -> T, T: Token> Sealed for F {}
}
<commit_msg>Document the Lookahead1 methods<commit_after>use std::cell::RefCell;

use proc_macro2::{Delimiter, Span};

use buffer::Cursor;
use error::{self, Error};
use span::IntoSpans;
use token::Token;

/// Support for checking the next token in a stream to decide how to parse.
///
/// An important advantage over [`ParseStream::peek`] is that here we
/// automatically construct an appropriate error message based on the token
/// alternatives that get peeked. If you are producing your own error message,
/// go ahead and use `ParseStream::peek` instead.
///
/// Use [`ParseStream::lookahead1`] to construct this object.
///
/// [`ParseStream::peek`]: struct.ParseBuffer.html#method.peek
/// [`ParseStream::lookahead1`]: struct.ParseBuffer.html#method.lookahead1
///
/// # Example
///
/// ```
/// # extern crate syn;
/// #
/// use syn::{ConstParam, Ident, Lifetime, LifetimeDef, Token, TypeParam};
/// use syn::parse::{Parse, ParseStream, Result};
///
/// // A generic parameter, a single one of the comma-separated elements inside
/// // angle brackets in:
/// //
/// //     fn f<T: Clone, 'a, 'b: 'a, const N: usize>() { ... }
/// //
/// // On invalid input, lookahead gives us a reasonable error message.
/// //
/// //     error: expected one of: identifier, lifetime, `const`
/// //       |
/// //     5 |     fn f<!Sized>() {}
/// //       |          ^
/// enum GenericParam {
///     Type(TypeParam),
///     Lifetime(LifetimeDef),
///     Const(ConstParam),
/// }
///
/// impl Parse for GenericParam {
///     fn parse(input: ParseStream) -> Result<Self> {
///         let lookahead = input.lookahead1();
///         if lookahead.peek(Ident) {
///             input.parse().map(GenericParam::Type)
///         } else if lookahead.peek(Lifetime) {
///             input.parse().map(GenericParam::Lifetime)
///         } else if lookahead.peek(Token![const]) {
///             input.parse().map(GenericParam::Const)
///         } else {
///             Err(lookahead.error())
///         }
///     }
/// }
/// #
/// # fn main() {}
/// ```
pub struct Lookahead1<'a> {
    scope: Span,
    cursor: Cursor<'a>,
    comparisons: RefCell<Vec<&'static str>>,
}

pub fn new(scope: Span, cursor: Cursor) -> Lookahead1 {
    Lookahead1 {
        scope: scope,
        cursor: cursor,
        comparisons: RefCell::new(Vec::new()),
    }
}

fn peek_impl(
    lookahead: &Lookahead1,
    peek: fn(Cursor) -> bool,
    display: fn() -> &'static str,
) -> bool {
    if peek(lookahead.cursor) {
        return true;
    }
    lookahead.comparisons.borrow_mut().push(display());
    false
}

impl<'a> Lookahead1<'a> {
    /// Looks at the next token in the parse stream to determine whether it
    /// matches the requested type of token.
    pub fn peek<T: Peek>(&self, token: T) -> bool {
        let _ = token;
        peek_impl(self, T::Token::peek, T::Token::display)
    }

    /// Triggers an error at the current position of the parse stream.
    ///
    /// The error message will identify all of the expected token types that
    /// have been peeked against this lookahead instance.
    pub fn error(self) -> Error {
        let comparisons = self.comparisons.borrow();
        match comparisons.len() {
            0 => if self.cursor.eof() {
                Error::new(self.scope, "unexpected end of input")
            } else {
                Error::new(self.cursor.span(), "unexpected token")
            },
            1 => {
                let message = format!("expected {}", comparisons[0]);
                error::new_at(self.scope, self.cursor, message)
            }
            _ => {
                let join = comparisons.join(", ");
                let message = format!("expected one of: {}", join);
                error::new_at(self.scope, self.cursor, message)
            }
        }
    }
}

/// Types that can be parsed by looking at just one token.
///
/// Use [`ParseStream::peek`] to peek one of these types in a parse stream
/// without consuming it from the stream.
///
/// This trait is sealed and cannot be implemented for types outside of Syn.
///
/// [`ParseStream::peek`]: struct.ParseBuffer.html#method.peek
pub trait Peek: private::Sealed {
    // Not public API.
    #[doc(hidden)]
    type Token: Token;
}

impl<F: FnOnce(TokenMarker) -> T, T: Token> Peek for F {
    type Token = T;
}

pub enum TokenMarker {}

impl<S> IntoSpans<S> for TokenMarker {
    fn into_spans(self) -> S {
        match self {}
    }
}

pub fn is_delimiter(cursor: Cursor, delimiter: Delimiter) -> bool {
    cursor.group(delimiter).is_some()
}

mod private {
    use super::{Token, TokenMarker};
    pub trait Sealed {}
    impl<F: FnOnce(TokenMarker) -> T, T: Token> Sealed for F {}
}
<|endoftext|>


<commit_before>// Copyright 2015-2018 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Import a new corpus into the framework
use glob::glob;
// use regex::Regex;
use crate::backend::Backend;
use crate::helpers::TaskStatus;
use crate::models::{Corpus, NewCorpus, NewTask};
use std::env;
use std::fs;
use std::io::Error;
use std::path::Path;
use std::path::PathBuf;
use Archive::*;

const BUFFER_SIZE: usize = 10_240;

/// Struct for performing corpus imports into `CorTeX`
pub struct Importer {
  /// a `Corpus` to be imported, containing all relevant metadata
  pub corpus: Corpus,
  /// a `Backend` on which to persist the import into the Task store
  pub backend: Backend,
  /// the current working directory, to resolve relative paths
  pub cwd: PathBuf,
}
impl Default for Importer {
  fn default() -> Importer {
    let default_backend = Backend::default();
    let name = "default";
    default_backend
      .add(&NewCorpus {
        path: ".".to_string(),
        name: name.to_string(),
        complex: false,
        description: String::new(),
      })
      .expect("Failed to create new corpus.");
    let registered_corpus = Corpus::find_by_name(name, &default_backend.connection).unwrap();

    Importer {
      corpus: registered_corpus,
      backend: default_backend,
      cwd: Importer::cwd(),
    }
  }
}

impl Importer {
  /// Convenience method for (recklessly?) obtaining the current working dir
  pub fn cwd() -> PathBuf { env::current_dir().unwrap() }
  /// Top-level method for unpacking an arxiv-toplogy corpus from its tar-ed form
  fn unpack(&self) -> Result<(), Error> {
    r#try!(self.unpack_arxiv_top());
    r#try!(self.unpack_arxiv_months());
    Ok(())
  }
  fn unpack_extend(&self) -> Result<(), Error> {
    r#try!(self.unpack_extend_arxiv_top());
    // We can reuse the monthly unpack, as it deletes all unpacked document archives
    // In other words, it always acts as a conservative extension
    r#try!(self.unpack_arxiv_months());
    Ok(())
  }

  /// Unpack the top-level tar files from an arxiv-topology corpus
  fn unpack_arxiv_top(&self) -> Result<(), Error> {
    println!("-- Starting top-level unpack process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // let base_name = path.file_stem().unwrap().to_str().unwrap();
          // If we wanted fine-grained control, we could infer the dir name:
          // let arxiv_name_re = Regex::new(r"arXiv_src_(\d+)_").unwrap();
          // let captures = arxiv_name_re.captures(base_name).unwrap();
          // let unpack_dirname = match captures.at(1) {
          //   Some(month) => month,
          //   None => base_name
          // };
          // --- but not for now

          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), BUFFER_SIZE)
            .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            if e.pathname().ends_with(".pdf") {
              continue;
            }
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                println!("To unpack: {:?}", full_extract_path);
                match e.extract_to(&full_extract_path, Vec::new()) {
                  Ok(_) => {},
                  _ => {
                    println!("Failed to extract {:?}", full_extract_path);
                  },
                }
              },
            }
          }
        },
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }
  /// Top-level extension unpacking for arxiv-topology corpora
  fn unpack_extend_arxiv_top(&self) -> Result<(), Error> {
    println!("-- Starting top-level unpack-extend process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), BUFFER_SIZE)
            .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => {}//println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                // Archive entries end in .gz, let's try that as well, to check if the directory is there
                let dir_extract_path = &full_extract_path[0..full_extract_path.len() - 3];
                match fs::metadata(dir_extract_path) {
                  Ok(_) => {}//println!("Directory for {:?} already exists, won't unpack.", e.pathname()),
                  Err(_) => {
                    // println!("To unpack: {:?}", full_extract_path);
                    match e.extract_to(&full_extract_path, Vec::new()) {
                      Ok(_) => {}
                      _ => {
                        println!("Failed to extract {:?}", full_extract_path);
                      }
                    }
                  }
                }
              }
            }
          }
        },
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Unpack the monthly sub-archives of an arxiv-topology corpus, into the CorTeX organization
  fn unpack_arxiv_months(&self) -> Result<(), Error> {
    println!("-- Starting to unpack monthly .gz archives");
    let path_str = self.corpus.path.clone();
    let gzs_path = path_str.to_string() + "/*/*.gz";
    for entry in glob(&gzs_path).unwrap() {
      match entry {
        Ok(path) => {
          let entry_path = path.to_str().unwrap();
          let entry_dir = path.parent().unwrap().to_str().unwrap();
          let base_name = path.file_stem().unwrap().to_str().unwrap();
          let default_tex_target = base_name.to_string() + ".tex";
          let entry_cp_dir = entry_dir.to_string() + "/" + base_name;
          fs::create_dir_all(entry_cp_dir.clone()).unwrap_or_else(|reason| {
            println!(
              "Failed to mkdir -p {:?} because: {:?}",
              entry_cp_dir.clone(),
              reason.kind()
            );
          });
          // We'll write out a ZIP file for each entry
          let full_extract_path = entry_cp_dir.to_string() + "/" + base_name + ".zip";
          let mut archive_writer_new = Writer::new()
            .unwrap()
            //.add_filter(ArchiveFilter::Lzip)
            // .set_compression(ArchiveFilter::None)
            .set_format(ArchiveFormat::Zip);
          archive_writer_new
            .open_filename(&full_extract_path.clone())
            .unwrap();

          // Careful here, some of arXiv's .gz files are really plain-text TeX files (surprise!!!)
          let mut raw_read_needed = false;
          match Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(entry_path, BUFFER_SIZE)
          {
            Err(_) => raw_read_needed = true,
            Ok(archive_reader) => {
              let mut file_count = 0;
              while let Ok(e) = archive_reader.next_header() {
                file_count += 1;
                match archive_writer_new.write_header(e) {
                  Ok(_) => {}, // TODO: If we need to print an error message, we can do so later.
                  Err(e2) => println!("Header write failed: {:?}", e2),
                };
                while let Ok(chunk) = archive_reader.read_data(BUFFER_SIZE) {
                  archive_writer_new.write_data(chunk).unwrap();
                }
              }
              if file_count == 0 {
                // Special case (bug? in libarchive crate), single file in .gz
                raw_read_needed = true;
              }
            },
          }

          if raw_read_needed {
            let raw_reader_new = Reader::new()
              .unwrap()
              .support_filter_all()
              .support_format_raw()
              .open_filename(entry_path, BUFFER_SIZE);
            match raw_reader_new {
              Ok(raw_reader) => match raw_reader.next_header() {
                Ok(_) => {
                  single_file_transfer(&default_tex_target, &raw_reader, &mut archive_writer_new);
                },
                Err(_) => println!("No content in archive: {:?}", entry_path),
              },
              Err(_) => println!("Unrecognizeable archive: {:?}", entry_path),
            }
          }
          // Done with this .gz , remove it:
          match fs::remove_file(path.clone()) {
            Ok(_) => {},
            Err(e) => println!("Can't remove source .gz: {:?}", e),
          };
        },
        Err(e) => println!("Failed gz glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Given a CorTeX-topology corpus, walk the file system and import it into the Task store
  pub fn walk_import(&self) -> Result<(), Error> {
    println!("-- Starting import walk");
    let import_extension = if self.corpus.complex { "zip" } else { "tex" };
    let mut walk_q: Vec<PathBuf> = vec![Path::new(&self.corpus.path).to_owned()];
    let mut import_q: Vec<NewTask> = Vec::new();
    let mut import_counter = 0;
    while !walk_q.is_empty() {
      let current_path = walk_q.pop().unwrap();
      let current_metadata = r#try!(fs::metadata(current_path.clone()));
      if current_metadata.is_dir() {
        println!("-- current path {:?}", current_path);
        // First, test if we just found an entry:
        let current_local_dir = current_path.file_name().unwrap();
        let current_entry =
          current_local_dir.to_str().unwrap().to_string() + "." + import_extension;
        let current_entry_path = current_path.to_str().unwrap().to_string() + "/" + &current_entry;
        match fs::metadata(&current_entry_path) {
          Ok(_) => {
            // Found the expected file, import this entry:
            println!("Found entry: {:?}", current_entry_path);
            import_counter += 1;
            import_q.push(self.new_task(&current_entry_path));
            if import_q.len() >= 1000 {
              // Flush the import queue to backend:
              println!("Checkpoint backend writer: job {:?}", import_counter);
              self.backend.mark_imported(&import_q).unwrap(); // TODO: Proper Error-handling
              import_q.clear();
            }
          },
          Err(_) => {
            // No such entry found, traversing into the directory:
            for subentry in r#try!(fs::read_dir(current_path.clone())) {
              let subentry = r#try!(subentry);
              walk_q.push(subentry.path());
            }
          },
        }
      }
    }
    if !import_q.is_empty() {
      println!("Checkpoint backend writer: job {:?}", import_q.len());
      self.backend.mark_imported(&import_q).unwrap();
    } // TODO: Proper Error-handling
    println!("--- Imported {:?} entries.", import_counter);
    Ok(())
  }

  /// Create a new NoProblem task for the "import" service and the Importer-specified corpus
  pub fn new_task(&self, entry: &str) -> NewTask {
    let abs_entry: String = if Path::new(&entry).is_relative() {
      let mut new_abs = self.cwd.clone();
      new_abs.push(entry);
      new_abs.to_str().unwrap().to_string()
    } else {
      entry.to_string()
    };

    NewTask {
      entry: abs_entry,
      status: TaskStatus::NoProblem.raw(),
      corpus_id: self.corpus.id,
      service_id: 2,
    }
  }
  /// Top-level import driver, performs an optional unpack, and then an import into the Task store
  pub fn process(&self) -> Result<(), Error> {
    // println!("Greetings from the import processor");
    if self.corpus.complex {
      // Complex setup has an unpack step:
      r#try!(self.unpack());
    }
    // Walk the directory tree and import the files in the Task store:
    r#try!(self.walk_import());

    Ok(())
  }

  /// Top-level corpus extension, performs a check for newly added documents and extracts+adds
  /// them to the existing corpus tasks
  pub fn extend_corpus(&self) -> Result<(), Error> {
    if self.corpus.complex {
      // Complex setup has an unpack step:
      r#try!(self.unpack_extend());
    }
    // Use the regular walk_import, at the cost of more database work,
    // the "Backend::mark_imported" ORM method allows us to insert only if new
    r#try!(self.walk_import());
    Ok(())
  }
}

/// Transfer the data contained within `Reader` to a `Writer`, assuming it was a single file
pub fn single_file_transfer(tex_target: &str, reader: &Reader, writer: &mut Writer) {
  // In a "raw" read, we don't know the data size in advance. So we bite the
  // bullet and read the usually tiny tex file in memory,
  // obtaining a size estimate
  let mut raw_data = Vec::new();
  while let Ok(chunk) = reader.read_data(BUFFER_SIZE) {
    raw_data.extend(chunk.into_iter());
  }
  let mut ok_header = false;
  match writer.write_header_new(&tex_target, raw_data.len() as i64) {
    Ok(_) => {
      ok_header = true;
    },
    Err(e) => {
      println!("Couldn't write header: {:?}", e);
    },
  }
  if ok_header {
    match writer.write_data(raw_data) {
      Ok(_) => {},
      Err(e) => println!("Failed to write data to {:?} because {:?}", tex_target, e),
    };
  }
}
<commit_msg>also ignore top-level PDF files when extending<commit_after>// Copyright 2015-2018 Deyan Ginev. See the LICENSE
// file at the top-level directory of this distribution.
//
// Licensed under the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>.
// This file may not be copied, modified, or distributed
// except according to those terms.

//! Import a new corpus into the framework
use glob::glob;
// use regex::Regex;
use crate::backend::Backend;
use crate::helpers::TaskStatus;
use crate::models::{Corpus, NewCorpus, NewTask};
use std::env;
use std::fs;
use std::io::Error;
use std::path::Path;
use std::path::PathBuf;
use Archive::*;

const BUFFER_SIZE: usize = 10_240;

/// Struct for performing corpus imports into `CorTeX`
pub struct Importer {
  /// a `Corpus` to be imported, containing all relevant metadata
  pub corpus: Corpus,
  /// a `Backend` on which to persist the import into the Task store
  pub backend: Backend,
  /// the current working directory, to resolve relative paths
  pub cwd: PathBuf,
}
impl Default for Importer {
  fn default() -> Importer {
    let default_backend = Backend::default();
    let name = "default";
    default_backend
      .add(&NewCorpus {
        path: ".".to_string(),
        name: name.to_string(),
        complex: false,
        description: String::new(),
      })
      .expect("Failed to create new corpus.");
    let registered_corpus = Corpus::find_by_name(name, &default_backend.connection).unwrap();

    Importer {
      corpus: registered_corpus,
      backend: default_backend,
      cwd: Importer::cwd(),
    }
  }
}

impl Importer {
  /// Convenience method for (recklessly?) obtaining the current working dir
  pub fn cwd() -> PathBuf { env::current_dir().unwrap() }
  /// Top-level method for unpacking an arxiv-toplogy corpus from its tar-ed form
  fn unpack(&self) -> Result<(), Error> {
    r#try!(self.unpack_arxiv_top());
    r#try!(self.unpack_arxiv_months());
    Ok(())
  }
  fn unpack_extend(&self) -> Result<(), Error> {
    r#try!(self.unpack_extend_arxiv_top());
    // We can reuse the monthly unpack, as it deletes all unpacked document archives
    // In other words, it always acts as a conservative extension
    r#try!(self.unpack_arxiv_months());
    Ok(())
  }

  /// Unpack the top-level tar files from an arxiv-topology corpus
  fn unpack_arxiv_top(&self) -> Result<(), Error> {
    println!("-- Starting top-level unpack process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // let base_name = path.file_stem().unwrap().to_str().unwrap();
          // If we wanted fine-grained control, we could infer the dir name:
          // let arxiv_name_re = Regex::new(r"arXiv_src_(\d+)_").unwrap();
          // let captures = arxiv_name_re.captures(base_name).unwrap();
          // let unpack_dirname = match captures.at(1) {
          //   Some(month) => month,
          //   None => base_name
          // };
          // --- but not for now

          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), BUFFER_SIZE)
            .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            if e.pathname().ends_with(".pdf") {
              continue;
            }
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                println!("To unpack: {:?}", full_extract_path);
                match e.extract_to(&full_extract_path, Vec::new()) {
                  Ok(_) => {},
                  _ => {
                    println!("Failed to extract {:?}", full_extract_path);
                  },
                }
              },
            }
          }
        },
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }
  /// Top-level extension unpacking for arxiv-topology corpora
  fn unpack_extend_arxiv_top(&self) -> Result<(), Error> {
    println!("-- Starting top-level unpack-extend process");
    let path_str = self.corpus.path.clone();
    let tars_path = path_str.to_string() + "/*.tar";
    for entry in glob(&tars_path).unwrap() {
      match entry {
        Ok(path) => {
          // Let's open the tar file and unpack it:
          let archive_reader = Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(path.to_str().unwrap(), BUFFER_SIZE)
            .unwrap();
          while let Ok(e) = archive_reader.next_header() {
            if e.pathname().ends_with(".pdf") {
              continue;
            }
            let full_extract_path = path_str.to_string() + &e.pathname();
            match fs::metadata(full_extract_path.clone()) {
              Ok(_) => {}//println!("File {:?} exists, won't unpack.", e.pathname()),
              Err(_) => {
                // Archive entries end in .gz, let's try that as well, to check if the directory is there
                let dir_extract_path = &full_extract_path[0..full_extract_path.len() - 3];
                match fs::metadata(dir_extract_path) {
                  Ok(_) => {}//println!("Directory for {:?} already exists, won't unpack.", e.pathname()),
                  Err(_) => {
                    // println!("To unpack: {:?}", full_extract_path);
                    match e.extract_to(&full_extract_path, Vec::new()) {
                      Ok(_) => {}
                      _ => {
                        println!("Failed to extract {:?}", full_extract_path);
                      }
                    }
                  }
                }
              }
            }
          }
        },
        Err(e) => println!("Failed tar glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Unpack the monthly sub-archives of an arxiv-topology corpus, into the CorTeX organization
  fn unpack_arxiv_months(&self) -> Result<(), Error> {
    println!("-- Starting to unpack monthly .gz archives");
    let path_str = self.corpus.path.clone();
    let gzs_path = path_str.to_string() + "/*/*.gz";
    for entry in glob(&gzs_path).unwrap() {
      match entry {
        Ok(path) => {
          let entry_path = path.to_str().unwrap();
          let entry_dir = path.parent().unwrap().to_str().unwrap();
          let base_name = path.file_stem().unwrap().to_str().unwrap();
          let default_tex_target = base_name.to_string() + ".tex";
          let entry_cp_dir = entry_dir.to_string() + "/" + base_name;
          fs::create_dir_all(entry_cp_dir.clone()).unwrap_or_else(|reason| {
            println!(
              "Failed to mkdir -p {:?} because: {:?}",
              entry_cp_dir.clone(),
              reason.kind()
            );
          });
          // We'll write out a ZIP file for each entry
          let full_extract_path = entry_cp_dir.to_string() + "/" + base_name + ".zip";
          let mut archive_writer_new = Writer::new()
            .unwrap()
            //.add_filter(ArchiveFilter::Lzip)
            // .set_compression(ArchiveFilter::None)
            .set_format(ArchiveFormat::Zip);
          archive_writer_new
            .open_filename(&full_extract_path.clone())
            .unwrap();

          // Careful here, some of arXiv's .gz files are really plain-text TeX files (surprise!!!)
          let mut raw_read_needed = false;
          match Reader::new()
            .unwrap()
            .support_filter_all()
            .support_format_all()
            .open_filename(entry_path, BUFFER_SIZE)
          {
            Err(_) => raw_read_needed = true,
            Ok(archive_reader) => {
              let mut file_count = 0;
              while let Ok(e) = archive_reader.next_header() {
                file_count += 1;
                match archive_writer_new.write_header(e) {
                  Ok(_) => {}, // TODO: If we need to print an error message, we can do so later.
                  Err(e2) => println!("Header write failed: {:?}", e2),
                };
                while let Ok(chunk) = archive_reader.read_data(BUFFER_SIZE) {
                  archive_writer_new.write_data(chunk).unwrap();
                }
              }
              if file_count == 0 {
                // Special case (bug? in libarchive crate), single file in .gz
                raw_read_needed = true;
              }
            },
          }

          if raw_read_needed {
            let raw_reader_new = Reader::new()
              .unwrap()
              .support_filter_all()
              .support_format_raw()
              .open_filename(entry_path, BUFFER_SIZE);
            match raw_reader_new {
              Ok(raw_reader) => match raw_reader.next_header() {
                Ok(_) => {
                  single_file_transfer(&default_tex_target, &raw_reader, &mut archive_writer_new);
                },
                Err(_) => println!("No content in archive: {:?}", entry_path),
              },
              Err(_) => println!("Unrecognizeable archive: {:?}", entry_path),
            }
          }
          // Done with this .gz , remove it:
          match fs::remove_file(path.clone()) {
            Ok(_) => {},
            Err(e) => println!("Can't remove source .gz: {:?}", e),
          };
        },
        Err(e) => println!("Failed gz glob: {:?}", e),
      }
    }
    Ok(())
  }

  /// Given a CorTeX-topology corpus, walk the file system and import it into the Task store
  pub fn walk_import(&self) -> Result<(), Error> {
    println!("-- Starting import walk");
    let import_extension = if self.corpus.complex { "zip" } else { "tex" };
    let mut walk_q: Vec<PathBuf> = vec![Path::new(&self.corpus.path).to_owned()];
    let mut import_q: Vec<NewTask> = Vec::new();
    let mut import_counter = 0;
    while !walk_q.is_empty() {
      let current_path = walk_q.pop().unwrap();
      let current_metadata = r#try!(fs::metadata(current_path.clone()));
      if current_metadata.is_dir() {
        println!("-- current path {:?}", current_path);
        // First, test if we just found an entry:
        let current_local_dir = current_path.file_name().unwrap();
        let current_entry =
          current_local_dir.to_str().unwrap().to_string() + "." + import_extension;
        let current_entry_path = current_path.to_str().unwrap().to_string() + "/" + &current_entry;
        match fs::metadata(&current_entry_path) {
          Ok(_) => {
            // Found the expected file, import this entry:
            println!("Found entry: {:?}", current_entry_path);
            import_counter += 1;
            import_q.push(self.new_task(&current_entry_path));
            if import_q.len() >= 1000 {
              // Flush the import queue to backend:
              println!("Checkpoint backend writer: job {:?}", import_counter);
              self.backend.mark_imported(&import_q).unwrap(); // TODO: Proper Error-handling
              import_q.clear();
            }
          },
          Err(_) => {
            // No such entry found, traversing into the directory:
            for subentry in r#try!(fs::read_dir(current_path.clone())) {
              let subentry = r#try!(subentry);
              walk_q.push(subentry.path());
            }
          },
        }
      }
    }
    if !import_q.is_empty() {
      println!("Checkpoint backend writer: job {:?}", import_q.len());
      self.backend.mark_imported(&import_q).unwrap();
    } // TODO: Proper Error-handling
    println!("--- Imported {:?} entries.", import_counter);
    Ok(())
  }

  /// Create a new NoProblem task for the "import" service and the Importer-specified corpus
  pub fn new_task(&self, entry: &str) -> NewTask {
    let abs_entry: String = if Path::new(&entry).is_relative() {
      let mut new_abs = self.cwd.clone();
      new_abs.push(entry);
      new_abs.to_str().unwrap().to_string()
    } else {
      entry.to_string()
    };

    NewTask {
      entry: abs_entry,
      status: TaskStatus::NoProblem.raw(),
      corpus_id: self.corpus.id,
      service_id: 2,
    }
  }
  /// Top-level import driver, performs an optional unpack, and then an import into the Task store
  pub fn process(&self) -> Result<(), Error> {
    // println!("Greetings from the import processor");
    if self.corpus.complex {
      // Complex setup has an unpack step:
      r#try!(self.unpack());
    }
    // Walk the directory tree and import the files in the Task store:
    r#try!(self.walk_import());

    Ok(())
  }

  /// Top-level corpus extension, performs a check for newly added documents and extracts+adds
  /// them to the existing corpus tasks
  pub fn extend_corpus(&self) -> Result<(), Error> {
    if self.corpus.complex {
      // Complex setup has an unpack step:
      r#try!(self.unpack_extend());
    }
    // Use the regular walk_import, at the cost of more database work,
    // the "Backend::mark_imported" ORM method allows us to insert only if new
    r#try!(self.walk_import());
    Ok(())
  }
}

/// Transfer the data contained within `Reader` to a `Writer`, assuming it was a single file
pub fn single_file_transfer(tex_target: &str, reader: &Reader, writer: &mut Writer) {
  // In a "raw" read, we don't know the data size in advance. So we bite the
  // bullet and read the usually tiny tex file in memory,
  // obtaining a size estimate
  let mut raw_data = Vec::new();
  while let Ok(chunk) = reader.read_data(BUFFER_SIZE) {
    raw_data.extend(chunk.into_iter());
  }
  let mut ok_header = false;
  match writer.write_header_new(&tex_target, raw_data.len() as i64) {
    Ok(_) => {
      ok_header = true;
    },
    Err(e) => {
      println!("Couldn't write header: {:?}", e);
    },
  }
  if ok_header {
    match writer.write_data(raw_data) {
      Ok(_) => {},
      Err(e) => println!("Failed to write data to {:?} because {:?}", tex_target, e),
    };
  }
}
<|endoftext|>


<commit_before>use time;
use calx::{Fonter, V2, Context};
use calx::text;
use calx::color;

struct Msg {
    pub text: String,
    /// Time point at which the Msg is phased out.
    pub timeout: f64,
}

impl Msg {
    fn new(text: String, timeout: f64) -> Msg {
        Msg {
            text: text,
            timeout: timeout,
        }
    }
}

pub struct MsgQueue {
    /// Regular messages that show up in the status panel
    msgs: Vec<Msg>,
    msg_done_time: Option<f64>,

    /// Special messages that show up one by one at the center of the screen.
    captions: Vec<Msg>,
    caption_done_time: Option<f64>,
}

impl MsgQueue {
    pub fn new() -> MsgQueue {
        MsgQueue {
            msgs: Vec::new(),
            msg_done_time: None,
            captions: Vec::new(),
            caption_done_time: None,
        }
    }

    pub fn msg(&mut self, text: String) {
        let timeout = add_time_to_read(self.msg_done_time, text.as_slice());
        self.msgs.push(Msg::new(text, timeout));
        self.msg_done_time = Some(timeout);
    }

    pub fn caption(&mut self, text: String) {
        let timeout = add_time_to_read(self.caption_done_time, text.as_slice());
        self.captions.push(Msg::new(text, timeout));
        self.caption_done_time = Some(timeout);
    }

    fn draw_msgs(&self, ctx: &mut Context) {
        let msg_columns = 32;
        let msg_rows = 16;
        let msg_origin = V2(0, 360 - (msg_rows - 1) * 8);

        let mut writer = ctx.text_writer(msg_origin, 0.1, color::LIGHTGRAY)
            .set_border(color::BLACK);

        for msg in self.msgs.iter() {
            let _ = write!(&mut writer, "{}", text::wrap_lines(msg_columns, msg.text.as_slice()).as_slice());
        }
    }

    fn draw_caption(&self, ctx: &mut Context) {
        if !self.captions.is_empty() {
            let width = self.captions[0].text.len() as int * 8;
            let origin = V2(320 - width / 2, 180);
            // TODO: Wrap and center-justify multiline caption

            let mut writer = ctx.text_writer(origin, 0.1, color::LIGHTGRAY)
                .set_border(color::BLACK);

            let _ = write!(&mut writer, "{}", self.captions[0].text.as_slice());
        }
    }

    pub fn draw(&self, ctx: &mut Context) {
        self.draw_msgs(ctx);
        self.draw_caption(ctx);
    }

    pub fn update(&mut self) {
        let t = time::precise_time_s();
        while !self.msgs.is_empty() {
            if self.msgs[0].timeout < t {
                self.msgs.remove(0);
            } else {
                break;
            }
        }
        if self.msgs.is_empty() { self.msg_done_time = None; }

        while !self.captions.is_empty() {
            if self.captions[0].timeout < t {
                self.captions.remove(0);
            } else {
                break;
            }
        }
        if self.captions.is_empty() { self.caption_done_time = None; }
    }
}

fn add_time_to_read(old_time: Option<f64>, text: &str) -> f64 {
    // Estimated time it takes the user to read one character in seconds.
    let letter_read_duration = 0.2;
    let read_time = text.len() as f64 * letter_read_duration;

    return match old_time {
        Some(t) => t,
        None => time::precise_time_s()
    } + read_time;
}

<commit_msg>Don't count trailing whitespace into time to read string<commit_after>use time;
use calx::{Fonter, V2, Context};
use calx::text;
use calx::color;

struct Msg {
    pub text: String,
    /// Time point at which the Msg is phased out.
    pub timeout: f64,
}

impl Msg {
    fn new(text: String, timeout: f64) -> Msg {
        Msg {
            text: text,
            timeout: timeout,
        }
    }
}

pub struct MsgQueue {
    /// Regular messages that show up in the status panel
    msgs: Vec<Msg>,
    msg_done_time: Option<f64>,

    /// Special messages that show up one by one at the center of the screen.
    captions: Vec<Msg>,
    caption_done_time: Option<f64>,
}

impl MsgQueue {
    pub fn new() -> MsgQueue {
        MsgQueue {
            msgs: Vec::new(),
            msg_done_time: None,
            captions: Vec::new(),
            caption_done_time: None,
        }
    }

    pub fn msg(&mut self, text: String) {
        let timeout = add_time_to_read(self.msg_done_time, text.as_slice());
        self.msgs.push(Msg::new(text, timeout));
        self.msg_done_time = Some(timeout);
    }

    pub fn caption(&mut self, text: String) {
        let timeout = add_time_to_read(self.caption_done_time, text.as_slice());
        self.captions.push(Msg::new(text, timeout));
        self.caption_done_time = Some(timeout);
    }

    fn draw_msgs(&self, ctx: &mut Context) {
        let msg_columns = 32;
        let msg_rows = 16;
        let msg_origin = V2(0, 360 - (msg_rows - 1) * 8);

        let mut writer = ctx.text_writer(msg_origin, 0.1, color::LIGHTGRAY)
            .set_border(color::BLACK);

        for msg in self.msgs.iter() {
            let _ = write!(&mut writer, "{}", text::wrap_lines(msg_columns, msg.text.as_slice()).as_slice());
        }
    }

    fn draw_caption(&self, ctx: &mut Context) {
        if !self.captions.is_empty() {
            let width = self.captions[0].text.len() as int * 8;
            let origin = V2(320 - width / 2, 180);
            // TODO: Wrap and center-justify multiline caption

            let mut writer = ctx.text_writer(origin, 0.1, color::LIGHTGRAY)
                .set_border(color::BLACK);

            let _ = write!(&mut writer, "{}", self.captions[0].text.as_slice());
        }
    }

    pub fn draw(&self, ctx: &mut Context) {
        self.draw_msgs(ctx);
        self.draw_caption(ctx);
    }

    pub fn update(&mut self) {
        let t = time::precise_time_s();
        while !self.msgs.is_empty() {
            if self.msgs[0].timeout < t {
                self.msgs.remove(0);
            } else {
                break;
            }
        }
        if self.msgs.is_empty() { self.msg_done_time = None; }

        while !self.captions.is_empty() {
            if self.captions[0].timeout < t {
                self.captions.remove(0);
            } else {
                break;
            }
        }
        if self.captions.is_empty() { self.caption_done_time = None; }
    }
}

fn add_time_to_read(old_time: Option<f64>, text: &str) -> f64 {
    // Estimated time it takes the user to read one character in seconds.
    let letter_read_duration = 0.2;
    let read_time = text.trim().len() as f64 * letter_read_duration;

    return match old_time {
        Some(t) => t,
        None => time::precise_time_s()
    } + read_time;
}

<|endoftext|>