<commit_before>use std::net::IpAddr;
use std::path::PathBuf;
use std::collections::{BTreeMap, HashMap, HashSet};
#[cfg(not(target_arch="wasm32"))] use std::os::unix::io::RawFd;
use std::ascii::AsciiExt;

use serde::de::{Deserializer, Deserialize, Error as DeError};
use serde::ser::{Serializer, Serialize};
use quire::validate::{Structure, Sequence, Scalar, Numeric, Enum};
use quire::validate::{Mapping, Nothing, Anything};
use id_map::{IdMap, IdMapExt, mapping_validator};

use sandbox_config::SandboxConfig;
use range::{in_range};
use child_config::ChildKind;

pub const DEFAULT_KILL_TIMEOUT: f32 = 5.;

#[cfg(target_arch="wasm32")] type RawFd = i32;

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct TmpfsInfo {
    pub size: usize,
    pub mode: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct PersistentInfo {
    pub path: PathBuf,
    pub mkdir: bool,
    pub mode: u32,
    pub user: u32,
    pub group: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct StatedirInfo {
    pub path: PathBuf,
    pub mode: u32,
    pub user: u32,
    pub group: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub enum Volume {
    Readonly(PathBuf),
    Persistent(PersistentInfo),
    Tmpfs(TmpfsInfo),
    Statedir(StatedirInfo),
}

#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy)]
pub enum ContainerKind {
    Daemon,
    Command,
    CommandOrDaemon,
}

impl ContainerKind {
    pub fn matches(self, child_kind: ChildKind) -> bool {
        use container_config::ContainerKind as L;
        use child_config::ChildKind as R;
        match (self, child_kind) {
            (L::Command, R::Command) => true,
            (L::Daemon, R::Daemon) => true,
            (L::CommandOrDaemon, R::Command) => true,
            (L::CommandOrDaemon, R::Daemon) => true,
            (L::Command, R::Daemon) => false,
            (L::Daemon, R::Command) => false,
        }
    }
}

#[derive(Deserialize, Serialize, Clone)]
pub struct ResolvConf {
    pub copy_from_host: bool,
}

#[derive(Deserialize, Serialize, Clone)]
pub struct HostsFile {
    pub copy_from_host: bool,
    pub localhost: Option<bool>,
    pub public_hostname: Option<bool>,
}

#[derive(Clone, Debug)]
pub struct Host(pub IpAddr);

#[derive(Deserialize, Serialize, Clone)]
pub struct TcpPort {
    pub host: Host,
    pub fd: RawFd,
    pub reuse_addr: bool,
    pub reuse_port: bool,
    pub listen_backlog: usize,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub enum Variable {
    TcpPort,
    Name,
    Choice(Vec<String>),
}

#[derive(Deserialize, Serialize)]
pub struct ContainerConfig {
    pub kind: ContainerKind,
    pub variables: BTreeMap<String, Variable>,
    pub volumes: BTreeMap<String, Volume>,
    pub user_id: u32,
    pub group_id: u32,
    pub restart_timeout: f32,
    pub kill_timeout: f32,
    pub memory_limit: u64,
    pub fileno_limit: u64,
    pub cpu_shares: usize,
    pub executable: String,
    pub arguments: Vec<String>,
    pub environ: BTreeMap<String, String>,
    pub workdir: PathBuf,
    pub resolv_conf: ResolvConf,
    pub hosts_file: HostsFile,
    pub uid_map: Vec<IdMap>,
    pub gid_map: Vec<IdMap>,
    pub stdout_stderr_file: Option<PathBuf>,
    pub interactive: bool,
    pub restart_process_only: bool,
    pub tcp_ports: HashMap<String, TcpPort>,
}

#[derive(Deserialize, Serialize)]
pub struct InstantiatedConfig {
    pub kind: ContainerKind,
    pub volumes: BTreeMap<String, Volume>,
    pub user_id: u32,
    pub group_id: u32,
    pub restart_timeout: f32,
    pub kill_timeout: f32,
    pub memory_limit: u64,
    pub fileno_limit: u64,
    pub cpu_shares: usize,
    pub executable: String,
    pub arguments: Vec<String>,
    pub environ: BTreeMap<String, String>,
    pub workdir: PathBuf,
    pub resolv_conf: ResolvConf,
    pub hosts_file: HostsFile,
    pub uid_map: Vec<IdMap>,
    pub gid_map: Vec<IdMap>,
    pub stdout_stderr_file: Option<PathBuf>,
    pub interactive: bool,
    pub restart_process_only: bool,
    pub tcp_ports: HashMap<u16, TcpPort>,
}

pub struct Variables<'a> {
    pub user_vars: &'a HashMap<String, String>,
    pub lithos_name: &'a str,
    pub lithos_config_filename: &'a str,
}

impl InstantiatedConfig {
    pub fn map_uid(&self, internal_uid: u32) -> Option<u32> {
        self.uid_map.map_id(internal_uid)
    }
    pub fn map_gid(&self, internal_gid: u32) -> Option<u32> {
        self.gid_map.map_id(internal_gid)
    }
}

impl ContainerConfig {
    pub fn validator<'x>() -> Structure<'x> {
        Structure::new()
        .member("kind", Scalar::new().default("Daemon"))
        .member("variables", Mapping::new(
            Scalar::new(),
            Enum::new()
                .option("TcpPort", Nothing)
                .option("Name", Nothing)
                .option("Choice", Sequence::new(Scalar::new()))
        ))
        .member("metadata", Anything)
        .member("volumes", Mapping::new(
                Scalar::new(),
                volume_validator()))
        .member("user_id", Numeric::new())
        .member("group_id", Numeric::new().default(0))
        .member("memory_limit", Numeric::new().default(0x7fffffffffffffffi64))
        .member("fileno_limit", Numeric::new().default(1024))
        .member("cpu_shares", Numeric::new().default(1024))
        .member("restart_timeout", Numeric::new().min(0).max(86400).default(1))
        .member("kill_timeout",
            Numeric::new().min(0).max(86400)
                .default(DEFAULT_KILL_TIMEOUT as i64))
        .member("executable", Scalar::new())
        .member("arguments", Sequence::new(Scalar::new()))
        .member("environ", Mapping::new(
                Scalar::new(),
                Scalar::new()))
        .member("workdir", Scalar::new().default("/"))
        .member("resolv_conf", Structure::new()
            .member("copy_from_host", Scalar::new().default(true)))
        .member("hosts_file", Structure::new()
            .member("copy_from_host", Scalar::new().default(true))
            .member("localhost", Scalar::new().optional())
            .member("public_hostname", Scalar::new().optional()))
        .member("uid_map", mapping_validator())
        .member("gid_map", mapping_validator())
        .member("stdout_stderr_file", Scalar::new().optional())
        .member("interactive", Scalar::new().default(false))
        .member("restart_process_only", Scalar::new().default(false))
        .member("tcp_ports", Mapping::new(
            Scalar::new(),
            Structure::new()
                .member("host", Scalar::new().default("0.0.0.0"))
                .member("fd", Numeric::new().min(0).optional())
                .member("reuse_addr", Scalar::new().default(true))
                .member("reuse_port", Scalar::new().default(false))
                .member("listen_backlog", Scalar::new().default(128))
            ))
    }
    pub fn instantiate(&self, variables: &Variables)
        -> Result<InstantiatedConfig, Vec<String>>
    {
        let mut errors1 = HashSet::new();
        let mut errors2 = HashSet::new();
        let result = {
            let mut replacer = |varname: &str| {
                let val = variables.user_vars.get(varname).map(|x| x.clone())
                    .or_else(|| match varname {
                        "lithos:name"
                        => Some(variables.lithos_name.to_string()),
                        "lithos:config_filename"
                        => Some(variables.lithos_config_filename.to_string()),
                        _ => None,
                    });
                match val {
                    Some(x) => x,
                    None => {
                        errors1.insert(format!("unknown variable {:?}", varname));
                        return format!("<<no var {:?}>>", varname);
                    }
                }
            };
            InstantiatedConfig {
                kind: self.kind.clone(),
                volumes: self.volumes.clone(),
                user_id: self.user_id.clone(),
                group_id: self.group_id.clone(),
                restart_timeout: self.restart_timeout.clone(),
                kill_timeout: self.kill_timeout.clone(),
                memory_limit: self.memory_limit.clone(),
                fileno_limit: self.fileno_limit.clone(),
                cpu_shares: self.cpu_shares.clone(),
                executable: self.executable.clone(),
                arguments: self.arguments.iter()
                    .map(|x| replace_vars(&x, &mut replacer).into())
                    .collect(),
                environ: self.environ.iter()
                    .map(|(key, val)| {
                        (key.clone(),
                         replace_vars(&val, &mut replacer).into())
                    })
                    .collect(),
                workdir: self.workdir.clone(),
                resolv_conf: self.resolv_conf.clone(),
                hosts_file: self.hosts_file.clone(),
                uid_map: self.uid_map.clone(),
                gid_map: self.gid_map.clone(),
                stdout_stderr_file: self.stdout_stderr_file.clone(),
                interactive: self.interactive.clone(),
                restart_process_only: self.restart_process_only.clone(),
                tcp_ports: self.tcp_ports.iter()
                    .map(|(key, val)| {
                        let s = replace_vars(&key, &mut replacer);
                        let port = match s.parse::<u16>() {
                            Ok(x) => x,
                            Err(e) => {
                                errors2.insert(format!("Bad port {:?}: {}",
                                    key, e));
                                return (0, val.clone());
                            }
                        };
                        (port, val.clone())
                    })
                    .collect(),
            }
        };
        if errors1.len() > 0 || errors2.len() > 0 {
            return Err(errors1.into_iter().chain(errors2.into_iter())
                       .collect());
        } else {
            return Ok(result);
        }
    }
}

pub fn volume_validator<'x>() -> Enum<'x> {
    Enum::new()
    .option("Persistent",  Structure::new()
        .member("path",  Scalar::new().default("/"))
        .member("mkdir",  Scalar::new().default(false))
        .member("mode",  Numeric::new().min(0).max(0o1777).default(0o777))
        .member("user",  Numeric::new().default(0))
        .member("group",  Numeric::new().default(0)))
    .option("Readonly", Scalar::new())
    .option("Tmpfs", Structure::new()
        .member("size", Numeric::new().min(0).default(100*1024*1024))
        .member("mode", Numeric::new().min(0).max(0o1777).default(0o777)))
    .option("Statedir", Structure::new()
        .member("path", Scalar::new().default("/"))
        .member("mode", Numeric::new().min(0).max(0o1777).default(0o777))
        .member("user", Numeric::new().default(0))
        .member("group", Numeric::new().default(0)))
}

impl<'a> Deserialize<'a> for Host {
    fn deserialize<D: Deserializer<'a>>(d: D) -> Result<Host, D::Error> {
        String::deserialize(d)?.parse().map(Host)
            .map_err(|x| D::Error::custom(format!("{}", x)))
    }
}

impl Serialize for Host {
    fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
        format!("{}", self.0).serialize(s)
    }
}

impl Variable {
    pub fn validate(&self, value: &str, sandbox: &SandboxConfig)
        -> Result<(), String>
    {
        match *self {
            Variable::TcpPort => {
                let port = value.parse::<u16>()
                    .map_err(|e| format!(
                        "invalid TcpPort {:?}: {}", value, e))?;
                if !in_range(&sandbox.allow_tcp_ports, port as u32) {
                    return Err(format!(
                        "TcpPort {:?} is not in allowed range", port));
                }
            }
            Variable::Name => {
                let chars_ok = value.chars().all(|x| {
                    x.is_ascii() && x.is_alphanumeric() || x == '-' || x == '_'
                });
                if !chars_ok {
                    return Err(format!("Value {:?} contains characters that \
                        are invalid for names (alphanumeric, `-` and `_`)",
                        value));
                }
            }
            Variable::Choice(ref choices) => {
                if !choices.iter().any(|x| x == value) {
                    return Err(format!("variable value {:?} \
                        is not one of {:?}", value, choices));
                }
            }
        }
        Ok(())
    }
}

fn replace_vars<F, S>(mut s: &str, mut f: F)
    -> String
    where F: FnMut(&str) -> S,
          S: AsRef<str>,
{
    let mut result = String::with_capacity(s.len());
    while let Some(vpos) = s.find("@{") {
        result.push_str(&s[..vpos]);
        s = &s[vpos..];
        if let Some(vend) = s.find('}') {
            let var = s[..vend].trim();
            result.push_str(f(var).as_ref());
            s = &s[vend+1..];
        } else {
            break;  // unclosed vars are just raw text
        }
    }
    result.push_str(s);
    return result;
}

#[cfg(test)]
mod test {
    use super::replace_vars;

    #[test]
    fn just_var() {
        assert_eq!(replace_vars("@{x}", |_| "1"), "1");
    }

    #[test]
    fn suffix() {
        assert_eq!(replace_vars("xxx@{x}", |_| "1"), "xxx1");
    }

    #[test]
    fn prefix() {
        assert_eq!(replace_vars("@{yy}zzz", |_| "1"), "1zzz");
    }

    #[test]
    fn middle() {
        assert_eq!(replace_vars("aaa@{yy}zzz", |_| "1"), "aaa1zzz");
    }
    #[test]
    fn two_vars() {
        assert_eq!(replace_vars("one @{x} two @{ y } three", |_| "1"),
            "one 1 two 1 three");
    }
}
<commit_msg>Parse metadata in container config as Json<commit_after>use std::net::IpAddr;
use std::path::PathBuf;
use std::collections::{BTreeMap, HashMap, HashSet};
#[cfg(not(target_arch="wasm32"))] use std::os::unix::io::RawFd;
use std::ascii::AsciiExt;

use serde::de::{Deserializer, Deserialize, Error as DeError};
use serde::ser::{Serializer, Serialize};
use serde_json::Value as Json;
use quire::validate::{Structure, Sequence, Scalar, Numeric, Enum};
use quire::validate::{Mapping, Nothing, Anything};
use id_map::{IdMap, IdMapExt, mapping_validator};

use sandbox_config::SandboxConfig;
use range::{in_range};
use child_config::ChildKind;

pub const DEFAULT_KILL_TIMEOUT: f32 = 5.;

#[cfg(target_arch="wasm32")] type RawFd = i32;

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct TmpfsInfo {
    pub size: usize,
    pub mode: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct PersistentInfo {
    pub path: PathBuf,
    pub mkdir: bool,
    pub mode: u32,
    pub user: u32,
    pub group: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct StatedirInfo {
    pub path: PathBuf,
    pub mode: u32,
    pub user: u32,
    pub group: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub enum Volume {
    Readonly(PathBuf),
    Persistent(PersistentInfo),
    Tmpfs(TmpfsInfo),
    Statedir(StatedirInfo),
}

#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy)]
pub enum ContainerKind {
    Daemon,
    Command,
    CommandOrDaemon,
}

impl ContainerKind {
    pub fn matches(self, child_kind: ChildKind) -> bool {
        use container_config::ContainerKind as L;
        use child_config::ChildKind as R;
        match (self, child_kind) {
            (L::Command, R::Command) => true,
            (L::Daemon, R::Daemon) => true,
            (L::CommandOrDaemon, R::Command) => true,
            (L::CommandOrDaemon, R::Daemon) => true,
            (L::Command, R::Daemon) => false,
            (L::Daemon, R::Command) => false,
        }
    }
}

#[derive(Deserialize, Serialize, Clone)]
pub struct ResolvConf {
    pub copy_from_host: bool,
}

#[derive(Deserialize, Serialize, Clone)]
pub struct HostsFile {
    pub copy_from_host: bool,
    pub localhost: Option<bool>,
    pub public_hostname: Option<bool>,
}

#[derive(Clone, Debug)]
pub struct Host(pub IpAddr);

#[derive(Deserialize, Serialize, Clone)]
pub struct TcpPort {
    pub host: Host,
    pub fd: RawFd,
    pub reuse_addr: bool,
    pub reuse_port: bool,
    pub listen_backlog: usize,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub enum Variable {
    TcpPort,
    Name,
    Choice(Vec<String>),
}

#[derive(Deserialize, Serialize)]
pub struct ContainerConfig {
    pub kind: ContainerKind,
    pub variables: BTreeMap<String, Variable>,
    pub metadata: Json,
    pub volumes: BTreeMap<String, Volume>,
    pub user_id: u32,
    pub group_id: u32,
    pub restart_timeout: f32,
    pub kill_timeout: f32,
    pub memory_limit: u64,
    pub fileno_limit: u64,
    pub cpu_shares: usize,
    pub executable: String,
    pub arguments: Vec<String>,
    pub environ: BTreeMap<String, String>,
    pub workdir: PathBuf,
    pub resolv_conf: ResolvConf,
    pub hosts_file: HostsFile,
    pub uid_map: Vec<IdMap>,
    pub gid_map: Vec<IdMap>,
    pub stdout_stderr_file: Option<PathBuf>,
    pub interactive: bool,
    pub restart_process_only: bool,
    pub tcp_ports: HashMap<String, TcpPort>,
}

#[derive(Deserialize, Serialize)]
pub struct InstantiatedConfig {
    pub kind: ContainerKind,
    pub volumes: BTreeMap<String, Volume>,
    pub user_id: u32,
    pub group_id: u32,
    pub restart_timeout: f32,
    pub kill_timeout: f32,
    pub memory_limit: u64,
    pub fileno_limit: u64,
    pub cpu_shares: usize,
    pub executable: String,
    pub arguments: Vec<String>,
    pub environ: BTreeMap<String, String>,
    pub workdir: PathBuf,
    pub resolv_conf: ResolvConf,
    pub hosts_file: HostsFile,
    pub uid_map: Vec<IdMap>,
    pub gid_map: Vec<IdMap>,
    pub stdout_stderr_file: Option<PathBuf>,
    pub interactive: bool,
    pub restart_process_only: bool,
    pub tcp_ports: HashMap<u16, TcpPort>,
}

pub struct Variables<'a> {
    pub user_vars: &'a HashMap<String, String>,
    pub lithos_name: &'a str,
    pub lithos_config_filename: &'a str,
}

impl InstantiatedConfig {
    pub fn map_uid(&self, internal_uid: u32) -> Option<u32> {
        self.uid_map.map_id(internal_uid)
    }
    pub fn map_gid(&self, internal_gid: u32) -> Option<u32> {
        self.gid_map.map_id(internal_gid)
    }
}

impl ContainerConfig {
    pub fn validator<'x>() -> Structure<'x> {
        Structure::new()
        .member("kind", Scalar::new().default("Daemon"))
        .member("variables", Mapping::new(
            Scalar::new(),
            Enum::new()
                .option("TcpPort", Nothing)
                .option("Name", Nothing)
                .option("Choice", Sequence::new(Scalar::new()))
        ))
        .member("metadata", Anything)
        .member("volumes", Mapping::new(
                Scalar::new(),
                volume_validator()))
        .member("user_id", Numeric::new())
        .member("group_id", Numeric::new().default(0))
        .member("memory_limit", Numeric::new().default(0x7fffffffffffffffi64))
        .member("fileno_limit", Numeric::new().default(1024))
        .member("cpu_shares", Numeric::new().default(1024))
        .member("restart_timeout", Numeric::new().min(0).max(86400).default(1))
        .member("kill_timeout",
            Numeric::new().min(0).max(86400)
                .default(DEFAULT_KILL_TIMEOUT as i64))
        .member("executable", Scalar::new())
        .member("arguments", Sequence::new(Scalar::new()))
        .member("environ", Mapping::new(
                Scalar::new(),
                Scalar::new()))
        .member("workdir", Scalar::new().default("/"))
        .member("resolv_conf", Structure::new()
            .member("copy_from_host", Scalar::new().default(true)))
        .member("hosts_file", Structure::new()
            .member("copy_from_host", Scalar::new().default(true))
            .member("localhost", Scalar::new().optional())
            .member("public_hostname", Scalar::new().optional()))
        .member("uid_map", mapping_validator())
        .member("gid_map", mapping_validator())
        .member("stdout_stderr_file", Scalar::new().optional())
        .member("interactive", Scalar::new().default(false))
        .member("restart_process_only", Scalar::new().default(false))
        .member("tcp_ports", Mapping::new(
            Scalar::new(),
            Structure::new()
                .member("host", Scalar::new().default("0.0.0.0"))
                .member("fd", Numeric::new().min(0).optional())
                .member("reuse_addr", Scalar::new().default(true))
                .member("reuse_port", Scalar::new().default(false))
                .member("listen_backlog", Scalar::new().default(128))
            ))
    }
    pub fn instantiate(&self, variables: &Variables)
        -> Result<InstantiatedConfig, Vec<String>>
    {
        let mut errors1 = HashSet::new();
        let mut errors2 = HashSet::new();
        let result = {
            let mut replacer = |varname: &str| {
                let val = variables.user_vars.get(varname).map(|x| x.clone())
                    .or_else(|| match varname {
                        "lithos:name"
                        => Some(variables.lithos_name.to_string()),
                        "lithos:config_filename"
                        => Some(variables.lithos_config_filename.to_string()),
                        _ => None,
                    });
                match val {
                    Some(x) => x,
                    None => {
                        errors1.insert(format!("unknown variable {:?}", varname));
                        return format!("<<no var {:?}>>", varname);
                    }
                }
            };
            InstantiatedConfig {
                kind: self.kind.clone(),
                volumes: self.volumes.clone(),
                user_id: self.user_id.clone(),
                group_id: self.group_id.clone(),
                restart_timeout: self.restart_timeout.clone(),
                kill_timeout: self.kill_timeout.clone(),
                memory_limit: self.memory_limit.clone(),
                fileno_limit: self.fileno_limit.clone(),
                cpu_shares: self.cpu_shares.clone(),
                executable: self.executable.clone(),
                arguments: self.arguments.iter()
                    .map(|x| replace_vars(&x, &mut replacer).into())
                    .collect(),
                environ: self.environ.iter()
                    .map(|(key, val)| {
                        (key.clone(),
                         replace_vars(&val, &mut replacer).into())
                    })
                    .collect(),
                workdir: self.workdir.clone(),
                resolv_conf: self.resolv_conf.clone(),
                hosts_file: self.hosts_file.clone(),
                uid_map: self.uid_map.clone(),
                gid_map: self.gid_map.clone(),
                stdout_stderr_file: self.stdout_stderr_file.clone(),
                interactive: self.interactive.clone(),
                restart_process_only: self.restart_process_only.clone(),
                tcp_ports: self.tcp_ports.iter()
                    .map(|(key, val)| {
                        let s = replace_vars(&key, &mut replacer);
                        let port = match s.parse::<u16>() {
                            Ok(x) => x,
                            Err(e) => {
                                errors2.insert(format!("Bad port {:?}: {}",
                                    key, e));
                                return (0, val.clone());
                            }
                        };
                        (port, val.clone())
                    })
                    .collect(),
            }
        };
        if errors1.len() > 0 || errors2.len() > 0 {
            return Err(errors1.into_iter().chain(errors2.into_iter())
                       .collect());
        } else {
            return Ok(result);
        }
    }
}

pub fn volume_validator<'x>() -> Enum<'x> {
    Enum::new()
    .option("Persistent",  Structure::new()
        .member("path",  Scalar::new().default("/"))
        .member("mkdir",  Scalar::new().default(false))
        .member("mode",  Numeric::new().min(0).max(0o1777).default(0o777))
        .member("user",  Numeric::new().default(0))
        .member("group",  Numeric::new().default(0)))
    .option("Readonly", Scalar::new())
    .option("Tmpfs", Structure::new()
        .member("size", Numeric::new().min(0).default(100*1024*1024))
        .member("mode", Numeric::new().min(0).max(0o1777).default(0o777)))
    .option("Statedir", Structure::new()
        .member("path", Scalar::new().default("/"))
        .member("mode", Numeric::new().min(0).max(0o1777).default(0o777))
        .member("user", Numeric::new().default(0))
        .member("group", Numeric::new().default(0)))
}

impl<'a> Deserialize<'a> for Host {
    fn deserialize<D: Deserializer<'a>>(d: D) -> Result<Host, D::Error> {
        String::deserialize(d)?.parse().map(Host)
            .map_err(|x| D::Error::custom(format!("{}", x)))
    }
}

impl Serialize for Host {
    fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
        format!("{}", self.0).serialize(s)
    }
}

impl Variable {
    pub fn validate(&self, value: &str, sandbox: &SandboxConfig)
        -> Result<(), String>
    {
        match *self {
            Variable::TcpPort => {
                let port = value.parse::<u16>()
                    .map_err(|e| format!(
                        "invalid TcpPort {:?}: {}", value, e))?;
                if !in_range(&sandbox.allow_tcp_ports, port as u32) {
                    return Err(format!(
                        "TcpPort {:?} is not in allowed range", port));
                }
            }
            Variable::Name => {
                let chars_ok = value.chars().all(|x| {
                    x.is_ascii() && x.is_alphanumeric() || x == '-' || x == '_'
                });
                if !chars_ok {
                    return Err(format!("Value {:?} contains characters that \
                        are invalid for names (alphanumeric, `-` and `_`)",
                        value));
                }
            }
            Variable::Choice(ref choices) => {
                if !choices.iter().any(|x| x == value) {
                    return Err(format!("variable value {:?} \
                        is not one of {:?}", value, choices));
                }
            }
        }
        Ok(())
    }
}

fn replace_vars<F, S>(mut s: &str, mut f: F)
    -> String
    where F: FnMut(&str) -> S,
          S: AsRef<str>,
{
    let mut result = String::with_capacity(s.len());
    while let Some(vpos) = s.find("@{") {
        result.push_str(&s[..vpos]);
        s = &s[vpos..];
        if let Some(vend) = s.find('}') {
            let var = s[..vend].trim();
            result.push_str(f(var).as_ref());
            s = &s[vend+1..];
        } else {
            break;  // unclosed vars are just raw text
        }
    }
    result.push_str(s);
    return result;
}

#[cfg(test)]
mod test {
    use super::replace_vars;

    #[test]
    fn just_var() {
        assert_eq!(replace_vars("@{x}", |_| "1"), "1");
    }

    #[test]
    fn suffix() {
        assert_eq!(replace_vars("xxx@{x}", |_| "1"), "xxx1");
    }

    #[test]
    fn prefix() {
        assert_eq!(replace_vars("@{yy}zzz", |_| "1"), "1zzz");
    }

    #[test]
    fn middle() {
        assert_eq!(replace_vars("aaa@{yy}zzz", |_| "1"), "aaa1zzz");
    }
    #[test]
    fn two_vars() {
        assert_eq!(replace_vars("one @{x} two @{ y } three", |_| "1"),
            "one 1 two 1 three");
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate kiss_ui;

use kiss_ui::callback::OnShow;
use kiss_ui::container::Vertical;
use kiss_ui::dialog::Dialog;
use kiss_ui::progress::ProgressBar;
use kiss_ui::text::Label;
use kiss_ui::timer::Timer;

fn main() {
    kiss_ui::show_gui(|| {
        let regular = ProgressBar::new();
        let dashed = ProgressBar::new().set_dashed(true);

        let dialog = Dialog::new(
            Vertical::new(
                children![
                    Label::new("Regular:"),
                    regular.clone(),
                    Label::new("Dashed:"),
                    dashed.clone(),
                    Label::new("Indefinite:"),
                    ProgressBar::new().set_indefinite(true),
                ]
            )
        );

        dialog
            .set_title("Progressbar Test")
            .set_on_show(move |_| {
                let regular = regular;
                let dashed = dashed;

                let on_timer_interval = move |timer: Timer|{
                    regular.add_value(0.1);
                    dashed.add_value(0.1);

                    if regular.get_value() == 1.0 {
                        timer.stop();
                    }
                };

                Timer::new()
                    .set_interval(1000)
                    .set_on_interval(on_timer_interval)
                    .start();
            });

        dialog
    });
}

<commit_msg>Cut a couple lines off progress_test<commit_after>#[macro_use]
extern crate kiss_ui;

use kiss_ui::callback::OnShow;
use kiss_ui::container::Vertical;
use kiss_ui::dialog::Dialog;
use kiss_ui::progress::ProgressBar;
use kiss_ui::text::Label;
use kiss_ui::timer::Timer;

fn main() {
    kiss_ui::show_gui(|| {
        let regular = ProgressBar::new();
        let dashed = ProgressBar::new().set_dashed(true);

        let dialog = Dialog::new(
            Vertical::new(
                children![
                    Label::new("Regular:"),
                    regular.clone(),
                    Label::new("Dashed:"),
                    dashed.clone(),
                    Label::new("Indefinite:"),
                    ProgressBar::new().set_indefinite(true),
                ]
            )
        );

        dialog
            .set_title("Progressbar Test")
            .set_on_show(move |_| {
                let on_timer_interval = move |timer: Timer|{
                    regular.add_value(0.1);
                    dashed.add_value(0.1);

                    if regular.get_value() == 1.0 {
                        timer.stop();
                    }
                };

                Timer::new()
                    .set_interval(1000)
                    .set_on_interval(on_timer_interval)
                    .start();
            });

        dialog
    });
}

<|endoftext|>


<commit_before>use crate::common::{BinarySerializable, CountingWriter, FixedSize, VInt};
use crate::directory::error::Incompatibility;
use crate::directory::read_only_source::ReadOnlySource;
use crate::directory::{AntiCallToken, TerminatingWrite};
use crate::Version;
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use crc32fast::Hasher;
use std::io;
use std::io::Write;

const FOOTER_MAX_LEN: usize = 10_000;

type CrcHashU32 = u32;

#[derive(Debug, Clone, PartialEq)]
pub struct Footer {
    pub version: Version,
    pub meta: String,
    pub versioned_footer: VersionedFooter,
}

/// Serialises the footer to a byte-array
/// - versioned_footer_len : 4 bytes
///-  versioned_footer: variable bytes
/// - meta_len: 4 bytes
/// - meta: variable bytes
/// - version_len: 4 bytes
/// - version json: variable bytes
impl BinarySerializable for Footer {
    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
        BinarySerializable::serialize(&self.versioned_footer, writer)?;
        BinarySerializable::serialize(&self.meta, writer)?;
        let version_string =
            serde_json::to_string(&self.version).map_err(|_err| io::ErrorKind::InvalidInput)?;
        BinarySerializable::serialize(&version_string, writer)?;
        Ok(())
    }

    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
        let versioned_footer = VersionedFooter::deserialize(reader)?;
        let meta = String::deserialize(reader)?;
        let version_json = String::deserialize(reader)?;
        let version = serde_json::from_str(&version_json)?;
        Ok(Footer {
            version,
            meta,
            versioned_footer,
        })
    }
}

impl Footer {
    pub fn new(versioned_footer: VersionedFooter) -> Self {
        let version = crate::VERSION.clone();
        let meta = version.to_string();
        Footer {
            version,
            meta,
            versioned_footer,
        }
    }

    pub fn append_footer<W: io::Write>(&self, mut write: &mut W) -> io::Result<()> {
        let mut counting_write = CountingWriter::wrap(&mut write);
        self.serialize(&mut counting_write)?;
        let written_len = counting_write.written_bytes();
        write.write_u32::<LittleEndian>(written_len as u32)?;
        Ok(())
    }

    pub fn extract_footer(source: ReadOnlySource) -> Result<(Footer, ReadOnlySource), io::Error> {
        if source.len() < 4 {
            return Err(io::Error::new(
                io::ErrorKind::UnexpectedEof,
                format!(
                    "File corrupted. The file is smaller than 4 bytes (len={}).",
                    source.len()
                ),
            ));
        }
        let (body_footer, footer_len_bytes) = source.split_from_end(u32::SIZE_IN_BYTES);
        let footer_len = LittleEndian::read_u32(footer_len_bytes.as_slice()) as usize;
        let body_len = body_footer.len() - footer_len;
        let (body, footer_data) = body_footer.split(body_len);
        let mut cursor = footer_data.as_slice();
        let footer = Footer::deserialize(&mut cursor)?;
        Ok((footer, body))
    }

    /// Confirms that the index will be read correctly by this version of tantivy
    /// Has to be called after `extract_footer` to make sure it's not accessing uninitialised memory
    pub fn is_compatible(&self) -> Result<(), Incompatibility> {
        let library_version = crate::version();
        match &self.versioned_footer {
            VersionedFooter::V1 {
                crc32: _crc,
                store_compression: compression,
            } => {
                if &library_version.store_compression != compression {
                    return Err(Incompatibility::CompressionMismatch {
                        library_compression_format: library_version.store_compression.to_string(),
                        index_compression_format: compression.to_string(),
                    });
                }
                Ok(())
            }
            VersionedFooter::UnknownVersion => Err(Incompatibility::IndexMismatch {
                library_version: library_version.clone(),
                index_version: self.version.clone(),
            }),
        }
    }
}

/// Footer that includes a crc32 hash that enables us to checksum files in the index
#[derive(Debug, Clone, PartialEq)]
pub enum VersionedFooter {
    UnknownVersion,
    V1 {
        crc32: CrcHashU32,
        store_compression: String,
    },
}

impl BinarySerializable for VersionedFooter {
    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
        let mut buf = Vec::new();
        match self {
            VersionedFooter::V1 {
                crc32,
                store_compression: compression,
            } => {
                // Serializes a valid `VersionedFooter` or panics if the version is unknown
                // [   version    |   crc_hash  | compression_mode ]
                // [    0..4      |     4..8    |     variable     ]
                BinarySerializable::serialize(&1u32, &mut buf)?;
                BinarySerializable::serialize(crc32, &mut buf)?;
                BinarySerializable::serialize(compression, &mut buf)?;
            }
            VersionedFooter::UnknownVersion => {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidInput,
                    "Cannot serialize an unknown versioned footer ",
                ));
            }
        }
        BinarySerializable::serialize(&VInt(buf.len() as u64), writer)?;
        assert!(buf.len() <= FOOTER_MAX_LEN);
        writer.write_all(&buf[..])?;
        Ok(())
    }

    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
        let len = VInt::deserialize(reader)?.0 as usize;
        if len > FOOTER_MAX_LEN {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!(
                    "Footer seems invalid as it suggests a footer len of {}. File is corrupted, \
            or the index was created with a different & old version of tantivy.",
                    len
                ),
            ));
        }
        let mut buf = vec![0u8; len];
        reader.read_exact(&mut buf[..])?;
        let mut cursor = &buf[..];
        let version = u32::deserialize(&mut cursor)?;
        if version == 1 {
            let crc32 = u32::deserialize(&mut cursor)?;
            let compression = String::deserialize(&mut cursor)?;
            Ok(VersionedFooter::V1 {
                crc32,
                store_compression: compression,
            })
        } else {
            Ok(VersionedFooter::UnknownVersion)
        }
    }
}

impl VersionedFooter {
    pub fn crc(&self) -> Option<CrcHashU32> {
        match self {
            VersionedFooter::V1 { crc32, .. } => Some(*crc32),
            VersionedFooter::UnknownVersion { .. } => None,
        }
    }
}

pub(crate) struct FooterProxy<W: TerminatingWrite> {
    /// always Some except after terminate call
    hasher: Option<Hasher>,
    /// always Some except after terminate call
    writer: Option<W>,
}

impl<W: TerminatingWrite> FooterProxy<W> {
    pub fn new(writer: W) -> Self {
        FooterProxy {
            hasher: Some(Hasher::new()),
            writer: Some(writer),
        }
    }
}

impl<W: TerminatingWrite> Write for FooterProxy<W> {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        let count = self.writer.as_mut().unwrap().write(buf)?;
        self.hasher.as_mut().unwrap().update(&buf[..count]);
        Ok(count)
    }

    fn flush(&mut self) -> io::Result<()> {
        self.writer.as_mut().unwrap().flush()
    }
}

impl<W: TerminatingWrite> TerminatingWrite for FooterProxy<W> {
    fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
        let crc32 = self.hasher.take().unwrap().finalize();
        let footer = Footer::new(VersionedFooter::V1 {
            crc32,
            store_compression: crate::store::COMPRESSION.to_string(),
        });
        let mut writer = self.writer.take().unwrap();
        footer.append_footer(&mut writer)?;
        writer.terminate()
    }
}

#[cfg(test)]
mod tests {

    use super::CrcHashU32;
    use super::FooterProxy;
    use crate::common::{BinarySerializable, VInt};
    use crate::directory::footer::{Footer, VersionedFooter};
    use crate::directory::TerminatingWrite;
    use byteorder::{ByteOrder, LittleEndian};
    use regex::Regex;
    use std::error::Error;
    use std::io;

    #[test]
    fn test_versioned_footer() {
        let mut vec = Vec::new();
        let footer_proxy = FooterProxy::new(&mut vec);
        assert!(footer_proxy.terminate().is_ok());
        assert_eq!(vec.len(), 167);
        let footer = Footer::deserialize(&mut &vec[..]).unwrap();
        if let VersionedFooter::V1 {
            crc32: _,
            store_compression,
        } = footer.versioned_footer
        {
            assert_eq!(store_compression, crate::store::COMPRESSION);
        } else {
            panic!("Versioned footer should be V1.");
        }
        assert_eq!(&footer.version, crate::version());
    }

    #[test]
    fn test_serialize_deserialize_footer() {
        let mut buffer = Vec::new();
        let crc32 = 123456u32;
        let footer: Footer = Footer::new(VersionedFooter::V1 {
            crc32,
            store_compression: "lz4".to_string(),
        });
        footer.serialize(&mut buffer).unwrap();
        let footer_deser = Footer::deserialize(&mut &buffer[..]).unwrap();
        assert_eq!(footer_deser, footer);
    }

    #[test]
    fn footer_length() {
        let crc32 = 1111111u32;
        let versioned_footer = VersionedFooter::V1 {
            crc32,
            store_compression: "lz4".to_string(),
        };
        let mut buf = Vec::new();
        versioned_footer.serialize(&mut buf).unwrap();
        assert_eq!(buf.len(), 13);
        let footer = Footer::new(versioned_footer);
        let regex_ptn = Regex::new(
            "tantivy v[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.{0,10}, index_format v[0-9]{1,5}",
        )
        .unwrap();
        assert!(regex_ptn.is_match(&footer.meta));
    }

    #[test]
    fn versioned_footer_from_bytes() {
        let v_footer_bytes = vec![
            // versionned footer length
            12 | 128,
            // index format version
            1,
            0,
            0,
            0,
            // crc 32
            12,
            35,
            89,
            18,
            // compression format
            3 | 128,
            b'l',
            b'z',
            b'4',
        ];
        let mut cursor = &v_footer_bytes[..];
        let versioned_footer = VersionedFooter::deserialize(&mut cursor).unwrap();
        assert!(cursor.is_empty());
        let expected_crc: u32 = LittleEndian::read_u32(&v_footer_bytes[5..9]) as CrcHashU32;
        let expected_versioned_footer: VersionedFooter = VersionedFooter::V1 {
            crc32: expected_crc,
            store_compression: "lz4".to_string(),
        };
        assert_eq!(versioned_footer, expected_versioned_footer);
        let mut buffer = Vec::new();
        assert!(versioned_footer.serialize(&mut buffer).is_ok());
        assert_eq!(&v_footer_bytes[..], &buffer[..]);
    }

    #[test]
    fn versioned_footer_panic() {
        let v_footer_bytes = vec![6u8 | 128u8, 3u8, 0u8, 0u8, 1u8, 0u8, 0u8];
        let mut b = &v_footer_bytes[..];
        let versioned_footer = VersionedFooter::deserialize(&mut b).unwrap();
        assert!(b.is_empty());
        let expected_versioned_footer = VersionedFooter::UnknownVersion;
        assert_eq!(versioned_footer, expected_versioned_footer);
        let mut buf = Vec::new();
        assert!(versioned_footer.serialize(&mut buf).is_err());
    }

    #[test]
    #[cfg(not(feature = "lz4"))]
    fn compression_mismatch() {
        let crc32 = 1111111u32;
        let versioned_footer = VersionedFooter::V1 {
            crc32,
            store_compression: "lz4".to_string(),
        };
        let footer = Footer::new(versioned_footer);
        let res = footer.is_compatible();
        assert!(res.is_err());
    }

    #[test]
    fn test_deserialize_too_large_footer() {
        let mut buf = vec![];
        assert!(FooterProxy::new(&mut buf).terminate().is_ok());
        let mut long_len_buf = [0u8; 10];
        let num_bytes = VInt(super::FOOTER_MAX_LEN as u64 + 1u64).serialize_into(&mut long_len_buf);
        buf[0..num_bytes].copy_from_slice(&long_len_buf[..num_bytes]);
        let err = Footer::deserialize(&mut &buf[..]).unwrap_err();
        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
        assert_eq!(
            err.description(),
            "Footer seems invalid as it suggests a footer len of 10001. File is corrupted, \
            or the index was created with a different & old version of tantivy."
        );
    }
}
<commit_msg>Removing `err.description()` before deprecation.<commit_after>use crate::common::{BinarySerializable, CountingWriter, FixedSize, VInt};
use crate::directory::error::Incompatibility;
use crate::directory::read_only_source::ReadOnlySource;
use crate::directory::{AntiCallToken, TerminatingWrite};
use crate::Version;
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use crc32fast::Hasher;
use std::io;
use std::io::Write;

const FOOTER_MAX_LEN: usize = 10_000;

type CrcHashU32 = u32;

#[derive(Debug, Clone, PartialEq)]
pub struct Footer {
    pub version: Version,
    pub meta: String,
    pub versioned_footer: VersionedFooter,
}

/// Serialises the footer to a byte-array
/// - versioned_footer_len : 4 bytes
///-  versioned_footer: variable bytes
/// - meta_len: 4 bytes
/// - meta: variable bytes
/// - version_len: 4 bytes
/// - version json: variable bytes
impl BinarySerializable for Footer {
    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
        BinarySerializable::serialize(&self.versioned_footer, writer)?;
        BinarySerializable::serialize(&self.meta, writer)?;
        let version_string =
            serde_json::to_string(&self.version).map_err(|_err| io::ErrorKind::InvalidInput)?;
        BinarySerializable::serialize(&version_string, writer)?;
        Ok(())
    }

    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
        let versioned_footer = VersionedFooter::deserialize(reader)?;
        let meta = String::deserialize(reader)?;
        let version_json = String::deserialize(reader)?;
        let version = serde_json::from_str(&version_json)?;
        Ok(Footer {
            version,
            meta,
            versioned_footer,
        })
    }
}

impl Footer {
    pub fn new(versioned_footer: VersionedFooter) -> Self {
        let version = crate::VERSION.clone();
        let meta = version.to_string();
        Footer {
            version,
            meta,
            versioned_footer,
        }
    }

    pub fn append_footer<W: io::Write>(&self, mut write: &mut W) -> io::Result<()> {
        let mut counting_write = CountingWriter::wrap(&mut write);
        self.serialize(&mut counting_write)?;
        let written_len = counting_write.written_bytes();
        write.write_u32::<LittleEndian>(written_len as u32)?;
        Ok(())
    }

    pub fn extract_footer(source: ReadOnlySource) -> Result<(Footer, ReadOnlySource), io::Error> {
        if source.len() < 4 {
            return Err(io::Error::new(
                io::ErrorKind::UnexpectedEof,
                format!(
                    "File corrupted. The file is smaller than 4 bytes (len={}).",
                    source.len()
                ),
            ));
        }
        let (body_footer, footer_len_bytes) = source.split_from_end(u32::SIZE_IN_BYTES);
        let footer_len = LittleEndian::read_u32(footer_len_bytes.as_slice()) as usize;
        let body_len = body_footer.len() - footer_len;
        let (body, footer_data) = body_footer.split(body_len);
        let mut cursor = footer_data.as_slice();
        let footer = Footer::deserialize(&mut cursor)?;
        Ok((footer, body))
    }

    /// Confirms that the index will be read correctly by this version of tantivy
    /// Has to be called after `extract_footer` to make sure it's not accessing uninitialised memory
    pub fn is_compatible(&self) -> Result<(), Incompatibility> {
        let library_version = crate::version();
        match &self.versioned_footer {
            VersionedFooter::V1 {
                crc32: _crc,
                store_compression: compression,
            } => {
                if &library_version.store_compression != compression {
                    return Err(Incompatibility::CompressionMismatch {
                        library_compression_format: library_version.store_compression.to_string(),
                        index_compression_format: compression.to_string(),
                    });
                }
                Ok(())
            }
            VersionedFooter::UnknownVersion => Err(Incompatibility::IndexMismatch {
                library_version: library_version.clone(),
                index_version: self.version.clone(),
            }),
        }
    }
}

/// Footer that includes a crc32 hash that enables us to checksum files in the index
#[derive(Debug, Clone, PartialEq)]
pub enum VersionedFooter {
    UnknownVersion,
    V1 {
        crc32: CrcHashU32,
        store_compression: String,
    },
}

impl BinarySerializable for VersionedFooter {
    fn serialize<W: io::Write>(&self, writer: &mut W) -> io::Result<()> {
        let mut buf = Vec::new();
        match self {
            VersionedFooter::V1 {
                crc32,
                store_compression: compression,
            } => {
                // Serializes a valid `VersionedFooter` or panics if the version is unknown
                // [   version    |   crc_hash  | compression_mode ]
                // [    0..4      |     4..8    |     variable     ]
                BinarySerializable::serialize(&1u32, &mut buf)?;
                BinarySerializable::serialize(crc32, &mut buf)?;
                BinarySerializable::serialize(compression, &mut buf)?;
            }
            VersionedFooter::UnknownVersion => {
                return Err(io::Error::new(
                    io::ErrorKind::InvalidInput,
                    "Cannot serialize an unknown versioned footer ",
                ));
            }
        }
        BinarySerializable::serialize(&VInt(buf.len() as u64), writer)?;
        assert!(buf.len() <= FOOTER_MAX_LEN);
        writer.write_all(&buf[..])?;
        Ok(())
    }

    fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
        let len = VInt::deserialize(reader)?.0 as usize;
        if len > FOOTER_MAX_LEN {
            return Err(io::Error::new(
                io::ErrorKind::InvalidData,
                format!(
                    "Footer seems invalid as it suggests a footer len of {}. File is corrupted, \
            or the index was created with a different & old version of tantivy.",
                    len
                ),
            ));
        }
        let mut buf = vec![0u8; len];
        reader.read_exact(&mut buf[..])?;
        let mut cursor = &buf[..];
        let version = u32::deserialize(&mut cursor)?;
        if version == 1 {
            let crc32 = u32::deserialize(&mut cursor)?;
            let compression = String::deserialize(&mut cursor)?;
            Ok(VersionedFooter::V1 {
                crc32,
                store_compression: compression,
            })
        } else {
            Ok(VersionedFooter::UnknownVersion)
        }
    }
}

impl VersionedFooter {
    pub fn crc(&self) -> Option<CrcHashU32> {
        match self {
            VersionedFooter::V1 { crc32, .. } => Some(*crc32),
            VersionedFooter::UnknownVersion { .. } => None,
        }
    }
}

pub(crate) struct FooterProxy<W: TerminatingWrite> {
    /// always Some except after terminate call
    hasher: Option<Hasher>,
    /// always Some except after terminate call
    writer: Option<W>,
}

impl<W: TerminatingWrite> FooterProxy<W> {
    pub fn new(writer: W) -> Self {
        FooterProxy {
            hasher: Some(Hasher::new()),
            writer: Some(writer),
        }
    }
}

impl<W: TerminatingWrite> Write for FooterProxy<W> {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        let count = self.writer.as_mut().unwrap().write(buf)?;
        self.hasher.as_mut().unwrap().update(&buf[..count]);
        Ok(count)
    }

    fn flush(&mut self) -> io::Result<()> {
        self.writer.as_mut().unwrap().flush()
    }
}

impl<W: TerminatingWrite> TerminatingWrite for FooterProxy<W> {
    fn terminate_ref(&mut self, _: AntiCallToken) -> io::Result<()> {
        let crc32 = self.hasher.take().unwrap().finalize();
        let footer = Footer::new(VersionedFooter::V1 {
            crc32,
            store_compression: crate::store::COMPRESSION.to_string(),
        });
        let mut writer = self.writer.take().unwrap();
        footer.append_footer(&mut writer)?;
        writer.terminate()
    }
}

#[cfg(test)]
mod tests {

    use super::CrcHashU32;
    use super::FooterProxy;
    use crate::common::{BinarySerializable, VInt};
    use crate::directory::footer::{Footer, VersionedFooter};
    use crate::directory::TerminatingWrite;
    use byteorder::{ByteOrder, LittleEndian};
    use regex::Regex;
    use std::io;

    #[test]
    fn test_versioned_footer() {
        let mut vec = Vec::new();
        let footer_proxy = FooterProxy::new(&mut vec);
        assert!(footer_proxy.terminate().is_ok());
        assert_eq!(vec.len(), 167);
        let footer = Footer::deserialize(&mut &vec[..]).unwrap();
        if let VersionedFooter::V1 {
            crc32: _,
            store_compression,
        } = footer.versioned_footer
        {
            assert_eq!(store_compression, crate::store::COMPRESSION);
        } else {
            panic!("Versioned footer should be V1.");
        }
        assert_eq!(&footer.version, crate::version());
    }

    #[test]
    fn test_serialize_deserialize_footer() {
        let mut buffer = Vec::new();
        let crc32 = 123456u32;
        let footer: Footer = Footer::new(VersionedFooter::V1 {
            crc32,
            store_compression: "lz4".to_string(),
        });
        footer.serialize(&mut buffer).unwrap();
        let footer_deser = Footer::deserialize(&mut &buffer[..]).unwrap();
        assert_eq!(footer_deser, footer);
    }

    #[test]
    fn footer_length() {
        let crc32 = 1111111u32;
        let versioned_footer = VersionedFooter::V1 {
            crc32,
            store_compression: "lz4".to_string(),
        };
        let mut buf = Vec::new();
        versioned_footer.serialize(&mut buf).unwrap();
        assert_eq!(buf.len(), 13);
        let footer = Footer::new(versioned_footer);
        let regex_ptn = Regex::new(
            "tantivy v[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.{0,10}, index_format v[0-9]{1,5}",
        )
        .unwrap();
        assert!(regex_ptn.is_match(&footer.meta));
    }

    #[test]
    fn versioned_footer_from_bytes() {
        let v_footer_bytes = vec![
            // versionned footer length
            12 | 128,
            // index format version
            1,
            0,
            0,
            0,
            // crc 32
            12,
            35,
            89,
            18,
            // compression format
            3 | 128,
            b'l',
            b'z',
            b'4',
        ];
        let mut cursor = &v_footer_bytes[..];
        let versioned_footer = VersionedFooter::deserialize(&mut cursor).unwrap();
        assert!(cursor.is_empty());
        let expected_crc: u32 = LittleEndian::read_u32(&v_footer_bytes[5..9]) as CrcHashU32;
        let expected_versioned_footer: VersionedFooter = VersionedFooter::V1 {
            crc32: expected_crc,
            store_compression: "lz4".to_string(),
        };
        assert_eq!(versioned_footer, expected_versioned_footer);
        let mut buffer = Vec::new();
        assert!(versioned_footer.serialize(&mut buffer).is_ok());
        assert_eq!(&v_footer_bytes[..], &buffer[..]);
    }

    #[test]
    fn versioned_footer_panic() {
        let v_footer_bytes = vec![6u8 | 128u8, 3u8, 0u8, 0u8, 1u8, 0u8, 0u8];
        let mut b = &v_footer_bytes[..];
        let versioned_footer = VersionedFooter::deserialize(&mut b).unwrap();
        assert!(b.is_empty());
        let expected_versioned_footer = VersionedFooter::UnknownVersion;
        assert_eq!(versioned_footer, expected_versioned_footer);
        let mut buf = Vec::new();
        assert!(versioned_footer.serialize(&mut buf).is_err());
    }

    #[test]
    #[cfg(not(feature = "lz4"))]
    fn compression_mismatch() {
        let crc32 = 1111111u32;
        let versioned_footer = VersionedFooter::V1 {
            crc32,
            store_compression: "lz4".to_string(),
        };
        let footer = Footer::new(versioned_footer);
        let res = footer.is_compatible();
        assert!(res.is_err());
    }

    #[test]
    fn test_deserialize_too_large_footer() {
        let mut buf = vec![];
        assert!(FooterProxy::new(&mut buf).terminate().is_ok());
        let mut long_len_buf = [0u8; 10];
        let num_bytes = VInt(super::FOOTER_MAX_LEN as u64 + 1u64).serialize_into(&mut long_len_buf);
        buf[0..num_bytes].copy_from_slice(&long_len_buf[..num_bytes]);
        let err = Footer::deserialize(&mut &buf[..]).unwrap_err();
        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
        assert_eq!(
            err.to_string(),
            "Footer seems invalid as it suggests a footer len of 10001. File is corrupted, \
            or the index was created with a different & old version of tantivy."
        );
    }
}
<|endoftext|>


<commit_before>extern crate tui;
extern crate termion;

use std::io;
use std::thread;
use std::time;
use std::sync::mpsc;

use termion::event;
use termion::input::TermRead;

use tui::Terminal;
use tui::backend::TermionBackend;
use tui::widgets::{Widget, Block, border, SelectableList, List};
use tui::layout::{Group, Direction, Size};
use tui::style::{Style, Color, Modifier};

struct App<'a> {
    items: Vec<&'a str>,
    selected: usize,
    events: Vec<(&'a str, &'a str)>,
    info_style: Style,
    warning_style: Style,
    error_style: Style,
    critical_style: Style,
}

impl<'a> App<'a> {
    fn new() -> App<'a> {
        App {
            items: vec!["Item1", "Item2", "Item3", "Item4", "Item5", "Item6", "Item7", "Item8",
                        "Item9", "Item10", "Item11", "Item12", "Item13", "Item14", "Item15",
                        "Item16", "Item17", "Item18", "Item19", "Item20", "Item21", "Item22",
                        "Item23", "Item24"],
            selected: 0,
            events: vec![("Event1", "INFO"),
                         ("Event2", "INFO"),
                         ("Event3", "CRITICAL"),
                         ("Event4", "ERROR"),
                         ("Event5", "INFO"),
                         ("Event6", "INFO"),
                         ("Event7", "WARNING"),
                         ("Event8", "INFO"),
                         ("Event9", "INFO"),
                         ("Event10", "INFO"),
                         ("Event11", "CRITICAL"),
                         ("Event12", "INFO"),
                         ("Event13", "INFO"),
                         ("Event14", "INFO"),
                         ("Event15", "INFO"),
                         ("Event16", "INFO"),
                         ("Event17", "ERROR"),
                         ("Event18", "ERROR"),
                         ("Event19", "INFO"),
                         ("Event20", "INFO"),
                         ("Event21", "WARNING"),
                         ("Event22", "INFO"),
                         ("Event23", "INFO"),
                         ("Event24", "WARNING"),
                         ("Event25", "INFO"),
                         ("Event26", "INFO")],
            info_style: Style::default().fg(Color::White),
            warning_style: Style::default().fg(Color::Yellow),
            error_style: Style::default().fg(Color::Magenta),
            critical_style: Style::default().fg(Color::Red),
        }
    }

    fn advance(&mut self) {
        let event = self.events.pop().unwrap();
        self.events.insert(0, event);
    }
}

enum Event {
    Input(event::Key),
    Tick,
}

fn main() {
    // Terminal initialization
    let backend = TermionBackend::new().unwrap();
    let mut terminal = Terminal::new(backend).unwrap();

    // Channels
    let (tx, rx) = mpsc::channel();
    let input_tx = tx.clone();
    let clock_tx = tx.clone();

    // Input
    thread::spawn(move || {
        let stdin = io::stdin();
        for c in stdin.keys() {
            let evt = c.unwrap();
            input_tx.send(Event::Input(evt)).unwrap();
            if evt == event::Key::Char('q') {
                break;
            }
        }
    });

    // Tick
    thread::spawn(move || {
        loop {
            clock_tx.send(Event::Tick).unwrap();
            thread::sleep(time::Duration::from_millis(500));
        }
    });

    // App
    let mut app = App::new();

    // First draw call
    terminal.clear().unwrap();
    terminal.hide_cursor().unwrap();
    draw(&mut terminal, &app);

    loop {
        let evt = rx.recv().unwrap();
        match evt {
            Event::Input(input) => {
                match input {
                    event::Key::Char('q') => {
                        break;
                    }
                    event::Key::Down => {
                        app.selected += 1;
                        if app.selected > app.items.len() - 1 {
                            app.selected = 0;
                        }
                    }
                    event::Key::Up => {
                        if app.selected > 0 {
                            app.selected -= 1;
                        } else {
                            app.selected = app.items.len() - 1;
                        }
                    }
                    _ => {}
                }
            }
            Event::Tick => {
                app.advance();
            }
        }
        draw(&mut terminal, &app);
    }

    terminal.show_cursor().unwrap();
}

fn draw(t: &mut Terminal<TermionBackend>, app: &App) {

    let size = t.size().unwrap();

    Group::default()
        .direction(Direction::Horizontal)
        .sizes(&[Size::Percent(50), Size::Percent(50)])
        .render(t, &size, |t, chunks| {
            SelectableList::default()
                .block(Block::default()
                    .borders(border::ALL)
                    .title("List"))
                .items(&app.items)
                .select(app.selected)
                .highlight_style(Style::default().fg(Color::Yellow).modifier(Modifier::Bold))
                .highlight_symbol(">")
                .render(t, &chunks[0]);
            List::default()
                .block(Block::default()
                    .borders(border::ALL)
                    .title("List"))
                .items(&app.events
                    .iter()
                    .map(|&(evt, level)| {
                        (format!("{}: {}", level, evt),
                         match level {
                            "ERROR" => &app.error_style,
                            "CRITICAL" => &app.critical_style,
                            "WARNING" => &app.warning_style,
                            _ => &app.info_style,
                        })
                    })
                    .collect::<Vec<(String, &Style)>>())
                .render(t, &chunks[1]);
        });

    t.draw().unwrap();
}
<commit_msg>test tui layout coded<commit_after>extern crate tui;
extern crate termion;

use std::io;
use std::thread;
use std::time;
use std::sync::mpsc;

use termion::event;
use termion::input::TermRead;

use tui::Terminal;
use tui::backend::TermionBackend;
use tui::widgets::{Widget, Block, border, SelectableList, List};
use tui::widgets::canvas::{Canvas, Map, MapResolution, Line};
use tui::layout::{Group, Direction, Size};
use tui::style::{Style, Color, Modifier};

struct App<'a> {
    items: Vec<&'a str>,
    selected: usize,
    events: Vec<(&'a str, &'a str)>,
    info_style: Style,
    warning_style: Style,
    error_style: Style,
    critical_style: Style,
}

impl<'a> App<'a> {
    fn new() -> App<'a> {
        App {
            items: vec!["Seek", "Flee", "Pursue"],
            selected: 0,
            events: vec![("Event1", "INFO"),
                         ("Event2", "INFO"),
                         ("Event3", "CRITICAL"),
                         ("Event4", "ERROR"),
                         ("Event5", "INFO"),
                         ("Event6", "INFO"),
                         ("Event7", "WARNING"),
                         ("Event8", "INFO"),
                         ("Event9", "INFO"),
                         ("Event10", "INFO"),
                         ("Event11", "CRITICAL"),
                         ("Event12", "INFO"),
                         ("Event13", "INFO"),
                         ("Event14", "INFO"),
                         ("Event15", "INFO"),
                         ("Event16", "INFO"),
                         ("Event17", "ERROR"),
                         ("Event18", "ERROR"),
                         ("Event19", "INFO"),
                         ("Event20", "INFO"),
                         ("Event21", "WARNING"),
                         ("Event22", "INFO"),
                         ("Event23", "INFO"),
                         ("Event24", "WARNING"),
                         ("Event25", "INFO"),
                         ("Event26", "INFO")],
            info_style: Style::default().fg(Color::White),
            warning_style: Style::default().fg(Color::Yellow),
            error_style: Style::default().fg(Color::Magenta),
            critical_style: Style::default().fg(Color::Red),
        }
    }

    fn advance(&mut self) {
        let event = self.events.pop().unwrap();
        self.events.insert(0, event);
    }
}

enum Event {
    Input(event::Key),
    Tick,
}

fn main() {
    // Terminal initialization
    let backend = TermionBackend::new().unwrap();
    let mut terminal = Terminal::new(backend).unwrap();

    // Channels
    let (tx, rx) = mpsc::channel();
    let input_tx = tx.clone();
    let clock_tx = tx.clone();

    // Input
    thread::spawn(move || {
        let stdin = io::stdin();
        for c in stdin.keys() {
            let evt = c.unwrap();
            input_tx.send(Event::Input(evt)).unwrap();
            if evt == event::Key::Char('q') {
                break;
            }
        }
    });

    // Tick
    thread::spawn(move || {
        loop {
            clock_tx.send(Event::Tick).unwrap();
            thread::sleep(time::Duration::from_millis(500));
        }
    });

    // App
    let mut app = App::new();

    // First draw call
    terminal.clear().unwrap();
    terminal.hide_cursor().unwrap();
    draw(&mut terminal, &app);

    loop {
        let evt = rx.recv().unwrap();
        match evt {
            Event::Input(input) => {
                match input {
                    event::Key::Char('q') => {
                        break;
                    }
                    event::Key::Down => {
                        app.selected += 1;
                        if app.selected > app.items.len() - 1 {
                            app.selected = 0;
                        }
                    }
                    event::Key::Up => {
                        if app.selected > 0 {
                            app.selected -= 1;
                        } else {
                            app.selected = app.items.len() - 1;
                        }
                    }
                    _ => {}
                }
            }
            Event::Tick => {
                app.advance();
            }
        }
        draw(&mut terminal, &app);
    }

    terminal.show_cursor().unwrap();
}

fn draw(t: &mut Terminal<TermionBackend>, app: &App) {

    let size = t.size().unwrap();

    Group::default()
        .direction(Direction::Vertical)
        .sizes(&[Size::Percent(90), Size::Percent(10)])
        .render(t, &size, |t, chunks| {
            Group::default()
                .direction(Direction::Horizontal)
                .sizes(&[Size::Percent(15), Size::Percent(85)])
                .render(t, &chunks[0], |t, chunks|{
                    SelectableList::default()
                        .block(Block::default()
                               .borders(border::ALL)
                               .title("Steering Behaviors")
                              )
                        .items(&app.items)
                        .select(app.selected)
                        .highlight_style(Style::default().fg(Color::Yellow).modifier(Modifier::Bold))
                        .highlight_symbol(">")
                        .render(t, &chunks[0]);

                    Canvas::default()
                        .block(Block::default()
                               .borders(border::ALL)
                               .title("Steering Actors")
                              )
                        .paint(|ctx|{
                            ctx.draw(&Map{
                                color : Color::Red,
                                resolution : MapResolution::High,
                            });
                        })
                        .x_bounds([-180.0, 180.0])
                        .y_bounds([-90.0,  90.0])
                        .render(t,&chunks[1]);
                });

            Block::default()
                .borders(border::ALL)
                .title("Instructions")
                .title_style(
                    Style::default()
                    .fg(Color::White)
                    .bg(Color::Red)
                    .modifier(Modifier::Bold)
                            )
                .render(t, &chunks[1]);
        });

    t.draw().unwrap();
}
<|endoftext|>


<commit_before>#![feature(test)]

extern crate test;
extern crate exonum;
extern crate time;
extern crate env_logger;

use std::thread;
use std::net::SocketAddr;

use time::Duration;
use test::Bencher;

use exonum::events::{Events, Reactor, EventsConfiguration, Event, Timeout};
use exonum::events::{Network, NetworkConfiguration};
use exonum::messages::{MessageWriter, RawMessage};
use exonum::crypto::gen_keypair;

struct BenchConfig {
    times: usize,
    len: usize,
    tcp_nodelay: bool
}

trait EventsBench {
    fn with_addr(addr: SocketAddr, cfg: &BenchConfig) -> Events;
    fn wait_for_msg(&mut self) -> Option<RawMessage>;
    fn gen_message(id: u16, len: usize) -> RawMessage;
    fn wait_for_messages(&mut self, mut count: usize, timeout: Duration) -> Result<(), String>;
    fn process_events(&mut self, timeout: Duration);
}

impl EventsBench for Events {
    fn with_addr(addr: SocketAddr, cfg: &BenchConfig) -> Events {
        let network = Network::with_config(NetworkConfiguration {
            listen_address: addr,
            max_connections: 128,
            tcp_nodelay: cfg.tcp_nodelay,
            tcp_keep_alive: None,
        });
        Events::with_config(EventsConfiguration::new(), network).unwrap()
    }

    fn wait_for_msg(&mut self) -> Option<RawMessage> {
        let time = self.get_time() + Duration::milliseconds(10000);
        self.add_timeout(Timeout::Status, time);
        loop {
            match self.poll() {
                Event::Incoming(msg) => return Some(msg),
                Event::Timeout(_) => return None,
                Event::Error(_) => return None,
                _ => {}
            }
        }
    }

    fn wait_for_messages(&mut self, mut count: usize, timeout: Duration) -> Result<(), String> {
        let time = self.get_time() + timeout;
        self.add_timeout(Timeout::Status, time);
        loop {
            match self.poll() {
                Event::Incoming(_) => {
                    count = count - 1;
                }
                Event::Timeout(_) => {
                    return Err(format!("Timeout exceeded, {} messages is not received", count))
                }
                Event::Error(_) => {
                    return Err(format!("An error occured, {} messages is not received", count))
                }
                _ => {}
            }
            if count == 0 {
                return Ok(());
            }
        }
    }

    fn gen_message(id: u16, len: usize) -> RawMessage {
        let writer = MessageWriter::new(id, len);
        RawMessage::new(writer.sign(&gen_keypair().1))
    }

    fn process_events(&mut self, timeout: Duration) {
        let time = self.get_time() + timeout;
        self.add_timeout(Timeout::Status, time);
        loop {
            match self.poll() {
                Event::Timeout(_) => break,
                Event::Error(_) => return,
                _ => {}
            }
        }
    }
}

fn bench_network(b: &mut Bencher, addrs: [SocketAddr; 2], cfg: BenchConfig) {
    b.iter(|| {
        let mut e1 = Events::with_addr(addrs[0], &cfg);
        let mut e2 = Events::with_addr(addrs[1], &cfg);
        e1.bind().unwrap();
        e2.bind().unwrap();

        let timeout = Duration::seconds(30);
        let len = cfg.len;
        let times = cfg.times;
        let t1 = thread::spawn(move || {
            for _ in 0..times {
                let msg = Events::gen_message(0, len);
                e1.send_to(&addrs[1], msg).unwrap();
                e1.wait_for_messages(1, timeout).unwrap();
            }
            e1.process_events(Duration::milliseconds(0));
        });
        let t2 = thread::spawn(move || {
            for _ in 0..times {
                let msg = Events::gen_message(1, len);
                e2.send_to(&addrs[0], msg).unwrap();
                e2.wait_for_messages(1, timeout).unwrap();
            }
            e2.process_events(Duration::milliseconds(0));
        });
        t1.join().unwrap();
        t2.join().unwrap();
    })
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_short_100(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100,
        times: 100
    };
    let addrs = ["127.0.0.1:9990".parse().unwrap(), "127.0.0.1:9991".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_short_1000(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100,
        times: 1000
    };
    let addrs = ["127.0.0.1:9992".parse().unwrap(), "127.0.0.1:9993".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_short_10000(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100,
        times: 10000
    };
    let addrs = ["127.0.0.1:9992".parse().unwrap(), "127.0.0.1:9993".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_short_100_nodelay(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: true,
        len: 100,
        times: 100
    };
    let addrs = ["127.0.0.1:9990".parse().unwrap(), "127.0.0.1:9991".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_short_10000_nodelay(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: true,
        len: 100,
        times: 10000
    };
    let addrs = ["127.0.0.1:9990".parse().unwrap(), "127.0.0.1:9991".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_long_10(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100000,
        times: 10
    };
    let addrs = ["127.0.0.1:9994".parse().unwrap(), "127.0.0.1:9995".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_long_100(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100000,
        times: 100
    };
    let addrs = ["127.0.0.1:9996".parse().unwrap(), "127.0.0.1:9997".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_long_1000(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100000,
        times: 1000
    };
    let addrs = ["127.0.0.1:9998".parse().unwrap(), "127.0.0.1:9999".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_long_1000_nodelay(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: true,
        len: 100000,
        times: 1000
    };
    let addrs = ["127.0.0.1:9998".parse().unwrap(), "127.0.0.1:9999".parse().unwrap()];
    bench_network(b, addrs, cfg);
}
<commit_msg>Fix bencmarks<commit_after>#![feature(test)]

extern crate test;
extern crate exonum;
extern crate time;
extern crate env_logger;

use std::thread;
use std::net::SocketAddr;

use time::Duration;
use test::Bencher;

use exonum::events::{Events, Reactor, EventsConfiguration, Event, NodeTimeout};
use exonum::events::{Network, NetworkConfiguration};
use exonum::messages::{MessageWriter, RawMessage};
use exonum::crypto::gen_keypair;

struct BenchConfig {
    times: usize,
    len: usize,
    tcp_nodelay: bool
}

trait EventsBench {
    fn with_addr(addr: SocketAddr, cfg: &BenchConfig) -> Events;
    fn wait_for_msg(&mut self) -> Option<RawMessage>;
    fn gen_message(id: u16, len: usize) -> RawMessage;
    fn wait_for_messages(&mut self, mut count: usize, timeout: Duration) -> Result<(), String>;
    fn process_events(&mut self, timeout: Duration);
}

impl EventsBench for Events {
    fn with_addr(addr: SocketAddr, cfg: &BenchConfig) -> Events {
        let network = Network::with_config(NetworkConfiguration {
            listen_address: addr,
            max_connections: 128,
            tcp_nodelay: cfg.tcp_nodelay,
            tcp_keep_alive: None,
            tcp_reconnect_timeout: 1000,
            tcp_reconnect_timeout_max: 600000,
        });
        Events::with_config(EventsConfiguration::new(), network).unwrap()
    }

    fn wait_for_msg(&mut self) -> Option<RawMessage> {
        let time = self.get_time() + Duration::milliseconds(10000);
        self.add_timeout(NodeTimeout::Status, time);
        loop {
            match self.poll() {
                Event::Incoming(msg) => return Some(msg),
                Event::Timeout(_) => return None,
                Event::Error(_) => return None,
                _ => {}
            }
        }
    }

    fn wait_for_messages(&mut self, mut count: usize, timeout: Duration) -> Result<(), String> {
        let time = self.get_time() + timeout;
        self.add_timeout(NodeTimeout::Status, time);
        loop {
            match self.poll() {
                Event::Incoming(_) => {
                    count = count - 1;
                }
                Event::Timeout(_) => {
                    return Err(format!("Timeout exceeded, {} messages is not received", count))
                }
                Event::Error(_) => {
                    return Err(format!("An error occured, {} messages is not received", count))
                }
                _ => {}
            }
            if count == 0 {
                return Ok(());
            }
        }
    }

    fn gen_message(id: u16, len: usize) -> RawMessage {
        let writer = MessageWriter::new(id, len);
        RawMessage::new(writer.sign(&gen_keypair().1))
    }

    fn process_events(&mut self, timeout: Duration) {
        let time = self.get_time() + timeout;
        self.add_timeout(NodeTimeout::Status, time);
        loop {
            match self.poll() {
                Event::Timeout(_) => break,
                Event::Error(_) => return,
                _ => {}
            }
        }
    }
}

fn bench_network(b: &mut Bencher, addrs: [SocketAddr; 2], cfg: BenchConfig) {
    b.iter(|| {
        let mut e1 = Events::with_addr(addrs[0], &cfg);
        let mut e2 = Events::with_addr(addrs[1], &cfg);
        e1.bind().unwrap();
        e2.bind().unwrap();

        let timeout = Duration::seconds(30);
        let len = cfg.len;
        let times = cfg.times;
        let t1 = thread::spawn(move || {
            for _ in 0..times {
                let msg = Events::gen_message(0, len);
                e1.send_to(&addrs[1], msg);
                e1.wait_for_messages(1, timeout).unwrap();
            }
            e1.process_events(Duration::milliseconds(0));
        });
        let t2 = thread::spawn(move || {
            for _ in 0..times {
                let msg = Events::gen_message(1, len);
                e2.send_to(&addrs[0], msg);
                e2.wait_for_messages(1, timeout).unwrap();
            }
            e2.process_events(Duration::milliseconds(0));
        });
        t1.join().unwrap();
        t2.join().unwrap();
    })
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_short_100(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100,
        times: 100
    };
    let addrs = ["127.0.0.1:9990".parse().unwrap(), "127.0.0.1:9991".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_short_1000(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100,
        times: 1000
    };
    let addrs = ["127.0.0.1:9992".parse().unwrap(), "127.0.0.1:9993".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_short_10000(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100,
        times: 10000
    };
    let addrs = ["127.0.0.1:9992".parse().unwrap(), "127.0.0.1:9993".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_short_100_nodelay(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: true,
        len: 100,
        times: 100
    };
    let addrs = ["127.0.0.1:9990".parse().unwrap(), "127.0.0.1:9991".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_short_10000_nodelay(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: true,
        len: 100,
        times: 10000
    };
    let addrs = ["127.0.0.1:9990".parse().unwrap(), "127.0.0.1:9991".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_long_10(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100000,
        times: 10
    };
    let addrs = ["127.0.0.1:9994".parse().unwrap(), "127.0.0.1:9995".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_long_100(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100000,
        times: 100
    };
    let addrs = ["127.0.0.1:9996".parse().unwrap(), "127.0.0.1:9997".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_long_1000(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: false,
        len: 100000,
        times: 1000
    };
    let addrs = ["127.0.0.1:9998".parse().unwrap(), "127.0.0.1:9999".parse().unwrap()];
    bench_network(b, addrs, cfg);
}

#[cfg(feature = "long_benchmarks")]
#[bench]
fn bench_msg_long_1000_nodelay(b: &mut Bencher) {
    let cfg = BenchConfig {
        tcp_nodelay: true,
        len: 100000,
        times: 1000
    };
    let addrs = ["127.0.0.1:9998".parse().unwrap(), "127.0.0.1:9999".parse().unwrap()];
    bench_network(b, addrs, cfg);
}
<|endoftext|>


<commit_before>use std::cmp::Ordering;
use std::collections::VecDeque;
use std::fs::{File, OpenOptions};
use std::io::SeekFrom;
use std::io::{BufRead, BufReader, BufWriter, Seek, Write};
use std::path::Path;

use tempdir::TempDir;

use super::{GlobalSettings, Line};

/// Iterator that provides sorted `T`s
pub struct ExtSortedIterator {
    buffers: Vec<VecDeque<Line>>,
    chunk_offsets: Vec<u64>,
    max_per_chunk: usize,
    chunks: usize,
    tmp_dir: TempDir,
    settings: GlobalSettings,
    failed: bool,
}

impl Iterator for ExtSortedIterator {
    type Item = Line;

    /// # Errors
    ///
    /// This method can fail due to issues reading intermediate sorted chunks
    /// from disk
    fn next(&mut self) -> Option<Self::Item> {
        if self.failed {
            return None;
        }
        // fill up any empty buffers
        let mut empty = true;
        for chunk_num in 0..self.chunks {
            if self.buffers[chunk_num as usize].is_empty() {
                let mut f = crash_if_err!(
                    1,
                    File::open(self.tmp_dir.path().join(chunk_num.to_string()))
                );
                crash_if_err!(1, f.seek(SeekFrom::Start(self.chunk_offsets[chunk_num])));
                let bytes_read = fill_buff(
                    &mut self.buffers[chunk_num as usize],
                    f,
                    self.max_per_chunk,
                    &self.settings,
                );
                self.chunk_offsets[chunk_num as usize] += bytes_read as u64;
                if !self.buffers[chunk_num as usize].is_empty() {
                    empty = false;
                }
            } else {
                empty = false;
            }
        }
        if empty {
            return None;
        }

        // find the next record to write
        // check is_empty() before unwrap()ing
        let mut idx = 0;
        for chunk_num in 0..self.chunks as usize {
            if !self.buffers[chunk_num].is_empty()
                && (self.buffers[idx].is_empty()
                    || super::compare_by(
                        self.buffers[chunk_num].front().unwrap(),
                        self.buffers[idx].front().unwrap(),
                        &self.settings,
                    ) == Ordering::Less)
            {
                idx = chunk_num;
            }
        }

        // unwrap due to checks above
        let r = self.buffers[idx].pop_front().unwrap();
        Some(r)
    }
}

/// Sort (based on `compare`) the `T`s provided by `unsorted` and return an
/// iterator
///
/// # Errors
///
/// This method can fail due to issues writing intermediate sorted chunks
/// to disk.
pub fn ext_sort(
    unsorted: impl Iterator<Item = Line>,
    settings: &GlobalSettings,
) -> ExtSortedIterator {
    let tmp_dir = crash_if_err!(1, TempDir::new_in(&settings.tmp_dir, "uutils_sort"));

    let mut iter = ExtSortedIterator {
        buffers: Vec::new(),
        chunk_offsets: Vec::new(),
        max_per_chunk: 0,
        chunks: 0,
        tmp_dir,
        settings: settings.clone(),
        failed: false,
    };

    let mut total_read = 0;
    let mut chunk = Vec::new();

    // make the initial chunks on disk
    for seq in unsorted {
        let seq_size = seq.estimate_size();
        total_read += seq_size;

        chunk.push(seq);

        if total_read >= settings.buffer_size {
            super::sort_by(&mut chunk, &settings);
            write_chunk(
                settings,
                &iter.tmp_dir.path().join(iter.chunks.to_string()),
                &mut chunk,
            );
            chunk.clear();
            total_read = 0;
            iter.chunks += 1;
        }
    }
    // write the last chunk
    if !chunk.is_empty() {
        super::sort_by(&mut chunk, &settings);
        write_chunk(
            settings,
            &iter.tmp_dir.path().join(iter.chunks.to_string()),
            &mut chunk,
        );
        iter.chunks += 1;
    }

    // initialize buffers for each chunk
    //
    // Having a right sized buffer for each chunk for smallish values seems silly to me?
    //
    // We will have to have the entire iter in memory sometime right?
    // Set minimum to the size of the writer buffer, ~8K

    const MINIMUM_READBACK_BUFFER: usize = 8200;
    let right_sized_buffer = settings
        .buffer_size
        .checked_div(iter.chunks)
        .unwrap_or(settings.buffer_size);
    iter.max_per_chunk = if right_sized_buffer > MINIMUM_READBACK_BUFFER {
        right_sized_buffer
    } else {
        MINIMUM_READBACK_BUFFER
    };
    iter.buffers = vec![VecDeque::new(); iter.chunks];
    iter.chunk_offsets = vec![0; iter.chunks];
    for chunk_num in 0..iter.chunks {
        let offset = fill_buff(
            &mut iter.buffers[chunk_num],
            crash_if_err!(
                1,
                File::open(iter.tmp_dir.path().join(chunk_num.to_string()))
            ),
            iter.max_per_chunk,
            &settings,
        );
        iter.chunk_offsets[chunk_num] = offset as u64;
    }

    iter
}

fn write_chunk(settings: &GlobalSettings, file: &Path, chunk: &mut Vec<Line>) {
    let new_file = crash_if_err!(1, OpenOptions::new().create(true).append(true).open(file));
    let mut buf_write = BufWriter::new(new_file);
    for s in chunk {
        crash_if_err!(1, buf_write.write_all(s.line.as_bytes()));
        crash_if_err!(
            1,
            buf_write.write_all(if settings.zero_terminated { "\0" } else { "\n" }.as_bytes(),)
        );
    }
    crash_if_err!(1, buf_write.flush());
}

fn fill_buff(
    vec: &mut VecDeque<Line>,
    file: File,
    max_bytes: usize,
    settings: &GlobalSettings,
) -> usize {
    let mut total_read = 0;
    let mut bytes_read = 0;
    for line in BufReader::new(file).split(if settings.zero_terminated {
        b'\0'
    } else {
        b'\n'
    }) {
        let line_s = String::from_utf8(crash_if_err!(1, line)).unwrap();
        bytes_read += line_s.len() + 1;
        let deserialized = Line::new(line_s, settings);
        total_read += deserialized.estimate_size();
        vec.push_back(deserialized);
        if total_read > max_bytes {
            break;
        }
    }

    bytes_read
}
<commit_msg>sort: use FileMerger for extsort merge step<commit_after>use std::fs::OpenOptions;
use std::io::{BufWriter, Write};
use std::path::Path;

use tempdir::TempDir;

use crate::{file_to_lines_iter, FileMerger};

use super::{GlobalSettings, Line};

/// Iterator that provides sorted `T`s
pub struct ExtSortedIterator<'a> {
    file_merger: FileMerger<'a>,
    // Keep tmp_dir around, it is deleted when dropped.
    _tmp_dir: TempDir,
}

impl<'a> Iterator for ExtSortedIterator<'a> {
    type Item = Line;
    fn next(&mut self) -> Option<Self::Item> {
        self.file_merger.next()
    }
}

/// Sort (based on `compare`) the `T`s provided by `unsorted` and return an
/// iterator
///
/// # Panics
///
/// This method can panic due to issues writing intermediate sorted chunks
/// to disk.
pub fn ext_sort(
    unsorted: impl Iterator<Item = Line>,
    settings: &GlobalSettings,
) -> ExtSortedIterator {
    let tmp_dir = crash_if_err!(1, TempDir::new_in(&settings.tmp_dir, "uutils_sort"));

    let mut total_read = 0;
    let mut chunk = Vec::new();

    let mut chunks_read = 0;
    let mut file_merger = FileMerger::new(settings);

    // make the initial chunks on disk
    for seq in unsorted {
        let seq_size = seq.estimate_size();
        total_read += seq_size;

        chunk.push(seq);

        if total_read >= settings.buffer_size && chunk.len() >= 2 {
            super::sort_by(&mut chunk, &settings);

            let file_path = tmp_dir.path().join(chunks_read.to_string());
            write_chunk(settings, &file_path, &mut chunk);
            chunk.clear();
            total_read = 0;
            chunks_read += 1;

            file_merger.push_file(Box::new(file_to_lines_iter(file_path, settings).unwrap()))
        }
    }
    // write the last chunk
    if !chunk.is_empty() {
        super::sort_by(&mut chunk, &settings);

        let file_path = tmp_dir.path().join(chunks_read.to_string());
        write_chunk(
            settings,
            &tmp_dir.path().join(chunks_read.to_string()),
            &mut chunk,
        );

        file_merger.push_file(Box::new(file_to_lines_iter(file_path, settings).unwrap()));
    }
    ExtSortedIterator {
        file_merger,
        _tmp_dir: tmp_dir,
    }
}

fn write_chunk(settings: &GlobalSettings, file: &Path, chunk: &mut Vec<Line>) {
    let new_file = crash_if_err!(1, OpenOptions::new().create(true).append(true).open(file));
    let mut buf_write = BufWriter::new(new_file);
    for s in chunk {
        crash_if_err!(1, buf_write.write_all(s.line.as_bytes()));
        crash_if_err!(
            1,
            buf_write.write_all(if settings.zero_terminated { "\0" } else { "\n" }.as_bytes(),)
        );
    }
    crash_if_err!(1, buf_write.flush());
}
<|endoftext|>