<commit_before>use std::net::IpAddr;
use std::path::PathBuf;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::os::unix::io::RawFd;

use regex::{Regex, Captures};
use serde::de::{Deserializer, Deserialize, Error as DeError};
use serde::ser::{Serializer, Serialize};
use quire::validate::{Structure, Sequence, Scalar, Numeric, Enum};
use quire::validate::{Mapping, Nothing, Anything};
use id_map::{IdMap, IdMapExt, mapping_validator};

use sandbox_config::SandboxConfig;
use utils::{in_range};
use child_config::ChildKind;

pub const DEFAULT_KILL_TIMEOUT: f32 = 5.;

lazy_static! {
    static ref VARIABLE_REGEX: Regex = Regex::new(r#"@\{([^}]*)\}"#).unwrap();
    static ref NAME_REGEX: Regex = Regex::new(r#"^[0-9a-zA-Z_-]+$"#).unwrap();
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct TmpfsInfo {
    pub size: usize,
    pub mode: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct PersistentInfo {
    pub path: PathBuf,
    pub mkdir: bool,
    pub mode: u32,
    pub user: u32,
    pub group: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct StatedirInfo {
    pub path: PathBuf,
    pub mode: u32,
    pub user: u32,
    pub group: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub enum Volume {
    Readonly(PathBuf),
    Persistent(PersistentInfo),
    Tmpfs(TmpfsInfo),
    Statedir(StatedirInfo),
}

#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy)]
pub enum ContainerKind {
    Daemon,
    Command,
    CommandOrDaemon,
}

impl ContainerKind {
    pub fn matches(self, child_kind: ChildKind) -> bool {
        use container_config::ContainerKind as L;
        use child_config::ChildKind as R;
        match (self, child_kind) {
            (L::Command, R::Command) => true,
            (L::Daemon, R::Daemon) => true,
            (L::CommandOrDaemon, R::Command) => true,
            (L::CommandOrDaemon, R::Daemon) => true,
            (L::Command, R::Daemon) => false,
            (L::Daemon, R::Command) => false,
        }
    }
}

#[derive(Deserialize, Serialize, Clone)]
pub struct ResolvConf {
    pub copy_from_host: bool,
}

#[derive(Deserialize, Serialize, Clone)]
pub struct HostsFile {
    pub copy_from_host: bool,
    pub localhost: Option<bool>,
    pub public_hostname: Option<bool>,
}

#[derive(Clone, Debug)]
pub struct Host(pub IpAddr);

#[derive(Deserialize, Serialize, Clone)]
pub struct TcpPort {
    pub host: Host,
    pub fd: RawFd,
    pub reuse_addr: bool,
    pub reuse_port: bool,
    pub listen_backlog: usize,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub enum Variable {
    TcpPort,
    Name,
    Choice(Vec<String>),
}

#[derive(Deserialize)]
pub struct ContainerConfig {
    pub kind: ContainerKind,
    pub variables: BTreeMap<String, Variable>,
    pub volumes: BTreeMap<String, Volume>,
    pub user_id: u32,
    pub group_id: u32,
    pub restart_timeout: f32,
    pub kill_timeout: f32,
    pub memory_limit: u64,
    pub fileno_limit: u64,
    pub cpu_shares: usize,
    pub executable: String,
    pub arguments: Vec<String>,
    pub environ: BTreeMap<String, String>,
    pub workdir: PathBuf,
    pub resolv_conf: ResolvConf,
    pub hosts_file: HostsFile,
    pub uid_map: Vec<IdMap>,
    pub gid_map: Vec<IdMap>,
    pub stdout_stderr_file: Option<PathBuf>,
    pub interactive: bool,
    pub restart_process_only: bool,
    pub tcp_ports: HashMap<String, TcpPort>,
}

#[derive(Deserialize, Serialize)]
pub struct InstantiatedConfig {
    pub kind: ContainerKind,
    pub volumes: BTreeMap<String, Volume>,
    pub user_id: u32,
    pub group_id: u32,
    pub restart_timeout: f32,
    pub kill_timeout: f32,
    pub memory_limit: u64,
    pub fileno_limit: u64,
    pub cpu_shares: usize,
    pub executable: String,
    pub arguments: Vec<String>,
    pub environ: BTreeMap<String, String>,
    pub workdir: PathBuf,
    pub resolv_conf: ResolvConf,
    pub hosts_file: HostsFile,
    pub uid_map: Vec<IdMap>,
    pub gid_map: Vec<IdMap>,
    pub stdout_stderr_file: Option<PathBuf>,
    pub interactive: bool,
    pub restart_process_only: bool,
    pub tcp_ports: HashMap<u16, TcpPort>,
}

pub struct Variables<'a> {
    pub user_vars: &'a HashMap<String, String>,
    pub lithos_name: &'a str,
    pub lithos_config_filename: &'a str,
}

impl InstantiatedConfig {
    pub fn map_uid(&self, internal_uid: u32) -> Option<u32> {
        self.uid_map.map_id(internal_uid)
    }
    pub fn map_gid(&self, internal_gid: u32) -> Option<u32> {
        self.gid_map.map_id(internal_gid)
    }
}

impl ContainerConfig {
    pub fn validator<'x>() -> Structure<'x> {
        Structure::new()
        .member("kind", Scalar::new().default("Daemon"))
        .member("variables", Mapping::new(
            Scalar::new(),
            Enum::new()
                .option("TcpPort", Nothing)
                .option("Name", Nothing)
                .option("Choice", Sequence::new(Scalar::new()))
        ))
        .member("metadata", Anything)
        .member("volumes", Mapping::new(
                Scalar::new(),
                volume_validator()))
        .member("user_id", Numeric::new())
        .member("group_id", Numeric::new().default(0))
        .member("memory_limit", Numeric::new().default(0x7fffffffffffffffi64))
        .member("fileno_limit", Numeric::new().default(1024))
        .member("cpu_shares", Numeric::new().default(1024))
        .member("restart_timeout", Numeric::new().min(0).max(86400).default(1))
        .member("kill_timeout",
            Numeric::new().min(0).max(86400)
                .default(DEFAULT_KILL_TIMEOUT as i64))
        .member("executable", Scalar::new())
        .member("arguments", Sequence::new(Scalar::new()))
        .member("environ", Mapping::new(
                Scalar::new(),
                Scalar::new()))
        .member("workdir", Scalar::new().default("/"))
        .member("resolv_conf", Structure::new()
            .member("copy_from_host", Scalar::new().default(true)))
        .member("hosts_file", Structure::new()
            .member("copy_from_host", Scalar::new().default(true))
            .member("localhost", Scalar::new().optional())
            .member("public_hostname", Scalar::new().optional()))
        .member("uid_map", mapping_validator())
        .member("gid_map", mapping_validator())
        .member("stdout_stderr_file", Scalar::new().optional())
        .member("interactive", Scalar::new().default(false))
        .member("restart_process_only", Scalar::new().default(false))
        .member("tcp_ports", Mapping::new(
            Scalar::new(),
            Structure::new()
                .member("host", Scalar::new().default("0.0.0.0"))
                .member("fd", Numeric::new().min(0).optional())
                .member("reuse_addr", Scalar::new().default(true))
                .member("reuse_port", Scalar::new().default(false))
                .member("listen_backlog", Scalar::new().default(128))
            ))
    }
    pub fn instantiate(&self, variables: &Variables)
        -> Result<InstantiatedConfig, Vec<String>>
    {
        let mut errors1 = HashSet::new();
        let mut errors2 = HashSet::new();
        let result = {
            let mut replacer = |capt: &Captures| {
                let varname = capt.get(1).unwrap().as_str();
                let val = variables.user_vars.get(varname).map(|x| x.clone())
                    .or_else(|| match varname {
                        "lithos:name"
                        => Some(variables.lithos_name.to_string()),
                        "lithos:config_filename"
                        => Some(variables.lithos_config_filename.to_string()),
                        _ => None,
                    });
                match val {
                    Some(x) => x,
                    None => {
                        errors1.insert(format!("unknown variable {:?}", varname));
                        return format!("<<no var {:?}>>", varname);
                    }
                }
            };
            InstantiatedConfig {
                kind: self.kind.clone(),
                volumes: self.volumes.clone(),
                user_id: self.user_id.clone(),
                group_id: self.group_id.clone(),
                restart_timeout: self.restart_timeout.clone(),
                kill_timeout: self.kill_timeout.clone(),
                memory_limit: self.memory_limit.clone(),
                fileno_limit: self.fileno_limit.clone(),
                cpu_shares: self.cpu_shares.clone(),
                executable: self.executable.clone(),
                arguments: self.arguments.iter()
                    .map(|x| VARIABLE_REGEX.replace(&x, &mut replacer).into())
                    .collect(),
                environ: self.environ.iter()
                    .map(|(key, val)| {
                        (key.clone(),
                         VARIABLE_REGEX.replace(&val, &mut replacer).into())
                    })
                    .collect(),
                workdir: self.workdir.clone(),
                resolv_conf: self.resolv_conf.clone(),
                hosts_file: self.hosts_file.clone(),
                uid_map: self.uid_map.clone(),
                gid_map: self.gid_map.clone(),
                stdout_stderr_file: self.stdout_stderr_file.clone(),
                interactive: self.interactive.clone(),
                restart_process_only: self.restart_process_only.clone(),
                tcp_ports: self.tcp_ports.iter()
                    .map(|(key, val)| {
                        let s = VARIABLE_REGEX.replace(&key, &mut replacer);
                        let port = match s.parse::<u16>() {
                            Ok(x) => x,
                            Err(e) => {
                                errors2.insert(format!("Bad port {:?}: {}",
                                    key, e));
                                return (0, val.clone());
                            }
                        };
                        (port, val.clone())
                    })
                    .collect(),
            }
        };
        if errors1.len() > 0 || errors2.len() > 0 {
            return Err(errors1.into_iter().chain(errors2.into_iter())
                       .collect());
        } else {
            return Ok(result);
        }
    }
}

pub fn volume_validator<'x>() -> Enum<'x> {
    Enum::new()
    .option("Persistent",  Structure::new()
        .member("path",  Scalar::new().default("/"))
        .member("mkdir",  Scalar::new().default(false))
        .member("mode",  Numeric::new().min(0).max(0o1777).default(0o777))
        .member("user",  Numeric::new().default(0))
        .member("group",  Numeric::new().default(0)))
    .option("Readonly", Scalar::new())
    .option("Tmpfs", Structure::new()
        .member("size", Numeric::new().min(0).default(100*1024*1024))
        .member("mode", Numeric::new().min(0).max(0o1777).default(0o777)))
    .option("Statedir", Structure::new()
        .member("path", Scalar::new().default("/"))
        .member("mode", Numeric::new().min(0).max(0o1777).default(0o777))
        .member("user", Numeric::new().default(0))
        .member("group", Numeric::new().default(0)))
}

impl<'a> Deserialize<'a> for Host {
    fn deserialize<D: Deserializer<'a>>(d: D) -> Result<Host, D::Error> {
        String::deserialize(d)?.parse().map(Host)
            .map_err(|x| D::Error::custom(format!("{}", x)))
    }
}

impl Serialize for Host {
    fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
        format!("{}", self.0).serialize(s)
    }
}

impl Variable {
    pub fn validate(&self, value: &str, sandbox: &SandboxConfig)
        -> Result<(), String>
    {
        match *self {
            Variable::TcpPort => {
                let port = value.parse::<u16>()
                    .map_err(|e| format!(
                        "invalid TcpPort {:?}: {}", value, e))?;
                if !in_range(&sandbox.allow_tcp_ports, port as u32) {
                    return Err(format!(
                        "TcpPort {:?} is not in allowed range", port));
                }
            }
            Variable::Name => {
                if !NAME_REGEX.is_match(value) {
                    return Err(format!("Value {:?} contains characters that \
                        are invalid for names (alphanumeric, `-` and `_`)",
                        value));
                }
            }
            Variable::Choice(ref choices) => {
                if !choices.iter().any(|x| x == value) {
                    return Err(format!("variable value {:?} \
                        is not one of {:?}", value, choices));
                }
            }
        }
        Ok(())
    }
}
<commit_msg>Add `Serialize` for container config (for future `lithos-config` crate)<commit_after>use std::net::IpAddr;
use std::path::PathBuf;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::os::unix::io::RawFd;

use regex::{Regex, Captures};
use serde::de::{Deserializer, Deserialize, Error as DeError};
use serde::ser::{Serializer, Serialize};
use quire::validate::{Structure, Sequence, Scalar, Numeric, Enum};
use quire::validate::{Mapping, Nothing, Anything};
use id_map::{IdMap, IdMapExt, mapping_validator};

use sandbox_config::SandboxConfig;
use utils::{in_range};
use child_config::ChildKind;

pub const DEFAULT_KILL_TIMEOUT: f32 = 5.;

lazy_static! {
    static ref VARIABLE_REGEX: Regex = Regex::new(r#"@\{([^}]*)\}"#).unwrap();
    static ref NAME_REGEX: Regex = Regex::new(r#"^[0-9a-zA-Z_-]+$"#).unwrap();
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct TmpfsInfo {
    pub size: usize,
    pub mode: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct PersistentInfo {
    pub path: PathBuf,
    pub mkdir: bool,
    pub mode: u32,
    pub user: u32,
    pub group: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct StatedirInfo {
    pub path: PathBuf,
    pub mode: u32,
    pub user: u32,
    pub group: u32,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub enum Volume {
    Readonly(PathBuf),
    Persistent(PersistentInfo),
    Tmpfs(TmpfsInfo),
    Statedir(StatedirInfo),
}

#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy)]
pub enum ContainerKind {
    Daemon,
    Command,
    CommandOrDaemon,
}

impl ContainerKind {
    pub fn matches(self, child_kind: ChildKind) -> bool {
        use container_config::ContainerKind as L;
        use child_config::ChildKind as R;
        match (self, child_kind) {
            (L::Command, R::Command) => true,
            (L::Daemon, R::Daemon) => true,
            (L::CommandOrDaemon, R::Command) => true,
            (L::CommandOrDaemon, R::Daemon) => true,
            (L::Command, R::Daemon) => false,
            (L::Daemon, R::Command) => false,
        }
    }
}

#[derive(Deserialize, Serialize, Clone)]
pub struct ResolvConf {
    pub copy_from_host: bool,
}

#[derive(Deserialize, Serialize, Clone)]
pub struct HostsFile {
    pub copy_from_host: bool,
    pub localhost: Option<bool>,
    pub public_hostname: Option<bool>,
}

#[derive(Clone, Debug)]
pub struct Host(pub IpAddr);

#[derive(Deserialize, Serialize, Clone)]
pub struct TcpPort {
    pub host: Host,
    pub fd: RawFd,
    pub reuse_addr: bool,
    pub reuse_port: bool,
    pub listen_backlog: usize,
}

#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub enum Variable {
    TcpPort,
    Name,
    Choice(Vec<String>),
}

#[derive(Deserialize, Serialize)]
pub struct ContainerConfig {
    pub kind: ContainerKind,
    pub variables: BTreeMap<String, Variable>,
    pub volumes: BTreeMap<String, Volume>,
    pub user_id: u32,
    pub group_id: u32,
    pub restart_timeout: f32,
    pub kill_timeout: f32,
    pub memory_limit: u64,
    pub fileno_limit: u64,
    pub cpu_shares: usize,
    pub executable: String,
    pub arguments: Vec<String>,
    pub environ: BTreeMap<String, String>,
    pub workdir: PathBuf,
    pub resolv_conf: ResolvConf,
    pub hosts_file: HostsFile,
    pub uid_map: Vec<IdMap>,
    pub gid_map: Vec<IdMap>,
    pub stdout_stderr_file: Option<PathBuf>,
    pub interactive: bool,
    pub restart_process_only: bool,
    pub tcp_ports: HashMap<String, TcpPort>,
}

#[derive(Deserialize, Serialize)]
pub struct InstantiatedConfig {
    pub kind: ContainerKind,
    pub volumes: BTreeMap<String, Volume>,
    pub user_id: u32,
    pub group_id: u32,
    pub restart_timeout: f32,
    pub kill_timeout: f32,
    pub memory_limit: u64,
    pub fileno_limit: u64,
    pub cpu_shares: usize,
    pub executable: String,
    pub arguments: Vec<String>,
    pub environ: BTreeMap<String, String>,
    pub workdir: PathBuf,
    pub resolv_conf: ResolvConf,
    pub hosts_file: HostsFile,
    pub uid_map: Vec<IdMap>,
    pub gid_map: Vec<IdMap>,
    pub stdout_stderr_file: Option<PathBuf>,
    pub interactive: bool,
    pub restart_process_only: bool,
    pub tcp_ports: HashMap<u16, TcpPort>,
}

pub struct Variables<'a> {
    pub user_vars: &'a HashMap<String, String>,
    pub lithos_name: &'a str,
    pub lithos_config_filename: &'a str,
}

impl InstantiatedConfig {
    pub fn map_uid(&self, internal_uid: u32) -> Option<u32> {
        self.uid_map.map_id(internal_uid)
    }
    pub fn map_gid(&self, internal_gid: u32) -> Option<u32> {
        self.gid_map.map_id(internal_gid)
    }
}

impl ContainerConfig {
    pub fn validator<'x>() -> Structure<'x> {
        Structure::new()
        .member("kind", Scalar::new().default("Daemon"))
        .member("variables", Mapping::new(
            Scalar::new(),
            Enum::new()
                .option("TcpPort", Nothing)
                .option("Name", Nothing)
                .option("Choice", Sequence::new(Scalar::new()))
        ))
        .member("metadata", Anything)
        .member("volumes", Mapping::new(
                Scalar::new(),
                volume_validator()))
        .member("user_id", Numeric::new())
        .member("group_id", Numeric::new().default(0))
        .member("memory_limit", Numeric::new().default(0x7fffffffffffffffi64))
        .member("fileno_limit", Numeric::new().default(1024))
        .member("cpu_shares", Numeric::new().default(1024))
        .member("restart_timeout", Numeric::new().min(0).max(86400).default(1))
        .member("kill_timeout",
            Numeric::new().min(0).max(86400)
                .default(DEFAULT_KILL_TIMEOUT as i64))
        .member("executable", Scalar::new())
        .member("arguments", Sequence::new(Scalar::new()))
        .member("environ", Mapping::new(
                Scalar::new(),
                Scalar::new()))
        .member("workdir", Scalar::new().default("/"))
        .member("resolv_conf", Structure::new()
            .member("copy_from_host", Scalar::new().default(true)))
        .member("hosts_file", Structure::new()
            .member("copy_from_host", Scalar::new().default(true))
            .member("localhost", Scalar::new().optional())
            .member("public_hostname", Scalar::new().optional()))
        .member("uid_map", mapping_validator())
        .member("gid_map", mapping_validator())
        .member("stdout_stderr_file", Scalar::new().optional())
        .member("interactive", Scalar::new().default(false))
        .member("restart_process_only", Scalar::new().default(false))
        .member("tcp_ports", Mapping::new(
            Scalar::new(),
            Structure::new()
                .member("host", Scalar::new().default("0.0.0.0"))
                .member("fd", Numeric::new().min(0).optional())
                .member("reuse_addr", Scalar::new().default(true))
                .member("reuse_port", Scalar::new().default(false))
                .member("listen_backlog", Scalar::new().default(128))
            ))
    }
    pub fn instantiate(&self, variables: &Variables)
        -> Result<InstantiatedConfig, Vec<String>>
    {
        let mut errors1 = HashSet::new();
        let mut errors2 = HashSet::new();
        let result = {
            let mut replacer = |capt: &Captures| {
                let varname = capt.get(1).unwrap().as_str();
                let val = variables.user_vars.get(varname).map(|x| x.clone())
                    .or_else(|| match varname {
                        "lithos:name"
                        => Some(variables.lithos_name.to_string()),
                        "lithos:config_filename"
                        => Some(variables.lithos_config_filename.to_string()),
                        _ => None,
                    });
                match val {
                    Some(x) => x,
                    None => {
                        errors1.insert(format!("unknown variable {:?}", varname));
                        return format!("<<no var {:?}>>", varname);
                    }
                }
            };
            InstantiatedConfig {
                kind: self.kind.clone(),
                volumes: self.volumes.clone(),
                user_id: self.user_id.clone(),
                group_id: self.group_id.clone(),
                restart_timeout: self.restart_timeout.clone(),
                kill_timeout: self.kill_timeout.clone(),
                memory_limit: self.memory_limit.clone(),
                fileno_limit: self.fileno_limit.clone(),
                cpu_shares: self.cpu_shares.clone(),
                executable: self.executable.clone(),
                arguments: self.arguments.iter()
                    .map(|x| VARIABLE_REGEX.replace(&x, &mut replacer).into())
                    .collect(),
                environ: self.environ.iter()
                    .map(|(key, val)| {
                        (key.clone(),
                         VARIABLE_REGEX.replace(&val, &mut replacer).into())
                    })
                    .collect(),
                workdir: self.workdir.clone(),
                resolv_conf: self.resolv_conf.clone(),
                hosts_file: self.hosts_file.clone(),
                uid_map: self.uid_map.clone(),
                gid_map: self.gid_map.clone(),
                stdout_stderr_file: self.stdout_stderr_file.clone(),
                interactive: self.interactive.clone(),
                restart_process_only: self.restart_process_only.clone(),
                tcp_ports: self.tcp_ports.iter()
                    .map(|(key, val)| {
                        let s = VARIABLE_REGEX.replace(&key, &mut replacer);
                        let port = match s.parse::<u16>() {
                            Ok(x) => x,
                            Err(e) => {
                                errors2.insert(format!("Bad port {:?}: {}",
                                    key, e));
                                return (0, val.clone());
                            }
                        };
                        (port, val.clone())
                    })
                    .collect(),
            }
        };
        if errors1.len() > 0 || errors2.len() > 0 {
            return Err(errors1.into_iter().chain(errors2.into_iter())
                       .collect());
        } else {
            return Ok(result);
        }
    }
}

pub fn volume_validator<'x>() -> Enum<'x> {
    Enum::new()
    .option("Persistent",  Structure::new()
        .member("path",  Scalar::new().default("/"))
        .member("mkdir",  Scalar::new().default(false))
        .member("mode",  Numeric::new().min(0).max(0o1777).default(0o777))
        .member("user",  Numeric::new().default(0))
        .member("group",  Numeric::new().default(0)))
    .option("Readonly", Scalar::new())
    .option("Tmpfs", Structure::new()
        .member("size", Numeric::new().min(0).default(100*1024*1024))
        .member("mode", Numeric::new().min(0).max(0o1777).default(0o777)))
    .option("Statedir", Structure::new()
        .member("path", Scalar::new().default("/"))
        .member("mode", Numeric::new().min(0).max(0o1777).default(0o777))
        .member("user", Numeric::new().default(0))
        .member("group", Numeric::new().default(0)))
}

impl<'a> Deserialize<'a> for Host {
    fn deserialize<D: Deserializer<'a>>(d: D) -> Result<Host, D::Error> {
        String::deserialize(d)?.parse().map(Host)
            .map_err(|x| D::Error::custom(format!("{}", x)))
    }
}

impl Serialize for Host {
    fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
        format!("{}", self.0).serialize(s)
    }
}

impl Variable {
    pub fn validate(&self, value: &str, sandbox: &SandboxConfig)
        -> Result<(), String>
    {
        match *self {
            Variable::TcpPort => {
                let port = value.parse::<u16>()
                    .map_err(|e| format!(
                        "invalid TcpPort {:?}: {}", value, e))?;
                if !in_range(&sandbox.allow_tcp_ports, port as u32) {
                    return Err(format!(
                        "TcpPort {:?} is not in allowed range", port));
                }
            }
            Variable::Name => {
                if !NAME_REGEX.is_match(value) {
                    return Err(format!("Value {:?} contains characters that \
                        are invalid for names (alphanumeric, `-` and `_`)",
                        value));
                }
            }
            Variable::Choice(ref choices) => {
                if !choices.iter().any(|x| x == value) {
                    return Err(format!("variable value {:?} \
                        is not one of {:?}", value, choices));
                }
            }
        }
        Ok(())
    }
}
<|endoftext|>


<commit_before>use std::collections::VecDeque;
use std::fmt;
use term;
use time::precise_time_s;

/// Keep track of this many past download amounts
const DOWNLOAD_TRACK_COUNT: usize = 5;

/// Tracks download progress and displays information about it to a terminal.
pub struct DownloadTracker {
    /// Content-Length of the to-be downloaded object.
    content_len: Option<u64>,
    /// Total data downloaded in bytes.
    total_downloaded: usize,
    /// Data downloaded this second.
    downloaded_this_sec: usize,
    /// Keeps track of amount of data downloaded every last few secs.
    /// Used for averaging the download speed.
    downloaded_last_few_secs: VecDeque<usize>,
    /// Time stamp of the last second
    last_sec: Option<f64>,
    /// How many seconds have elapsed since the download started
    seconds_elapsed: u32,
    /// The terminal we write the information to.
    term: Box<term::StdoutTerminal>,
    /// Whether we displayed progress for the download or not.
    ///
    /// If the download is quick enough, we don't have time to
    /// display the progress info.
    /// In that case, we do not want to do some cleanup stuff we normally do.
    displayed_progress: bool,
}

impl DownloadTracker {
    /// Creates a new DownloadTracker.
    pub fn new() -> Self {
        DownloadTracker {
            content_len: None,
            total_downloaded: 0,
            downloaded_this_sec: 0,
            downloaded_last_few_secs: VecDeque::with_capacity(DOWNLOAD_TRACK_COUNT),
            seconds_elapsed: 0,
            last_sec: None,
            term: term::stdout().expect("Failed to open terminal"),
            displayed_progress: false,
        }
    }
    /// Notifies self that Content-Length information has been received.
    pub fn content_length_received(&mut self, content_len: u64) {
        self.content_len = Some(content_len);
    }
    /// Notifies self that data of size `len` has been received.
    pub fn data_received(&mut self, len: usize) {
        self.total_downloaded += len;
        self.downloaded_this_sec += len;

        match self.last_sec {
            None => self.last_sec = Some(precise_time_s()),
            Some(start) => {
                let elapsed = precise_time_s() - start;
                if elapsed >= 1.0 {
                    self.seconds_elapsed += 1;

                    self.display();
                    self.last_sec = Some(precise_time_s());
                    if self.downloaded_last_few_secs.len() == DOWNLOAD_TRACK_COUNT {
                        self.downloaded_last_few_secs.pop_back();
                    }
                    self.downloaded_last_few_secs.push_front(self.downloaded_this_sec);
                    self.downloaded_this_sec = 0;
                }
            }
        }
    }
    /// Notifies self that the download has finished.
    pub fn download_finished(&mut self) {
        if self.displayed_progress {
            // Display the finished state
            self.display();
            let _ = writeln!(&mut self.term, "");
        }
        self.prepare_for_new_download();
    }
    /// Resets the state to be ready for a new download.
    fn prepare_for_new_download(&mut self) {
        self.content_len = None;
        self.total_downloaded = 0;
        self.downloaded_this_sec = 0;
        self.downloaded_last_few_secs.clear();
        self.seconds_elapsed = 0;
        self.last_sec = None;
        self.displayed_progress = false;
    }
    /// Display the tracked download information to the terminal.
    fn display(&mut self) {
        let total_h = HumanReadable(self.total_downloaded as f64);
        let sum = self.downloaded_last_few_secs
                      .iter()
                      .fold(0usize, |a, &v| a + v);
        let len = self.downloaded_last_few_secs.len();
        let speed = if len > 0 {
            (sum / len) as u64
        } else {
            0
        };
        let speed_h = HumanReadable(speed as f64);

        match self.content_len {
            Some(content_len) => {
                use std::borrow::Cow;

                let percent = (self.total_downloaded as f64 / content_len as f64) * 100.;
                let content_len_h = HumanReadable(content_len as f64);
                let remaining = content_len - self.total_downloaded as u64;
                let eta = if speed > 0 {
                    Cow::Owned(format!("{}s", remaining / speed))
                } else {
                    Cow::Borrowed("Unknown")
                };
                let _ = write!(&mut self.term,
                               "{} / {} ({:.2}%) {}/s ETA: {}",
                               total_h,
                               content_len_h,
                               percent,
                               speed_h,
                               eta);
            }
            None => {
                let _ = write!(&mut self.term, "Total: {} Speed: {}/s", total_h, speed_h);
            }
        }
        // delete_line() doesn't seem to clear the line properly.
        // Instead, let's just print some whitespace to clear it.
        let _ = write!(&mut self.term, "                ");
        let _ = self.term.flush();
        let _ = self.term.carriage_return();
        self.displayed_progress = true;
    }
}

/// Human readable representation of data size in bytes
struct HumanReadable<T>(T);

impl<T: Into<f64> + Clone> fmt::Display for HumanReadable<T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        const KIB: f64 = 1024.0;
        const MIB: f64 = 1048576.0;
        let size: f64 = self.0.clone().into();

        if size >= MIB {
            write!(f, "{:.2} MiB", size / MIB)
        } else if size >= KIB {
            write!(f, "{:.2} KiB", size / KIB)
        } else {
            write!(f, "{} B", size)
        }
    }
}
<commit_msg>Make HumanReadable less (needlessly, confusingly) generic<commit_after>use std::collections::VecDeque;
use std::fmt;
use term;
use time::precise_time_s;

/// Keep track of this many past download amounts
const DOWNLOAD_TRACK_COUNT: usize = 5;

/// Tracks download progress and displays information about it to a terminal.
pub struct DownloadTracker {
    /// Content-Length of the to-be downloaded object.
    content_len: Option<u64>,
    /// Total data downloaded in bytes.
    total_downloaded: usize,
    /// Data downloaded this second.
    downloaded_this_sec: usize,
    /// Keeps track of amount of data downloaded every last few secs.
    /// Used for averaging the download speed.
    downloaded_last_few_secs: VecDeque<usize>,
    /// Time stamp of the last second
    last_sec: Option<f64>,
    /// How many seconds have elapsed since the download started
    seconds_elapsed: u32,
    /// The terminal we write the information to.
    term: Box<term::StdoutTerminal>,
    /// Whether we displayed progress for the download or not.
    ///
    /// If the download is quick enough, we don't have time to
    /// display the progress info.
    /// In that case, we do not want to do some cleanup stuff we normally do.
    displayed_progress: bool,
}

impl DownloadTracker {
    /// Creates a new DownloadTracker.
    pub fn new() -> Self {
        DownloadTracker {
            content_len: None,
            total_downloaded: 0,
            downloaded_this_sec: 0,
            downloaded_last_few_secs: VecDeque::with_capacity(DOWNLOAD_TRACK_COUNT),
            seconds_elapsed: 0,
            last_sec: None,
            term: term::stdout().expect("Failed to open terminal"),
            displayed_progress: false,
        }
    }
    /// Notifies self that Content-Length information has been received.
    pub fn content_length_received(&mut self, content_len: u64) {
        self.content_len = Some(content_len);
    }
    /// Notifies self that data of size `len` has been received.
    pub fn data_received(&mut self, len: usize) {
        self.total_downloaded += len;
        self.downloaded_this_sec += len;

        match self.last_sec {
            None => self.last_sec = Some(precise_time_s()),
            Some(start) => {
                let elapsed = precise_time_s() - start;
                if elapsed >= 1.0 {
                    self.seconds_elapsed += 1;

                    self.display();
                    self.last_sec = Some(precise_time_s());
                    if self.downloaded_last_few_secs.len() == DOWNLOAD_TRACK_COUNT {
                        self.downloaded_last_few_secs.pop_back();
                    }
                    self.downloaded_last_few_secs.push_front(self.downloaded_this_sec);
                    self.downloaded_this_sec = 0;
                }
            }
        }
    }
    /// Notifies self that the download has finished.
    pub fn download_finished(&mut self) {
        if self.displayed_progress {
            // Display the finished state
            self.display();
            let _ = writeln!(&mut self.term, "");
        }
        self.prepare_for_new_download();
    }
    /// Resets the state to be ready for a new download.
    fn prepare_for_new_download(&mut self) {
        self.content_len = None;
        self.total_downloaded = 0;
        self.downloaded_this_sec = 0;
        self.downloaded_last_few_secs.clear();
        self.seconds_elapsed = 0;
        self.last_sec = None;
        self.displayed_progress = false;
    }
    /// Display the tracked download information to the terminal.
    fn display(&mut self) {
        let total_h = HumanReadable(self.total_downloaded as u64);
        let sum = self.downloaded_last_few_secs
                      .iter()
                      .fold(0usize, |a, &v| a + v);
        let len = self.downloaded_last_few_secs.len();
        let speed = if len > 0 {
            (sum / len) as u64
        } else {
            0
        };
        let speed_h = HumanReadable(speed);

        match self.content_len {
            Some(content_len) => {
                use std::borrow::Cow;

                let percent = (self.total_downloaded as f64 / content_len as f64) * 100.;
                let content_len_h = HumanReadable(content_len);
                let remaining = content_len - self.total_downloaded as u64;
                let eta = if speed > 0 {
                    Cow::Owned(format!("{}s", remaining / speed))
                } else {
                    Cow::Borrowed("Unknown")
                };
                let _ = write!(&mut self.term,
                               "{} / {} ({:.2}%) {}/s ETA: {}",
                               total_h,
                               content_len_h,
                               percent,
                               speed_h,
                               eta);
            }
            None => {
                let _ = write!(&mut self.term, "Total: {} Speed: {}/s", total_h, speed_h);
            }
        }
        // delete_line() doesn't seem to clear the line properly.
        // Instead, let's just print some whitespace to clear it.
        let _ = write!(&mut self.term, "                ");
        let _ = self.term.flush();
        let _ = self.term.carriage_return();
        self.displayed_progress = true;
    }
}

/// Human readable representation of data size in bytes
struct HumanReadable(u64);

impl fmt::Display for HumanReadable {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        const KIB: f64 = 1024.0;
        const MIB: f64 = 1048576.0;
        let size = self.0 as f64;

        if size >= MIB {
            write!(f, "{:.2} MiB", size / MIB)
        } else if size >= KIB {
            write!(f, "{:.2} KiB", size / KIB)
        } else {
            write!(f, "{} B", size)
        }
    }
}
<|endoftext|>


<commit_before>//! This module is UNSTABLE, the reason is - recently added.

use std::convert::From;
use std::io::Write;

use super::super::value::{Float, Integer, ValueRef};
use super::*;

// TODO: Make pub sometimes.
use super::{write_str, write_bin};

#[derive(Debug)]
pub struct Error(WriteError);

impl From<FixedValueWriteError> for Error {
    fn from(err: FixedValueWriteError) -> Error {
        match err {
            FixedValueWriteError(err) => Error(err)
        }
    }
}

impl From<ValueWriteError> for Error {
    fn from(err: ValueWriteError) -> Error {
        match err {
            ValueWriteError::InvalidMarkerWrite(err) => Error(err),
            ValueWriteError::InvalidDataWrite(err) => Error(err)
        }
    }
}

pub fn write_value_ref<W>(wr: &mut W, val: &ValueRef) -> Result<(), Error>
    where W: Write
{
    match val {
        &ValueRef::Nil => try!(write_nil(wr)),
        &ValueRef::Boolean(val) => try!(write_bool(wr, val)),
        &ValueRef::Integer(Integer::U64(val)) => {
            try!(write_uint(wr, val));
        }
        &ValueRef::Integer(Integer::I64(val)) => {
            try!(write_sint(wr, val));
        }
        // TODO: Replace with generic write_float(...).
        &ValueRef::Float(Float::F32(val)) => try!(write_f32(wr, val)),
        &ValueRef::Float(Float::F64(val)) => try!(write_f64(wr, val)),
        &ValueRef::String(val) => {
            try!(write_str(wr, val));
        }
        &ValueRef::Binary(val) => {
            try!(write_bin(wr, val));
        }
        &ValueRef::Array(ref val) => {
            let len = val.len() as u32;

            try!(write_array_len(wr, len));

            for item in val {
                try!(write_value_ref(wr, item));
            }
        }
        &ValueRef::Map(ref val) => {
            let len = val.len() as u32;

            try!(write_map_len(wr, len));

            for &(ref key, ref val) in val {
                try!(write_value_ref(wr, key));
                try!(write_value_ref(wr, val));
            }
        }
        &ValueRef::Ext(ty, data) => {
            try!(write_ext_meta(wr, data.len() as u32, ty));
            try!(wr.write_all(data).map_err(|err| ValueWriteError::InvalidDataWrite(WriteError(err))));
        }
    }

    Ok(())
}
<commit_msg>ValueRef: fixed includes for PR.<commit_after>//! This module is UNSTABLE, the reason is - recently added.

use std::convert::From;
use std::io::Write;

use super::super::value::{Float, Integer, ValueRef};
use super::FixedValueWriteError;
use super::ValueWriteError;
use super::WriteError;
use super::write_array_len;
use super::write_bin;
use super::write_bool;
use super::write_ext_meta;
use super::write_f32;
use super::write_f64;
use super::write_map_len;
use super::write_nil;
use super::write_sint;
use super::write_str;
use super::write_uint;

#[derive(Debug)]
pub struct Error(WriteError);

impl From<FixedValueWriteError> for Error {
    fn from(err: FixedValueWriteError) -> Error {
        match err {
            FixedValueWriteError(err) => Error(err)
        }
    }
}

impl From<ValueWriteError> for Error {
    fn from(err: ValueWriteError) -> Error {
        match err {
            ValueWriteError::InvalidMarkerWrite(err) => Error(err),
            ValueWriteError::InvalidDataWrite(err) => Error(err)
        }
    }
}

pub fn write_value_ref<W>(wr: &mut W, val: &ValueRef) -> Result<(), Error>
    where W: Write
{
    match val {
        &ValueRef::Nil => try!(write_nil(wr)),
        &ValueRef::Boolean(val) => try!(write_bool(wr, val)),
        &ValueRef::Integer(Integer::U64(val)) => {
            try!(write_uint(wr, val));
        }
        &ValueRef::Integer(Integer::I64(val)) => {
            try!(write_sint(wr, val));
        }
        // TODO: Replace with generic write_float(...).
        &ValueRef::Float(Float::F32(val)) => try!(write_f32(wr, val)),
        &ValueRef::Float(Float::F64(val)) => try!(write_f64(wr, val)),
        &ValueRef::String(val) => {
            try!(write_str(wr, val));
        }
        &ValueRef::Binary(val) => {
            try!(write_bin(wr, val));
        }
        &ValueRef::Array(ref val) => {
            let len = val.len() as u32;

            try!(write_array_len(wr, len));

            for item in val {
                try!(write_value_ref(wr, item));
            }
        }
        &ValueRef::Map(ref val) => {
            let len = val.len() as u32;

            try!(write_map_len(wr, len));

            for &(ref key, ref val) in val {
                try!(write_value_ref(wr, key));
                try!(write_value_ref(wr, val));
            }
        }
        &ValueRef::Ext(ty, data) => {
            try!(write_ext_meta(wr, data.len() as u32, ty));
            try!(wr.write_all(data).map_err(|err| ValueWriteError::InvalidDataWrite(WriteError(err))));
        }
    }

    Ok(())
}
<|endoftext|>


<commit_before>use schema::{Schema, Field, Document};
use fastfield::FastFieldSerializer;
use std::io;
use schema::Value;
use DocId;
use common;
use schema::FieldType;

/// The fastfieldswriter regroup all of the fast field writers.
pub struct FastFieldsWriter {
    field_writers: Vec<IntFastFieldWriter>,
}

impl FastFieldsWriter {
    /// Create all `FastFieldWriter` required by the schema.
    pub fn from_schema(schema: &Schema) -> FastFieldsWriter {
        let field_writers: Vec<IntFastFieldWriter> = schema
            .fields()
            .iter()
            .enumerate()
            .flat_map(|(field_id, field_entry)| {
                let field = Field(field_id as u32);
                match *field_entry.field_type() {
                    FieldType::I64(ref int_options) => {
                        if int_options.is_fast() {
                            let mut fast_field_writer = IntFastFieldWriter::new(field);
                            fast_field_writer.set_val_if_missing(common::i64_to_u64(0i64));
                            Some(fast_field_writer)
                        } else {
                            None
                        }
                    }
                    FieldType::U64(ref int_options) => {
                        if int_options.is_fast() {
                            Some(IntFastFieldWriter::new(field))
                        } else {
                            None
                        }
                    }
                    _ => None,
                }
            })
            .collect();
        FastFieldsWriter { field_writers: field_writers }
    }

    /// Returns a `FastFieldsWriter`
    /// with a `IntFastFieldWriter` for each
    /// of the field given in argument.
    pub fn new(fields: Vec<Field>) -> FastFieldsWriter {
        FastFieldsWriter {
            field_writers: fields.into_iter().map(IntFastFieldWriter::new).collect(),
        }
    }

    /// Get the `FastFieldWriter` associated to a field.
    pub fn get_field_writer(&mut self, field: Field) -> Option<&mut IntFastFieldWriter> {
        // TODO optimize
        self.field_writers
            .iter_mut()
            .find(|field_writer| field_writer.field == field)
    }

    /// Indexes all of the fastfields of a new document.
    pub fn add_document(&mut self, doc: &Document) {
        for field_writer in &mut self.field_writers {
            field_writer.add_document(doc);
        }
    }

    /// Serializes all of the `FastFieldWriter`s by pushing them in
    /// order to the fast field serializer.
    pub fn serialize(&self, serializer: &mut FastFieldSerializer) -> io::Result<()> {
        for field_writer in &self.field_writers {
            field_writer.serialize(serializer)?;
        }
        Ok(())
    }

    /// Ensures all of the fast field writers have
    /// reached `doc`. (included)
    ///
    /// The missing values will be filled with 0.
    pub fn fill_val_up_to(&mut self, doc: DocId) {
        for field_writer in &mut self.field_writers {
            field_writer.fill_val_up_to(doc);
        }
    }
}

/// Fast field writer for ints.
/// The fast field writer just keeps the values in memory.
///
/// Only when the segment writer can be closed and
/// persisted on disc, the fast field writer is
/// sent to a `FastFieldSerializer` via the `.serialize(...)`
/// method.
///
/// We cannot serialize earlier as the values are
/// bitpacked and the number of bits required for bitpacking
/// can only been known once we have seen all of the values.
///
/// Both u64, and i64 use the same writer.
/// i64 are just remapped to the `0..2^64 - 1`
/// using `common::i64_to_u64`.
pub struct IntFastFieldWriter {
    field: Field,
    vals: Vec<u64>,
    val_if_missing: u64,
}

impl IntFastFieldWriter {
    /// Creates a new `IntFastFieldWriter`
    pub fn new(field: Field) -> IntFastFieldWriter {
        IntFastFieldWriter {
            field: field,
            vals: Vec::new(),
            val_if_missing: 0u64,
        }
    }

    /// Sets the default value.
    ///
    /// This default value is recorded for documents if
    /// a document does not have any value.
    fn set_val_if_missing(&mut self, val_if_missing: u64) {
        self.val_if_missing = val_if_missing;
    }

    /// Ensures all of the fast field writer have
    /// reached `doc`. (included)
    ///
    /// The missing values will be filled with 0.
    fn fill_val_up_to(&mut self, doc: DocId) {
        let target = doc as usize + 1;
        debug_assert!(self.vals.len() <= target);
        let val_if_missing = self.val_if_missing;
        while self.vals.len() < target {
            self.add_val(val_if_missing);
        }
    }

    /// Records a new value.
    ///
    /// The n-th value being recorded is implicitely
    /// associated to the document with the `DocId` n.
    /// (Well, `n-1` actually because of 0-indexing)
    pub fn add_val(&mut self, val: u64) {
        self.vals.push(val);
    }

    /// Extract the value associated to the fast field for
    /// this document.
    ///
    /// i64 are remapped to u64 using the logic
    /// in `common::i64_to_u64`.
    ///
    /// If the value is missing, then the default value is used
    /// instead.
    /// If the document has more than one value for the given field,
    /// only the first one is taken in account.
    fn extract_val(&self, doc: &Document) -> u64 {
        match doc.get_first(self.field) {
            Some(v) => {
                match *v {
                    Value::U64(ref val) => *val,
                    Value::I64(ref val) => common::i64_to_u64(*val),
                    _ => panic!("Expected a u64field, got {:?} ", v),
                }
            }
            None => self.val_if_missing,
        }
    }

    /// Extract the fast field value from the document
    /// (or use the default value) and records it.
    pub fn add_document(&mut self, doc: &Document) {
        let val = self.extract_val(doc);
        self.add_val(val);
    }

    /// Push the fast fields value to the `FastFieldWriter`.
    pub fn serialize(&self, serializer: &mut FastFieldSerializer) -> io::Result<()> {
        let zero = 0;
        let min = *self.vals.iter().min().unwrap_or(&zero);
        let max = *self.vals.iter().max().unwrap_or(&min);
        serializer.new_u64_fast_field(self.field, min, max)?;
        for &val in &self.vals {
            serializer.add_val(val)?;
        }
        serializer.close_field()
    }
}
<commit_msg>VInt encode values in IntFastFieldWriter<commit_after>use schema::{Schema, Field, Document};
use fastfield::FastFieldSerializer;
use std::io;
use schema::Value;
use DocId;
use schema::FieldType;
use common;
use common::VInt;
use common::BinarySerializable;

/// The fastfieldswriter regroup all of the fast field writers.
pub struct FastFieldsWriter {
    field_writers: Vec<IntFastFieldWriter>,
}

impl FastFieldsWriter {
    /// Create all `FastFieldWriter` required by the schema.
    pub fn from_schema(schema: &Schema) -> FastFieldsWriter {
        let field_writers: Vec<IntFastFieldWriter> = schema
            .fields()
            .iter()
            .enumerate()
            .flat_map(|(field_id, field_entry)| {
                let field = Field(field_id as u32);
                match *field_entry.field_type() {
                    FieldType::I64(ref int_options) => {
                        if int_options.is_fast() {
                            let mut fast_field_writer = IntFastFieldWriter::new(field);
                            fast_field_writer.set_val_if_missing(common::i64_to_u64(0i64));
                            Some(fast_field_writer)
                        } else {
                            None
                        }
                    }
                    FieldType::U64(ref int_options) => {
                        if int_options.is_fast() {
                            Some(IntFastFieldWriter::new(field))
                        } else {
                            None
                        }
                    }
                    _ => None,
                }
            })
            .collect();
        FastFieldsWriter { field_writers: field_writers }
    }

    /// Returns a `FastFieldsWriter`
    /// with a `IntFastFieldWriter` for each
    /// of the field given in argument.
    pub fn new(fields: Vec<Field>) -> FastFieldsWriter {
        FastFieldsWriter {
            field_writers: fields.into_iter().map(IntFastFieldWriter::new).collect(),
        }
    }

    /// Get the `FastFieldWriter` associated to a field.
    pub fn get_field_writer(&mut self, field: Field) -> Option<&mut IntFastFieldWriter> {
        // TODO optimize
        self.field_writers
            .iter_mut()
            .find(|field_writer| field_writer.field == field)
    }

    /// Indexes all of the fastfields of a new document.
    pub fn add_document(&mut self, doc: &Document) {
        for field_writer in &mut self.field_writers {
            field_writer.add_document(doc);
        }
    }

    /// Serializes all of the `FastFieldWriter`s by pushing them in
    /// order to the fast field serializer.
    pub fn serialize(&self, serializer: &mut FastFieldSerializer) -> io::Result<()> {
        for field_writer in &self.field_writers {
            field_writer.serialize(serializer)?;
        }
        Ok(())
    }

    /// Ensures all of the fast field writers have
    /// reached `doc`. (included)
    ///
    /// The missing values will be filled with 0.
    pub fn fill_val_up_to(&mut self, doc: DocId) {
        for field_writer in &mut self.field_writers {
            field_writer.fill_val_up_to(doc);
        }
    }
}

/// Fast field writer for ints.
/// The fast field writer just keeps the values in memory.
///
/// Only when the segment writer can be closed and
/// persisted on disc, the fast field writer is
/// sent to a `FastFieldSerializer` via the `.serialize(...)`
/// method.
///
/// We cannot serialize earlier as the values are
/// bitpacked and the number of bits required for bitpacking
/// can only been known once we have seen all of the values.
///
/// Both u64, and i64 use the same writer.
/// i64 are just remapped to the `0..2^64 - 1`
/// using `common::i64_to_u64`.
pub struct IntFastFieldWriter {
    field: Field,
    vals: Vec<u8>,
    val_count: usize,
    val_if_missing: u64,
    val_min: u64,
    val_max: u64,
}

impl IntFastFieldWriter {
    /// Creates a new `IntFastFieldWriter`
    pub fn new(field: Field) -> IntFastFieldWriter {
        IntFastFieldWriter {
            field: field,
            vals: Vec::new(),
            val_count: 0,
            val_if_missing: 0u64,
            val_min: u64::max_value(),
            val_max: 0,
        }
    }

    /// Sets the default value.
    ///
    /// This default value is recorded for documents if
    /// a document does not have any value.
    fn set_val_if_missing(&mut self, val_if_missing: u64) {
        self.val_if_missing = val_if_missing;
    }

    /// Ensures all of the fast field writer have
    /// reached `doc`. (included)
    ///
    /// The missing values will be filled with 0.
    fn fill_val_up_to(&mut self, doc: DocId) {
        let target = doc as usize + 1;
        debug_assert!(self.val_count <= target);
        let val_if_missing = self.val_if_missing;
        while self.val_count < target {
            self.add_val(val_if_missing);
        }
    }

    /// Records a new value.
    ///
    /// The n-th value being recorded is implicitely
    /// associated to the document with the `DocId` n.
    /// (Well, `n-1` actually because of 0-indexing)
    pub fn add_val(&mut self, val: u64) {
        VInt(val)
            .serialize(&mut self.vals)
            .expect("unable to serialize VInt to Vec");

        if val > self.val_max {
            self.val_max = val;
        }
        if val < self.val_min {
            self.val_min = val;
        }

        self.val_count += 1;
    }

    /// Extract the value associated to the fast field for
    /// this document.
    ///
    /// i64 are remapped to u64 using the logic
    /// in `common::i64_to_u64`.
    ///
    /// If the value is missing, then the default value is used
    /// instead.
    /// If the document has more than one value for the given field,
    /// only the first one is taken in account.
    fn extract_val(&self, doc: &Document) -> u64 {
        match doc.get_first(self.field) {
            Some(v) => {
                match *v {
                    Value::U64(ref val) => *val,
                    Value::I64(ref val) => common::i64_to_u64(*val),
                    _ => panic!("Expected a u64field, got {:?} ", v),
                }
            }
            None => self.val_if_missing,
        }
    }

    /// Extract the fast field value from the document
    /// (or use the default value) and records it.
    pub fn add_document(&mut self, doc: &Document) {
        let val = self.extract_val(doc);
        self.add_val(val);
    }

    /// Push the fast fields value to the `FastFieldWriter`.
    pub fn serialize(&self, serializer: &mut FastFieldSerializer) -> io::Result<()> {
        let (min, max) = if self.val_min > self.val_max {
            (0, 0)
        } else {
            (self.val_min, self.val_max)
        };

        serializer.new_u64_fast_field(self.field, min, max)?;

        let mut cursor = self.vals.as_slice();
        while let Ok(VInt(val)) = VInt::deserialize(&mut cursor) {
            serializer.add_val(val)?;
        }

        serializer.close_field()
    }
}
<|endoftext|>


<commit_before>use std::collections::HashSet;
use std::convert::AsRef;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;
use std::sync::Arc;
use toml;
use ::util::errors::*;
use ::flota::Cypherable;

macro_rules! unfold {
    ( $toml:ident, $key:expr, $ty:tt, optional, $default:expr ) => {{
        match stringify!($ty) {
            "bool"|"IPv4"|"i32"|"PathBuf"|"String"|"Url" => {
                unfold!($toml, $key, $ty, optional)
                    .or(Some($default)).unwrap()
            },
            _ => panic!("unsupported type")
        }
    }};
    ( $toml:ident, $key:expr, $ty:tt ) => {{
        match stringify!($ty) {
            "bool"| "IPv4"|"i32"|"PathBuf"|"String"|"Url" => {
                unfold!($toml, $key, $ty, optional)
                     .ok_or(format!("`{}` must be specified", $key).as_str()).unwrap()
            },
            _ => panic!("unsupported type")
        }
    }};
    ( $toml:ident, $key:expr, String, optional ) => {{
        $toml.lookup($key)
             .map(|val| val.as_str().unwrap().to_string())
    }};
    ( $toml:ident, $key:expr, IPv4, optional ) => {{
        $toml.lookup($key)
             .map(|val| {
                 IPv4::from_cidr_notation(val.as_str().unwrap())
                     .unwrap()
             })
    }};
    ( $toml:ident, $key:expr, i32, optional ) => {{
        $toml.lookup($key)
             .map(|val| val.as_integer().unwrap() as i32)
    }};
    ( $toml:ident, $key:expr, PathBuf, optional ) => {{
        $toml.lookup($key)
             .map(|val| PathBuf::from(val.as_str().unwrap()))
    }};
    ( $toml:ident, $key:expr, Url, optional ) => {{
        $toml.lookup($key)
             .map(|val| {
                 Url::parse(val.as_str().unwrap()).unwrap()
             })
    }};
    ( $toml:ident, $key:expr, bool, optional ) => {{
        $toml.lookup($key)
             .map(|val| val.as_bool().unwrap())
    }};
}

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
// XXX: local/remote choices might probably be sufficient
pub enum ExecType {
    Console,
    Local,
    Ssh,
}

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Exec {
    /// of enum ExecType
    pub exec_type: ExecType,
    /// Hostname on which this Exec will be executed.
    pub host: Option<String>,
    /// For the time being this is supposed to be directly
    /// executed on the guest side.
    pub command: String,
    /// Optionally you can set an expected stdout.
    pub expect_stdout: Option<String>,
    /// Optionally you can set an expected stderr.
    pub expect_stderr: Option<String>,
    /// Optionally you can set an expected exit code.
    pub expect_status: Option<i32>,
    /// If either expect_stdout or expect_status set,
    /// and if this is set true, all the following
    /// executions would be skipped on an unexpecte result.
    pub abort_on_failure: bool,
}

impl Cypherable for Exec {
    fn cypher_ident(&self) -> String {
        format!("Exec {{ exec_type: '{exec_type:?}',
                         host: '{host:?}',
                         command: '{command}',
                         expect_stdout: '{expect_stdout:?}',
                         expect_stderr: '{expect_stderr:?}',
                         expect_status: '{expect_status:?}',
                         abort_on_failure: '{abort_on_failure}' }}",
                exec_type = self.exec_type,
                host = self.host,
                command = self.command,
                expect_stdout = self.expect_stdout,
                expect_stderr = self.expect_stderr,
                expect_status = self.expect_status,
                abort_on_failure = self.abort_on_failure)
    }
}

impl Exec {
    pub fn from_toml(tml: &toml::Value) -> Result<Exec> {
        let exec_type = unfold!(tml, "type", String);
        let command = unfold!(tml, "command", String);
        let expect_stdout = unfold!(tml, "stdout", String, optional);
        let expect_stderr = unfold!(tml, "stderr", String, optional);
        let expect_status = unfold!(tml, "status", i32, optional);
        let abort_on_failure = unfold!(tml, "abort_on_failure", bool, optional, false);
        match &*exec_type {
            "console" => Ok(Exec {
                exec_type: ExecType::Console,
                host: unfold!(tml, "host", String, optional),
                command: command,
                expect_stdout: expect_stdout,
                expect_stderr: expect_stderr,
                expect_status: expect_status,
                abort_on_failure: abort_on_failure
            }),
            "local" => Ok(Exec {
                exec_type: ExecType::Console,
                host: Some("localhost".to_string()),
                command: command,
                expect_stdout: expect_stdout,
                expect_stderr: expect_stderr,
                expect_status: expect_status,
                abort_on_failure: abort_on_failure
            }),
            "ssh" => Ok(Exec {
                exec_type: ExecType::Ssh,
                host: unfold!(tml, "host", String, optional),
                command: command,
                expect_stdout: expect_stdout,
                expect_stderr: expect_stderr,
                expect_status: expect_status,
                abort_on_failure: abort_on_failure,
            }),
            _ => Err("failed to build exec".into())
        }
    }
}

pub mod setting;
pub mod template;
pub mod cluster;

use self::setting::Setting;
use self::template::Template;
use self::cluster::Cluster;

#[derive(Debug)]
pub struct Config {
    pub setting: Arc<Setting>,
    pub templates: HashSet<Arc<Template>>,
    pub clusters: HashSet<Arc<Cluster>>,
}

impl Config {
    pub fn from_toml_file(path: &Path) -> Result<Config> {
        let mut file = File::open(path.to_str().unwrap())
            .expect(format!("Cannot open toml file: {}", path.display()).as_ref());
        let mut buf = String::new();
        file.read_to_string(&mut buf)
            .expect(format!("Cannot read toml file: {}", path.display()).as_ref());
        match buf.parse() {
            Ok(tml) => {
                Config::from_toml(&tml)
            },
            Err(err) => {
                for e in err.iter() {
                    error!("{}", e);
                }
                Err("invalid toml".into())
            }
        }
    }
    pub fn from_toml(tml: &toml::Value) -> Result<Config> {
        // global setting
        let setting = if let Some(val) = tml.lookup("setting") {
            Arc::new(Setting::from_toml(&val))
        } else {
            // if blank, default setting applied.
            Arc::new(Setting::default())
        };

        // templates
        let templates = if let Some(&toml::Value::Array(ref vals)) = tml.lookup("template") {
            vals.iter().map(|val| {
                // XXX: should we just overlook broken config?
                Arc::new(Template::from_toml(&val, &setting).unwrap())
            }).collect::<HashSet<_>>()
        } else {
            // there must be at least one template definition
            return Err("no template found".into())
        };

        // clusters
        let clusters = if let Some(&toml::Value::Array(ref vals)) = tml.lookup("cluster") {
            vals.iter().map(|val| {
                // XXX: should we just overlook broken config?
                Arc::new(Cluster::from_toml(&val, &templates.clone()).unwrap())
            }).collect::<HashSet<_>>()
        } else {
            // there must be at least one cluster definition
            return Err("no cluster found".into())
        };

        Ok(Config {
            setting: setting,
            templates: templates,
            clusters: clusters,
        })
    }
}
<commit_msg>fix bug<commit_after>use std::collections::HashSet;
use std::convert::AsRef;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;
use std::sync::Arc;
use toml;
use ::util::errors::*;
use ::flota::Cypherable;

macro_rules! unfold {
    ( $toml:ident, $key:expr, $ty:tt, optional, $default:expr ) => {{
        match stringify!($ty) {
            "bool"|"IPv4"|"i32"|"PathBuf"|"String"|"Url" => {
                unfold!($toml, $key, $ty, optional)
                    .or(Some($default)).unwrap()
            },
            _ => panic!("unsupported type")
        }
    }};
    ( $toml:ident, $key:expr, $ty:tt ) => {{
        match stringify!($ty) {
            "bool"| "IPv4"|"i32"|"PathBuf"|"String"|"Url" => {
                unfold!($toml, $key, $ty, optional)
                     .ok_or(format!("`{}` must be specified", $key).as_str()).unwrap()
            },
            _ => panic!("unsupported type")
        }
    }};
    ( $toml:ident, $key:expr, String, optional ) => {{
        $toml.lookup($key)
             .map(|val| val.as_str().unwrap().to_string())
    }};
    ( $toml:ident, $key:expr, IPv4, optional ) => {{
        $toml.lookup($key)
             .map(|val| {
                 IPv4::from_cidr_notation(val.as_str().unwrap())
                     .unwrap()
             })
    }};
    ( $toml:ident, $key:expr, i32, optional ) => {{
        $toml.lookup($key)
             .map(|val| val.as_integer().unwrap() as i32)
    }};
    ( $toml:ident, $key:expr, PathBuf, optional ) => {{
        $toml.lookup($key)
             .map(|val| PathBuf::from(val.as_str().unwrap()))
    }};
    ( $toml:ident, $key:expr, Url, optional ) => {{
        $toml.lookup($key)
             .map(|val| {
                 Url::parse(val.as_str().unwrap()).unwrap()
             })
    }};
    ( $toml:ident, $key:expr, bool, optional ) => {{
        $toml.lookup($key)
             .map(|val| val.as_bool().unwrap())
    }};
}

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
// XXX: local/remote choices might probably be sufficient
pub enum ExecType {
    Console,
    Local,
    Ssh,
}

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Exec {
    /// of enum ExecType
    pub exec_type: ExecType,
    /// Hostname on which this Exec will be executed.
    pub host: Option<String>,
    /// For the time being this is supposed to be directly
    /// executed on the guest side.
    pub command: String,
    /// Optionally you can set an expected stdout.
    pub expect_stdout: Option<String>,
    /// Optionally you can set an expected stderr.
    pub expect_stderr: Option<String>,
    /// Optionally you can set an expected exit code.
    pub expect_status: Option<i32>,
    /// If either expect_stdout or expect_status set,
    /// and if this is set true, all the following
    /// executions would be skipped on an unexpecte result.
    pub abort_on_failure: bool,
}

impl Cypherable for Exec {
    fn cypher_ident(&self) -> String {
        format!("Exec {{ exec_type: '{exec_type:?}',
                         host: '{host:?}',
                         command: '{command}',
                         expect_stdout: '{expect_stdout:?}',
                         expect_stderr: '{expect_stderr:?}',
                         expect_status: '{expect_status:?}',
                         abort_on_failure: '{abort_on_failure}' }}",
                exec_type = self.exec_type,
                host = self.host,
                command = self.command,
                expect_stdout = self.expect_stdout,
                expect_stderr = self.expect_stderr,
                expect_status = self.expect_status,
                abort_on_failure = self.abort_on_failure)
    }
}

impl Exec {
    pub fn from_toml(tml: &toml::Value) -> Result<Exec> {
        let exec_type = unfold!(tml, "type", String);
        let command = unfold!(tml, "command", String);
        let expect_stdout = unfold!(tml, "stdout", String, optional);
        let expect_stderr = unfold!(tml, "stderr", String, optional);
        let expect_status = unfold!(tml, "status", i32, optional);
        let abort_on_failure = unfold!(tml, "abort_on_failure", bool, optional, false);
        match &*exec_type {
            "console" => Ok(Exec {
                exec_type: ExecType::Console,
                host: unfold!(tml, "host", String, optional),
                command: command,
                expect_stdout: expect_stdout,
                expect_stderr: expect_stderr,
                expect_status: expect_status,
                abort_on_failure: abort_on_failure
            }),
            "local" => Ok(Exec {
                exec_type: ExecType::Local,
                host: Some("localhost".to_string()),
                command: command,
                expect_stdout: expect_stdout,
                expect_stderr: expect_stderr,
                expect_status: expect_status,
                abort_on_failure: abort_on_failure
            }),
            "ssh" => Ok(Exec {
                exec_type: ExecType::Ssh,
                host: unfold!(tml, "host", String, optional),
                command: command,
                expect_stdout: expect_stdout,
                expect_stderr: expect_stderr,
                expect_status: expect_status,
                abort_on_failure: abort_on_failure,
            }),
            _ => Err("failed to build exec".into())
        }
    }
}

pub mod setting;
pub mod template;
pub mod cluster;

use self::setting::Setting;
use self::template::Template;
use self::cluster::Cluster;

#[derive(Debug)]
pub struct Config {
    pub setting: Arc<Setting>,
    pub templates: HashSet<Arc<Template>>,
    pub clusters: HashSet<Arc<Cluster>>,
}

impl Config {
    pub fn from_toml_file(path: &Path) -> Result<Config> {
        let mut file = File::open(path.to_str().unwrap())
            .expect(format!("Cannot open toml file: {}", path.display()).as_ref());
        let mut buf = String::new();
        file.read_to_string(&mut buf)
            .expect(format!("Cannot read toml file: {}", path.display()).as_ref());
        match buf.parse() {
            Ok(tml) => {
                Config::from_toml(&tml)
            },
            Err(err) => {
                for e in err.iter() {
                    error!("{}", e);
                }
                Err("invalid toml".into())
            }
        }
    }
    pub fn from_toml(tml: &toml::Value) -> Result<Config> {
        // global setting
        let setting = if let Some(val) = tml.lookup("setting") {
            Arc::new(Setting::from_toml(&val))
        } else {
            // if blank, default setting applied.
            Arc::new(Setting::default())
        };

        // templates
        let templates = if let Some(&toml::Value::Array(ref vals)) = tml.lookup("template") {
            vals.iter().map(|val| {
                // XXX: should we just overlook broken config?
                Arc::new(Template::from_toml(&val, &setting).unwrap())
            }).collect::<HashSet<_>>()
        } else {
            // there must be at least one template definition
            return Err("no template found".into())
        };

        // clusters
        let clusters = if let Some(&toml::Value::Array(ref vals)) = tml.lookup("cluster") {
            vals.iter().map(|val| {
                // XXX: should we just overlook broken config?
                Arc::new(Cluster::from_toml(&val, &templates.clone()).unwrap())
            }).collect::<HashSet<_>>()
        } else {
            // there must be at least one cluster definition
            return Err("no cluster found".into())
        };

        Ok(Config {
            setting: setting,
            templates: templates,
            clusters: clusters,
        })
    }
}
<|endoftext|>


<commit_before>/// optimal pareto front (f_1, 1 - sqrt(f_1))

/// 0 <= x[i] <= 1.0
fn zdt1(x: &[f32]) -> (f32, f32) {
    let n = x.len();
    assert!(n >= 2);

    let f1 = x[0];
    let g = 1.0 + (9.0 / (n - 1) as f32) * x[1..].iter().fold(0.0, |b, &i| b + i);
    let f2 = g * (1.0 - (f1 / g).sqrt());

    (f1, f2)
}

trait Dominate<Rhs=Self> {
    fn dominates(&self, other: &Rhs) -> bool;
}

trait MultiObjective {
    fn num_objectives() -> usize;
    fn get_f32_objective(&self, i: usize) -> f32;
}

#[derive(Debug)]
struct MultiObjective2<T>
    where T: Sized + PartialOrd
{
    objectives: [T; 2],
}

impl MultiObjective for MultiObjective2<f32> {
    fn num_objectives() -> usize {
        2
    }
    fn get_f32_objective(&self, i: usize) -> f32 {
        self.objectives[i]
    }
}

#[derive(Debug)]
struct MultiObjective3<T>
    where T: Sized + PartialOrd
{
    objectives: [T; 3],
}

fn dominates_slices<T: Sized + PartialOrd>(sa: &[T], sb: &[T]) -> bool {
    assert!(sa.len() == sb.len());
    assert!(sa.len() > 0);

    let mut less_cnt = 0;
    for (a, b) in sa.iter().zip(sb.iter()) {
        if a > b {
            return false;
        } else if a < b {
            less_cnt += 1;
        }
    }

    return less_cnt > 0;
}

impl<T:Sized+PartialOrd> Dominate for MultiObjective2<T> {
    fn dominates(&self, other: &Self) -> bool {
        dominates_slices(&self.objectives[..], &other.objectives[..])
    }
}

/// `n` stop after we have found this number of solutions (we include the whole pareto front).
fn fast_non_dominated_sort<P: Dominate>(solutions: &[P], n: usize) -> Vec<Vec<usize>> {
    let mut fronts: Vec<Vec<usize>> = Vec::new();
    let mut current_front = Vec::new();

    let mut domination_count: Vec<usize> = (0..solutions.len()).map(|_| 0).collect();
    let mut dominated_solutions: Vec<Vec<usize>> = (0..solutions.len())
                                                       .map(|_| Vec::new())
                                                       .collect();
    let mut rank: Vec<Option<usize>> = (0..solutions.len()).map(|_| None).collect();
    let mut found_solutions: usize = 0;

    for (i, p) in solutions.iter().enumerate() {
        for (j, q) in solutions.iter().enumerate() {
            if i == j {
                continue;
            }
            if p.dominates(q) {
                // Add `q` to the set of solutions dominated by `p`.
                dominated_solutions[i].push(j);
            } else if q.dominates(p) {
                // Increment domination counter of `p`.
                domination_count[i] += 1;
            }
        }

        if domination_count[i] == 0 {
            // `p` belongs to the first front as it is not dominated by any
            // other solution.
            current_front.push(i);
            assert!(rank[i].is_none());
            rank[i] = Some(fronts.len());
        }
    }

    while !current_front.is_empty() {
        found_solutions += current_front.len();
        if found_solutions >= n {
            fronts.push(current_front);
            break;
        } else {
            // we haven't found enough solutions yet.
            let mut next_front = Vec::new();
            for &p_i in current_front.iter() {
                for &q_i in dominated_solutions[p_i].iter() {
                    domination_count[q_i] -= 1;
                    if domination_count[q_i] == 0 {
                        // q belongs to the next front
                        assert!(rank[q_i].is_none());
                        rank[q_i] = Some(fronts.len() + 1);
                        next_front.push(q_i);
                    }
                }
            }
            fronts.push(current_front);
            current_front = next_front;
        }
    }

    for i in 0..solutions.len() {
        assert!(rank[i].is_some());
    }

    return fronts;
}

/// returns: (individual_idx, distance)
fn crowding_distance_assignment<P: MultiObjective>(solutions: &[P],
                                                   individuals_idx: &[usize])
                                                   -> Vec<(usize, f32)> {
    let l = individuals_idx.len();
    let mut distance: Vec<f32> = (0..l).map(|_| 0.0).collect();

    // maps index into distance array to index into solutions.
    let mut map: Vec<(usize, usize)> = (0..l).map(|i| (i, individuals_idx[i])).collect();

    for m in 0..P::num_objectives() {
        // sort using objective `m`
        map.sort_by(|a, b| {
            solutions[a.1]
                .get_f32_objective(m)
                .partial_cmp(&solutions[b.1].get_f32_objective(m))
                .unwrap()
        });
        distance[map[0].0] = std::f32::INFINITY;
        distance[map[l - 1].0] = std::f32::INFINITY;
        let min_f = solutions[map[0].1].get_f32_objective(m);
        let max_f = solutions[map[l - 1].1].get_f32_objective(m);
        if min_f != max_f {
            let norm = P::num_objectives() as f32 * (max_f - min_f);
            debug_assert!(norm != 0.0);
            for i in 1..(l - 1) {
                distance[map[i].0] += (solutions[map[i + 1].1].get_f32_objective(m) -
                                       solutions[map[i - 1].1].get_f32_objective(m)) /
                                      norm;
            }
        }
    }

    let mut res: Vec<(usize, f32)> = (0..l).map(|_| (0, -1.0)).collect();
    for &(i, idx) in map.iter() {
        debug_assert!(res[i].1 == -1.0);
        debug_assert!(res[i].0 == 0);
        res[i] = (idx, distance[i]);
    }

    return res;
}

#[test]
fn test_dominates() {
    let a = MultiObjective2 { objectives: [1.0, 0.1] };
    let b = MultiObjective2 { objectives: [0.1, 0.1] };
    let c = MultiObjective2 { objectives: [0.1, 1.0] };

    assert_eq!(false, a.dominates(&a));
    assert_eq!(false, a.dominates(&b));
    assert_eq!(false, a.dominates(&c));

    assert_eq!(true, b.dominates(&a));
    assert_eq!(false, b.dominates(&b));
    assert_eq!(true, b.dominates(&c));

    assert_eq!(false, c.dominates(&a));
    assert_eq!(false, c.dominates(&b));
    assert_eq!(false, c.dominates(&c));
}

fn main() {
    let mut solutions: Vec<MultiObjective2<f32>> = Vec::new();
    solutions.push(MultiObjective2 { objectives: [1.0, 0.1] });
    solutions.push(MultiObjective2 { objectives: [0.1, 0.1] });
    solutions.push(MultiObjective2 { objectives: [0.1, 1.0] });
    solutions.push(MultiObjective2 { objectives: [0.5, 0.5] });

    let fronts = fast_non_dominated_sort(&solutions[..], 10);
    println!("solutions: {:?}", solutions);
    println!("fronts: {:?}", fronts);

    for front in fronts.iter() {
        let distances = crowding_distance_assignment(&solutions[..], &front[..]);
        println!("front: {:?}", front);
        println!("distances: {:?}", distances);
    }
}
<commit_msg>Remove rank. Rank is implicit by containment in pareto front<commit_after>/// optimal pareto front (f_1, 1 - sqrt(f_1))

/// 0 <= x[i] <= 1.0
fn zdt1(x: &[f32]) -> (f32, f32) {
    let n = x.len();
    assert!(n >= 2);

    let f1 = x[0];
    let g = 1.0 + (9.0 / (n - 1) as f32) * x[1..].iter().fold(0.0, |b, &i| b + i);
    let f2 = g * (1.0 - (f1 / g).sqrt());

    (f1, f2)
}

trait Dominate<Rhs=Self> {
    fn dominates(&self, other: &Rhs) -> bool;
}

trait MultiObjective {
    fn num_objectives() -> usize;
    fn get_f32_objective(&self, i: usize) -> f32;
}

#[derive(Debug)]
struct MultiObjective2<T>
    where T: Sized + PartialOrd
{
    objectives: [T; 2],
}

impl MultiObjective for MultiObjective2<f32> {
    fn num_objectives() -> usize {
        2
    }
    fn get_f32_objective(&self, i: usize) -> f32 {
        self.objectives[i]
    }
}

#[derive(Debug)]
struct MultiObjective3<T>
    where T: Sized + PartialOrd
{
    objectives: [T; 3],
}

fn dominates_slices<T: Sized + PartialOrd>(sa: &[T], sb: &[T]) -> bool {
    assert!(sa.len() == sb.len());
    assert!(sa.len() > 0);

    let mut less_cnt = 0;
    for (a, b) in sa.iter().zip(sb.iter()) {
        if a > b {
            return false;
        } else if a < b {
            less_cnt += 1;
        }
    }

    return less_cnt > 0;
}

impl<T:Sized+PartialOrd> Dominate for MultiObjective2<T> {
    fn dominates(&self, other: &Self) -> bool {
        dominates_slices(&self.objectives[..], &other.objectives[..])
    }
}

/// `n` stop after we have found this number of solutions (we include the whole pareto front).
fn fast_non_dominated_sort<P: Dominate>(solutions: &[P], n: usize) -> Vec<Vec<usize>> {
    let mut fronts: Vec<Vec<usize>> = Vec::new();
    let mut current_front = Vec::new();

    let mut domination_count: Vec<usize> = (0..solutions.len()).map(|_| 0).collect();
    let mut dominated_solutions: Vec<Vec<usize>> = (0..solutions.len())
                                                       .map(|_| Vec::new())
                                                       .collect();
    let mut found_solutions: usize = 0;

    for (i, p) in solutions.iter().enumerate() {
        for (j, q) in solutions.iter().enumerate() {
            if i == j {
                continue;
            }
            if p.dominates(q) {
                // Add `q` to the set of solutions dominated by `p`.
                dominated_solutions[i].push(j);
            } else if q.dominates(p) {
                // Increment domination counter of `p`.
                domination_count[i] += 1;
            }
        }

        if domination_count[i] == 0 {
            // `p` belongs to the first front as it is not dominated by any
            // other solution.
            current_front.push(i);
        }
    }

    while !current_front.is_empty() {
        found_solutions += current_front.len();
        if found_solutions >= n {
            fronts.push(current_front);
            break;
        } else {
            // we haven't found enough solutions yet.
            let mut next_front = Vec::new();
            for &p_i in current_front.iter() {
                for &q_i in dominated_solutions[p_i].iter() {
                    domination_count[q_i] -= 1;
                    if domination_count[q_i] == 0 {
                        // q belongs to the next front
                        next_front.push(q_i);
                    }
                }
            }
            fronts.push(current_front);
            current_front = next_front;
        }
    }

    return fronts;
}

/// returns: (individual_idx, distance)
fn crowding_distance_assignment<P: MultiObjective>(solutions: &[P],
                                                   individuals_idx: &[usize])
                                                   -> Vec<(usize, f32)> {
    let l = individuals_idx.len();
    let mut distance: Vec<f32> = (0..l).map(|_| 0.0).collect();

    // maps index into distance array to index into solutions.
    let mut map: Vec<(usize, usize)> = (0..l).map(|i| (i, individuals_idx[i])).collect();

    for m in 0..P::num_objectives() {
        // sort using objective `m`
        map.sort_by(|a, b| {
            solutions[a.1]
                .get_f32_objective(m)
                .partial_cmp(&solutions[b.1].get_f32_objective(m))
                .unwrap()
        });
        distance[map[0].0] = std::f32::INFINITY;
        distance[map[l - 1].0] = std::f32::INFINITY;
        let min_f = solutions[map[0].1].get_f32_objective(m);
        let max_f = solutions[map[l - 1].1].get_f32_objective(m);
        if min_f != max_f {
            let norm = P::num_objectives() as f32 * (max_f - min_f);
            debug_assert!(norm != 0.0);
            for i in 1..(l - 1) {
                distance[map[i].0] += (solutions[map[i + 1].1].get_f32_objective(m) -
                                       solutions[map[i - 1].1].get_f32_objective(m)) /
                                      norm;
            }
        }
    }

    let mut res: Vec<(usize, f32)> = (0..l).map(|_| (0, -1.0)).collect();
    for &(i, idx) in map.iter() {
        debug_assert!(res[i].1 == -1.0);
        debug_assert!(res[i].0 == 0);
        res[i] = (idx, distance[i]);
    }

    return res;
}

#[test]
fn test_dominates() {
    let a = MultiObjective2 { objectives: [1.0, 0.1] };
    let b = MultiObjective2 { objectives: [0.1, 0.1] };
    let c = MultiObjective2 { objectives: [0.1, 1.0] };

    assert_eq!(false, a.dominates(&a));
    assert_eq!(false, a.dominates(&b));
    assert_eq!(false, a.dominates(&c));

    assert_eq!(true, b.dominates(&a));
    assert_eq!(false, b.dominates(&b));
    assert_eq!(true, b.dominates(&c));

    assert_eq!(false, c.dominates(&a));
    assert_eq!(false, c.dominates(&b));
    assert_eq!(false, c.dominates(&c));
}

fn main() {
    let mut solutions: Vec<MultiObjective2<f32>> = Vec::new();
    solutions.push(MultiObjective2 { objectives: [1.0, 0.1] });
    solutions.push(MultiObjective2 { objectives: [0.1, 0.1] });
    solutions.push(MultiObjective2 { objectives: [0.1, 1.0] });
    solutions.push(MultiObjective2 { objectives: [0.5, 0.5] });

    let fronts = fast_non_dominated_sort(&solutions[..], 10);
    println!("solutions: {:?}", solutions);
    println!("fronts: {:?}", fronts);

    for front in fronts.iter() {
        let distances = crowding_distance_assignment(&solutions[..], &front[..]);
        println!("front: {:?}", front);
        println!("distances: {:?}", distances);
    }
}
<|endoftext|>


<commit_before>// TODO: COPYRIGHT, USE & AUTHORS

use super::ga_core::GASolution;

use std::cmp::Ordering;
use std::iter::FromIterator;

// Better name than 'Basis'?
pub enum GAPopulationSortBasis
{
    Raw,
    Scaled,
}

// Isn't a boolean enough?
pub enum GAPopulationSortOrder
{
    LowIsBest,
    HighIsBest,
}

/// Genetic Algorithm Population
// TODO: RUST DOCS!
pub struct GAPopulation<T: GASolution>
{
    population: Vec<T>,

    sort_order: GAPopulationSortOrder,

    // 'population' ordered by Raw score.
    population_order_raw: Vec<usize>,
    // Is 'population_order_raw' sorted?
    is_raw_sorted: bool,

    // 'population' ordered by Scaled score.
    population_order_scaled: Vec<usize>,
    // Is 'population_scaled' sorted?
    is_scaled_sorted: bool,

    // We keep 2 lists of indexes to the population vector.
    // One sorted by Raw Score and one by Scaled Score (Fitness).
}
impl<T: GASolution> GAPopulation<T>
{
    // Need best() and sort() for Selectors.

    // TODO: New should use some parameters, maybe a Config
    pub fn new(p: Vec<T>, order: GAPopulationSortOrder) -> GAPopulation<T>
    {
        let gap = GAPopulation
                  {
                      population: p,
                      sort_order: order,
                      population_order_raw: vec![],
                      is_raw_sorted: false,
                      population_order_scaled: vec![],
                      is_scaled_sorted: false
                  };

        gap
    }

    pub fn population(&mut self) -> &mut Vec<T>
    {
        return &mut self.population
    }

    pub fn evaluate(&mut self)
    {
        for ref mut ind in &mut self.population
        {
            ind.evaluate();
        }
    }

    pub fn size(&self) -> usize
    {
        self.population.len()
    }

    //TODO: this is a temporary implementation
    pub fn select(&self) -> &T
    {
        self.individual(0, GAPopulationSortBasis::Scaled)
    }

    //TODO: This is a temporary implementation
    pub fn best(&self) -> &T
    {
        self.individual(0, GAPopulationSortBasis::Scaled)
    }

    //TODO: This is a temporary implementation
    pub fn worst(&self) -> &T
    {
        self.individual(self.size()-1, GAPopulationSortBasis::Scaled)
    }

    pub fn individual(&self, i : usize, sort_basis : GAPopulationSortBasis) -> &T
    {
        // TODO: Check that i makes sense
        match sort_basis
        {
            GAPopulationSortBasis::Raw
            => { &self.population[self.population_order_raw[i]] },
            GAPopulationSortBasis::Scaled
            => { &self.population[self.population_order_scaled[i]] },
        }
    }

    pub fn sort(&mut self)
    {
        self.sort_int(false, GAPopulationSortBasis::Scaled);
        self.sort_int(false, GAPopulationSortBasis::Raw);
    }

    //TODO: I hate this name
    pub fn sort_int(&mut self, force_sort: bool, sort_basis: GAPopulationSortBasis)
    {
        let mut ordered : Vec<usize> = Vec::from_iter(0..self.size());
        match sort_basis
        {
            GAPopulationSortBasis::Raw
            =>  if (!self.is_raw_sorted) || force_sort
                {
                    match self.sort_order
                    {
                        GAPopulationSortOrder::LowIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s1].score()
                                                .partial_cmp(&self.population[*s2].score()).unwrap_or(Ordering::Equal));

                        },
                        GAPopulationSortOrder::HighIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s2].score()
                                                .partial_cmp(&self.population[*s1].score()).unwrap_or(Ordering::Equal));

                        },
                    };
                    self.population_order_raw = ordered;
                    self.is_raw_sorted = true;
                },

            GAPopulationSortBasis::Scaled
            =>  if (!self.is_scaled_sorted) || force_sort
                {
                    match self.sort_order
                    {
                        GAPopulationSortOrder::LowIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s1].fitness()
                                                .partial_cmp(&self.population[*s2].fitness()).unwrap_or(Ordering::Equal));
                        },

                        GAPopulationSortOrder::HighIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s2].fitness()
                                                .partial_cmp(&self.population[*s1].fitness()).unwrap_or(Ordering::Equal));
                        }
                    };
                    self.population_order_scaled = ordered;
                    self.is_scaled_sorted = true;
                },
        };
    }
}

#[cfg(test)]
mod test
{
}
<commit_msg>individual() looks more like galib::best(), it returns the ith-best.<commit_after>// TODO: COPYRIGHT, USE & AUTHORS

use super::ga_core::GASolution;

use std::cmp::Ordering;
use std::iter::FromIterator;

// Better name than 'Basis'?
pub enum GAPopulationSortBasis
{
    Raw,
    Scaled,
}

// Isn't a boolean enough?
pub enum GAPopulationSortOrder
{
    LowIsBest,
    HighIsBest,
}

/// Genetic Algorithm Population
// TODO: RUST DOCS!
pub struct GAPopulation<T: GASolution>
{
    population: Vec<T>,

    sort_order: GAPopulationSortOrder,

    // 'population' ordered by Raw score.
    population_order_raw: Vec<usize>,
    // Is 'population_order_raw' sorted?
    is_raw_sorted: bool,

    // 'population' ordered by Scaled score.
    population_order_scaled: Vec<usize>,
    // Is 'population_scaled' sorted?
    is_scaled_sorted: bool,

    // We keep 2 lists of indexes to the population vector.
    // One sorted by Raw Score and one by Scaled Score (Fitness).
}
impl<T: GASolution> GAPopulation<T>
{
    // Need best() and sort() for Selectors.

    // TODO: New should use some parameters, maybe a Config
    pub fn new(p: Vec<T>, order: GAPopulationSortOrder) -> GAPopulation<T>
    {
        let gap = GAPopulation
                  {
                      population: p,
                      sort_order: order,
                      population_order_raw: vec![],
                      is_raw_sorted: false,
                      population_order_scaled: vec![],
                      is_scaled_sorted: false
                  };

        gap
    }

    pub fn population(&mut self) -> &mut Vec<T>
    {
        return &mut self.population
    }

    pub fn evaluate(&mut self)
    {
        for ref mut ind in &mut self.population
        {
            ind.evaluate();
        }
    }

    pub fn size(&self) -> usize
    {
        self.population.len()
    }

    //TODO: this is a temporary implementation
    pub fn select(&self) -> &T
    {
        self.individual(0, GAPopulationSortBasis::Scaled)
    }

    //TODO: This is a temporary implementation
    pub fn best(&self) -> &T
    {
        self.individual(0, GAPopulationSortBasis::Scaled)
    }

    //TODO: This is a temporary implementation
    pub fn worst(&self) -> &T
    {
        self.individual(self.size()-1, GAPopulationSortBasis::Scaled)
    }

    // Q: Seems like this could be ith_best() instead of individual().
    //    indvidual() could be used to iterate through the population in no
    //    particular order of score.
    pub fn individual(&self, i : usize, sort_basis : GAPopulationSortBasis) -> &T
    {
        // TODO: Check that i makes sense
        match sort_basis
        {
            GAPopulationSortBasis::Raw
            => { &self.population[self.population_order_raw[i]] },
            GAPopulationSortBasis::Scaled
            => { &self.population[self.population_order_scaled[i]] },
        }
    }

    pub fn sort(&mut self)
    {
        self.sort_int(false, GAPopulationSortBasis::Scaled);
        self.sort_int(false, GAPopulationSortBasis::Raw);

        // Q: Selector needs to be updated after sorting?
    }

    //TODO: I hate this name
    pub fn sort_int(&mut self, force_sort: bool, sort_basis: GAPopulationSortBasis)
    {
        let mut ordered : Vec<usize> = Vec::from_iter(0..self.size());
        match sort_basis
        {
            GAPopulationSortBasis::Raw
            =>  if (!self.is_raw_sorted) || force_sort
                {
                    match self.sort_order
                    {
                        GAPopulationSortOrder::LowIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s1].score()
                                                .partial_cmp(&self.population[*s2].score()).unwrap_or(Ordering::Equal));

                        },
                        GAPopulationSortOrder::HighIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s2].score()
                                                .partial_cmp(&self.population[*s1].score()).unwrap_or(Ordering::Equal));

                        },
                    };
                    self.population_order_raw = ordered;
                    self.is_raw_sorted = true;
                },

            GAPopulationSortBasis::Scaled
            =>  if (!self.is_scaled_sorted) || force_sort
                {
                    match self.sort_order
                    {
                        GAPopulationSortOrder::LowIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s1].fitness()
                                                .partial_cmp(&self.population[*s2].fitness()).unwrap_or(Ordering::Equal));
                        },

                        GAPopulationSortOrder::HighIsBest =>
                        {
                            ordered.sort_by(|s1: &usize, s2: &usize|
                                            self.population[*s2].fitness()
                                                .partial_cmp(&self.population[*s1].fitness()).unwrap_or(Ordering::Equal));
                        }
                    };
                    self.population_order_scaled = ordered;
                    self.is_scaled_sorted = true;
                },
        };
    }
}

#[cfg(test)]
mod test
{
}
<|endoftext|>


<commit_before>//
//  SOS: the Stupid Operating System
//  by Hawk Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015 Hawk Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! Panic handling and stack unwinding

use core::fmt::{Arguments, Write};
use super::io::term;
use super::arch::drivers::vga::Color;

#[lang = "panic_fmt"]
#[no_mangle] #[inline(never)] #[cold]
pub extern fn rust_begin_unwind( args: Arguments
                               , file: &'static str
                               , line: usize ) -> !
{
    write!(term::CONSOLE.lock()
                        .set_colors(Color::White, Color::Red)
                        .clear()
          , "\nKERNEL PANIC in {}:{}\n\t{}"
          , file, line, args
          );
    loop { }
}

/// Required for Rust stack unwinding
#[lang = "eh_personality"]
#[no_mangle] #[inline(never)] #[cold]
pub extern fn eh_personality() {
    // TODO: add support for stack unwinding
}

#[lang = "stack_exhausted"]
#[no_mangle] #[inline(never)] #[cold]
pub extern "C" fn __morestack() -> ! {
    loop { }
}

#[allow(non_snake_case)]
#[no_mangle] #[inline(never)] #[cold]
pub extern "C" fn _Unwind_Resume() -> ! {
    loop { }
}

// #[lang = "begin_unwind"]
// #[no_mangle] #[inline(never)] #[cold]
// pub fn begin_unwind<M: Send>(msg: M
//                             , file_line: &(&'static str, usize))  -> !
// { loop { } }
<commit_msg>Kernel panic moderately prettier<commit_after>//
//  SOS: the Stupid Operating System
//  by Hawk Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015 Hawk Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! Panic handling and stack unwinding

use core::fmt::{Arguments, Write};
use super::io::term;
use super::arch::drivers::vga::Color;

#[lang = "panic_fmt"]
#[no_mangle] #[inline(never)] #[cold]
pub extern fn rust_begin_unwind( args: Arguments
                               , file: &'static str
                               , line: usize ) -> !
{
    write!(term::CONSOLE.lock()
                        .set_colors(Color::White, Color::Red)
                        .clear()
          , "KERNEL PANIC \
            \nSomething has gone horribly wrong in {} at line {} \
            \n\t{}"
          , file, line, args
          );
    loop { }
}

/// Required for Rust stack unwinding
#[lang = "eh_personality"]
#[no_mangle] #[inline(never)] #[cold]
pub extern fn eh_personality() {
    // TODO: add support for stack unwinding
}

#[lang = "stack_exhausted"]
#[no_mangle] #[inline(never)] #[cold]
pub extern "C" fn __morestack() -> ! {
    loop { }
}

#[allow(non_snake_case)]
#[no_mangle] #[inline(never)] #[cold]
pub extern "C" fn _Unwind_Resume() -> ! {
    loop { }
}

// #[lang = "begin_unwind"]
// #[no_mangle] #[inline(never)] #[cold]
// pub fn begin_unwind<M: Send>(msg: M
//                             , file_line: &(&'static str, usize))  -> !
// { loop { } }
<|endoftext|>