<commit_before>pub mod loggingmem;
mod pagedmem;

// The m68k had a 24 bit external address bus with
// (2^24 bytes = ) 16 MB addressable space
pub const ADDRBUS_MASK: u32 = 0xffffff;

#[derive(Copy, Clone, Hash, Eq, PartialEq)]
pub struct AddressSpace(Mode, Segment);

impl AddressSpace {
    pub fn fc(&self) -> u32 {
        match *self {
            USER_DATA => 1,
            USER_PROGRAM => 2,
            SUPERVISOR_DATA => 5,
            SUPERVISOR_PROGRAM => 6,
        }
    }
}
use std::fmt;
impl fmt::Debug for AddressSpace {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            AddressSpace(mode, segment) => write!(f, "[{:?}/{:?}]", mode, segment),
        }
    }
}

#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)]
enum Segment {
    Program, Data
}
#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)]
enum Mode {
    User, Supervisor
}

pub const SUPERVISOR_PROGRAM: AddressSpace = AddressSpace(Mode::Supervisor, Segment::Program);
pub const SUPERVISOR_DATA: AddressSpace = AddressSpace(Mode::Supervisor, Segment::Data);
pub const USER_PROGRAM: AddressSpace = AddressSpace(Mode::User, Segment::Program);
pub const USER_DATA: AddressSpace = AddressSpace(Mode::User, Segment::Data);

pub trait AddressBus {
    fn copy_from(&mut self, other: &Self);
    fn read_byte(&self, address_space: AddressSpace, address: u32) -> u32;
    fn read_word(&self, address_space: AddressSpace, address: u32) -> u32;
    fn read_long(&self, address_space: AddressSpace, address: u32) -> u32;
    fn write_byte(&mut self, address_space: AddressSpace, address: u32, value: u32);
    fn write_word(&mut self, address_space: AddressSpace, address: u32, value: u32);
    fn write_long(&mut self, address_space: AddressSpace, address: u32, value: u32);
}

<commit_msg>Make pagedmem a public module<commit_after>pub mod loggingmem;
pub mod pagedmem;

// The m68k had a 24 bit external address bus with
// (2^24 bytes = ) 16 MB addressable space
pub const ADDRBUS_MASK: u32 = 0xffffff;

#[derive(Copy, Clone, Hash, Eq, PartialEq)]
pub struct AddressSpace(Mode, Segment);

impl AddressSpace {
    pub fn fc(&self) -> u32 {
        match *self {
            USER_DATA => 1,
            USER_PROGRAM => 2,
            SUPERVISOR_DATA => 5,
            SUPERVISOR_PROGRAM => 6,
        }
    }
}
use std::fmt;
impl fmt::Debug for AddressSpace {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            AddressSpace(mode, segment) => write!(f, "[{:?}/{:?}]", mode, segment),
        }
    }
}

#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)]
enum Segment {
    Program, Data
}
#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)]
enum Mode {
    User, Supervisor
}

pub const SUPERVISOR_PROGRAM: AddressSpace = AddressSpace(Mode::Supervisor, Segment::Program);
pub const SUPERVISOR_DATA: AddressSpace = AddressSpace(Mode::Supervisor, Segment::Data);
pub const USER_PROGRAM: AddressSpace = AddressSpace(Mode::User, Segment::Program);
pub const USER_DATA: AddressSpace = AddressSpace(Mode::User, Segment::Data);

pub trait AddressBus {
    fn copy_from(&mut self, other: &Self);
    fn read_byte(&self, address_space: AddressSpace, address: u32) -> u32;
    fn read_word(&self, address_space: AddressSpace, address: u32) -> u32;
    fn read_long(&self, address_space: AddressSpace, address: u32) -> u32;
    fn write_byte(&mut self, address_space: AddressSpace, address: u32, value: u32);
    fn write_word(&mut self, address_space: AddressSpace, address: u32, value: u32);
    fn write_long(&mut self, address_space: AddressSpace, address: u32, value: u32);
}

<|endoftext|>


<commit_before>#![cfg_attr(feature = "cargo-clippy", allow(unused_parens))]

use std::collections::{BTreeMap, HashSet};
use std::env;
use std::fs;
use std::path::{Path, PathBuf};

#[cfg(feature = "serde")]
use serde::Deserialize;
#[cfg(feature = "config")]
use toml;

pub use askama_escape::MarkupDisplay;

mod error;
pub use crate::error::{Error, Result};
pub mod filters;
pub mod helpers;

#[derive(Debug)]
pub struct Config<'a> {
    pub dirs: Vec<PathBuf>,
    pub syntaxes: BTreeMap<String, Syntax<'a>>,
    pub default_syntax: &'a str,
    pub escapers: Vec<(HashSet<String>, String)>,
}

impl<'a> Config<'a> {
    pub fn new(s: &str) -> Config<'_> {
        let root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
        let default_dirs = vec![root.join("templates")];

        let mut syntaxes = BTreeMap::new();
        syntaxes.insert(DEFAULT_SYNTAX_NAME.to_string(), Syntax::default());

        let raw = if s.is_empty() {
            RawConfig::default()
        } else {
            RawConfig::from_toml_str(s)
        };

        let (dirs, default_syntax) = match raw.general {
            Some(General {
                dirs,
                default_syntax,
            }) => (
                dirs.map_or(default_dirs, |v| {
                    v.into_iter().map(|dir| root.join(dir)).collect()
                }),
                default_syntax.unwrap_or(DEFAULT_SYNTAX_NAME),
            ),
            None => (default_dirs, DEFAULT_SYNTAX_NAME),
        };

        if let Some(raw_syntaxes) = raw.syntax {
            for raw_s in raw_syntaxes {
                let name = raw_s.name;

                if syntaxes
                    .insert(name.to_string(), Syntax::from(raw_s))
                    .is_some()
                {
                    panic!("syntax \"{}\" is already defined", name)
                }
            }
        }

        if !syntaxes.contains_key(default_syntax) {
            panic!("default syntax \"{}\" not found", default_syntax)
        }

        let mut escapers = Vec::new();
        if let Some(configured) = raw.escaper {
            for escaper in configured {
                escapers.push((
                    escaper
                        .extensions
                        .iter()
                        .map(|ext| (*ext).to_string())
                        .collect(),
                    escaper.path.to_string(),
                ));
            }
        }
        for (extensions, path) in DEFAULT_ESCAPERS {
            escapers.push((str_set(extensions), (*path).to_string()));
        }

        Config {
            dirs,
            syntaxes,
            default_syntax,
            escapers,
        }
    }

    pub fn find_template(&self, path: &str, start_at: Option<&Path>) -> PathBuf {
        if let Some(root) = start_at {
            let relative = root.with_file_name(path);
            if relative.exists() {
                return relative;
            }
        }

        for dir in &self.dirs {
            let rooted = dir.join(path);
            if rooted.exists() {
                return rooted;
            }
        }

        panic!(
            "template {:?} not found in directories {:?}",
            path, self.dirs
        )
    }
}

#[derive(Debug)]
pub struct Syntax<'a> {
    pub block_start: &'a str,
    pub block_end: &'a str,
    pub expr_start: &'a str,
    pub expr_end: &'a str,
    pub comment_start: &'a str,
    pub comment_end: &'a str,
}

impl<'a> Default for Syntax<'a> {
    fn default() -> Self {
        Self {
            block_start: "{%",
            block_end: "%}",
            expr_start: "{{",
            expr_end: "}}",
            comment_start: "{#",
            comment_end: "#}",
        }
    }
}

impl<'a> From<RawSyntax<'a>> for Syntax<'a> {
    fn from(raw: RawSyntax<'a>) -> Self {
        let default = Self::default();
        let syntax = Self {
            block_start: raw.block_start.unwrap_or(default.block_start),
            block_end: raw.block_end.unwrap_or(default.block_end),
            expr_start: raw.expr_start.unwrap_or(default.expr_start),
            expr_end: raw.expr_end.unwrap_or(default.expr_end),
            comment_start: raw.comment_start.unwrap_or(default.comment_start),
            comment_end: raw.comment_end.unwrap_or(default.comment_end),
        };

        if syntax.block_start.len() != 2
            || syntax.block_end.len() != 2
            || syntax.expr_start.len() != 2
            || syntax.expr_end.len() != 2
            || syntax.comment_start.len() != 2
            || syntax.comment_end.len() != 2
        {
            panic!("length of delimiters must be two")
        }

        let bs = syntax.block_start.as_bytes()[0];
        let be = syntax.block_start.as_bytes()[1];
        let cs = syntax.comment_start.as_bytes()[0];
        let ce = syntax.comment_start.as_bytes()[1];
        let es = syntax.block_start.as_bytes()[0];
        let ee = syntax.block_start.as_bytes()[1];
        if !((bs == cs && bs == es) || (be == ce && be == ee)) {
            panic!("bad delimiters block_start: {}, comment_start: {}, expr_start: {}, needs one of the two characters in common", syntax.block_start, syntax.comment_start, syntax.expr_start);
        }

        syntax
    }
}

#[cfg_attr(feature = "serde", derive(Deserialize))]
#[derive(Default)]
struct RawConfig<'d> {
    #[cfg_attr(feature = "serde", serde(borrow))]
    general: Option<General<'d>>,
    syntax: Option<Vec<RawSyntax<'d>>>,
    escaper: Option<Vec<RawEscaper<'d>>>,
}

impl<'d> RawConfig<'d> {
    #[cfg(feature = "config")]
    fn from_toml_str<'n>(s: &'n str) -> RawConfig<'n> {
        toml::from_str(&s).unwrap_or_else(|_| panic!("invalid TOML in {}", CONFIG_FILE_NAME))
    }

    #[cfg(not(feature = "config"))]
    fn from_toml_str<'n>(_: &'n str) -> RawConfig<'n> {
        panic!("toml support not available")
    }
}

#[cfg_attr(feature = "serde", derive(Deserialize))]
struct General<'a> {
    #[cfg_attr(feature = "serde", serde(borrow))]
    dirs: Option<Vec<&'a str>>,
    default_syntax: Option<&'a str>,
}

#[cfg_attr(feature = "serde", derive(Deserialize))]
struct RawSyntax<'a> {
    name: &'a str,
    block_start: Option<&'a str>,
    block_end: Option<&'a str>,
    expr_start: Option<&'a str>,
    expr_end: Option<&'a str>,
    comment_start: Option<&'a str>,
    comment_end: Option<&'a str>,
}

#[cfg_attr(feature = "serde", derive(Deserialize))]
struct RawEscaper<'a> {
    path: &'a str,
    extensions: Vec<&'a str>,
}

pub fn read_config_file() -> String {
    let root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
    let filename = root.join(CONFIG_FILE_NAME);
    if filename.exists() {
        fs::read_to_string(&filename)
            .unwrap_or_else(|_| panic!("unable to read {}", filename.to_str().unwrap()))
    } else {
        "".to_string()
    }
}

fn str_set<T>(vals: &[T]) -> HashSet<String>
where
    T: ToString,
{
    vals.iter().map(|s| s.to_string()).collect()
}

static CONFIG_FILE_NAME: &str = "askama.toml";
static DEFAULT_SYNTAX_NAME: &str = "default";
static DEFAULT_ESCAPERS: &[(&[&str], &str)] = &[
    (&["html", "htm", "xml"], "::askama::Html"),
    (&["md", "none", "txt", "yml", ""], "::askama::Text"),
    (&["j2", "jinja", "jinja2"], "::askama::Html"),
];

#[cfg(test)]
#[allow(clippy::blacklisted_name)]
mod tests {
    use super::*;
    use std::env;
    use std::path::{Path, PathBuf};

    #[test]
    fn test_default_config() {
        let mut root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
        root.push("templates");
        let config = Config::new("");
        assert_eq!(config.dirs, vec![root]);
    }

    #[cfg(feature = "config")]
    #[test]
    fn test_config_dirs() {
        let mut root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
        root.push("tpl");
        let config = Config::new("[general]\ndirs = [\"tpl\"]");
        assert_eq!(config.dirs, vec![root]);
    }

    fn assert_eq_rooted(actual: &Path, expected: &str) {
        let mut root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
        root.push("templates");
        let mut inner = PathBuf::new();
        inner.push(expected);
        assert_eq!(actual.strip_prefix(root).unwrap(), inner);
    }

    #[test]
    fn find_absolute() {
        let config = Config::new("");
        let root = config.find_template("a.html", None);
        let path = config.find_template("sub/b.html", Some(&root));
        assert_eq_rooted(&path, "sub/b.html");
    }

    #[test]
    #[should_panic]
    fn find_relative_nonexistent() {
        let config = Config::new("");
        let root = config.find_template("a.html", None);
        config.find_template("b.html", Some(&root));
    }

    #[test]
    fn find_relative() {
        let config = Config::new("");
        let root = config.find_template("sub/b.html", None);
        let path = config.find_template("c.html", Some(&root));
        assert_eq_rooted(&path, "sub/c.html");
    }

    #[test]
    fn find_relative_sub() {
        let config = Config::new("");
        let root = config.find_template("sub/b.html", None);
        let path = config.find_template("sub1/d.html", Some(&root));
        assert_eq_rooted(&path, "sub/sub1/d.html");
    }

    #[cfg(feature = "config")]
    #[test]
    fn add_syntax() {
        let raw_config = r#"
        [general]
        default_syntax = "foo"

        [[syntax]]
        name = "foo"
        block_start = "{<"

        [[syntax]]
        name = "bar"
        expr_start = "{!"
        "#;

        let default_syntax = Syntax::default();
        let config = Config::new(raw_config);
        assert_eq!(config.default_syntax, "foo");

        let foo = config.syntaxes.get("foo").unwrap();
        assert_eq!(foo.block_start, "{<");
        assert_eq!(foo.block_end, default_syntax.block_end);
        assert_eq!(foo.expr_start, default_syntax.expr_start);
        assert_eq!(foo.expr_end, default_syntax.expr_end);
        assert_eq!(foo.comment_start, default_syntax.comment_start);
        assert_eq!(foo.comment_end, default_syntax.comment_end);

        let bar = config.syntaxes.get("bar").unwrap();
        assert_eq!(bar.block_start, default_syntax.block_start);
        assert_eq!(bar.block_end, default_syntax.block_end);
        assert_eq!(bar.expr_start, "{!");
        assert_eq!(bar.expr_end, default_syntax.expr_end);
        assert_eq!(bar.comment_start, default_syntax.comment_start);
        assert_eq!(bar.comment_end, default_syntax.comment_end);
    }

    #[cfg(feature = "config")]
    #[test]
    fn add_syntax_two() {
        let raw_config = r#"
        syntax = [{ name = "foo", block_start = "{<" },
                  { name = "bar", expr_start = "{!" } ]

        [general]
        default_syntax = "foo"
        "#;

        let default_syntax = Syntax::default();
        let config = Config::new(raw_config);
        assert_eq!(config.default_syntax, "foo");

        let foo = config.syntaxes.get("foo").unwrap();
        assert_eq!(foo.block_start, "{<");
        assert_eq!(foo.block_end, default_syntax.block_end);
        assert_eq!(foo.expr_start, default_syntax.expr_start);
        assert_eq!(foo.expr_end, default_syntax.expr_end);
        assert_eq!(foo.comment_start, default_syntax.comment_start);
        assert_eq!(foo.comment_end, default_syntax.comment_end);

        let bar = config.syntaxes.get("bar").unwrap();
        assert_eq!(bar.block_start, default_syntax.block_start);
        assert_eq!(bar.block_end, default_syntax.block_end);
        assert_eq!(bar.expr_start, "{!");
        assert_eq!(bar.expr_end, default_syntax.expr_end);
        assert_eq!(bar.comment_start, default_syntax.comment_start);
        assert_eq!(bar.comment_end, default_syntax.comment_end);
    }

    #[cfg(feature = "toml")]
    #[should_panic]
    #[test]
    fn use_default_at_syntax_name() {
        let raw_config = r#"
        syntax = [{ name = "default" }]
        "#;

        let _config = Config::new(raw_config);
    }

    #[cfg(feature = "toml")]
    #[should_panic]
    #[test]
    fn duplicated_syntax_name_on_list() {
        let raw_config = r#"
        syntax = [{ name = "foo", block_start = "~<" },
                  { name = "foo", block_start = "%%" } ]
        "#;

        let _config = Config::new(raw_config);
    }

    #[cfg(feature = "toml")]
    #[should_panic]
    #[test]
    fn is_not_exist_default_syntax() {
        let raw_config = r#"
        [general]
        default_syntax = "foo"
        "#;

        let _config = Config::new(raw_config);
    }

    #[cfg(feature = "config")]
    #[test]
    fn escape_modes() {
        let config = Config::new(
            r#"
            [[escaper]]
            path = "::askama::Js"
            extensions = ["js"]
        "#,
        );
        assert_eq!(
            config.escapers,
            vec![
                (str_set(&["js"]), "::askama::Js".into()),
                (str_set(&["html", "htm", "xml"]), "::askama::Html".into()),
                (
                    str_set(&["md", "none", "txt", "yml", ""]),
                    "::askama::Text".into()
                ),
                (str_set(&["j2", "jinja", "jinja2"]), "::askama::Html".into()),
            ]
        );
    }
}
<commit_msg>Remove unnecessary explicit lifetimes<commit_after>#![cfg_attr(feature = "cargo-clippy", allow(unused_parens))]

use std::collections::{BTreeMap, HashSet};
use std::env;
use std::fs;
use std::path::{Path, PathBuf};

#[cfg(feature = "serde")]
use serde::Deserialize;
#[cfg(feature = "config")]
use toml;

pub use askama_escape::MarkupDisplay;

mod error;
pub use crate::error::{Error, Result};
pub mod filters;
pub mod helpers;

#[derive(Debug)]
pub struct Config<'a> {
    pub dirs: Vec<PathBuf>,
    pub syntaxes: BTreeMap<String, Syntax<'a>>,
    pub default_syntax: &'a str,
    pub escapers: Vec<(HashSet<String>, String)>,
}

impl<'a> Config<'a> {
    pub fn new(s: &str) -> Config<'_> {
        let root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
        let default_dirs = vec![root.join("templates")];

        let mut syntaxes = BTreeMap::new();
        syntaxes.insert(DEFAULT_SYNTAX_NAME.to_string(), Syntax::default());

        let raw = if s.is_empty() {
            RawConfig::default()
        } else {
            RawConfig::from_toml_str(s)
        };

        let (dirs, default_syntax) = match raw.general {
            Some(General {
                dirs,
                default_syntax,
            }) => (
                dirs.map_or(default_dirs, |v| {
                    v.into_iter().map(|dir| root.join(dir)).collect()
                }),
                default_syntax.unwrap_or(DEFAULT_SYNTAX_NAME),
            ),
            None => (default_dirs, DEFAULT_SYNTAX_NAME),
        };

        if let Some(raw_syntaxes) = raw.syntax {
            for raw_s in raw_syntaxes {
                let name = raw_s.name;

                if syntaxes
                    .insert(name.to_string(), Syntax::from(raw_s))
                    .is_some()
                {
                    panic!("syntax \"{}\" is already defined", name)
                }
            }
        }

        if !syntaxes.contains_key(default_syntax) {
            panic!("default syntax \"{}\" not found", default_syntax)
        }

        let mut escapers = Vec::new();
        if let Some(configured) = raw.escaper {
            for escaper in configured {
                escapers.push((
                    escaper
                        .extensions
                        .iter()
                        .map(|ext| (*ext).to_string())
                        .collect(),
                    escaper.path.to_string(),
                ));
            }
        }
        for (extensions, path) in DEFAULT_ESCAPERS {
            escapers.push((str_set(extensions), (*path).to_string()));
        }

        Config {
            dirs,
            syntaxes,
            default_syntax,
            escapers,
        }
    }

    pub fn find_template(&self, path: &str, start_at: Option<&Path>) -> PathBuf {
        if let Some(root) = start_at {
            let relative = root.with_file_name(path);
            if relative.exists() {
                return relative;
            }
        }

        for dir in &self.dirs {
            let rooted = dir.join(path);
            if rooted.exists() {
                return rooted;
            }
        }

        panic!(
            "template {:?} not found in directories {:?}",
            path, self.dirs
        )
    }
}

#[derive(Debug)]
pub struct Syntax<'a> {
    pub block_start: &'a str,
    pub block_end: &'a str,
    pub expr_start: &'a str,
    pub expr_end: &'a str,
    pub comment_start: &'a str,
    pub comment_end: &'a str,
}

impl<'a> Default for Syntax<'a> {
    fn default() -> Self {
        Self {
            block_start: "{%",
            block_end: "%}",
            expr_start: "{{",
            expr_end: "}}",
            comment_start: "{#",
            comment_end: "#}",
        }
    }
}

impl<'a> From<RawSyntax<'a>> for Syntax<'a> {
    fn from(raw: RawSyntax<'a>) -> Self {
        let default = Self::default();
        let syntax = Self {
            block_start: raw.block_start.unwrap_or(default.block_start),
            block_end: raw.block_end.unwrap_or(default.block_end),
            expr_start: raw.expr_start.unwrap_or(default.expr_start),
            expr_end: raw.expr_end.unwrap_or(default.expr_end),
            comment_start: raw.comment_start.unwrap_or(default.comment_start),
            comment_end: raw.comment_end.unwrap_or(default.comment_end),
        };

        if syntax.block_start.len() != 2
            || syntax.block_end.len() != 2
            || syntax.expr_start.len() != 2
            || syntax.expr_end.len() != 2
            || syntax.comment_start.len() != 2
            || syntax.comment_end.len() != 2
        {
            panic!("length of delimiters must be two")
        }

        let bs = syntax.block_start.as_bytes()[0];
        let be = syntax.block_start.as_bytes()[1];
        let cs = syntax.comment_start.as_bytes()[0];
        let ce = syntax.comment_start.as_bytes()[1];
        let es = syntax.block_start.as_bytes()[0];
        let ee = syntax.block_start.as_bytes()[1];
        if !((bs == cs && bs == es) || (be == ce && be == ee)) {
            panic!("bad delimiters block_start: {}, comment_start: {}, expr_start: {}, needs one of the two characters in common", syntax.block_start, syntax.comment_start, syntax.expr_start);
        }

        syntax
    }
}

#[cfg_attr(feature = "serde", derive(Deserialize))]
#[derive(Default)]
struct RawConfig<'d> {
    #[cfg_attr(feature = "serde", serde(borrow))]
    general: Option<General<'d>>,
    syntax: Option<Vec<RawSyntax<'d>>>,
    escaper: Option<Vec<RawEscaper<'d>>>,
}

impl<'d> RawConfig<'d> {
    #[cfg(feature = "config")]
    fn from_toml_str(s: &str) -> RawConfig<'_> {
        toml::from_str(&s).unwrap_or_else(|_| panic!("invalid TOML in {}", CONFIG_FILE_NAME))
    }

    #[cfg(not(feature = "config"))]
    fn from_toml_str<'n>(_: &'n str) -> RawConfig<'n> {
        panic!("toml support not available")
    }
}

#[cfg_attr(feature = "serde", derive(Deserialize))]
struct General<'a> {
    #[cfg_attr(feature = "serde", serde(borrow))]
    dirs: Option<Vec<&'a str>>,
    default_syntax: Option<&'a str>,
}

#[cfg_attr(feature = "serde", derive(Deserialize))]
struct RawSyntax<'a> {
    name: &'a str,
    block_start: Option<&'a str>,
    block_end: Option<&'a str>,
    expr_start: Option<&'a str>,
    expr_end: Option<&'a str>,
    comment_start: Option<&'a str>,
    comment_end: Option<&'a str>,
}

#[cfg_attr(feature = "serde", derive(Deserialize))]
struct RawEscaper<'a> {
    path: &'a str,
    extensions: Vec<&'a str>,
}

pub fn read_config_file() -> String {
    let root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
    let filename = root.join(CONFIG_FILE_NAME);
    if filename.exists() {
        fs::read_to_string(&filename)
            .unwrap_or_else(|_| panic!("unable to read {}", filename.to_str().unwrap()))
    } else {
        "".to_string()
    }
}

fn str_set<T>(vals: &[T]) -> HashSet<String>
where
    T: ToString,
{
    vals.iter().map(|s| s.to_string()).collect()
}

static CONFIG_FILE_NAME: &str = "askama.toml";
static DEFAULT_SYNTAX_NAME: &str = "default";
static DEFAULT_ESCAPERS: &[(&[&str], &str)] = &[
    (&["html", "htm", "xml"], "::askama::Html"),
    (&["md", "none", "txt", "yml", ""], "::askama::Text"),
    (&["j2", "jinja", "jinja2"], "::askama::Html"),
];

#[cfg(test)]
#[allow(clippy::blacklisted_name)]
mod tests {
    use super::*;
    use std::env;
    use std::path::{Path, PathBuf};

    #[test]
    fn test_default_config() {
        let mut root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
        root.push("templates");
        let config = Config::new("");
        assert_eq!(config.dirs, vec![root]);
    }

    #[cfg(feature = "config")]
    #[test]
    fn test_config_dirs() {
        let mut root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
        root.push("tpl");
        let config = Config::new("[general]\ndirs = [\"tpl\"]");
        assert_eq!(config.dirs, vec![root]);
    }

    fn assert_eq_rooted(actual: &Path, expected: &str) {
        let mut root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
        root.push("templates");
        let mut inner = PathBuf::new();
        inner.push(expected);
        assert_eq!(actual.strip_prefix(root).unwrap(), inner);
    }

    #[test]
    fn find_absolute() {
        let config = Config::new("");
        let root = config.find_template("a.html", None);
        let path = config.find_template("sub/b.html", Some(&root));
        assert_eq_rooted(&path, "sub/b.html");
    }

    #[test]
    #[should_panic]
    fn find_relative_nonexistent() {
        let config = Config::new("");
        let root = config.find_template("a.html", None);
        config.find_template("b.html", Some(&root));
    }

    #[test]
    fn find_relative() {
        let config = Config::new("");
        let root = config.find_template("sub/b.html", None);
        let path = config.find_template("c.html", Some(&root));
        assert_eq_rooted(&path, "sub/c.html");
    }

    #[test]
    fn find_relative_sub() {
        let config = Config::new("");
        let root = config.find_template("sub/b.html", None);
        let path = config.find_template("sub1/d.html", Some(&root));
        assert_eq_rooted(&path, "sub/sub1/d.html");
    }

    #[cfg(feature = "config")]
    #[test]
    fn add_syntax() {
        let raw_config = r#"
        [general]
        default_syntax = "foo"

        [[syntax]]
        name = "foo"
        block_start = "{<"

        [[syntax]]
        name = "bar"
        expr_start = "{!"
        "#;

        let default_syntax = Syntax::default();
        let config = Config::new(raw_config);
        assert_eq!(config.default_syntax, "foo");

        let foo = config.syntaxes.get("foo").unwrap();
        assert_eq!(foo.block_start, "{<");
        assert_eq!(foo.block_end, default_syntax.block_end);
        assert_eq!(foo.expr_start, default_syntax.expr_start);
        assert_eq!(foo.expr_end, default_syntax.expr_end);
        assert_eq!(foo.comment_start, default_syntax.comment_start);
        assert_eq!(foo.comment_end, default_syntax.comment_end);

        let bar = config.syntaxes.get("bar").unwrap();
        assert_eq!(bar.block_start, default_syntax.block_start);
        assert_eq!(bar.block_end, default_syntax.block_end);
        assert_eq!(bar.expr_start, "{!");
        assert_eq!(bar.expr_end, default_syntax.expr_end);
        assert_eq!(bar.comment_start, default_syntax.comment_start);
        assert_eq!(bar.comment_end, default_syntax.comment_end);
    }

    #[cfg(feature = "config")]
    #[test]
    fn add_syntax_two() {
        let raw_config = r#"
        syntax = [{ name = "foo", block_start = "{<" },
                  { name = "bar", expr_start = "{!" } ]

        [general]
        default_syntax = "foo"
        "#;

        let default_syntax = Syntax::default();
        let config = Config::new(raw_config);
        assert_eq!(config.default_syntax, "foo");

        let foo = config.syntaxes.get("foo").unwrap();
        assert_eq!(foo.block_start, "{<");
        assert_eq!(foo.block_end, default_syntax.block_end);
        assert_eq!(foo.expr_start, default_syntax.expr_start);
        assert_eq!(foo.expr_end, default_syntax.expr_end);
        assert_eq!(foo.comment_start, default_syntax.comment_start);
        assert_eq!(foo.comment_end, default_syntax.comment_end);

        let bar = config.syntaxes.get("bar").unwrap();
        assert_eq!(bar.block_start, default_syntax.block_start);
        assert_eq!(bar.block_end, default_syntax.block_end);
        assert_eq!(bar.expr_start, "{!");
        assert_eq!(bar.expr_end, default_syntax.expr_end);
        assert_eq!(bar.comment_start, default_syntax.comment_start);
        assert_eq!(bar.comment_end, default_syntax.comment_end);
    }

    #[cfg(feature = "toml")]
    #[should_panic]
    #[test]
    fn use_default_at_syntax_name() {
        let raw_config = r#"
        syntax = [{ name = "default" }]
        "#;

        let _config = Config::new(raw_config);
    }

    #[cfg(feature = "toml")]
    #[should_panic]
    #[test]
    fn duplicated_syntax_name_on_list() {
        let raw_config = r#"
        syntax = [{ name = "foo", block_start = "~<" },
                  { name = "foo", block_start = "%%" } ]
        "#;

        let _config = Config::new(raw_config);
    }

    #[cfg(feature = "toml")]
    #[should_panic]
    #[test]
    fn is_not_exist_default_syntax() {
        let raw_config = r#"
        [general]
        default_syntax = "foo"
        "#;

        let _config = Config::new(raw_config);
    }

    #[cfg(feature = "config")]
    #[test]
    fn escape_modes() {
        let config = Config::new(
            r#"
            [[escaper]]
            path = "::askama::Js"
            extensions = ["js"]
        "#,
        );
        assert_eq!(
            config.escapers,
            vec![
                (str_set(&["js"]), "::askama::Js".into()),
                (str_set(&["html", "htm", "xml"]), "::askama::Html".into()),
                (
                    str_set(&["md", "none", "txt", "yml", ""]),
                    "::askama::Text".into()
                ),
                (str_set(&["j2", "jinja", "jinja2"]), "::askama::Html".into()),
            ]
        );
    }
}
<|endoftext|>


<commit_before>// External modules:
use darwin_rs::{Individual, SimulationBuilder, Population, PopulationBuilder};
use image;
// use image::{GenericImage, FilterType, DynamicImage, imageops};
use image::{GenericImage, DynamicImage, Rgba};
use rand::Rng;
use rand;
use walkdir::{WalkDir};

// use std::path::Path;
use std::f64::consts::PI;
use std::cmp;
use std::sync::Arc;

// Internal modules:
use config::PanolutionConfig;

#[derive(Clone)]
pub struct ImageArrangement {
    pub image: Arc<DynamicImage>,
    pub samples: u32,
    pub x0: u32,
    pub y0: u32,
    pub x1: u32,
    pub y1: u32,
    pub angle: f64,
    // TODO: add more image operations
}

#[derive(Clone)]
pub struct Solution {
    pub arrangement: Vec<ImageArrangement>,
}

struct SamplePixel{
    x: u32,
    y: u32,
    pixels: Vec<(u8, u8, u8)>
}

fn valid_image_file(file_name: &str) -> bool {
    let extension = file_name.split(".").last().unwrap_or("").to_lowercase();

    let supported: Vec<String> = vec!["jpg", "jpeg", "gif", "png", "tif", "tiff"].iter().map(|s| s.to_string()).collect();

    supported.contains(&extension)
}

fn calc_canvas_size(arrangement: &Vec<ImageArrangement>) -> (u32, u32, u32, u32) {
    arrangement.iter().fold((u32::max_value(), u32::max_value(), 0, 0), |(cx0, cy0, cx1, cy1), im_ar| {
        (cmp::min(cx0, im_ar.x0), cmp::min(cy0, im_ar.y0), cmp::max(cx1, im_ar.x1), cmp::max(cy1, im_ar.y1))
    })
}

fn get_pixel(cx: f64, cy: f64, im_ar: &ImageArrangement) -> Option<(u8, u8, u8)> {
    // Center of image
    let mx = ((im_ar.x0 + im_ar.x1) / 2) as f64;
    let my = ((im_ar.y0 + im_ar.y1) / 2) as f64;

    // Move center of rotation to origin (0,0)
    let ox = cx - mx;
    let oy = cy - my;

    // Rotate canvas point in reverse of image orientation (clockwise)
    let cos_a = im_ar.angle.cos();
    let sin_a = im_ar.angle.sin();

    let rx = (ox * cos_a) + (oy * sin_a);
    let ry = (-ox * sin_a) + (oy * cos_a);

    // Move back to image coordinate system
    let imx = ((rx + mx) as u32) - im_ar.x0;
    let imy = ((ry + my) as u32) - im_ar.y0;

    // Check if point is inside the image:

    if (imx >= 0) && (imx < im_ar.image.width()) && (imy >= 0) && (imy < im_ar.image.height()) {
        let pixel = im_ar.image.get_pixel(imx, imy) as Rgba<u8>;
        return Some((pixel.data[0], pixel.data[1], pixel.data[2]));
    }

    None
}

fn make_all_populations(num_of_individuals: u32, num_of_populations: u32, initial_solution: &Solution) -> Vec<Population<Solution>> {
    let mut result = Vec::new();

    let mut initial_population = Vec::new();

    for _ in 0..num_of_individuals {
        initial_population.push(initial_solution.clone());
    }

    for i in 0..num_of_populations {
        let pop = PopulationBuilder::<Solution>::new()
            .set_id(i + 1)
            .initial_population(&initial_population)
            .reset_limit_end(0)
            .increasing_mutation_rate()
            .finalize().unwrap();

        result.push(pop);
    }

    result
}

impl Individual for Solution {
    fn mutate(&mut self) {
        let mut rng = rand::thread_rng();
        let num_of_operations: u8 = 4;

        let index1: usize = rng.gen_range(0, self.arrangement.len());

        // Probably use this create and enums for operation:
        // https://github.com/andersk/enum_primitive-rs
        let operation: u8 = rng.gen_range(0, num_of_operations);

        match operation {
            0 => {
                // Change image order
                let mut index2: usize = rng.gen_range(0, self.arrangement.len());
                while index1 == index2 {
                    index2 = rng.gen_range(0, self.arrangement.len());
                }

                let px0 = self.arrangement[index1].x0;
                let py0 = self.arrangement[index1].y0;
                let px1 = self.arrangement[index1].x1;
                let py1 = self.arrangement[index1].y1;

                self.arrangement[index1].x0 = self.arrangement[index2].x0;
                self.arrangement[index1].y0 = self.arrangement[index2].y0;
                self.arrangement[index1].x1 = self.arrangement[index2].x1;
                self.arrangement[index1].y1 = self.arrangement[index2].y1;

                self.arrangement[index2].x0 = px0;
                self.arrangement[index2].y0 = py0;
                self.arrangement[index2].x1 = px1;
                self.arrangement[index2].y1 = py1;
            },
            1 => {
                // Move x
                let dx = rng.gen_range::<i32>(0, 500) - 250;
                let new_x = (self.arrangement[index1].x0 as i32) + dx;

                if new_x >= 0 {
                    self.arrangement[index1].x0 = new_x as u32;
                    self.arrangement[index1].x1 = ((self.arrangement[index1].x1 as i32) + dx) as u32;
                }
            },
            2 => {
                // Move y
                let dy = rng.gen_range::<i32>(0, 500) - 250;
                let new_y = (self.arrangement[index1].y0 as i32) + dy;

                if new_y >= 0 {
                    self.arrangement[index1].y0 = new_y as u32;
                    self.arrangement[index1].y1 = ((self.arrangement[index1].y1 as i32) + dy) as u32;
                }
            },
            3 => {
                // Rotate
                self.arrangement[index1].angle = self.arrangement[index1].angle + (rng.next_f64() * 0.1);
                if self.arrangement[index1].angle < 0.0 {
                    self.arrangement[index1].angle = 0.0;
                } else if self.arrangement[index1].angle > 2.0 * PI {
                    self.arrangement[index1].angle = 0.0;
                }
            },
            op => {
                warn!("mutate(): unknown operation: {}", op)
            }
        }
    }

    fn calculate_fitness(&mut self) -> f64 {
        let num_of_samples = self.arrangement[0].samples;

        let (cx0, cy0, cx1, cy1) = calc_canvas_size(&self.arrangement);

        // Create sample points on canvas:
        let mut rng = rand::thread_rng();
        let mut sample_pixels = Vec::new();

        for _ in 0..num_of_samples {
            sample_pixels.push(SamplePixel{
                x: rng.gen_range::<u32>(cx0, cx1),
                y: rng.gen_range::<u32>(cy0, cy1),
                pixels: Vec::new()
            });
        }

        for im_ar in &self.arrangement {
            for sp in &mut sample_pixels {
                if let Some(pixel) = get_pixel(sp.x as f64, sp.y as f64, &im_ar) {
                    sp.pixels.push(pixel);
                }
            }
        }

        let error_sum = sample_pixels.iter().fold(0, |sum, sp| {
            if sp.pixels.len() > 0 {
                let r0 = sp.pixels[0].0;
                let g0 = sp.pixels[0].1;
                let b0 = sp.pixels[0].2;
                let max_diff = sp.pixels.iter().fold((0, r0, g0, b0), |(diff, r1, g1, b1), &(r2, g2, b2)| {
                    let rdiff = if r1 > r2 {r1 - r2} else {r2 - r1};
                    let gdiff = if g1 > g2 {g1 - g2} else {g2 - g1};
                    let bdiff = if b1 > b2 {b1 - b2} else {b2 - b1};

                    (cmp::max(diff, rdiff + gdiff + bdiff), r2, g2, b2)
                });
                sum + max_diff.0
            } else {
                sum
            }

        });

        (error_sum as f64) / (num_of_samples as f64)
    }

    fn reset(&mut self) {
        for arrangement in &mut self.arrangement {
            // Reset all image operation to original image
            arrangement.x1 = arrangement.x1 - arrangement.x0;
            arrangement.y1 = arrangement.y1 - arrangement.y0;
            arrangement.x0 = 0;
            arrangement.y0 = 0;
            arrangement.angle = 0.0;
        }
    }
}

fn run_darwin(solution: &Solution, config: &PanolutionConfig) -> Solution {
    let pano = SimulationBuilder::<Solution>::new()
        .iterations(config.max_iteration)
        .threads(4)
        .add_multiple_populations(make_all_populations(20, 8, &solution))
        .finalize();

    match pano {
        Err(e) => {
            error!("An error occured");

            for e in e.iter().skip(1) {
                error!("Caused by '{}'", e)
            }

            solution.clone()
        },
        Ok(mut pano_simulation) => {
            pano_simulation.run();

            info!("Total run time: {} ms", pano_simulation.total_time_in_ms);
            info!("Improvement factor: {}", pano_simulation.simulation_result.improvement_factor);
            info!("Number of iterations: {}", pano_simulation.simulation_result.iteration_counter);
            info!("New fitness: {}", pano_simulation.simulation_result.fittest[0].fitness);

            pano_simulation.simulation_result.fittest[0].individual.clone()
        }
    }
}

pub fn optimize(solution: Option<&Solution>, config: &PanolutionConfig, sample_index: usize) -> Solution {
    match solution {
        None => {
            let mut arrangement = Vec::new();

            for entry in WalkDir::new(&config.input_path) {
                if let Ok(entry) = entry {
                    if entry.file_type().is_file() {
                        if let Some(file_name) = entry.file_name().to_str() {
                            if valid_image_file(file_name) {
                                if let Some(full_path) = entry.path().to_str() {

                                    if let Ok(img) = image::open(&full_path) {
                                        arrangement.push(ImageArrangement{
                                            image: Arc::new(img.clone()),
                                            samples: config.num_of_samples[sample_index],
                                            x0: 0,
                                            y0: 0,
                                            x1: img.width(),
                                            y1: img.height(),
                                            angle: 0.0,
                                        });
                                    } else {
                                        error!("Could not open image: {}", full_path)
                                    }
                                } else {
                                    error!("Could not convert full path to str: {:?}", entry);
                                }
                            } else {
                                error!("Image format currently not supported: {}", file_name);
                            }
                        } else {
                            error!("Could not convert file name to str: {:?}", entry);
                        }
                    } else {
                        info!("Ignore non-file: {:?}", entry);
                    }
                } else {
                    error!("Error in WalkDir");
                }
            }

            let solution = Solution{arrangement: arrangement};

            if solution.arrangement.len() > 0 {
                run_darwin(&solution, &config)
            } else {
                // An error occured
                // TODO: return Result<Soultion> and use chain_err
                solution
            }
        },
        Some(solution) => {
            let mut updated_arrangement = solution.arrangement.clone();

            for im_ar in &mut updated_arrangement {
                im_ar.samples = config.num_of_samples[sample_index];
            }

            run_darwin(&Solution{arrangement: updated_arrangement}, &config)
        }
    }
}
<commit_msg>fix overflow bug<commit_after>// External modules:
use darwin_rs::{Individual, SimulationBuilder, Population, PopulationBuilder};
use image;
// use image::{GenericImage, FilterType, DynamicImage, imageops};
use image::{GenericImage, DynamicImage, Rgba};
use rand::Rng;
use rand;
use walkdir::{WalkDir};

// use std::path::Path;
use std::f64::consts::PI;
use std::cmp;
use std::sync::Arc;

// Internal modules:
use config::PanolutionConfig;

#[derive(Clone)]
pub struct ImageArrangement {
    pub image: Arc<DynamicImage>,
    pub samples: u32,
    pub x0: u32,
    pub y0: u32,
    pub x1: u32,
    pub y1: u32,
    pub angle: f64,
    // TODO: add more image operations
}

#[derive(Clone)]
pub struct Solution {
    pub arrangement: Vec<ImageArrangement>,
}

struct SamplePixel{
    x: u32,
    y: u32,
    pixels: Vec<(u8, u8, u8)>
}

fn valid_image_file(file_name: &str) -> bool {
    let extension = file_name.split(".").last().unwrap_or("").to_lowercase();

    let supported: Vec<String> = vec!["jpg", "jpeg", "gif", "png", "tif", "tiff"].iter().map(|s| s.to_string()).collect();

    supported.contains(&extension)
}

fn calc_canvas_size(arrangement: &Vec<ImageArrangement>) -> (u32, u32, u32, u32) {
    arrangement.iter().fold((u32::max_value(), u32::max_value(), 0, 0), |(cx0, cy0, cx1, cy1), im_ar| {
        (cmp::min(cx0, im_ar.x0), cmp::min(cy0, im_ar.y0), cmp::max(cx1, im_ar.x1), cmp::max(cy1, im_ar.y1))
    })
}

fn get_pixel(cx: f64, cy: f64, im_ar: &ImageArrangement) -> Option<(u8, u8, u8)> {
    // Center of image
    let mx = ((im_ar.x0 + im_ar.x1) / 2) as f64;
    let my = ((im_ar.y0 + im_ar.y1) / 2) as f64;

    // Move center of rotation to origin (0,0)
    let ox = cx - mx;
    let oy = cy - my;

    // Rotate canvas point in reverse of image orientation (clockwise)
    let cos_a = im_ar.angle.cos();
    let sin_a = im_ar.angle.sin();

    let rx = (ox * cos_a) + (oy * sin_a);
    let ry = (-ox * sin_a) + (oy * cos_a);

    // Move back to image coordinate system
    let imx = ((rx + mx) as u32) - im_ar.x0;
    let imy = ((ry + my) as u32) - im_ar.y0;

    // Check if point is inside the image:

    if (imx >= 0) && (imx < im_ar.image.width()) && (imy >= 0) && (imy < im_ar.image.height()) {
        let pixel = im_ar.image.get_pixel(imx, imy) as Rgba<u8>;
        return Some((pixel.data[0], pixel.data[1], pixel.data[2]));
    }

    None
}

fn make_all_populations(num_of_individuals: u32, num_of_populations: u32, initial_solution: &Solution) -> Vec<Population<Solution>> {
    let mut result = Vec::new();

    let mut initial_population = Vec::new();

    for _ in 0..num_of_individuals {
        initial_population.push(initial_solution.clone());
    }

    for i in 0..num_of_populations {
        let pop = PopulationBuilder::<Solution>::new()
            .set_id(i + 1)
            .initial_population(&initial_population)
            .reset_limit_end(0)
            .increasing_mutation_rate()
            .finalize().unwrap();

        result.push(pop);
    }

    result
}

impl Individual for Solution {
    fn mutate(&mut self) {
        let mut rng = rand::thread_rng();
        let num_of_operations: u8 = 4;

        let index1: usize = rng.gen_range(0, self.arrangement.len());

        // Probably use this create and enums for operation:
        // https://github.com/andersk/enum_primitive-rs
        let operation: u8 = rng.gen_range(0, num_of_operations);

        match operation {
            0 => {
                // Change image order
                let mut index2: usize = rng.gen_range(0, self.arrangement.len());
                while index1 == index2 {
                    index2 = rng.gen_range(0, self.arrangement.len());
                }

                let px0 = self.arrangement[index1].x0;
                let py0 = self.arrangement[index1].y0;
                let px1 = self.arrangement[index1].x1;
                let py1 = self.arrangement[index1].y1;

                self.arrangement[index1].x0 = self.arrangement[index2].x0;
                self.arrangement[index1].y0 = self.arrangement[index2].y0;
                self.arrangement[index1].x1 = self.arrangement[index2].x1;
                self.arrangement[index1].y1 = self.arrangement[index2].y1;

                self.arrangement[index2].x0 = px0;
                self.arrangement[index2].y0 = py0;
                self.arrangement[index2].x1 = px1;
                self.arrangement[index2].y1 = py1;
            },
            1 => {
                // Move x
                let dx = rng.gen_range::<i32>(0, 500) - 250;
                let new_x = (self.arrangement[index1].x0 as i32) + dx;

                if new_x >= 0 {
                    self.arrangement[index1].x0 = new_x as u32;
                    self.arrangement[index1].x1 = ((self.arrangement[index1].x1 as i32) + dx) as u32;
                }
            },
            2 => {
                // Move y
                let dy = rng.gen_range::<i32>(0, 500) - 250;
                let new_y = (self.arrangement[index1].y0 as i32) + dy;

                if new_y >= 0 {
                    self.arrangement[index1].y0 = new_y as u32;
                    self.arrangement[index1].y1 = ((self.arrangement[index1].y1 as i32) + dy) as u32;
                }
            },
            3 => {
                // Rotate
                self.arrangement[index1].angle = self.arrangement[index1].angle + (rng.next_f64() * 0.1);
                if self.arrangement[index1].angle < 0.0 {
                    self.arrangement[index1].angle = 0.0;
                } else if self.arrangement[index1].angle > 2.0 * PI {
                    self.arrangement[index1].angle = 0.0;
                }
            },
            op => {
                warn!("mutate(): unknown operation: {}", op)
            }
        }
    }

    fn calculate_fitness(&mut self) -> f64 {
        let num_of_samples = self.arrangement[0].samples;

        let (cx0, cy0, cx1, cy1) = calc_canvas_size(&self.arrangement);

        // Create sample points on canvas:
        let mut rng = rand::thread_rng();
        let mut sample_pixels = Vec::new();

        for _ in 0..num_of_samples {
            sample_pixels.push(SamplePixel{
                x: rng.gen_range::<u32>(cx0, cx1),
                y: rng.gen_range::<u32>(cy0, cy1),
                pixels: Vec::new()
            });
        }

        for im_ar in &self.arrangement {
            for sp in &mut sample_pixels {
                if let Some(pixel) = get_pixel(sp.x as f64, sp.y as f64, &im_ar) {
                    sp.pixels.push(pixel);
                }
            }
        }

        let error_sum = sample_pixels.iter().fold(0, |sum, sp| {
            if sp.pixels.len() > 0 {
                let r0 = sp.pixels[0].0;
                let g0 = sp.pixels[0].1;
                let b0 = sp.pixels[0].2;
                let max_diff = sp.pixels.iter().fold((0_u32, r0, g0, b0), |(diff, r1, g1, b1), &(r2, g2, b2)| {
                    let rdiff = if r1 > r2 {r1 - r2} else {r2 - r1};
                    let gdiff  = if g1 > g2 {g1 - g2} else {g2 - g1};
                    let bdiff  = if b1 > b2 {b1 - b2} else {b2 - b1};

                    (cmp::max(diff, (rdiff as u32) + (gdiff as u32) + (bdiff as u32)), r2, g2, b2)
                });
                sum + max_diff.0
            } else {
                sum
            }

        });

        (error_sum as f64) / (num_of_samples as f64)
    }

    fn reset(&mut self) {
        for arrangement in &mut self.arrangement {
            // Reset all image operation to original image
            arrangement.x1 = arrangement.x1 - arrangement.x0;
            arrangement.y1 = arrangement.y1 - arrangement.y0;
            arrangement.x0 = 0;
            arrangement.y0 = 0;
            arrangement.angle = 0.0;
        }
    }
}

fn run_darwin(solution: &Solution, config: &PanolutionConfig) -> Solution {
    let pano = SimulationBuilder::<Solution>::new()
        .iterations(config.max_iteration)
        .threads(4)
        .add_multiple_populations(make_all_populations(20, 8, &solution))
        .finalize();

    match pano {
        Err(e) => {
            error!("An error occured");

            for e in e.iter().skip(1) {
                error!("Caused by '{}'", e)
            }

            solution.clone()
        },
        Ok(mut pano_simulation) => {
            pano_simulation.run();

            info!("Total run time: {} ms", pano_simulation.total_time_in_ms);
            info!("Improvement factor: {}", pano_simulation.simulation_result.improvement_factor);
            info!("Number of iterations: {}", pano_simulation.simulation_result.iteration_counter);
            info!("New fitness: {}", pano_simulation.simulation_result.fittest[0].fitness);

            pano_simulation.simulation_result.fittest[0].individual.clone()
        }
    }
}

pub fn optimize(solution: Option<&Solution>, config: &PanolutionConfig, sample_index: usize) -> Solution {
    match solution {
        None => {
            let mut arrangement = Vec::new();

            for entry in WalkDir::new(&config.input_path) {
                if let Ok(entry) = entry {
                    if entry.file_type().is_file() {
                        if let Some(file_name) = entry.file_name().to_str() {
                            if valid_image_file(file_name) {
                                if let Some(full_path) = entry.path().to_str() {

                                    if let Ok(img) = image::open(&full_path) {
                                        arrangement.push(ImageArrangement{
                                            image: Arc::new(img.clone()),
                                            samples: config.num_of_samples[sample_index],
                                            x0: 0,
                                            y0: 0,
                                            x1: img.width(),
                                            y1: img.height(),
                                            angle: 0.0,
                                        });
                                    } else {
                                        error!("Could not open image: {}", full_path)
                                    }
                                } else {
                                    error!("Could not convert full path to str: {:?}", entry);
                                }
                            } else {
                                error!("Image format currently not supported: {}", file_name);
                            }
                        } else {
                            error!("Could not convert file name to str: {:?}", entry);
                        }
                    } else {
                        info!("Ignore non-file: {:?}", entry);
                    }
                } else {
                    error!("Error in WalkDir");
                }
            }

            let solution = Solution{arrangement: arrangement};

            if solution.arrangement.len() > 0 {
                run_darwin(&solution, &config)
            } else {
                // An error occured
                // TODO: return Result<Soultion> and use chain_err
                solution
            }
        },
        Some(solution) => {
            let mut updated_arrangement = solution.arrangement.clone();

            for im_ar in &mut updated_arrangement {
                im_ar.samples = config.num_of_samples[sample_index];
            }

            run_darwin(&Solution{arrangement: updated_arrangement}, &config)
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 TiKV Project Authors. Licensed under Apache-2.0.

use std::path::Path;
use std::sync::{Arc, Mutex};

use tempfile::Builder;

use kvproto::metapb;
use kvproto::raft_serverpb::RegionLocalState;

use engine::rocks;
use engine::Engines;
use engine::*;
use test_raftstore::*;
use tikv::import::SSTImporter;
use tikv::raftstore::coprocessor::CoprocessorHost;
use tikv::raftstore::store::fsm::store::StoreMeta;
use tikv::raftstore::store::{bootstrap_store, fsm, keys, SnapManager};
use tikv::server::Node;
use tikv_util::worker::FutureWorker;

fn test_bootstrap_idempotent<T: Simulator>(cluster: &mut Cluster<T>) {
    // assume that there is a node  bootstrap the cluster and add region in pd successfully
    cluster.add_first_region().unwrap();
    // now  at same time start the another node, and will recive cluster is not bootstrap
    // it will try to bootstrap with a new region, but will failed
    // the region number still 1
    cluster.start().unwrap();
    cluster.check_regions_number(1);
    cluster.shutdown();
    sleep_ms(500);
    cluster.start().unwrap();
    cluster.check_regions_number(1);
}

#[test]
fn test_node_bootstrap_with_prepared_data() {
    // create a node
    let pd_client = Arc::new(TestPdClient::new(0, false));
    let cfg = new_tikv_config(0);

    let (_, system) = fsm::create_raft_batch_system(&cfg.raft_store);
    let simulate_trans = SimulateTransport::new(ChannelTransport::new());
    let tmp_path = Builder::new().prefix("test_cluster").tempdir().unwrap();
    let engine = Arc::new(
        rocks::util::new_engine(tmp_path.path().to_str().unwrap(), None, ALL_CFS, None).unwrap(),
    );
    let tmp_path_raft = tmp_path.path().join(Path::new("raft"));
    let raft_engine = Arc::new(
        rocks::util::new_engine(tmp_path_raft.to_str().unwrap(), None, &[], None).unwrap(),
    );
    let shared_block_cache = false;
    let engines = Engines::new(
        Arc::clone(&engine),
        Arc::clone(&raft_engine),
        shared_block_cache,
    );
    let tmp_mgr = Builder::new().prefix("test_cluster").tempdir().unwrap();

    let mut node = Node::new(system, &cfg.server, &cfg.raft_store, Arc::clone(&pd_client));
    let snap_mgr = SnapManager::new(tmp_mgr.path().to_str().unwrap(), Some(node.get_router()));
    let pd_worker = FutureWorker::new("test-pd-worker");

    // assume there is a node has bootstrapped the cluster and add region in pd successfully
    bootstrap_with_first_region(Arc::clone(&pd_client)).unwrap();

    // now anthoer node at same time begin bootstrap node, but panic after prepared bootstrap
    // now rocksDB must have some prepare data
    bootstrap_store(&engines, 0, 1).unwrap();
    let region = node.prepare_bootstrap_cluster(&engines, 1).unwrap();
    assert!(engine
        .get_msg::<metapb::Region>(keys::PREPARE_BOOTSTRAP_KEY)
        .unwrap()
        .is_some());
    let region_state_key = keys::region_state_key(region.get_id());
    assert!(engine
        .get_msg_cf::<RegionLocalState>(CF_RAFT, &region_state_key)
        .unwrap()
        .is_some());

    // Create coprocessor.
    let coprocessor_host = CoprocessorHost::new(cfg.coprocessor, node.get_router());

    let importer = {
        let dir = tmp_path.path().join("import-sst");
        Arc::new(SSTImporter::new(dir).unwrap())
    };

    // try to restart this node, will clear the prepare data
    node.start(
        engines,
        simulate_trans,
        snap_mgr,
        pd_worker,
        Arc::new(Mutex::new(StoreMeta::new(0))),
        coprocessor_host,
        importer,
    )
    .unwrap();
    assert!(Arc::clone(&engine)
        .get_msg::<metapb::Region>(keys::PREPARE_BOOTSTRAP_KEY)
        .unwrap()
        .is_none());
    assert!(engine
        .get_msg_cf::<RegionLocalState>(CF_RAFT, &region_state_key)
        .unwrap()
        .is_none());
    assert_eq!(pd_client.get_regions_number() as u32, 1);
    node.stop();
}

#[test]
fn test_node_bootstrap_idempotent() {
    let mut cluster = new_node_cluster(0, 3);
    test_bootstrap_idempotent(&mut cluster);
}
<commit_msg>cleanup: fix typos in   test_bootstrap.rs (#5300)<commit_after>// Copyright 2017 TiKV Project Authors. Licensed under Apache-2.0.

use std::path::Path;
use std::sync::{Arc, Mutex};

use tempfile::Builder;

use kvproto::metapb;
use kvproto::raft_serverpb::RegionLocalState;

use engine::rocks;
use engine::Engines;
use engine::*;
use test_raftstore::*;
use tikv::import::SSTImporter;
use tikv::raftstore::coprocessor::CoprocessorHost;
use tikv::raftstore::store::fsm::store::StoreMeta;
use tikv::raftstore::store::{bootstrap_store, fsm, keys, SnapManager};
use tikv::server::Node;
use tikv_util::worker::FutureWorker;

fn test_bootstrap_idempotent<T: Simulator>(cluster: &mut Cluster<T>) {
    // assume that there is a node  bootstrap the cluster and add region in pd successfully
    cluster.add_first_region().unwrap();
    // now  at same time start the another node, and will recive cluster is not bootstrap
    // it will try to bootstrap with a new region, but will failed
    // the region number still 1
    cluster.start().unwrap();
    cluster.check_regions_number(1);
    cluster.shutdown();
    sleep_ms(500);
    cluster.start().unwrap();
    cluster.check_regions_number(1);
}

#[test]
fn test_node_bootstrap_with_prepared_data() {
    // create a node
    let pd_client = Arc::new(TestPdClient::new(0, false));
    let cfg = new_tikv_config(0);

    let (_, system) = fsm::create_raft_batch_system(&cfg.raft_store);
    let simulate_trans = SimulateTransport::new(ChannelTransport::new());
    let tmp_path = Builder::new().prefix("test_cluster").tempdir().unwrap();
    let engine = Arc::new(
        rocks::util::new_engine(tmp_path.path().to_str().unwrap(), None, ALL_CFS, None).unwrap(),
    );
    let tmp_path_raft = tmp_path.path().join(Path::new("raft"));
    let raft_engine = Arc::new(
        rocks::util::new_engine(tmp_path_raft.to_str().unwrap(), None, &[], None).unwrap(),
    );
    let shared_block_cache = false;
    let engines = Engines::new(
        Arc::clone(&engine),
        Arc::clone(&raft_engine),
        shared_block_cache,
    );
    let tmp_mgr = Builder::new().prefix("test_cluster").tempdir().unwrap();

    let mut node = Node::new(system, &cfg.server, &cfg.raft_store, Arc::clone(&pd_client));
    let snap_mgr = SnapManager::new(tmp_mgr.path().to_str().unwrap(), Some(node.get_router()));
    let pd_worker = FutureWorker::new("test-pd-worker");

    // assume there is a node has bootstrapped the cluster and add region in pd successfully
    bootstrap_with_first_region(Arc::clone(&pd_client)).unwrap();

    // now another node at same time begin bootstrap node, but panic after prepared bootstrap
    // now rocksDB must have some prepare data
    bootstrap_store(&engines, 0, 1).unwrap();
    let region = node.prepare_bootstrap_cluster(&engines, 1).unwrap();
    assert!(engine
        .get_msg::<metapb::Region>(keys::PREPARE_BOOTSTRAP_KEY)
        .unwrap()
        .is_some());
    let region_state_key = keys::region_state_key(region.get_id());
    assert!(engine
        .get_msg_cf::<RegionLocalState>(CF_RAFT, &region_state_key)
        .unwrap()
        .is_some());

    // Create coprocessor.
    let coprocessor_host = CoprocessorHost::new(cfg.coprocessor, node.get_router());

    let importer = {
        let dir = tmp_path.path().join("import-sst");
        Arc::new(SSTImporter::new(dir).unwrap())
    };

    // try to restart this node, will clear the prepare data
    node.start(
        engines,
        simulate_trans,
        snap_mgr,
        pd_worker,
        Arc::new(Mutex::new(StoreMeta::new(0))),
        coprocessor_host,
        importer,
    )
    .unwrap();
    assert!(Arc::clone(&engine)
        .get_msg::<metapb::Region>(keys::PREPARE_BOOTSTRAP_KEY)
        .unwrap()
        .is_none());
    assert!(engine
        .get_msg_cf::<RegionLocalState>(CF_RAFT, &region_state_key)
        .unwrap()
        .is_none());
    assert_eq!(pd_client.get_regions_number() as u32, 1);
    node.stop();
}

#[test]
fn test_node_bootstrap_idempotent() {
    let mut cluster = new_node_cluster(0, 3);
    test_bootstrap_idempotent(&mut cluster);
}
<|endoftext|>


<commit_before>#![deny(rust_2018_idioms)]

use cargo::{
    core::{
        package::PackageSet,
        registry::PackageRegistry,
        resolver::{self, Method},
        source::SourceMap,
        Dependency, Package, PackageId, Source, SourceId, TargetKind,
    },
    sources::RegistrySource,
    util::Config,
};
use itertools::Itertools;
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use std::{
    collections::{BTreeMap, BTreeSet, HashSet},
    fs::File,
    io::{Read, Write},
};

/// The list of crates from crates.io
#[derive(Debug, Deserialize)]
struct TopCrates {
    crates: Vec<Crate>,
}

/// A single crate from crates.io
#[derive(Debug, Deserialize)]
struct OneCrate {
    #[serde(rename="crate")]
    krate: Crate,
}

/// The shared description of a crate
#[derive(Debug, Deserialize)]
struct Crate {
    #[serde(rename="id")]
    name: String,
}

/// A Cargo.toml file.
#[derive(Serialize)]
struct TomlManifest {
    package: TomlPackage,
    profile: Profiles,
    #[serde(serialize_with = "toml::ser::tables_last")]
    dependencies: BTreeMap<String, DependencySpec>,
}

/// Header of Cargo.toml file.
#[derive(Serialize)]
struct TomlPackage {
    name: String,
    version: String,
    authors: Vec<String>,
}

/// A mapping of a crates name to its identifier used in source code
#[derive(Debug, Serialize)]
struct CrateInformation {
    name: String,
    version: String,
    id: String,
}

/// Hand-curated changes to the crate list
#[derive(Debug, Deserialize)]
struct Modifications {
    #[serde(default)]
    blacklist: Vec<String>,
    #[serde(default)]
    additions: BTreeSet<String>,
}

/// A profile section in a Cargo.toml file
#[derive(Serialize)]
#[serde(rename_all="kebab-case")]
struct Profile {
    codegen_units: u32,
    incremental: bool,
}

/// Available profile types
#[derive(Serialize)]
struct Profiles {
    dev: Profile,
    release: Profile,
}

#[derive(Debug, Serialize, Clone)]
#[serde(rename_all = "kebab-case")]
struct DependencySpec {
    #[serde(skip_serializing_if = "String::is_empty")]
    package: String,
    #[serde(serialize_with = "exact_version")]
    version: String,
    #[serde(skip_serializing_if = "Vec::is_empty")]
    features: Vec<String>,
    #[serde(skip_serializing_if = "is_true")]
    default_features: bool,
}

fn exact_version<S>(version: &String, serializer: S) -> Result<S::Ok, S::Error>
where
    S: serde::Serializer,
{
    format!("={}", version).serialize(serializer)
}

fn is_true(b: &bool) -> bool {
    *b
}

impl Modifications {
    fn blacklisted(&self, name: &str) -> bool {
        self.blacklist.iter().any(|n| n == name)
    }
}

lazy_static! {
    static ref MODIFICATIONS: Modifications = {
        let mut f = File::open("crate-modifications.toml")
            .expect("unable to open crate modifications file");

        let mut d = Vec::new();
        f.read_to_end(&mut d)
            .expect("unable to read crate modifications file");

        toml::from_slice(&d)
            .expect("unable to parse crate modifications file")
    };
}

impl TopCrates {
    /// List top 100 crates by number of downloads on crates.io.
    fn download() -> TopCrates {
        let resp =
            reqwest::get("https://crates.io/api/v1/crates?page=1&per_page=100&sort=downloads")
            .expect("Could not fetch top crates");
        assert!(resp.status().is_success());

        serde_json::from_reader(resp).expect("Invalid JSON")
    }

    /// Add crates that have been hand-picked
    fn add_curated_crates(&mut self) {
        self.crates.extend({
            MODIFICATIONS
                .additions
                .iter()
                .cloned()
                .map(|name| Crate { name })
        });
    }
}

/// Finds the features specified by the custom metadata of `pkg`.
///
/// Our custom metadata format looks like:
///
///     [package.metadata.playground]
///     default-features = true
///     features = ["std", "extra-traits"]
///     all-features = false
///
/// All fields are optional.
fn playground_metadata_features(pkg: &Package) -> Option<(Vec<String>, bool)> {
    let custom_metadata = pkg.manifest().custom_metadata()?;
    let playground_metadata = custom_metadata.get("playground")?;

    #[derive(Deserialize)]
    #[serde(default, rename_all = "kebab-case")]
    struct Metadata {
        features: Vec<String>,
        default_features: bool,
        all_features: bool,
    }

    impl Default for Metadata {
        fn default() -> Self {
            Metadata {
                features: Vec::new(),
                default_features: true,
                all_features: false,
            }
        }
    }

    let metadata = match playground_metadata.clone().try_into::<Metadata>() {
        Ok(metadata) => metadata,
        Err(err) => {
            eprintln!(
                "Failed to parse custom metadata for {} {}: {}",
                pkg.name(), pkg.version(), err);
            return None;
        }
    };

    // If `all-features` is set then we ignore `features`.
    let summary = pkg.summary();
    let mut enabled_features: BTreeSet<String> = if metadata.all_features {
        summary.features().keys().map(ToString::to_string).collect()
    } else {
        metadata.features.into_iter().collect()
    };

    // If not opting out of default features, remove default features from the
    // explicit features list. This avoids ongoing spurious diffs in our
    // generated Cargo.toml as default features are added to a library.
    if metadata.default_features {
        if let Some(default_feature_names) = summary.features().get("default") {
            enabled_features.remove("default");
            for feature in default_feature_names {
                enabled_features.remove(&feature.to_string(summary));
            }
        }
    }

    if !enabled_features.is_empty() || !metadata.default_features {
        Some((
            enabled_features.into_iter().collect(),
            metadata.default_features,
        ))
    } else {
        None
    }
}

fn write_manifest(manifest: TomlManifest, path: &str) {
    let mut f = File::create(path).expect("Unable to create Cargo.toml");
    let content = toml::to_vec(&manifest).expect("Couldn't serialize TOML");
    f.write_all(&content).expect("Couldn't write Cargo.toml");
}

fn main() {
    // Setup to interact with cargo.
    let config = Config::default().expect("Unable to create default Cargo config");
    let _lock = config.acquire_package_cache_lock();
    let crates_io = SourceId::crates_io(&config).expect("Unable to create crates.io source ID");
    let mut source = RegistrySource::remote(crates_io, &HashSet::new(), &config);
    source.update().expect("Unable to update registry");

    let mut top = TopCrates::download();
    top.add_curated_crates();

    // Find the newest (non-prerelease, non-yanked) versions of all
    // the interesting crates.
    let mut summaries = Vec::new();
    for Crate { ref name } in top.crates {
        if MODIFICATIONS.blacklisted(name) {
            continue;
        }

        // Query the registry for a summary of this crate.
        // Usefully, this doesn't seem to include yanked versions
        let dep = Dependency::parse_no_deprecated(name, None, crates_io)
            .unwrap_or_else(|e| panic!("Unable to parse dependency for {}: {}", name, e));

        let matches = source.query_vec(&dep).unwrap_or_else(|e| {
            panic!("Unable to query registry for {}: {}", name, e);
        });

        // Find the newest non-prelease version
        let summary = matches.into_iter()
            .filter(|summary| !summary.version().is_prerelease())
            .max_by_key(|summary| summary.version().clone())
            .unwrap_or_else(|| panic!("Registry has no viable versions of {}", name));

        // Add a dependency on this crate.
        summaries.push((summary, Method::Required {
            dev_deps: false,
            features: Default::default(),
            uses_default_features: true,
            all_features: false,
        }));
    }

    // Resolve transitive dependencies.
    let mut registry = PackageRegistry::new(&config)
        .expect("Unable to create package registry");
    registry.lock_patches();
    let try_to_use = HashSet::new();
    let resolve = resolver::resolve(&summaries, &[], &mut registry, &try_to_use, None, true)
        .expect("Unable to resolve dependencies");

    // Get the package information for all dependencies.
    let package_ids: Vec<_> = resolve
        .iter()
        .filter(|pkg| !MODIFICATIONS.blacklisted(pkg.name().as_str()))
        .map(|pkg| {
            PackageId::new(&pkg.name(), pkg.version(), crates_io).unwrap_or_else(|e| {
                panic!(
                    "Unable to build PackageId for {} {}: {}",
                    pkg.name(),
                    pkg.version(),
                    e
                )
            })
        })
        .collect();

    let mut sources = SourceMap::new();
    sources.insert(Box::new(source));

    let package_set =
        PackageSet::new(&package_ids, sources, &config).expect("Unable to create a PackageSet");

    let mut packages = package_set
        .get_many(package_set.package_ids())
        .expect("Unable to download packages");

    // Sort all packages by name then version (descending), so that
    // when we group them we know we get all the same crates together
    // and the newest version first.
    packages.sort_by(|a, b| {
        a.name()
            .cmp(&b.name())
            .then(a.version().cmp(&b.version()).reverse())
    });

    let mut dependencies = BTreeMap::new();
    let mut infos = Vec::new();

    for (name, pkgs) in &packages.into_iter().group_by(|pkg| pkg.name()) {
        for pkg in pkgs {
            let version = pkg.version();

            let crate_name = pkg
                .targets()
                .iter()
                .flat_map(|target| match target.kind() {
                    TargetKind::Lib(_) => Some(target.crate_name()),
                    _ => None,
                })
                .next()
                .unwrap_or_else(|| panic!("{} did not have a library", name));

            // We see the newest version first.
            let exposed_name = crate_name.clone();

            let (features, default_features) =
                playground_metadata_features(&pkg).unwrap_or_else(|| (Vec::new(), true));

            dependencies.insert(
                exposed_name.clone(),
                DependencySpec {
                    package: name.to_string(),
                    version: version.to_string(),
                    features,
                    default_features,
                },
            );

            infos.push(CrateInformation {
                name: name.to_string(),
                version: version.to_string(),
                id: exposed_name,
            });

            break; // Don't include additional versions of this crate
        }
    }

    // Construct playground's Cargo.toml.
    let manifest = TomlManifest {
        package: TomlPackage {
            name: "playground".to_owned(),
            version: "0.0.1".to_owned(),
            authors: vec!["The Rust Playground".to_owned()],
        },
        profile: Profiles {
            dev: Profile { codegen_units: 1, incremental: false },
            release: Profile { codegen_units: 1, incremental: false },
        },
        dependencies,
    };

    // Write manifest file.
    let cargo_toml = "../compiler/base/Cargo.toml";
    write_manifest(manifest, cargo_toml);
    println!("wrote {}", cargo_toml);

    let path = "../compiler/base/crate-information.json";
    let mut f = File::create(path)
        .unwrap_or_else(|e| panic!("Unable to create {}: {}", path, e));
    serde_json::to_writer_pretty(&mut f, &infos)
        .unwrap_or_else(|e| panic!("Unable to write {}: {}", path, e));
    println!("Wrote {}", path);
}
<commit_msg>Expose all versions of crates that have been compiled<commit_after>#![deny(rust_2018_idioms)]

use cargo::{
    core::{
        package::PackageSet,
        registry::PackageRegistry,
        resolver::{self, Method},
        source::SourceMap,
        Dependency, Package, PackageId, Source, SourceId, TargetKind,
    },
    sources::RegistrySource,
    util::Config,
};
use itertools::Itertools;
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use std::{
    collections::{BTreeMap, BTreeSet, HashSet},
    fs::File,
    io::{Read, Write},
};

/// The list of crates from crates.io
#[derive(Debug, Deserialize)]
struct TopCrates {
    crates: Vec<Crate>,
}

/// A single crate from crates.io
#[derive(Debug, Deserialize)]
struct OneCrate {
    #[serde(rename="crate")]
    krate: Crate,
}

/// The shared description of a crate
#[derive(Debug, Deserialize)]
struct Crate {
    #[serde(rename="id")]
    name: String,
}

/// A Cargo.toml file.
#[derive(Serialize)]
struct TomlManifest {
    package: TomlPackage,
    profile: Profiles,
    #[serde(serialize_with = "toml::ser::tables_last")]
    dependencies: BTreeMap<String, DependencySpec>,
}

/// Header of Cargo.toml file.
#[derive(Serialize)]
struct TomlPackage {
    name: String,
    version: String,
    authors: Vec<String>,
}

/// A mapping of a crates name to its identifier used in source code
#[derive(Debug, Serialize)]
struct CrateInformation {
    name: String,
    version: String,
    id: String,
}

/// Hand-curated changes to the crate list
#[derive(Debug, Deserialize)]
struct Modifications {
    #[serde(default)]
    blacklist: Vec<String>,
    #[serde(default)]
    additions: BTreeSet<String>,
}

/// A profile section in a Cargo.toml file
#[derive(Serialize)]
#[serde(rename_all="kebab-case")]
struct Profile {
    codegen_units: u32,
    incremental: bool,
}

/// Available profile types
#[derive(Serialize)]
struct Profiles {
    dev: Profile,
    release: Profile,
}

#[derive(Debug, Serialize, Clone)]
#[serde(rename_all = "kebab-case")]
struct DependencySpec {
    #[serde(skip_serializing_if = "String::is_empty")]
    package: String,
    #[serde(serialize_with = "exact_version")]
    version: String,
    #[serde(skip_serializing_if = "Vec::is_empty")]
    features: Vec<String>,
    #[serde(skip_serializing_if = "is_true")]
    default_features: bool,
}

fn exact_version<S>(version: &String, serializer: S) -> Result<S::Ok, S::Error>
where
    S: serde::Serializer,
{
    format!("={}", version).serialize(serializer)
}

fn is_true(b: &bool) -> bool {
    *b
}

impl Modifications {
    fn blacklisted(&self, name: &str) -> bool {
        self.blacklist.iter().any(|n| n == name)
    }
}

lazy_static! {
    static ref MODIFICATIONS: Modifications = {
        let mut f = File::open("crate-modifications.toml")
            .expect("unable to open crate modifications file");

        let mut d = Vec::new();
        f.read_to_end(&mut d)
            .expect("unable to read crate modifications file");

        toml::from_slice(&d)
            .expect("unable to parse crate modifications file")
    };
}

impl TopCrates {
    /// List top 100 crates by number of downloads on crates.io.
    fn download() -> TopCrates {
        let resp =
            reqwest::get("https://crates.io/api/v1/crates?page=1&per_page=100&sort=downloads")
            .expect("Could not fetch top crates");
        assert!(resp.status().is_success());

        serde_json::from_reader(resp).expect("Invalid JSON")
    }

    /// Add crates that have been hand-picked
    fn add_curated_crates(&mut self) {
        self.crates.extend({
            MODIFICATIONS
                .additions
                .iter()
                .cloned()
                .map(|name| Crate { name })
        });
    }
}

/// Finds the features specified by the custom metadata of `pkg`.
///
/// Our custom metadata format looks like:
///
///     [package.metadata.playground]
///     default-features = true
///     features = ["std", "extra-traits"]
///     all-features = false
///
/// All fields are optional.
fn playground_metadata_features(pkg: &Package) -> Option<(Vec<String>, bool)> {
    let custom_metadata = pkg.manifest().custom_metadata()?;
    let playground_metadata = custom_metadata.get("playground")?;

    #[derive(Deserialize)]
    #[serde(default, rename_all = "kebab-case")]
    struct Metadata {
        features: Vec<String>,
        default_features: bool,
        all_features: bool,
    }

    impl Default for Metadata {
        fn default() -> Self {
            Metadata {
                features: Vec::new(),
                default_features: true,
                all_features: false,
            }
        }
    }

    let metadata = match playground_metadata.clone().try_into::<Metadata>() {
        Ok(metadata) => metadata,
        Err(err) => {
            eprintln!(
                "Failed to parse custom metadata for {} {}: {}",
                pkg.name(), pkg.version(), err);
            return None;
        }
    };

    // If `all-features` is set then we ignore `features`.
    let summary = pkg.summary();
    let mut enabled_features: BTreeSet<String> = if metadata.all_features {
        summary.features().keys().map(ToString::to_string).collect()
    } else {
        metadata.features.into_iter().collect()
    };

    // If not opting out of default features, remove default features from the
    // explicit features list. This avoids ongoing spurious diffs in our
    // generated Cargo.toml as default features are added to a library.
    if metadata.default_features {
        if let Some(default_feature_names) = summary.features().get("default") {
            enabled_features.remove("default");
            for feature in default_feature_names {
                enabled_features.remove(&feature.to_string(summary));
            }
        }
    }

    if !enabled_features.is_empty() || !metadata.default_features {
        Some((
            enabled_features.into_iter().collect(),
            metadata.default_features,
        ))
    } else {
        None
    }
}

fn write_manifest(manifest: TomlManifest, path: &str) {
    let mut f = File::create(path).expect("Unable to create Cargo.toml");
    let content = toml::to_vec(&manifest).expect("Couldn't serialize TOML");
    f.write_all(&content).expect("Couldn't write Cargo.toml");
}

fn main() {
    // Setup to interact with cargo.
    let config = Config::default().expect("Unable to create default Cargo config");
    let _lock = config.acquire_package_cache_lock();
    let crates_io = SourceId::crates_io(&config).expect("Unable to create crates.io source ID");
    let mut source = RegistrySource::remote(crates_io, &HashSet::new(), &config);
    source.update().expect("Unable to update registry");

    let mut top = TopCrates::download();
    top.add_curated_crates();

    // Find the newest (non-prerelease, non-yanked) versions of all
    // the interesting crates.
    let mut summaries = Vec::new();
    for Crate { ref name } in top.crates {
        if MODIFICATIONS.blacklisted(name) {
            continue;
        }

        // Query the registry for a summary of this crate.
        // Usefully, this doesn't seem to include yanked versions
        let dep = Dependency::parse_no_deprecated(name, None, crates_io)
            .unwrap_or_else(|e| panic!("Unable to parse dependency for {}: {}", name, e));

        let matches = source.query_vec(&dep).unwrap_or_else(|e| {
            panic!("Unable to query registry for {}: {}", name, e);
        });

        // Find the newest non-prelease version
        let summary = matches.into_iter()
            .filter(|summary| !summary.version().is_prerelease())
            .max_by_key(|summary| summary.version().clone())
            .unwrap_or_else(|| panic!("Registry has no viable versions of {}", name));

        // Add a dependency on this crate.
        summaries.push((summary, Method::Required {
            dev_deps: false,
            features: Default::default(),
            uses_default_features: true,
            all_features: false,
        }));
    }

    // Resolve transitive dependencies.
    let mut registry = PackageRegistry::new(&config)
        .expect("Unable to create package registry");
    registry.lock_patches();
    let try_to_use = HashSet::new();
    let resolve = resolver::resolve(&summaries, &[], &mut registry, &try_to_use, None, true)
        .expect("Unable to resolve dependencies");

    // Get the package information for all dependencies.
    let package_ids: Vec<_> = resolve
        .iter()
        .filter(|pkg| !MODIFICATIONS.blacklisted(pkg.name().as_str()))
        .map(|pkg| {
            PackageId::new(&pkg.name(), pkg.version(), crates_io).unwrap_or_else(|e| {
                panic!(
                    "Unable to build PackageId for {} {}: {}",
                    pkg.name(),
                    pkg.version(),
                    e
                )
            })
        })
        .collect();

    let mut sources = SourceMap::new();
    sources.insert(Box::new(source));

    let package_set =
        PackageSet::new(&package_ids, sources, &config).expect("Unable to create a PackageSet");

    let mut packages = package_set
        .get_many(package_set.package_ids())
        .expect("Unable to download packages");

    // Sort all packages by name then version (descending), so that
    // when we group them we know we get all the same crates together
    // and the newest version first.
    packages.sort_by(|a, b| {
        a.name()
            .cmp(&b.name())
            .then(a.version().cmp(&b.version()).reverse())
    });

    let mut dependencies = BTreeMap::new();
    let mut infos = Vec::new();

    for (name, pkgs) in &packages.into_iter().group_by(|pkg| pkg.name()) {
        let mut first = true;

        for pkg in pkgs {
            let version = pkg.version();

            let crate_name = pkg
                .targets()
                .iter()
                .flat_map(|target| match target.kind() {
                    TargetKind::Lib(_) => Some(target.crate_name()),
                    _ => None,
                })
                .next()
                .unwrap_or_else(|| panic!("{} did not have a library", name));

            // We see the newest version first. Any subsequent
            // versions will have their version appended so that they
            // are uniquely named
            let exposed_name = if first {
                crate_name.clone()
            } else {
                format!(
                    "{}_{}_{}_{}",
                    crate_name, version.major, version.minor, version.patch
                )
            };

            let (features, default_features) =
                playground_metadata_features(&pkg).unwrap_or_else(|| (Vec::new(), true));

            dependencies.insert(
                exposed_name.clone(),
                DependencySpec {
                    package: name.to_string(),
                    version: version.to_string(),
                    features,
                    default_features,
                },
            );

            infos.push(CrateInformation {
                name: name.to_string(),
                version: version.to_string(),
                id: exposed_name,
            });

            first = false;
        }
    }

    // Construct playground's Cargo.toml.
    let manifest = TomlManifest {
        package: TomlPackage {
            name: "playground".to_owned(),
            version: "0.0.1".to_owned(),
            authors: vec!["The Rust Playground".to_owned()],
        },
        profile: Profiles {
            dev: Profile { codegen_units: 1, incremental: false },
            release: Profile { codegen_units: 1, incremental: false },
        },
        dependencies,
    };

    // Write manifest file.
    let cargo_toml = "../compiler/base/Cargo.toml";
    write_manifest(manifest, cargo_toml);
    println!("wrote {}", cargo_toml);

    let path = "../compiler/base/crate-information.json";
    let mut f = File::create(path)
        .unwrap_or_else(|e| panic!("Unable to create {}: {}", path, e));
    serde_json::to_writer_pretty(&mut f, &infos)
        .unwrap_or_else(|e| panic!("Unable to write {}: {}", path, e));
    println!("Wrote {}", path);
}
<|endoftext|>