<commit_before>use std::env;
use std::fs::{self, File};
use std::io::prelude::*;
use std::path::{Path, PathBuf};
use std::str;

use cargo;
use support::cargo_process;
use support::paths::{self, CargoPathExt};
use support::registry::Package;
use support::{basic_bin_manifest, basic_manifest, cargo_exe, project, Project};

#[cfg_attr(windows, allow(dead_code))]
enum FakeKind<'a> {
    Executable,
    Symlink { target: &'a Path },
}

/// Add an empty file with executable flags (and platform-dependent suffix).
/// TODO: move this to `Project` if other cases using this emerge.
fn fake_file(proj: Project, dir: &Path, name: &str, kind: &FakeKind) -> Project {
    let path = proj
        .root()
        .join(dir)
        .join(&format!("{}{}", name, env::consts::EXE_SUFFIX));
    path.parent().unwrap().mkdir_p();
    match *kind {
        FakeKind::Executable => {
            File::create(&path).unwrap();
            make_executable(&path);
        }
        FakeKind::Symlink { target } => {
            make_symlink(&path, target);
        }
    }
    return proj;

    #[cfg(unix)]
    fn make_executable(p: &Path) {
        use std::os::unix::prelude::*;

        let mut perms = fs::metadata(p).unwrap().permissions();
        let mode = perms.mode();
        perms.set_mode(mode | 0o111);
        fs::set_permissions(p, perms).unwrap();
    }
    #[cfg(windows)]
    fn make_executable(_: &Path) {}
    #[cfg(unix)]
    fn make_symlink(p: &Path, t: &Path) {
        ::std::os::unix::fs::symlink(t, p).expect("Failed to create symlink");
    }
    #[cfg(windows)]
    fn make_symlink(_: &Path, _: &Path) {
        panic!("Not supported")
    }
}

fn path() -> Vec<PathBuf> {
    env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect()
}

#[test]
fn list_commands_with_descriptions() {
    let p = project().build();
    p.cargo("--list")
        .with_stdout_contains("    build                Compile a local package and all of its dependencies")
        // assert read-manifest prints the right one-line description followed by another command, indented.
        .with_stdout_contains("    read-manifest        Print a JSON representation of a Cargo.toml manifest.")
        .run();
}

#[test]
fn list_command_looks_at_path() {
    let proj = project().build();
    let proj = fake_file(
        proj,
        Path::new("path-test"),
        "cargo-1",
        &FakeKind::Executable,
    );

    let mut path = path();
    path.push(proj.root().join("path-test"));
    let path = env::join_paths(path.iter()).unwrap();
    let output = cargo_process("-v --list").env("PATH", &path).exec_with_output().unwrap();
    let output = str::from_utf8(&output.stdout).unwrap();
    assert!(
        output.contains("\n    1                   "),
        "missing 1: {}",
        output
    );
}

// windows and symlinks don't currently agree that well
#[cfg(unix)]
#[test]
fn list_command_resolves_symlinks() {
    let proj = project().build();
    let proj = fake_file(
        proj,
        Path::new("path-test"),
        "cargo-2",
        &FakeKind::Symlink {
            target: &cargo_exe(),
        },
    );

    let mut path = path();
    path.push(proj.root().join("path-test"));
    let path = env::join_paths(path.iter()).unwrap();
    let output = cargo_process("-v --list").env("PATH", &path).exec_with_output().unwrap();
    let output = str::from_utf8(&output.stdout).unwrap();
    assert!(
        output.contains("\n    2                   "),
        "missing 2: {}",
        output
    );
}

#[test]
fn find_closest_biuld_to_build() {
    cargo_process("biuld")
        .with_status(101)
        .with_stderr_contains(
            "\
error: no such subcommand: `biuld`

<tab>Did you mean `build`?
",
        ).run();

    // But, if we actually have `biuld`, it must work!
    // https://github.com/rust-lang/cargo/issues/5201
    Package::new("cargo-biuld", "1.0.0")
        .file(
            "src/main.rs",
            r#"
            fn main() {
                println!("Similar, but not identical to, build");
            }
        "#,
        ).publish();

    cargo_process("install cargo-biuld").run();
    cargo_process("biuld")
        .with_stdout("Similar, but not identical to, build\n")
        .run();
    cargo_process("--list")
        .with_stdout_contains(
            "    build                Compile a local package and all of its dependencies\n",
        ).with_stdout_contains("    biuld\n")
        .run();
}

// if a subcommand is more than 3 edit distance away, we don't make a suggestion
#[test]
fn find_closest_dont_correct_nonsense() {
    cargo_process("there-is-no-way-that-there-is-a-command-close-to-this")
        .cwd(&paths::root())
        .with_status(101)
        .with_stderr(
            "[ERROR] no such subcommand: \
                        `there-is-no-way-that-there-is-a-command-close-to-this`
",
        ).run();
}

#[test]
fn displays_subcommand_on_error() {
    cargo_process("invalid-command")
        .with_status(101)
        .with_stderr("[ERROR] no such subcommand: `invalid-command`\n")
        .run();
}

#[test]
fn override_cargo_home() {
    let root = paths::root();
    let my_home = root.join("my_home");
    fs::create_dir(&my_home).unwrap();
    File::create(&my_home.join("config"))
        .unwrap()
        .write_all(
            br#"
        [cargo-new]
        name = "foo"
        email = "bar"
        git = false
    "#,
        ).unwrap();

    cargo_process("new foo")
        .env("USER", "foo")
        .env("CARGO_HOME", &my_home)
        .run();

    let toml = paths::root().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml)
        .unwrap()
        .read_to_string(&mut contents)
        .unwrap();
    assert!(contents.contains(r#"authors = ["foo <bar>"]"#));
}

#[test]
fn cargo_subcommand_env() {
    let src = format!(
        r#"
        use std::env;

        fn main() {{
            println!("{{}}", env::var("{}").unwrap());
        }}
        "#,
        cargo::CARGO_ENV
    );

    let p = project()
        .at("cargo-envtest")
        .file("Cargo.toml", &basic_bin_manifest("cargo-envtest"))
        .file("src/main.rs", &src)
        .build();

    let target_dir = p.target_debug_dir();

    p.cargo("build").run();
    assert!(p.bin("cargo-envtest").is_file());

    let cargo = cargo_exe().canonicalize().unwrap();
    let mut path = path();
    path.push(target_dir);
    let path = env::join_paths(path.iter()).unwrap();

    cargo_process("envtest")
        .env("PATH", &path)
        .with_stdout(cargo.to_str().unwrap())
        .run();
}

#[test]
fn cargo_subcommand_args() {
    let p = project()
        .at("cargo-foo")
        .file("Cargo.toml", &basic_manifest("cargo-foo", "0.0.1"))
        .file(
            "src/main.rs",
            r#"
            fn main() {
                let args: Vec<_> = ::std::env::args().collect();
                println!("{:?}", args);
            }
        "#,
        ).build();

    p.cargo("build").run();
    let cargo_foo_bin = p.bin("cargo-foo");
    assert!(cargo_foo_bin.is_file());

    let mut path = path();
    path.push(p.target_debug_dir());
    let path = env::join_paths(path.iter()).unwrap();

    cargo_process("foo bar -v --help")
        .env("PATH", &path)
        .with_stdout(
            if cfg!(windows) { // weird edge-case w/ CWD & (windows vs unix)
                format!(r#"[{:?}, "foo", "bar", "-v", "--help"]"#, cargo_foo_bin)
            } else {
                r#"["[CWD]/cargo-foo/target/debug/cargo-foo", "foo", "bar", "-v", "--help"]"#.to_string()
            }
        ).run();
}

#[test]
fn cargo_help() {
    cargo_process("").run();
    cargo_process("help").run();
    cargo_process("-h").run();
    cargo_process("help build").run();
    cargo_process("build -h").run();
    cargo_process("help help").run();
}

#[test]
fn cargo_help_external_subcommand() {
    Package::new("cargo-fake-help", "1.0.0")
        .file(
            "src/main.rs",
            r#"
            fn main() {
                if ::std::env::args().nth(2) == Some(String::from("--help")) {
                    println!("fancy help output");
                }
            }"#,
        ).publish();
    cargo_process("install cargo-fake-help").run();
    cargo_process("help fake-help")
        .with_stdout("fancy help output\n")
        .run();
}

#[test]
fn explain() {
    cargo_process("--explain E0001")
        .with_stdout_contains(
            "This error suggests that the expression arm corresponding to the noted pattern",
        ).run();
}

// Test that the output of 'cargo -Z help' shows a different help screen with
// all the -Z flags.
#[test]
fn z_flags_help() {
    cargo_process("-Z help")
        .with_stdout_contains(
            "    -Z unstable-options -- Allow the usage of unstable options such as --registry",
        ).run();
}
<commit_msg>Simplify cargo_command::cargo_subcommand_args test<commit_after>use std::env;
use std::fs::{self, File};
use std::io::prelude::*;
use std::path::{Path, PathBuf};
use std::str;

use cargo;
use support::cargo_process;
use support::paths::{self, CargoPathExt};
use support::registry::Package;
use support::{basic_bin_manifest, basic_manifest, cargo_exe, project, Project};

#[cfg_attr(windows, allow(dead_code))]
enum FakeKind<'a> {
    Executable,
    Symlink { target: &'a Path },
}

/// Add an empty file with executable flags (and platform-dependent suffix).
/// TODO: move this to `Project` if other cases using this emerge.
fn fake_file(proj: Project, dir: &Path, name: &str, kind: &FakeKind) -> Project {
    let path = proj
        .root()
        .join(dir)
        .join(&format!("{}{}", name, env::consts::EXE_SUFFIX));
    path.parent().unwrap().mkdir_p();
    match *kind {
        FakeKind::Executable => {
            File::create(&path).unwrap();
            make_executable(&path);
        }
        FakeKind::Symlink { target } => {
            make_symlink(&path, target);
        }
    }
    return proj;

    #[cfg(unix)]
    fn make_executable(p: &Path) {
        use std::os::unix::prelude::*;

        let mut perms = fs::metadata(p).unwrap().permissions();
        let mode = perms.mode();
        perms.set_mode(mode | 0o111);
        fs::set_permissions(p, perms).unwrap();
    }
    #[cfg(windows)]
    fn make_executable(_: &Path) {}
    #[cfg(unix)]
    fn make_symlink(p: &Path, t: &Path) {
        ::std::os::unix::fs::symlink(t, p).expect("Failed to create symlink");
    }
    #[cfg(windows)]
    fn make_symlink(_: &Path, _: &Path) {
        panic!("Not supported")
    }
}

fn path() -> Vec<PathBuf> {
    env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect()
}

#[test]
fn list_commands_with_descriptions() {
    let p = project().build();
    p.cargo("--list")
        .with_stdout_contains("    build                Compile a local package and all of its dependencies")
        // assert read-manifest prints the right one-line description followed by another command, indented.
        .with_stdout_contains("    read-manifest        Print a JSON representation of a Cargo.toml manifest.")
        .run();
}

#[test]
fn list_command_looks_at_path() {
    let proj = project().build();
    let proj = fake_file(
        proj,
        Path::new("path-test"),
        "cargo-1",
        &FakeKind::Executable,
    );

    let mut path = path();
    path.push(proj.root().join("path-test"));
    let path = env::join_paths(path.iter()).unwrap();
    let output = cargo_process("-v --list").env("PATH", &path).exec_with_output().unwrap();
    let output = str::from_utf8(&output.stdout).unwrap();
    assert!(
        output.contains("\n    1                   "),
        "missing 1: {}",
        output
    );
}

// windows and symlinks don't currently agree that well
#[cfg(unix)]
#[test]
fn list_command_resolves_symlinks() {
    let proj = project().build();
    let proj = fake_file(
        proj,
        Path::new("path-test"),
        "cargo-2",
        &FakeKind::Symlink {
            target: &cargo_exe(),
        },
    );

    let mut path = path();
    path.push(proj.root().join("path-test"));
    let path = env::join_paths(path.iter()).unwrap();
    let output = cargo_process("-v --list").env("PATH", &path).exec_with_output().unwrap();
    let output = str::from_utf8(&output.stdout).unwrap();
    assert!(
        output.contains("\n    2                   "),
        "missing 2: {}",
        output
    );
}

#[test]
fn find_closest_biuld_to_build() {
    cargo_process("biuld")
        .with_status(101)
        .with_stderr_contains(
            "\
error: no such subcommand: `biuld`

<tab>Did you mean `build`?
",
        ).run();

    // But, if we actually have `biuld`, it must work!
    // https://github.com/rust-lang/cargo/issues/5201
    Package::new("cargo-biuld", "1.0.0")
        .file(
            "src/main.rs",
            r#"
            fn main() {
                println!("Similar, but not identical to, build");
            }
        "#,
        ).publish();

    cargo_process("install cargo-biuld").run();
    cargo_process("biuld")
        .with_stdout("Similar, but not identical to, build\n")
        .run();
    cargo_process("--list")
        .with_stdout_contains(
            "    build                Compile a local package and all of its dependencies\n",
        ).with_stdout_contains("    biuld\n")
        .run();
}

// if a subcommand is more than 3 edit distance away, we don't make a suggestion
#[test]
fn find_closest_dont_correct_nonsense() {
    cargo_process("there-is-no-way-that-there-is-a-command-close-to-this")
        .cwd(&paths::root())
        .with_status(101)
        .with_stderr(
            "[ERROR] no such subcommand: \
                        `there-is-no-way-that-there-is-a-command-close-to-this`
",
        ).run();
}

#[test]
fn displays_subcommand_on_error() {
    cargo_process("invalid-command")
        .with_status(101)
        .with_stderr("[ERROR] no such subcommand: `invalid-command`\n")
        .run();
}

#[test]
fn override_cargo_home() {
    let root = paths::root();
    let my_home = root.join("my_home");
    fs::create_dir(&my_home).unwrap();
    File::create(&my_home.join("config"))
        .unwrap()
        .write_all(
            br#"
        [cargo-new]
        name = "foo"
        email = "bar"
        git = false
    "#,
        ).unwrap();

    cargo_process("new foo")
        .env("USER", "foo")
        .env("CARGO_HOME", &my_home)
        .run();

    let toml = paths::root().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml)
        .unwrap()
        .read_to_string(&mut contents)
        .unwrap();
    assert!(contents.contains(r#"authors = ["foo <bar>"]"#));
}

#[test]
fn cargo_subcommand_env() {
    let src = format!(
        r#"
        use std::env;

        fn main() {{
            println!("{{}}", env::var("{}").unwrap());
        }}
        "#,
        cargo::CARGO_ENV
    );

    let p = project()
        .at("cargo-envtest")
        .file("Cargo.toml", &basic_bin_manifest("cargo-envtest"))
        .file("src/main.rs", &src)
        .build();

    let target_dir = p.target_debug_dir();

    p.cargo("build").run();
    assert!(p.bin("cargo-envtest").is_file());

    let cargo = cargo_exe().canonicalize().unwrap();
    let mut path = path();
    path.push(target_dir);
    let path = env::join_paths(path.iter()).unwrap();

    cargo_process("envtest")
        .env("PATH", &path)
        .with_stdout(cargo.to_str().unwrap())
        .run();
}

#[test]
fn cargo_subcommand_args() {
    let p = project()
        .at("cargo-foo")
        .file("Cargo.toml", &basic_manifest("cargo-foo", "0.0.1"))
        .file(
            "src/main.rs",
            r#"
            fn main() {
                let args: Vec<_> = ::std::env::args().collect();
                println!("{:?}", args);
            }
        "#,
        ).build();

    p.cargo("build").run();
    let cargo_foo_bin = p.bin("cargo-foo");
    assert!(cargo_foo_bin.is_file());

    let mut path = path();
    path.push(p.target_debug_dir());
    let path = env::join_paths(path.iter()).unwrap();

    cargo_process("foo bar -v --help")
        .env("PATH", &path)
        .with_stdout(r#"["[CWD]/cargo-foo/target/debug/cargo-foo", "foo", "bar", "-v", "--help"]"#)
        .run();
}

#[test]
fn cargo_help() {
    cargo_process("").run();
    cargo_process("help").run();
    cargo_process("-h").run();
    cargo_process("help build").run();
    cargo_process("build -h").run();
    cargo_process("help help").run();
}

#[test]
fn cargo_help_external_subcommand() {
    Package::new("cargo-fake-help", "1.0.0")
        .file(
            "src/main.rs",
            r#"
            fn main() {
                if ::std::env::args().nth(2) == Some(String::from("--help")) {
                    println!("fancy help output");
                }
            }"#,
        ).publish();
    cargo_process("install cargo-fake-help").run();
    cargo_process("help fake-help")
        .with_stdout("fancy help output\n")
        .run();
}

#[test]
fn explain() {
    cargo_process("--explain E0001")
        .with_stdout_contains(
            "This error suggests that the expression arm corresponding to the noted pattern",
        ).run();
}

// Test that the output of 'cargo -Z help' shows a different help screen with
// all the -Z flags.
#[test]
fn z_flags_help() {
    cargo_process("-Z help")
        .with_stdout_contains(
            "    -Z unstable-options -- Allow the usage of unstable options such as --registry",
        ).run();
}
<|endoftext|>


<commit_before>// This example implements an OCR (optical character recognition):
// https://en.wikipedia.org/wiki/Optical_character_recognition
// using an evolutionary algorithm.
//
// Note that evolutionary algorithms do no guarantee to always find the optimal solution.
// But they can get very close.

extern crate rand;
extern crate image;
extern crate imageproc;
extern crate rusttype;
#[macro_use] extern crate lazy_static;
extern crate simplelog;

// internal crates
extern crate darwin_rs;

use rand::Rng;
use std::fs::File;
use std::io::Read;
use std::path::Path;
use image::{ImageBuffer, Luma};
use imageproc::stats::root_mean_squared_error;
use simplelog::{SimpleLogger, LogLevelFilter};
use std::sync::Mutex;
use std::str;

// internal modules
use darwin_rs::{Individual, SimulationBuilder, PopulationBuilder, SimError};

lazy_static!{
    static ref FONT: rusttype::Font<'static>  = {
        // TODO: use fontconfig-rs in the future: https://github.com/abonander/fontconfig-rs
        let mut file = File::open("/usr/share/fonts/truetype/liberation/LiberationMono-Regular.ttf").unwrap();
        let mut font_data: Vec<u8> = Vec::new();
        let bytes_read = file.read_to_end(&mut font_data).unwrap();
        println!("bytes read: {}", bytes_read);

        let collection = rusttype::FontCollection::from_bytes(font_data);
        let font = collection.into_font().unwrap();
        font
    };
}

lazy_static! {
    static ref ORIGINAL_IMG: Mutex<ImageBuffer<Luma<u8>, Vec<u8>>> = {
        let original_img: ImageBuffer<Luma<u8>, Vec<u8>> = ImageBuffer::new(640, 70);
        Mutex::new(original_img)
    };
}

#[derive(Debug, Clone)]
struct TextBox {
    x: u32,
    y: u32,
    text: Vec<u8>,
}

#[derive(Debug, Clone)]
struct OCRItem {
    content: Vec<TextBox>,
}

impl Individual for OCRItem {
    fn new() -> OCRItem {
        OCRItem { content: Vec::new() }
    }

    fn mutate(&mut self) {
        let mut rng = rand::thread_rng();

        let operation: u8 = rng.gen_range(0, 6);

        match operation {
            0 => { // Add a new empty text box
                self.content.push( TextBox{ x: 0, y: 0, text: Vec::new() } );
            },
            1 => { // Remove a text box
                if self.content.len() > 1 { // Leave at least one text box.
                    let remove_index = rng.gen_range(0, self.content.len());
                    let _ = self.content.remove(remove_index as usize);
                }
            },
            2 => { // Move the text box around
                if self.content.len() > 0 {
                    let move_index = rng.gen_range(0, self.content.len());
                    let canvas = ORIGINAL_IMG.lock().unwrap();
                    let new_x = rng.gen_range(0, canvas.width());
                    let new_y = rng.gen_range(0, canvas.height());

                    self.content[move_index].x = new_x;
                    self.content[move_index].y = new_y;
                }
            },
            3 => { // Add a random character to the text box
                if self.content.len() > 0 {
                    let content_index = rng.gen_range(0, self.content.len());
                    let add_char_index = if self.content[content_index].text.is_empty() { 0 }
                        else { rng.gen_range(0, self.content[content_index].text.len()) };
                    let random_ascii_value: u8 = rng.gen_range(33, 127);

                    self.content[content_index].text.insert(add_char_index, random_ascii_value);
                }
            },
            4 => { // Remove a character from the text box
                if self.content.len() > 0 {
                    let content_index = rng.gen_range(0, self.content.len());
                    if self.content[content_index].text.len() > 1 { // Leave at least one char
                        let remove_char_index = rng.gen_range(0, self.content[content_index].text.len());
                        self.content[content_index].text.remove(remove_char_index);

                    }
                }
            },
            5 => { // Swap two characters inside a text box
                if self.content.len() > 0 {
                    let content_index = rng.gen_range(0, self.content.len());
                    if self.content[content_index].text.len() > 1 { // Need at least two chars to swap
                        let index1: usize = rng.gen_range(0, self.content[content_index].text.len());
                        let index2: usize = rng.gen_range(0, self.content[content_index].text.len());
                        let tmp_char = self.content[content_index].text[index1];
                        self.content[content_index].text[index1] = self.content[content_index].text[index2];
                        self.content[content_index].text[index2] = tmp_char;
                    }
                }
            },
            _ => println!("unknown operation: {}", operation),
        }
    }

    fn calculate_fitness(&self) -> f64 {
        let mut constructed_img: ImageBuffer<Luma<u8>, Vec<u8>> = ImageBuffer::new(640, 70);

        for text_box in self.content.iter() {
            match str::from_utf8(&text_box.text) {
                Ok(text) => draw_text_line(&mut constructed_img, &FONT, text_box.x as i32,
                    text_box.y as i32, text),
                _ => {} // Do nothing for now
            }
        }

        let canvas = ORIGINAL_IMG.lock().unwrap();

        root_mean_squared_error(&*canvas, &constructed_img)
    }
}

fn draw_text_line(canvas: &mut ImageBuffer<Luma<u8>, Vec<u8>>,
                  font: &rusttype::Font,
                  pos_x: i32,
                  pos_y: i32,
                  text: &str) {

    // println!("text: {}", text);

    let height: f32 = 18.0;

    let scale = rusttype::Scale { x: height * 1.0, y: height };
    let v_metrics = font.v_metrics(scale);
    let offset = rusttype::point(0.0, v_metrics.ascent);
    let glyphs: Vec<rusttype::PositionedGlyph> = font.layout(text, scale, offset).collect();

    for g in glyphs {
        if let Some(bb) = g.pixel_bounding_box() {
            g.draw(|x, y, v| {
                let x = ((x as i32) + bb.min.x + pos_x) as u32;
                let y = ((y as i32) + bb.min.y + pos_y) as u32;
                if x < canvas.width() && y < canvas.height() {
                    canvas.put_pixel(x, y, Luma::<u8>{ data: [(v * 255.0) as u8] } );
                }
            })
        }
    }

}

fn main() {
    println!("Darwin test: optical character recognition");

    let _ = SimpleLogger::init(LogLevelFilter::Info);

    {
        let mut canvas = ORIGINAL_IMG.lock().unwrap();

        draw_text_line(&mut canvas, &FONT, 10, 10, "This is a test text!");
        draw_text_line(&mut canvas, &FONT, 10, 40, "Just to see how good OCR works...");

        let img_file = Path::new("rendered_text.png");
        let _ = canvas.save(&img_file);
    }

    let population1 = PopulationBuilder::<OCRItem>::new()
        .set_id(1)
        .individuals(30)
        .increasing_exp_mutation_rate(1.1)
        .reset_limit_increment(100)
        .reset_limit_start(100)
        .reset_limit_end(1000)
        .finalize().unwrap();

    let population2 = PopulationBuilder::<OCRItem>::new()
        .set_id(2)
        .individuals(30)
        .increasing_exp_mutation_rate(1.15)
        .reset_limit_increment(200)
        .reset_limit_start(100)
        .reset_limit_end(2000)
        .finalize().unwrap();

    let population3 = PopulationBuilder::<OCRItem>::new()
        .set_id(3)
        .individuals(30)
        .increasing_exp_mutation_rate(1.2)
        .reset_limit_increment(300)
        .reset_limit_start(100)
        .reset_limit_end(3000)
        .finalize().unwrap();

    let ocr_builder = SimulationBuilder::<OCRItem>::new()
        .fitness(10.0)
        .threads(2)
        .add_population(population1)
        .add_population(population2)
        .add_population(population3)
        .finalize();

    match ocr_builder {
        Err(SimError::EndIterationTooLow) => println!("more than 10 iteratons needed"),
        Ok(mut ocr_simulation) => {
            ocr_simulation.run();

            println!("total run time: {} ms", ocr_simulation.total_time_in_ms);
            println!("improvement factor: {}", ocr_simulation.simulation_result.improvement_factor);
            println!("number of iterations: {}", ocr_simulation.simulation_result.iteration_counter);

            ocr_simulation.print_fitness();

            for content in ocr_simulation.simulation_result.fittest[0].individual.content.iter() {
                println!("x: {}, y: {}, text: {}", content.x, content.y, str::from_utf8(&content.text).unwrap())
            }
        }
    }
}
<commit_msg>add weights to operation, remove one mutex<commit_after>// This example implements an OCR (optical character recognition):
// https://en.wikipedia.org/wiki/Optical_character_recognition
// using an evolutionary algorithm.
//
// Note that evolutionary algorithms do no guarantee to always find the optimal solution.
// But they can get very close.

extern crate rand;
extern crate image;
extern crate imageproc;
extern crate rusttype;
#[macro_use] extern crate lazy_static;
extern crate simplelog;

// internal crates
extern crate darwin_rs;

use rand::Rng;
use rand::distributions::{Weighted, WeightedChoice, IndependentSample};
use std::fs::File;
use std::io::Read;
use std::path::Path;
use image::{ImageBuffer, Luma};
use imageproc::stats::root_mean_squared_error;
use simplelog::{SimpleLogger, LogLevelFilter};
use std::str;

// internal modules
use darwin_rs::{Individual, SimulationBuilder, PopulationBuilder, SimError};

lazy_static!{
    static ref FONT: rusttype::Font<'static>  = {
        // TODO: use fontconfig-rs in the future: https://github.com/abonander/fontconfig-rs
        let mut file = File::open("/usr/share/fonts/truetype/liberation/LiberationMono-Regular.ttf").unwrap();
        let mut font_data: Vec<u8> = Vec::new();
        let bytes_read = file.read_to_end(&mut font_data).unwrap();
        println!("bytes read: {}", bytes_read);

        let collection = rusttype::FontCollection::from_bytes(font_data);
        collection.into_font().unwrap()
    };
}

lazy_static! {
    static ref ORIGINAL_IMG: ImageBuffer<Luma<u8>, Vec<u8>> = {
        let mut original_img: ImageBuffer<Luma<u8>, Vec<u8>> = ImageBuffer::new(640, 70);
        draw_text_line(&mut original_img, &FONT, 10, 10, "This is a test text!");
        draw_text_line(&mut original_img, &FONT, 10, 40, "Just to see how good OCR works...");
        original_img
    };
}

#[derive(Debug, Clone)]
enum MOperation {
    AddEmptyTextBox,
    RemoveTextBox,
    MoveTextAround,
    AddRandomCharacter,
    RemoveCharacter,
    SwapTwoCharacters
}

#[derive(Debug, Clone)]
struct TextBox {
    x: u32,
    y: u32,
    text: Vec<u8>,
}

#[derive(Debug, Clone)]
struct OCRItem {
    content: Vec<TextBox>,
}

impl Individual for OCRItem {
    fn new() -> OCRItem {
        OCRItem { content: Vec::new() }
    }

    fn mutate(&mut self) {
        let mut rng = rand::thread_rng();

        let mut items = vec!(
            Weighted { weight: 1, item: MOperation::AddEmptyTextBox },
            Weighted { weight: 1, item: MOperation::RemoveTextBox },
            Weighted { weight: 2, item: MOperation::MoveTextAround },
            Weighted { weight: 10, item: MOperation::AddRandomCharacter },
            Weighted { weight: 10, item: MOperation::RemoveCharacter },
            Weighted { weight: 10, item: MOperation::SwapTwoCharacters }
        );
        let weighted_choice = WeightedChoice::new(&mut items);

        match weighted_choice.ind_sample(&mut rng) {
            MOperation::AddEmptyTextBox => { // Add a new empty text box
                self.content.push( TextBox{ x: 0, y: 0, text: Vec::new() } );
            },
            MOperation::RemoveTextBox => { // Remove a text box
                if self.content.len() > 1 { // Leave at least one text box.
                    let remove_index = rng.gen_range(0, self.content.len());
                    let _ = self.content.remove(remove_index as usize);
                }
            },
            MOperation::MoveTextAround => { // Move the text box around
                if self.content.len() > 0 {
                    let move_index = rng.gen_range(0, self.content.len());
                    let new_x = rng.gen_range(0, ORIGINAL_IMG.width());
                    let new_y = rng.gen_range(0, ORIGINAL_IMG.height());

                    self.content[move_index].x = new_x;
                    self.content[move_index].y = new_y;
                }
            },
            MOperation::AddRandomCharacter => { // Add a random character to the text box
                if self.content.len() > 0 {
                    let content_index = rng.gen_range(0, self.content.len());
                    let add_char_index = if self.content[content_index].text.is_empty() { 0 }
                        else { rng.gen_range(0, self.content[content_index].text.len()) };
                    let random_ascii_value: u8 = rng.gen_range(33, 127);

                    self.content[content_index].text.insert(add_char_index, random_ascii_value);
                }
            },
            MOperation::RemoveCharacter => { // Remove a character from the text box
                if self.content.len() > 0 {
                    let content_index = rng.gen_range(0, self.content.len());
                    if self.content[content_index].text.len() > 1 { // Leave at least one char
                        let remove_char_index = rng.gen_range(0, self.content[content_index].text.len());
                        self.content[content_index].text.remove(remove_char_index);

                    }
                }
            },
            MOperation::SwapTwoCharacters => { // Swap two characters inside a text box
                if self.content.len() > 0 {
                    let content_index = rng.gen_range(0, self.content.len());
                    if self.content[content_index].text.len() > 1 { // Need at least two chars to swap
                        let index1: usize = rng.gen_range(0, self.content[content_index].text.len());
                        let index2: usize = rng.gen_range(0, self.content[content_index].text.len());
                        let tmp_char = self.content[content_index].text[index1];
                        self.content[content_index].text[index1] = self.content[content_index].text[index2];
                        self.content[content_index].text[index2] = tmp_char;
                    }
                }
            },
        }
    }

    fn calculate_fitness(&self) -> f64 {
        let mut constructed_img: ImageBuffer<Luma<u8>, Vec<u8>> = ImageBuffer::new(640, 70);

        for text_box in self.content.iter() {
            match str::from_utf8(&text_box.text) {
                Ok(text) => draw_text_line(&mut constructed_img, &FONT, text_box.x as i32,
                    text_box.y as i32, text),
                _ => {} // Do nothing for now
            }
        }

        root_mean_squared_error(&*ORIGINAL_IMG, &constructed_img)
    }
}

fn draw_text_line(canvas: &mut ImageBuffer<Luma<u8>, Vec<u8>>,
                  font: &rusttype::Font,
                  pos_x: i32,
                  pos_y: i32,
                  text: &str) {

    // println!("text: {}", text);

    let height: f32 = 18.0;

    let scale = rusttype::Scale { x: height * 1.0, y: height };
    let v_metrics = font.v_metrics(scale);
    let offset = rusttype::point(0.0, v_metrics.ascent);
    let glyphs: Vec<rusttype::PositionedGlyph> = font.layout(text, scale, offset).collect();

    for g in glyphs {
        if let Some(bb) = g.pixel_bounding_box() {
            g.draw(|x, y, v| {
                let x = ((x as i32) + bb.min.x + pos_x) as u32;
                let y = ((y as i32) + bb.min.y + pos_y) as u32;
                if x < canvas.width() && y < canvas.height() {
                    canvas.put_pixel(x, y, Luma::<u8>{ data: [(v * 255.0) as u8] } );
                }
            })
        }
    }

}

fn main() {
    println!("Darwin test: optical character recognition");

    let _ = SimpleLogger::init(LogLevelFilter::Info);

    let img_file = Path::new("rendered_text.png");
    let _ = ORIGINAL_IMG.save(&img_file);

    let population1 = PopulationBuilder::<OCRItem>::new()
        .set_id(1)
        .individuals(40)
        .increasing_exp_mutation_rate(1.10)
        .reset_limit_increment(100)
        .reset_limit_start(100)
        .reset_limit_end(1000)
        .finalize().unwrap();

    let population2 = PopulationBuilder::<OCRItem>::new()
        .set_id(2)
        .individuals(40)
        .increasing_exp_mutation_rate(1.13)
        .reset_limit_increment(200)
        .reset_limit_start(100)
        .reset_limit_end(2000)
        .finalize().unwrap();

    let population3 = PopulationBuilder::<OCRItem>::new()
        .set_id(3)
        .individuals(40)
        .increasing_exp_mutation_rate(1.16)
        .reset_limit_increment(300)
        .reset_limit_start(100)
        .reset_limit_end(3000)
        .finalize().unwrap();

    let population4 = PopulationBuilder::<OCRItem>::new()
        .set_id(4)
        .individuals(40)
        .increasing_exp_mutation_rate(1.19)
        .reset_limit_increment(300)
        .reset_limit_start(100)
        .reset_limit_end(3000)
        .finalize().unwrap();

    let ocr_builder = SimulationBuilder::<OCRItem>::new()
        .fitness(10.0)
        .threads(3)
        .add_population(population1)
        .add_population(population2)
        .add_population(population3)
        .add_population(population4)
        .finalize();

    match ocr_builder {
        Err(SimError::EndIterationTooLow) => println!("more than 10 iteratons needed"),
        Ok(mut ocr_simulation) => {
            ocr_simulation.run();

            println!("total run time: {} ms", ocr_simulation.total_time_in_ms);
            println!("improvement factor: {}", ocr_simulation.simulation_result.improvement_factor);
            println!("number of iterations: {}", ocr_simulation.simulation_result.iteration_counter);

            ocr_simulation.print_fitness();

            for content in ocr_simulation.simulation_result.fittest[0].individual.content.iter() {
                println!("x: {}, y: {}, text: {}", content.x, content.y, str::from_utf8(&content.text).unwrap())
            }
        }
    }
}
<|endoftext|>


<commit_before>/* diosix hypervisor physical memory management
 *
 * allocate/free contiguous regions of physical memory.
 * these regions are categorized into two groups,
 * depending on the region size.
 *
 * large: >= PHYS_RAM_LARGE_REGION_MIN_SIZE
 * large regions are sized in multiples of
 * PHYS_RAM_LARGE_REGION_MIN_SIZE and are allocated
 * from the top of free region blocks, descending.
 * these are aimed at large blocks of contiguous
 * memory for guest supervisor OSes.
 *
 * small: < PHYS_RAM_LARGE_REGION_MIN_SIZE
 * small regions are sized in multiples of
 * PHYS_RAM_SMALL_REGION_MIN_SIZE and are allocated
 * from the bottom of free region blocks, ascending.
 * these are aimed at small blocks of memory
 * for the hypervisor's private per-CPU heaps.
 *
 * this arrangement is to avoid large and small
 * allocations fragmenting free region blocks
 *
 * (c) Chris Williams, 2019-2021.
 *
 * See LICENSE for usage and copying.
 */

use platform;
use super::lock::Mutex;
use alloc::vec::Vec;
use platform::physmem::{PhysMemBase, PhysMemEnd, PhysMemSize, AccessPermissions, validate_ram};
use super::error::Cause;
use super::hardware;

/* needed to convert a region into a slice */
use core::slice;

/* to avoid fragmentation, round up physical memory region allocations into multiples of these totals,
depending on the region type. this only applies when creating regions with alloc_region() */
const PHYS_RAM_LARGE_REGION_MIN_SIZE: PhysMemSize = 64 * 1024 * 1024; /* 64MB ought to be enough for anyone */
const PHYS_RAM_SMALL_REGION_MIN_SIZE: PhysMemSize =  1 * 1024 * 1024; /* smaller blocks are multiples of 1MB in size */

/* ensure large region bases are aligned down to multiples of this value
   note: region minimum size must be a non-zero multiple of region base alignment */
const PHYS_RAM_LARGE_REGION_ALIGNMENT: PhysMemSize = 4 * 1024 * 1024; /* 4MB alignment */

/* define whether to split a region N bytes from the top or from the bottom */
#[derive(Clone, Copy, Debug)]
pub enum RegionSplit
{
    FromBottom,
    FromTop
}

/* define whether a region is dirty or clean */
#[derive(Clone, Copy, Debug)]
pub enum RegionHygiene
{
    DontClean, /* don't zero this region */
    CanClean
}

/* describe a physical memory region */
#[derive(Copy, Clone)]
pub struct Region
{
    base: PhysMemBase,
    size: PhysMemSize,
    hygiene: RegionHygiene
}

impl Region
{
    /* create a new region */
    pub fn new(base: PhysMemBase, size: PhysMemSize, hygiene: RegionHygiene) -> Region
    {
        Region
        {
            base,
            size,
            hygiene
        }
    }

    /* scrub a whole region. FIXME: make this fast and efficient!
    Note: this only zeroes the region in release mode to avoid delays
    in debugging/development with slow region zeroing */
    pub fn clean(&mut self)
    {
        match self.hygiene
        {
            RegionHygiene::DontClean =>
            {
                hvalert!("BUG: Tried to scrub don't-clean region 0x{:x}", self.base);
                return;
            },
            RegionHygiene::CanClean =>
            {
                #[cfg(not(debug_assertions))]
                self.as_u8_slice().fill(0x0);
            }
        }
    }

    /* fill the end of a region with an array of bytes. thus if the array is 10 bytes long,
    the final 10 bytes of the region will be filled from that array, ascending
    => bytes = array to write into the region
    <= physical address for start of array in the region, or error code */
    pub fn fill_end(&self, bytes: Vec<u8>) -> Result<PhysMemBase, Cause>
    {
        let array_size = bytes.len();

        /* ensure we've got enough space to accomdate the array */
        if self.size < array_size
        {
            return Err(Cause::PhysRegionTooSmall)
        }

        for index in 0..array_size
        {
            unsafe
            {
                *(((self.base + self.size) - (array_size - index)) as *mut u8) = bytes[index];
            }
        }

        Ok((self.base + self.size) - array_size)
    }

    /* allow the currently running supervisor kernel to access this region of physical memory */
    pub fn grant_access(&self)
    {
        platform::physmem::protect(self.base, self.base + self.size, AccessPermissions::ReadWriteExecute);
    }

    /* return or change attributes */
    pub fn base(&self) -> PhysMemBase { self.base }
    pub fn end(&self) -> PhysMemEnd { self.base + self.size }
    pub fn size(&self) -> PhysMemSize { self.size }

    /* represent the region as a word-size or byte-size slice
    **use carefully** don't hold a slice over an IRQ, for example */
    pub fn as_usize_slice(&self) -> &mut [usize]
    {
        unsafe { slice::from_raw_parts_mut(self.base as *mut usize, self.size) }
    }
    pub fn as_u8_slice(&self) -> &mut [u8]
    {
        unsafe { slice::from_raw_parts_mut(self.base as *mut u8, self.size) }
    }

    /* split the region into two portions, lower and upper, and return the two portions.
    maintain the region's hygiene.
    => count = split the region this number of bytes into the block
       measure_from = FromBottom: count is number of bytes from bottom of the block, ascending
                      FromTop: count is number of bytes from the top of the block, descending
    <= return two portions as regions, lower and upper, or a failure code */
    pub fn split(&self, count: PhysMemSize, measure_from: RegionSplit) -> Result<(Region, Region), Cause>
    {
        /* check the split mark is within bounds */
        if count > self.size
        {
            return Err(Cause::PhysRegionSplitOutOfBounds);
        }

        /* return (lower, upper) */
        Ok(match measure_from
        {
            RegionSplit::FromBottom =>
            (
                Region::new(self.base, count, self.hygiene),
                Region::new(self.base + count, self.size - count, self.hygiene)
            ),

            RegionSplit::FromTop =>
            (
                Region::new(self.base, self.size - count, self.hygiene),
                Region::new(self.base + self.size - count, count, self.hygiene)
            ),
        })
    }
}

/* gather up all physical RAM areas from which future capsule and heap physical
RAM allocations will be drawn into the REGIONS list. this list is built from
available, free physical RAM: it must *not* include any RAM areas already in use by
the hypervisor, boot supervisor image, peripherals, etc. the underlying
platform code needs to exclude those off-limits areas.

this list must also be sorted, by base address, lowest first. this is so that
adjoining regions can be merged into one. this list also contains only free
and available regions. if a region is in use, it must be removed from the list. */
lazy_static!
{
    /* acquire REGIONS lock before accessing any physical RAM regions */
    static ref REGIONS: Mutex<SortedRegions> = Mutex::new(SortedRegions::new());
}

/* implement a sorted list of regions */
struct SortedRegions
{
    regions: Vec<Region>
}

impl SortedRegions
{
    /* create an empty list */
    pub fn new() -> SortedRegions
    {
        SortedRegions
        {
            regions: Vec::new()
        }
    }

    /* find a region that has a size equal to or greater than the required size.
       if one is found, remove the region and return it. if one can't be found,
       return an error code. */
    pub fn find(&mut self, required_size: PhysMemSize) -> Result<Region, Cause>
    {
        for index in 0..self.regions.len()
        {
            if self.regions[index].size() >= required_size
            {
                /* remove from the list and return */
                return Ok(self.regions.remove(index));
            }
        }

        Err(Cause::PhysRegionNoMatch) /* can't find a region large enough */
    }

    /* insert a region into the list, sorted by base addresses, lowest first */
    pub fn insert(&mut self, to_insert: Region) -> Result<(), Cause>
    {
        /* ignore zero-size inserts */
        if to_insert.size() == 0
        {
            return Ok(())
        }

        for index in 0..self.regions.len()
        {
            if to_insert.end() <= self.regions[index].base()
            {
                self.regions.insert(index, to_insert);
                return Ok(())
            }

            /* check to make sure we're not adding a region that will collide with another */
            if to_insert.base() >= self.regions[index].base() && to_insert.base() < self.regions[index].end()
            {
                return Err(Cause::PhysRegionCollision);
            }
        }

        /* insert at the end: region greater than all others */
        self.regions.push(to_insert);
        Ok(())
    }

    /* merge all adjoining free regions. this requires the list to be sorted by base address ascending */
    pub fn merge(&mut self)
    {
        let mut cursor = 0;
        loop
        {
            /* prevent search from going out of bounds */
            if (cursor + 1) >= self.regions.len()
            {
                break;
            }

            if self.regions[cursor].end() == self.regions[cursor + 1].base()
            {
                /* absorb the next region's size into this region */
                self.regions[cursor].size = self.regions[cursor].size() + self.regions.remove(cursor + 1).size();
            }
            else
            {
                /* move onto next region */
                cursor = cursor + 1;
            }
        }
    }
}

/* initialize the physical memory system by registering all physical RAM available for use as allocatable regions */
pub fn init() -> Result<(), Cause>
{
    /* we need to know the CPU count so that any memory preallocated or reserved for the cores can be skipped */
    let nr_cpu_cores = match hardware::get_nr_cpu_cores()
    {
        Some(c) => c,
        None => return Err(Cause::PhysicalCoreCountUnknown)
    };

    /* the device tree defines chunks of memory that may or may not be entirely available for use */
    let chunks = match hardware::get_phys_ram_chunks()
    {
        Some(c) => c,
        None => return Err(Cause::PhysNoRAMFound)
    };

    /* iterate over the physical memory chunks... */
    let mut regions = REGIONS.lock();
    for chunk in chunks
    {
        /* ...and let validate_ram break each chunk in sections we can safely use.
        assume the RAM is clean: the firmware or boot code should have wiped it,
        or it should contain random values */
        for section in validate_ram(nr_cpu_cores, chunk)
        {
            regions.insert(Region::new(section.base, section.size, RegionHygiene::CanClean))?;
        }
    }

    Ok(())
}

/* perform housekeeping duties on idle physical CPU cores */
macro_rules! physmemhousekeeper
{
    () => ($crate::physmem::coalesce_regions());
}

pub fn coalesce_regions()
{
    REGIONS.lock().merge();
}

/* allocate a region of available physical memory for guest capsule or hypervisor heap use.
   capsules should use large regions, and the heap should use small, ideally.
   => size = number of bytes for the region, which will be rounded up to next multiple of:
     PHYS_RAM_LARGE_REGION_MIN_SIZE if the size >= PHYS_RAM_LARGE_REGION_MIN_SIZE (large type)
     PHYS_RAM_SMALL_REGION_MIN_SIZE if the size < PHYS_RAM_LARGE_REGION_MIN_SIZE (small type)

     note, large type regions will have a base address aligned down to PHYS_RAM_LARGE_REGION_ALIGNMENT
     this is so that guests that require 2MB or 4MB kernel alignment (eg RV64GC Linux) work as expected
     see: https://patchwork.kernel.org/patch/10868465/
     this code assumes the top of physically available RAM is aligned to PHYS_RAM_LARGE_REGION_ALIGNMENT

   <= Region structure for the space, or an error code */
pub fn alloc_region(size: PhysMemSize) -> Result<Region, Cause>
{
    /* determine where to split the free region block, and the region type */
    let (split_from, region_multiple) = if size >= PHYS_RAM_LARGE_REGION_MIN_SIZE
    {
        (RegionSplit::FromTop, PHYS_RAM_LARGE_REGION_MIN_SIZE)
    }
    else
    {
        (RegionSplit::FromBottom, PHYS_RAM_SMALL_REGION_MIN_SIZE)
    };

    /* round up to a multiple of the minimum size of a region type to avoid fragmentation */
    let adjusted_size = match size % region_multiple
    {
        0 => size,
        d => (size - d) + region_multiple
    };

    let mut regions = REGIONS.lock();
    match regions.find(adjusted_size) // find will remove found region from free list if successful
    {
        Ok(found) =>
        {
            /* split the found region into two parts: one portion for the newly
            allocated region, and the remaining portion is returned to the free list.
            adjusted_size defines whwre in the region the split point occurs.
            split_from defines whether adjusted_size is measured from the top or
            bottom of the region block */
            match (found.split(adjusted_size, split_from), split_from)
            {
                /* split so that the lower portion is allocated, and the upper portion is returned to the free list */
                (Ok((mut lower, upper)), RegionSplit::FromBottom) =>
                {
                    regions.insert(upper)?;
                    lower.clean();
                    Ok(lower)
                },

                /* split so that the upper portion is allocated, and the lower portion is returned to the free list */
                (Ok((lower, upper)), RegionSplit::FromTop) =>
                {
                    /* bring the base of the upper portion down to alignment mark */
                    let mut aligned_upper = match upper.base % PHYS_RAM_LARGE_REGION_ALIGNMENT
                    {
                        0 => Region::new(upper.base, upper.size, found.hygiene),
                        d => Region::new(upper.base - d, upper.size + d, found.hygiene)
                    };

                    /* fail out if upper portion crashes through the lower portion base after alignment */
                    if lower.size < aligned_upper.size - upper.size
                    {
                        return Err(Cause::PhysRegionRegionAlignmentFailure)
                    }

                    /* adjust the size of the lower portion if the upper portion was aligned down */
                    let adjusted_lower = match aligned_upper.size - upper.size
                    {
                        0 => lower,
                        d => Region::new(lower.base, lower.size - d, found.hygiene)
                    };

                    regions.insert(adjusted_lower)?;
                    aligned_upper.clean();
                    Ok(aligned_upper)
                },

                (Err(e), _) => Err(e)
            }
        },
        Err(_) => Err(Cause::PhysNotEnoughFreeRAM)
    }
}

/* deallocate a region so that its physical RAM can be reallocated.
   only accept samll regions that are multiples of PHYS_RAM_SMALL_REGION_MIN_SIZE
   and large regions that are multiples of PHYS_RAM_LARGE_REGION_MIN_SIZE
   => to_free = region to deallocate
   <= Ok for success, or an error code for failure */
pub fn dealloc_region(to_free: Region) -> Result<(), Cause>
{
    let size = to_free.size();

    /* police the size of the region */
    if size < PHYS_RAM_LARGE_REGION_MIN_SIZE
    {
        if size % PHYS_RAM_SMALL_REGION_MIN_SIZE != 0
        {
            return Err(Cause::PhysRegionSmallNotMultiple);
        }
    }
    else
    {
        if size % PHYS_RAM_LARGE_REGION_MIN_SIZE != 0
        {
            return Err(Cause::PhysRegionLargeNotMultiple);
        }
    }

    REGIONS.lock().insert(to_free)
}
<commit_msg>Use named mutex<commit_after>/* diosix hypervisor physical memory management
 *
 * allocate/free contiguous regions of physical memory.
 * these regions are categorized into two groups,
 * depending on the region size.
 *
 * large: >= PHYS_RAM_LARGE_REGION_MIN_SIZE
 * large regions are sized in multiples of
 * PHYS_RAM_LARGE_REGION_MIN_SIZE and are allocated
 * from the top of free region blocks, descending.
 * these are aimed at large blocks of contiguous
 * memory for guest supervisor OSes.
 *
 * small: < PHYS_RAM_LARGE_REGION_MIN_SIZE
 * small regions are sized in multiples of
 * PHYS_RAM_SMALL_REGION_MIN_SIZE and are allocated
 * from the bottom of free region blocks, ascending.
 * these are aimed at small blocks of memory
 * for the hypervisor's private per-CPU heaps.
 *
 * this arrangement is to avoid large and small
 * allocations fragmenting free region blocks
 *
 * (c) Chris Williams, 2019-2021.
 *
 * See LICENSE for usage and copying.
 */

use platform;
use super::lock::Mutex;
use alloc::vec::Vec;
use platform::physmem::{PhysMemBase, PhysMemEnd, PhysMemSize, AccessPermissions, validate_ram};
use super::error::Cause;
use super::hardware;

/* needed to convert a region into a slice */
use core::slice;

/* to avoid fragmentation, round up physical memory region allocations into multiples of these totals,
depending on the region type. this only applies when creating regions with alloc_region() */
const PHYS_RAM_LARGE_REGION_MIN_SIZE: PhysMemSize = 64 * 1024 * 1024; /* 64MB ought to be enough for anyone */
const PHYS_RAM_SMALL_REGION_MIN_SIZE: PhysMemSize =  1 * 1024 * 1024; /* smaller blocks are multiples of 1MB in size */

/* ensure large region bases are aligned down to multiples of this value
   note: region minimum size must be a non-zero multiple of region base alignment */
const PHYS_RAM_LARGE_REGION_ALIGNMENT: PhysMemSize = 4 * 1024 * 1024; /* 4MB alignment */

/* define whether to split a region N bytes from the top or from the bottom */
#[derive(Clone, Copy, Debug)]
pub enum RegionSplit
{
    FromBottom,
    FromTop
}

/* define whether a region is dirty or clean */
#[derive(Clone, Copy, Debug)]
pub enum RegionHygiene
{
    DontClean, /* don't zero this region */
    CanClean
}

/* describe a physical memory region */
#[derive(Copy, Clone)]
pub struct Region
{
    base: PhysMemBase,
    size: PhysMemSize,
    hygiene: RegionHygiene
}

impl Region
{
    /* create a new region */
    pub fn new(base: PhysMemBase, size: PhysMemSize, hygiene: RegionHygiene) -> Region
    {
        Region
        {
            base,
            size,
            hygiene
        }
    }

    /* scrub a whole region. FIXME: make this fast and efficient!
    Note: this only zeroes the region in release mode to avoid delays
    in debugging/development with slow region zeroing */
    pub fn clean(&mut self)
    {
        match self.hygiene
        {
            RegionHygiene::DontClean =>
            {
                hvalert!("BUG: Tried to scrub don't-clean region 0x{:x}", self.base);
                return;
            },
            RegionHygiene::CanClean =>
            {
                #[cfg(not(debug_assertions))]
                self.as_u8_slice().fill(0x0);
            }
        }
    }

    /* fill the end of a region with an array of bytes. thus if the array is 10 bytes long,
    the final 10 bytes of the region will be filled from that array, ascending
    => bytes = array to write into the region
    <= physical address for start of array in the region, or error code */
    pub fn fill_end(&self, bytes: Vec<u8>) -> Result<PhysMemBase, Cause>
    {
        let array_size = bytes.len();

        /* ensure we've got enough space to accomdate the array */
        if self.size < array_size
        {
            return Err(Cause::PhysRegionTooSmall)
        }

        for index in 0..array_size
        {
            unsafe
            {
                *(((self.base + self.size) - (array_size - index)) as *mut u8) = bytes[index];
            }
        }

        Ok((self.base + self.size) - array_size)
    }

    /* allow the currently running supervisor kernel to access this region of physical memory */
    pub fn grant_access(&self)
    {
        platform::physmem::protect(self.base, self.base + self.size, AccessPermissions::ReadWriteExecute);
    }

    /* return or change attributes */
    pub fn base(&self) -> PhysMemBase { self.base }
    pub fn end(&self) -> PhysMemEnd { self.base + self.size }
    pub fn size(&self) -> PhysMemSize { self.size }

    /* represent the region as a word-size or byte-size slice
    **use carefully** don't hold a slice over an IRQ, for example */
    pub fn as_usize_slice(&self) -> &mut [usize]
    {
        unsafe { slice::from_raw_parts_mut(self.base as *mut usize, self.size) }
    }
    pub fn as_u8_slice(&self) -> &mut [u8]
    {
        unsafe { slice::from_raw_parts_mut(self.base as *mut u8, self.size) }
    }

    /* split the region into two portions, lower and upper, and return the two portions.
    maintain the region's hygiene.
    => count = split the region this number of bytes into the block
       measure_from = FromBottom: count is number of bytes from bottom of the block, ascending
                      FromTop: count is number of bytes from the top of the block, descending
    <= return two portions as regions, lower and upper, or a failure code */
    pub fn split(&self, count: PhysMemSize, measure_from: RegionSplit) -> Result<(Region, Region), Cause>
    {
        /* check the split mark is within bounds */
        if count > self.size
        {
            return Err(Cause::PhysRegionSplitOutOfBounds);
        }

        /* return (lower, upper) */
        Ok(match measure_from
        {
            RegionSplit::FromBottom =>
            (
                Region::new(self.base, count, self.hygiene),
                Region::new(self.base + count, self.size - count, self.hygiene)
            ),

            RegionSplit::FromTop =>
            (
                Region::new(self.base, self.size - count, self.hygiene),
                Region::new(self.base + self.size - count, count, self.hygiene)
            ),
        })
    }
}

/* gather up all physical RAM areas from which future capsule and heap physical
RAM allocations will be drawn into the REGIONS list. this list is built from
available, free physical RAM: it must *not* include any RAM areas already in use by
the hypervisor, boot supervisor image, peripherals, etc. the underlying
platform code needs to exclude those off-limits areas.

this list must also be sorted, by base address, lowest first. this is so that
adjoining regions can be merged into one. this list also contains only free
and available regions. if a region is in use, it must be removed from the list. */
lazy_static!
{
    /* acquire REGIONS lock before accessing any physical RAM regions */
    static ref REGIONS: Mutex<SortedRegions> = Mutex::new("RAM regions", SortedRegions::new());
}

/* implement a sorted list of regions */
struct SortedRegions
{
    regions: Vec<Region>
}

impl SortedRegions
{
    /* create an empty list */
    pub fn new() -> SortedRegions
    {
        SortedRegions
        {
            regions: Vec::new()
        }
    }

    /* find a region that has a size equal to or greater than the required size.
       if one is found, remove the region and return it. if one can't be found,
       return an error code. */
    pub fn find(&mut self, required_size: PhysMemSize) -> Result<Region, Cause>
    {
        for index in 0..self.regions.len()
        {
            if self.regions[index].size() >= required_size
            {
                /* remove from the list and return */
                return Ok(self.regions.remove(index));
            }
        }

        Err(Cause::PhysRegionNoMatch) /* can't find a region large enough */
    }

    /* insert a region into the list, sorted by base addresses, lowest first */
    pub fn insert(&mut self, to_insert: Region) -> Result<(), Cause>
    {
        /* ignore zero-size inserts */
        if to_insert.size() == 0
        {
            return Ok(())
        }

        for index in 0..self.regions.len()
        {
            if to_insert.end() <= self.regions[index].base()
            {
                self.regions.insert(index, to_insert);
                return Ok(())
            }

            /* check to make sure we're not adding a region that will collide with another */
            if to_insert.base() >= self.regions[index].base() && to_insert.base() < self.regions[index].end()
            {
                return Err(Cause::PhysRegionCollision);
            }
        }

        /* insert at the end: region greater than all others */
        self.regions.push(to_insert);
        Ok(())
    }

    /* merge all adjoining free regions. this requires the list to be sorted by base address ascending */
    pub fn merge(&mut self)
    {
        let mut cursor = 0;
        loop
        {
            /* prevent search from going out of bounds */
            if (cursor + 1) >= self.regions.len()
            {
                break;
            }

            if self.regions[cursor].end() == self.regions[cursor + 1].base()
            {
                /* absorb the next region's size into this region */
                self.regions[cursor].size = self.regions[cursor].size() + self.regions.remove(cursor + 1).size();
            }
            else
            {
                /* move onto next region */
                cursor = cursor + 1;
            }
        }
    }
}

/* initialize the physical memory system by registering all physical RAM available for use as allocatable regions */
pub fn init() -> Result<(), Cause>
{
    /* we need to know the CPU count so that any memory preallocated or reserved for the cores can be skipped */
    let nr_cpu_cores = match hardware::get_nr_cpu_cores()
    {
        Some(c) => c,
        None => return Err(Cause::PhysicalCoreCountUnknown)
    };

    /* the device tree defines chunks of memory that may or may not be entirely available for use */
    let chunks = match hardware::get_phys_ram_chunks()
    {
        Some(c) => c,
        None => return Err(Cause::PhysNoRAMFound)
    };

    /* iterate over the physical memory chunks... */
    let mut regions = REGIONS.lock();
    for chunk in chunks
    {
        /* ...and let validate_ram break each chunk in sections we can safely use.
        assume the RAM is clean: the firmware or boot code should have wiped it,
        or it should contain random values */
        for section in validate_ram(nr_cpu_cores, chunk)
        {
            regions.insert(Region::new(section.base, section.size, RegionHygiene::CanClean))?;
        }
    }

    Ok(())
}

/* perform housekeeping duties on idle physical CPU cores */
macro_rules! physmemhousekeeper
{
    () => ($crate::physmem::coalesce_regions());
}

pub fn coalesce_regions()
{
    REGIONS.lock().merge();
}

/* allocate a region of available physical memory for guest capsule or hypervisor heap use.
   capsules should use large regions, and the heap should use small, ideally.
   => size = number of bytes for the region, which will be rounded up to next multiple of:
     PHYS_RAM_LARGE_REGION_MIN_SIZE if the size >= PHYS_RAM_LARGE_REGION_MIN_SIZE (large type)
     PHYS_RAM_SMALL_REGION_MIN_SIZE if the size < PHYS_RAM_LARGE_REGION_MIN_SIZE (small type)

     note, large type regions will have a base address aligned down to PHYS_RAM_LARGE_REGION_ALIGNMENT
     this is so that guests that require 2MB or 4MB kernel alignment (eg RV64GC Linux) work as expected
     see: https://patchwork.kernel.org/patch/10868465/
     this code assumes the top of physically available RAM is aligned to PHYS_RAM_LARGE_REGION_ALIGNMENT

   <= Region structure for the space, or an error code */
pub fn alloc_region(size: PhysMemSize) -> Result<Region, Cause>
{
    /* determine where to split the free region block, and the region type */
    let (split_from, region_multiple) = if size >= PHYS_RAM_LARGE_REGION_MIN_SIZE
    {
        (RegionSplit::FromTop, PHYS_RAM_LARGE_REGION_MIN_SIZE)
    }
    else
    {
        (RegionSplit::FromBottom, PHYS_RAM_SMALL_REGION_MIN_SIZE)
    };

    /* round up to a multiple of the minimum size of a region type to avoid fragmentation */
    let adjusted_size = match size % region_multiple
    {
        0 => size,
        d => (size - d) + region_multiple
    };

    let mut regions = REGIONS.lock();
    match regions.find(adjusted_size) // find will remove found region from free list if successful
    {
        Ok(found) =>
        {
            /* split the found region into two parts: one portion for the newly
            allocated region, and the remaining portion is returned to the free list.
            adjusted_size defines whwre in the region the split point occurs.
            split_from defines whether adjusted_size is measured from the top or
            bottom of the region block */
            match (found.split(adjusted_size, split_from), split_from)
            {
                /* split so that the lower portion is allocated, and the upper portion is returned to the free list */
                (Ok((mut lower, upper)), RegionSplit::FromBottom) =>
                {
                    regions.insert(upper)?;
                    lower.clean();
                    Ok(lower)
                },

                /* split so that the upper portion is allocated, and the lower portion is returned to the free list */
                (Ok((lower, upper)), RegionSplit::FromTop) =>
                {
                    /* bring the base of the upper portion down to alignment mark */
                    let mut aligned_upper = match upper.base % PHYS_RAM_LARGE_REGION_ALIGNMENT
                    {
                        0 => Region::new(upper.base, upper.size, found.hygiene),
                        d => Region::new(upper.base - d, upper.size + d, found.hygiene)
                    };

                    /* fail out if upper portion crashes through the lower portion base after alignment */
                    if lower.size < aligned_upper.size - upper.size
                    {
                        return Err(Cause::PhysRegionRegionAlignmentFailure)
                    }

                    /* adjust the size of the lower portion if the upper portion was aligned down */
                    let adjusted_lower = match aligned_upper.size - upper.size
                    {
                        0 => lower,
                        d => Region::new(lower.base, lower.size - d, found.hygiene)
                    };

                    regions.insert(adjusted_lower)?;
                    aligned_upper.clean();
                    Ok(aligned_upper)
                },

                (Err(e), _) => Err(e)
            }
        },
        Err(_) => Err(Cause::PhysNotEnoughFreeRAM)
    }
}

/* deallocate a region so that its physical RAM can be reallocated.
   only accept samll regions that are multiples of PHYS_RAM_SMALL_REGION_MIN_SIZE
   and large regions that are multiples of PHYS_RAM_LARGE_REGION_MIN_SIZE
   => to_free = region to deallocate
   <= Ok for success, or an error code for failure */
pub fn dealloc_region(to_free: Region) -> Result<(), Cause>
{
    let size = to_free.size();

    /* police the size of the region */
    if size < PHYS_RAM_LARGE_REGION_MIN_SIZE
    {
        if size % PHYS_RAM_SMALL_REGION_MIN_SIZE != 0
        {
            return Err(Cause::PhysRegionSmallNotMultiple);
        }
    }
    else
    {
        if size % PHYS_RAM_LARGE_REGION_MIN_SIZE != 0
        {
            return Err(Cause::PhysRegionLargeNotMultiple);
        }
    }

    REGIONS.lock().insert(to_free)
}
<|endoftext|>