<commit_before>use std::{
    collections::HashMap,
    path::{Path, PathBuf},
};

use xshell::{cmd, pushd, pushenv, read_file};

use crate::{cargo_files, codegen, project_root, rust_files};

#[test]
fn generate_grammar() {
    codegen::generate_syntax().unwrap()
}

#[test]
fn generate_parser_tests() {
    codegen::generate_parser_tests().unwrap()
}

#[test]
fn generate_assists_tests() {
    codegen::generate_assists_tests().unwrap();
}

/// This clones rustc repo, and so is not worth to keep up-to-date. We update
/// manually by un-ignoring the test from time to time.
#[test]
#[ignore]
fn generate_lint_completions() {
    codegen::generate_lint_completions().unwrap()
}

#[test]
fn check_code_formatting() {
    let _dir = pushd(project_root()).unwrap();
    let _e = pushenv("RUSTUP_TOOLCHAIN", "stable");
    crate::ensure_rustfmt().unwrap();
    let res = cmd!("cargo fmt -- --check").run();
    if !res.is_ok() {
        let _ = cmd!("cargo fmt").run();
    }
    res.unwrap()
}

#[test]
fn smoke_test_generate_documentation() {
    codegen::docs().unwrap()
}

#[test]
fn check_lsp_extensions_docs() {
    let expected_hash = {
        let lsp_ext_rs =
            read_file(project_root().join("crates/rust-analyzer/src/lsp_ext.rs")).unwrap();
        stable_hash(lsp_ext_rs.as_str())
    };

    let actual_hash = {
        let lsp_extensions_md =
            read_file(project_root().join("docs/dev/lsp-extensions.md")).unwrap();
        let text = lsp_extensions_md
            .lines()
            .find_map(|line| line.strip_prefix("lsp_ext.rs hash:"))
            .unwrap()
            .trim();
        u64::from_str_radix(text, 16).unwrap()
    };

    if actual_hash != expected_hash {
        panic!(
            "
lsp_ext.rs was changed without touching lsp-extensions.md.

Expected hash: {:x}
Actual hash:   {:x}

Please adjust docs/dev/lsp-extensions.md.
",
            expected_hash, actual_hash
        )
    }
}

#[test]
fn rust_files_are_tidy() {
    let mut tidy_docs = TidyDocs::default();
    for path in rust_files() {
        let text = read_file(&path).unwrap();
        check_todo(&path, &text);
        check_dbg(&path, &text);
        check_trailing_ws(&path, &text);
        deny_clippy(&path, &text);
        tidy_docs.visit(&path, &text);
    }
    tidy_docs.finish();
}

#[test]
fn cargo_files_are_tidy() {
    for cargo in cargo_files() {
        let mut section = None;
        for (line_no, text) in read_file(&cargo).unwrap().lines().enumerate() {
            let text = text.trim();
            if text.starts_with('[') {
                if !text.ends_with(']') {
                    panic!(
                        "\nplease don't add comments or trailing whitespace in section lines.\n\
                            {}:{}\n",
                        cargo.display(),
                        line_no + 1
                    )
                }
                section = Some(text);
                continue;
            }
            let text: String = text.split_whitespace().collect();
            if !text.contains("path=") {
                continue;
            }
            match section {
                Some(s) if s.contains("dev-dependencies") => {
                    if text.contains("version") {
                        panic!(
                            "\ncargo internal dev-dependencies should not have a version.\n\
                            {}:{}\n",
                            cargo.display(),
                            line_no + 1
                        );
                    }
                }
                Some(s) if s.contains("dependencies") => {
                    if !text.contains("version") {
                        panic!(
                            "\ncargo internal dependencies should have a version.\n\
                            {}:{}\n",
                            cargo.display(),
                            line_no + 1
                        );
                    }
                }
                _ => {}
            }
        }
    }
}

#[test]
fn check_merge_commits() {
    let stdout = cmd!("git rev-list --merges --invert-grep --author 'bors\\[bot\\]' HEAD~19..")
        .read()
        .unwrap();
    if !stdout.is_empty() {
        panic!(
            "
Merge commits are not allowed in the history.

When updating a pull-request, please rebase your feature branch
on top of master by running `git rebase master`. If rebase fails,
you can re-apply your changes like this:

  # Just look around to see the current state.
  $ git status
  $ git log

  # Abort in-progress rebase and merges, if any.
  $ git rebase --abort
  $ git merge --abort

  # Make the branch point to the latest commit from master,
  # while maintaining your local changes uncommited.
  $ git reset --soft origin/master

  # Commit all changes in a single batch.
  $ git commit -am'My changes'

  # Verify that everything looks alright.
  $ git status
  $ git log

  # Push the changes. We did a rebase, so we need `--force` option.
  # `--force-with-lease` is a more safe (Rusty) version of `--force`.
  $ git push --force-with-lease

  # Verify that both local and remote branch point to the same commit.
  $ git log

And don't fear to mess something up during a rebase -- you can
always restore the previous state using `git ref-log`:

https://github.blog/2015-06-08-how-to-undo-almost-anything-with-git/#redo-after-undo-local
"
        );
    }
}

fn deny_clippy(path: &Path, text: &str) {
    let ignore = &[
        // The documentation in string literals may contain anything for its own purposes
        "ide_completion/src/generated_lint_completions.rs",
    ];
    if ignore.iter().any(|p| path.ends_with(p)) {
        return;
    }

    if text.contains("\u{61}llow(clippy") {
        panic!(
            "\n\nallowing lints is forbidden: {}.
rust-analyzer intentionally doesn't check clippy on CI.
You can allow lint globally via `xtask clippy`.
See https://github.com/rust-lang/rust-clippy/issues/5537 for discussion.

",
            path.display()
        )
    }
}

#[test]
fn check_licenses() {
    let expected = "
0BSD OR MIT OR Apache-2.0
Apache-2.0
Apache-2.0 OR BSL-1.0
Apache-2.0 OR MIT
Apache-2.0/MIT
BSD-3-Clause
CC0-1.0
ISC
MIT
MIT / Apache-2.0
MIT OR Apache-2.0
MIT OR Apache-2.0 OR Zlib
MIT OR Zlib OR Apache-2.0
MIT/Apache-2.0
Unlicense OR MIT
Unlicense/MIT
Zlib OR Apache-2.0 OR MIT
"
    .lines()
    .filter(|it| !it.is_empty())
    .collect::<Vec<_>>();

    let meta = cmd!("cargo metadata --format-version 1").read().unwrap();
    let mut licenses = meta
        .split(|c| c == ',' || c == '{' || c == '}')
        .filter(|it| it.contains(r#""license""#))
        .map(|it| it.trim())
        .map(|it| it[r#""license":"#.len()..].trim_matches('"'))
        .collect::<Vec<_>>();
    licenses.sort();
    licenses.dedup();
    if licenses != expected {
        let mut diff = String::new();

        diff += &format!("New Licenses:\n");
        for &l in licenses.iter() {
            if !expected.contains(&l) {
                diff += &format!("  {}\n", l)
            }
        }

        diff += &format!("\nMissing Licenses:\n");
        for &l in expected.iter() {
            if !licenses.contains(&l) {
                diff += &format!("  {}\n", l)
            }
        }

        panic!("different set of licenses!\n{}", diff);
    }
    assert_eq!(licenses, expected);
}

fn check_todo(path: &Path, text: &str) {
    let need_todo = &[
        // This file itself obviously needs to use todo (<- like this!).
        "tests/tidy.rs",
        // Some of our assists generate `todo!()`.
        "handlers/add_turbo_fish.rs",
        "handlers/generate_function.rs",
        // To support generating `todo!()` in assists, we have `expr_todo()` in
        // `ast::make`.
        "ast/make.rs",
        // The documentation in string literals may contain anything for its own purposes
        "ide_completion/src/generated_lint_completions.rs",
    ];
    if need_todo.iter().any(|p| path.ends_with(p)) {
        return;
    }
    if text.contains("TODO") || text.contains("TOOD") || text.contains("todo!") {
        // Generated by an assist
        if text.contains("${0:todo!()}") {
            return;
        }

        panic!(
            "\nTODO markers or todo! macros should not be committed to the master branch,\n\
             use FIXME instead\n\
             {}\n",
            path.display(),
        )
    }
}

fn check_dbg(path: &Path, text: &str) {
    let need_dbg = &[
        // This file itself obviously needs to use dbg.
        "tests/tidy.rs",
        // Assists to remove `dbg!()`
        "handlers/remove_dbg.rs",
        // We have .dbg postfix
        "ide_completion/src/completions/postfix.rs",
        // The documentation in string literals may contain anything for its own purposes
        "ide_completion/src/lib.rs",
        "ide_completion/src/generated_lint_completions.rs",
        // test for doc test for remove_dbg
        "src/tests/generated.rs",
    ];
    if need_dbg.iter().any(|p| path.ends_with(p)) {
        return;
    }
    if text.contains("dbg!") {
        panic!(
            "\ndbg! macros should not be committed to the master branch,\n\
             {}\n",
            path.display(),
        )
    }
}

fn check_trailing_ws(path: &Path, text: &str) {
    if is_exclude_dir(path, &["test_data"]) {
        return;
    }
    for (line_number, line) in text.lines().enumerate() {
        if line.chars().last().map(char::is_whitespace) == Some(true) {
            panic!("Trailing whitespace in {} at line {}", path.display(), line_number)
        }
    }
}

#[derive(Default)]
struct TidyDocs {
    missing_docs: Vec<String>,
    contains_fixme: Vec<PathBuf>,
}

impl TidyDocs {
    fn visit(&mut self, path: &Path, text: &str) {
        // Test hopefully don't really need comments, and for assists we already
        // have special comments which are source of doc tests and user docs.
        if is_exclude_dir(path, &["tests", "test_data"]) {
            return;
        }

        if is_exclude_file(path) {
            return;
        }

        let first_line = match text.lines().next() {
            Some(it) => it,
            None => return,
        };

        if first_line.starts_with("//!") {
            if first_line.contains("FIXME") {
                self.contains_fixme.push(path.to_path_buf());
            }
        } else {
            if text.contains("// Feature:") || text.contains("// Assist:") {
                return;
            }
            self.missing_docs.push(path.display().to_string());
        }

        fn is_exclude_file(d: &Path) -> bool {
            let file_names = ["tests.rs", "famous_defs_fixture.rs"];

            d.file_name()
                .unwrap_or_default()
                .to_str()
                .map(|f_n| file_names.iter().any(|name| *name == f_n))
                .unwrap_or(false)
        }
    }

    fn finish(self) {
        if !self.missing_docs.is_empty() {
            panic!(
                "\nMissing docs strings\n\n\
                 modules:\n{}\n\n",
                self.missing_docs.join("\n")
            )
        }

        let poorly_documented = [
            "hir",
            "hir_expand",
            "ide",
            "mbe",
            "parser",
            "profile",
            "project_model",
            "syntax",
            "tt",
            "hir_ty",
        ];

        let mut has_fixmes =
            poorly_documented.iter().map(|it| (*it, false)).collect::<HashMap<&str, bool>>();
        'outer: for path in self.contains_fixme {
            for krate in poorly_documented.iter() {
                if path.components().any(|it| it.as_os_str() == *krate) {
                    has_fixmes.insert(krate, true);
                    continue 'outer;
                }
            }
            panic!("FIXME doc in a fully-documented crate: {}", path.display())
        }

        for (krate, has_fixme) in has_fixmes.iter() {
            if !has_fixme {
                panic!("crate {} is fully documented :tada:, remove it from the list of poorly documented crates", krate)
            }
        }
    }
}

fn is_exclude_dir(p: &Path, dirs_to_exclude: &[&str]) -> bool {
    p.strip_prefix(project_root())
        .unwrap()
        .components()
        .rev()
        .skip(1)
        .filter_map(|it| it.as_os_str().to_str())
        .any(|it| dirs_to_exclude.contains(&it))
}

#[allow(deprecated)]
fn stable_hash(text: &str) -> u64 {
    use std::hash::{Hash, Hasher, SipHasher};

    let text = text.replace('\r', "");
    let mut hasher = SipHasher::default();
    text.hash(&mut hasher);
    hasher.finish()
}
<commit_msg>Update license check<commit_after>use std::{
    collections::HashMap,
    path::{Path, PathBuf},
};

use xshell::{cmd, pushd, pushenv, read_file};

use crate::{cargo_files, codegen, project_root, rust_files};

#[test]
fn generate_grammar() {
    codegen::generate_syntax().unwrap()
}

#[test]
fn generate_parser_tests() {
    codegen::generate_parser_tests().unwrap()
}

#[test]
fn generate_assists_tests() {
    codegen::generate_assists_tests().unwrap();
}

/// This clones rustc repo, and so is not worth to keep up-to-date. We update
/// manually by un-ignoring the test from time to time.
#[test]
#[ignore]
fn generate_lint_completions() {
    codegen::generate_lint_completions().unwrap()
}

#[test]
fn check_code_formatting() {
    let _dir = pushd(project_root()).unwrap();
    let _e = pushenv("RUSTUP_TOOLCHAIN", "stable");
    crate::ensure_rustfmt().unwrap();
    let res = cmd!("cargo fmt -- --check").run();
    if !res.is_ok() {
        let _ = cmd!("cargo fmt").run();
    }
    res.unwrap()
}

#[test]
fn smoke_test_generate_documentation() {
    codegen::docs().unwrap()
}

#[test]
fn check_lsp_extensions_docs() {
    let expected_hash = {
        let lsp_ext_rs =
            read_file(project_root().join("crates/rust-analyzer/src/lsp_ext.rs")).unwrap();
        stable_hash(lsp_ext_rs.as_str())
    };

    let actual_hash = {
        let lsp_extensions_md =
            read_file(project_root().join("docs/dev/lsp-extensions.md")).unwrap();
        let text = lsp_extensions_md
            .lines()
            .find_map(|line| line.strip_prefix("lsp_ext.rs hash:"))
            .unwrap()
            .trim();
        u64::from_str_radix(text, 16).unwrap()
    };

    if actual_hash != expected_hash {
        panic!(
            "
lsp_ext.rs was changed without touching lsp-extensions.md.

Expected hash: {:x}
Actual hash:   {:x}

Please adjust docs/dev/lsp-extensions.md.
",
            expected_hash, actual_hash
        )
    }
}

#[test]
fn rust_files_are_tidy() {
    let mut tidy_docs = TidyDocs::default();
    for path in rust_files() {
        let text = read_file(&path).unwrap();
        check_todo(&path, &text);
        check_dbg(&path, &text);
        check_trailing_ws(&path, &text);
        deny_clippy(&path, &text);
        tidy_docs.visit(&path, &text);
    }
    tidy_docs.finish();
}

#[test]
fn cargo_files_are_tidy() {
    for cargo in cargo_files() {
        let mut section = None;
        for (line_no, text) in read_file(&cargo).unwrap().lines().enumerate() {
            let text = text.trim();
            if text.starts_with('[') {
                if !text.ends_with(']') {
                    panic!(
                        "\nplease don't add comments or trailing whitespace in section lines.\n\
                            {}:{}\n",
                        cargo.display(),
                        line_no + 1
                    )
                }
                section = Some(text);
                continue;
            }
            let text: String = text.split_whitespace().collect();
            if !text.contains("path=") {
                continue;
            }
            match section {
                Some(s) if s.contains("dev-dependencies") => {
                    if text.contains("version") {
                        panic!(
                            "\ncargo internal dev-dependencies should not have a version.\n\
                            {}:{}\n",
                            cargo.display(),
                            line_no + 1
                        );
                    }
                }
                Some(s) if s.contains("dependencies") => {
                    if !text.contains("version") {
                        panic!(
                            "\ncargo internal dependencies should have a version.\n\
                            {}:{}\n",
                            cargo.display(),
                            line_no + 1
                        );
                    }
                }
                _ => {}
            }
        }
    }
}

#[test]
fn check_merge_commits() {
    let stdout = cmd!("git rev-list --merges --invert-grep --author 'bors\\[bot\\]' HEAD~19..")
        .read()
        .unwrap();
    if !stdout.is_empty() {
        panic!(
            "
Merge commits are not allowed in the history.

When updating a pull-request, please rebase your feature branch
on top of master by running `git rebase master`. If rebase fails,
you can re-apply your changes like this:

  # Just look around to see the current state.
  $ git status
  $ git log

  # Abort in-progress rebase and merges, if any.
  $ git rebase --abort
  $ git merge --abort

  # Make the branch point to the latest commit from master,
  # while maintaining your local changes uncommited.
  $ git reset --soft origin/master

  # Commit all changes in a single batch.
  $ git commit -am'My changes'

  # Verify that everything looks alright.
  $ git status
  $ git log

  # Push the changes. We did a rebase, so we need `--force` option.
  # `--force-with-lease` is a more safe (Rusty) version of `--force`.
  $ git push --force-with-lease

  # Verify that both local and remote branch point to the same commit.
  $ git log

And don't fear to mess something up during a rebase -- you can
always restore the previous state using `git ref-log`:

https://github.blog/2015-06-08-how-to-undo-almost-anything-with-git/#redo-after-undo-local
"
        );
    }
}

fn deny_clippy(path: &Path, text: &str) {
    let ignore = &[
        // The documentation in string literals may contain anything for its own purposes
        "ide_completion/src/generated_lint_completions.rs",
    ];
    if ignore.iter().any(|p| path.ends_with(p)) {
        return;
    }

    if text.contains("\u{61}llow(clippy") {
        panic!(
            "\n\nallowing lints is forbidden: {}.
rust-analyzer intentionally doesn't check clippy on CI.
You can allow lint globally via `xtask clippy`.
See https://github.com/rust-lang/rust-clippy/issues/5537 for discussion.

",
            path.display()
        )
    }
}

#[test]
fn check_licenses() {
    let expected = "
0BSD OR MIT OR Apache-2.0
Apache-2.0
Apache-2.0 OR BSL-1.0
Apache-2.0 OR MIT
Apache-2.0/MIT
BSD-3-Clause
CC0-1.0 OR Artistic-2.0
ISC
MIT
MIT / Apache-2.0
MIT OR Apache-2.0
MIT OR Apache-2.0 OR Zlib
MIT OR Zlib OR Apache-2.0
MIT/Apache-2.0
Unlicense OR MIT
Unlicense/MIT
Zlib OR Apache-2.0 OR MIT
"
    .lines()
    .filter(|it| !it.is_empty())
    .collect::<Vec<_>>();

    let meta = cmd!("cargo metadata --format-version 1").read().unwrap();
    let mut licenses = meta
        .split(|c| c == ',' || c == '{' || c == '}')
        .filter(|it| it.contains(r#""license""#))
        .map(|it| it.trim())
        .map(|it| it[r#""license":"#.len()..].trim_matches('"'))
        .collect::<Vec<_>>();
    licenses.sort();
    licenses.dedup();
    if licenses != expected {
        let mut diff = String::new();

        diff += &format!("New Licenses:\n");
        for &l in licenses.iter() {
            if !expected.contains(&l) {
                diff += &format!("  {}\n", l)
            }
        }

        diff += &format!("\nMissing Licenses:\n");
        for &l in expected.iter() {
            if !licenses.contains(&l) {
                diff += &format!("  {}\n", l)
            }
        }

        panic!("different set of licenses!\n{}", diff);
    }
    assert_eq!(licenses, expected);
}

fn check_todo(path: &Path, text: &str) {
    let need_todo = &[
        // This file itself obviously needs to use todo (<- like this!).
        "tests/tidy.rs",
        // Some of our assists generate `todo!()`.
        "handlers/add_turbo_fish.rs",
        "handlers/generate_function.rs",
        // To support generating `todo!()` in assists, we have `expr_todo()` in
        // `ast::make`.
        "ast/make.rs",
        // The documentation in string literals may contain anything for its own purposes
        "ide_completion/src/generated_lint_completions.rs",
    ];
    if need_todo.iter().any(|p| path.ends_with(p)) {
        return;
    }
    if text.contains("TODO") || text.contains("TOOD") || text.contains("todo!") {
        // Generated by an assist
        if text.contains("${0:todo!()}") {
            return;
        }

        panic!(
            "\nTODO markers or todo! macros should not be committed to the master branch,\n\
             use FIXME instead\n\
             {}\n",
            path.display(),
        )
    }
}

fn check_dbg(path: &Path, text: &str) {
    let need_dbg = &[
        // This file itself obviously needs to use dbg.
        "tests/tidy.rs",
        // Assists to remove `dbg!()`
        "handlers/remove_dbg.rs",
        // We have .dbg postfix
        "ide_completion/src/completions/postfix.rs",
        // The documentation in string literals may contain anything for its own purposes
        "ide_completion/src/lib.rs",
        "ide_completion/src/generated_lint_completions.rs",
        // test for doc test for remove_dbg
        "src/tests/generated.rs",
    ];
    if need_dbg.iter().any(|p| path.ends_with(p)) {
        return;
    }
    if text.contains("dbg!") {
        panic!(
            "\ndbg! macros should not be committed to the master branch,\n\
             {}\n",
            path.display(),
        )
    }
}

fn check_trailing_ws(path: &Path, text: &str) {
    if is_exclude_dir(path, &["test_data"]) {
        return;
    }
    for (line_number, line) in text.lines().enumerate() {
        if line.chars().last().map(char::is_whitespace) == Some(true) {
            panic!("Trailing whitespace in {} at line {}", path.display(), line_number)
        }
    }
}

#[derive(Default)]
struct TidyDocs {
    missing_docs: Vec<String>,
    contains_fixme: Vec<PathBuf>,
}

impl TidyDocs {
    fn visit(&mut self, path: &Path, text: &str) {
        // Test hopefully don't really need comments, and for assists we already
        // have special comments which are source of doc tests and user docs.
        if is_exclude_dir(path, &["tests", "test_data"]) {
            return;
        }

        if is_exclude_file(path) {
            return;
        }

        let first_line = match text.lines().next() {
            Some(it) => it,
            None => return,
        };

        if first_line.starts_with("//!") {
            if first_line.contains("FIXME") {
                self.contains_fixme.push(path.to_path_buf());
            }
        } else {
            if text.contains("// Feature:") || text.contains("// Assist:") {
                return;
            }
            self.missing_docs.push(path.display().to_string());
        }

        fn is_exclude_file(d: &Path) -> bool {
            let file_names = ["tests.rs", "famous_defs_fixture.rs"];

            d.file_name()
                .unwrap_or_default()
                .to_str()
                .map(|f_n| file_names.iter().any(|name| *name == f_n))
                .unwrap_or(false)
        }
    }

    fn finish(self) {
        if !self.missing_docs.is_empty() {
            panic!(
                "\nMissing docs strings\n\n\
                 modules:\n{}\n\n",
                self.missing_docs.join("\n")
            )
        }

        let poorly_documented = [
            "hir",
            "hir_expand",
            "ide",
            "mbe",
            "parser",
            "profile",
            "project_model",
            "syntax",
            "tt",
            "hir_ty",
        ];

        let mut has_fixmes =
            poorly_documented.iter().map(|it| (*it, false)).collect::<HashMap<&str, bool>>();
        'outer: for path in self.contains_fixme {
            for krate in poorly_documented.iter() {
                if path.components().any(|it| it.as_os_str() == *krate) {
                    has_fixmes.insert(krate, true);
                    continue 'outer;
                }
            }
            panic!("FIXME doc in a fully-documented crate: {}", path.display())
        }

        for (krate, has_fixme) in has_fixmes.iter() {
            if !has_fixme {
                panic!("crate {} is fully documented :tada:, remove it from the list of poorly documented crates", krate)
            }
        }
    }
}

fn is_exclude_dir(p: &Path, dirs_to_exclude: &[&str]) -> bool {
    p.strip_prefix(project_root())
        .unwrap()
        .components()
        .rev()
        .skip(1)
        .filter_map(|it| it.as_os_str().to_str())
        .any(|it| dirs_to_exclude.contains(&it))
}

#[allow(deprecated)]
fn stable_hash(text: &str) -> u64 {
    use std::hash::{Hash, Hasher, SipHasher};

    let text = text.replace('\r', "");
    let mut hasher = SipHasher::default();
    text.hash(&mut hasher);
    hasher.finish()
}
<|endoftext|>


<commit_before>use spirv_cross::{glsl, spirv};

mod common;
use crate::common::words_from_bytes;

#[test]
fn glsl_compiler_options_has_default() {
    let compiler_options = glsl::CompilerOptions::default();
    assert_eq!(compiler_options.vertex.invert_y, false);
    assert_eq!(compiler_options.vertex.transform_clip_space, false);
}

#[test]
fn ast_compiles_to_glsl() {
    let mut ast = spirv::Ast::<glsl::Target>::parse(&spirv::Module::from_words(words_from_bytes(
        include_bytes!("shaders/simple.vert.spv"),
    )))
    .unwrap();
    ast.set_compiler_options(&glsl::CompilerOptions {
        version: glsl::Version::V4_60,
        no_420_pack_extension: false,
        vertex: glsl::CompilerVertexOptions::default(),
    })
    .unwrap();

    assert_eq!(
        ast.compile().unwrap(),
        "\
#version 460

layout(std140) uniform uniform_buffer_object
{
    mat4 u_model_view_projection;
    float u_scale;
} _22;

layout(location = 0) out vec3 v_normal;
layout(location = 1) in vec3 a_normal;
layout(location = 0) in vec4 a_position;

void main()
{
    v_normal = a_normal;
    gl_Position = (_22.u_model_view_projection * a_position) * _22.u_scale;
}

"
    );
}

#[test]
fn ast_compiles_all_versions_to_glsl() {
    use spirv_cross::glsl::Version::*;

    let module =
        spirv::Module::from_words(words_from_bytes(include_bytes!("shaders/simple.vert.spv")));
    let mut ast = spirv::Ast::<glsl::Target>::parse(&module).unwrap();

    let versions = [
        V1_10, V1_20, V1_30, V1_40, V1_50, V3_30, V4_00, V4_10, V4_20, V4_30, V4_40, V4_50, V4_60,
        V1_00Es, V3_00Es,
    ];
    for &version in versions.iter() {
        if ast
            .set_compiler_options(&glsl::CompilerOptions {
                version,
                no_420_pack_extension: false,
                vertex: glsl::CompilerVertexOptions::default(),
            })
            .is_err()
        {
            panic!("Did not compile");
        }
    }
}

#[test]
fn ast_renames_interface_variables() {
    let vert =
        spirv::Module::from_words(words_from_bytes(include_bytes!("shaders/struct.vert.spv")));
    let mut vert_ast = spirv::Ast::<glsl::Target>::parse(&vert).unwrap();
    vert_ast
        .set_compiler_options(&glsl::CompilerOptions {
            version: glsl::Version::V1_00Es,
            no_420_pack_extension: false,
            vertex: glsl::CompilerVertexOptions::default(),
        })
        .unwrap();
    let vert_stage_outputs = vert_ast.get_shader_resources().unwrap().stage_outputs;
    vert_ast
        .rename_interface_variable(&vert_stage_outputs, 0, "renamed")
        .unwrap();

    let vert_output = vert_ast.compile().unwrap();

    let frag =
        spirv::Module::from_words(words_from_bytes(include_bytes!("shaders/struct.frag.spv")));
    let mut frag_ast = spirv::Ast::<glsl::Target>::parse(&frag).unwrap();
    frag_ast
        .set_compiler_options(&glsl::CompilerOptions {
            version: glsl::Version::V1_00Es,
            no_420_pack_extension: false,
            vertex: glsl::CompilerVertexOptions::default(),
        })
        .unwrap();
    let frag_stage_inputs = frag_ast.get_shader_resources().unwrap().stage_inputs;
    frag_ast
        .rename_interface_variable(&frag_stage_inputs, 0, "renamed")
        .unwrap();
    let frag_output = frag_ast.compile().unwrap();

    assert_eq!(
        vert_output,
        "\
#version 100

struct SPIRV_Cross_Interface_Location0
{
    vec4 InterfaceMember0;
    vec4 InterfaceMember1;
    vec4 InterfaceMember2;
    vec4 InterfaceMember3;
};

varying vec4 renamed_InterfaceMember0;
varying vec4 renamed_InterfaceMember1;
varying vec4 renamed_InterfaceMember2;
varying vec4 renamed_InterfaceMember3;
attribute vec4 a;
attribute vec4 b;
attribute vec4 c;
attribute vec4 d;

void main()
{
    {
        SPIRV_Cross_Interface_Location0 renamed = SPIRV_Cross_Interface_Location0(a, b, c, d);
        renamed_InterfaceMember0 = renamed.InterfaceMember0;
        renamed_InterfaceMember1 = renamed.InterfaceMember1;
        renamed_InterfaceMember2 = renamed.InterfaceMember2;
        renamed_InterfaceMember3 = renamed.InterfaceMember3;
    }
}

"
    );

    assert_eq!(
        frag_output,
        "\
#version 100
precision mediump float;
precision highp int;

struct SPIRV_Cross_Interface_Location0
{
    vec4 InterfaceMember0;
    vec4 InterfaceMember1;
    vec4 InterfaceMember2;
    vec4 InterfaceMember3;
};

varying vec4 renamed_InterfaceMember0;
varying vec4 renamed_InterfaceMember1;
varying vec4 renamed_InterfaceMember2;
varying vec4 renamed_InterfaceMember3;

void main()
{
    gl_FragData[0] = vec4(renamed_InterfaceMember0.x, renamed_InterfaceMember1.y, renamed_InterfaceMember2.z, renamed_InterfaceMember3.w);
}

"
    );
}

#[test]
fn ast_can_rename_combined_image_samplers() {
    let mut ast = spirv::Ast::<glsl::Target>::parse(&spirv::Module::from_words(words_from_bytes(
        include_bytes!("shaders/sampler.frag.spv"),
    )))
    .unwrap();
    ast.set_compiler_options(&glsl::CompilerOptions {
        version: glsl::Version::V4_10,
        no_420_pack_extension: false,
        vertex: glsl::CompilerVertexOptions::default(),
    })
    .unwrap();
    for cis in ast.get_combined_image_samplers().unwrap() {
        let new_name = "combined_sampler".to_string()
            + "_"
            + &cis.sampler_id.to_string()
            + "_"
            + &cis.image_id.to_string()
            + "_"
            + &cis.combined_id.to_string();
        ast.set_name(cis.combined_id, &new_name).unwrap();
        assert_eq!(new_name, ast.get_name(cis.combined_id).unwrap());
    }

    assert_eq!(
        ast.compile().unwrap(),
        "\
#version 410
#ifdef GL_ARB_shading_language_420pack
#extension GL_ARB_shading_language_420pack : require
#endif

uniform sampler2D combined_sampler_16_12_26;

layout(location = 0) out vec4 target0;
layout(location = 0) in vec2 v_uv;

void main()
{
    target0 = texture(combined_sampler_16_12_26, v_uv);
}

"
    );
}
<commit_msg>added forgotten glsl_tests file for 420pack-extension option<commit_after>use spirv_cross::{glsl, spirv};

mod common;
use crate::common::words_from_bytes;

#[test]
fn glsl_compiler_options_has_default() {
    let compiler_options = glsl::CompilerOptions::default();
    assert_eq!(compiler_options.vertex.invert_y, false);
    assert_eq!(compiler_options.vertex.transform_clip_space, false);
}

#[test]
fn ast_compiles_to_glsl() {
    let mut ast = spirv::Ast::<glsl::Target>::parse(&spirv::Module::from_words(words_from_bytes(
        include_bytes!("shaders/simple.vert.spv"),
    )))
    .unwrap();
    ast.set_compiler_options(&glsl::CompilerOptions {
        version: glsl::Version::V4_60,
        enable_420_pack_extension: true,
        vertex: glsl::CompilerVertexOptions::default(),
    })
    .unwrap();

    assert_eq!(
        ast.compile().unwrap(),
        "\
#version 460

layout(std140) uniform uniform_buffer_object
{
    mat4 u_model_view_projection;
    float u_scale;
} _22;

layout(location = 0) out vec3 v_normal;
layout(location = 1) in vec3 a_normal;
layout(location = 0) in vec4 a_position;

void main()
{
    v_normal = a_normal;
    gl_Position = (_22.u_model_view_projection * a_position) * _22.u_scale;
}

"
    );
}

#[test]
fn ast_compiles_all_versions_to_glsl() {
    use spirv_cross::glsl::Version::*;

    let module =
        spirv::Module::from_words(words_from_bytes(include_bytes!("shaders/simple.vert.spv")));
    let mut ast = spirv::Ast::<glsl::Target>::parse(&module).unwrap();

    let versions = [
        V1_10, V1_20, V1_30, V1_40, V1_50, V3_30, V4_00, V4_10, V4_20, V4_30, V4_40, V4_50, V4_60,
        V1_00Es, V3_00Es,
    ];
    for &version in versions.iter() {
        if ast
            .set_compiler_options(&glsl::CompilerOptions {
                version,
                enable_420_pack_extension: true,
                vertex: glsl::CompilerVertexOptions::default(),
            })
            .is_err()
        {
            panic!("Did not compile");
        }
    }
}

#[test]
fn ast_renames_interface_variables() {
    let vert =
        spirv::Module::from_words(words_from_bytes(include_bytes!("shaders/struct.vert.spv")));
    let mut vert_ast = spirv::Ast::<glsl::Target>::parse(&vert).unwrap();
    vert_ast
        .set_compiler_options(&glsl::CompilerOptions {
            version: glsl::Version::V1_00Es,
            enable_420_pack_extension: true,
            vertex: glsl::CompilerVertexOptions::default(),
        })
        .unwrap();
    let vert_stage_outputs = vert_ast.get_shader_resources().unwrap().stage_outputs;
    vert_ast
        .rename_interface_variable(&vert_stage_outputs, 0, "renamed")
        .unwrap();

    let vert_output = vert_ast.compile().unwrap();

    let frag =
        spirv::Module::from_words(words_from_bytes(include_bytes!("shaders/struct.frag.spv")));
    let mut frag_ast = spirv::Ast::<glsl::Target>::parse(&frag).unwrap();
    frag_ast
        .set_compiler_options(&glsl::CompilerOptions {
            version: glsl::Version::V1_00Es,
            enable_420_pack_extension: true,
            vertex: glsl::CompilerVertexOptions::default(),
        })
        .unwrap();
    let frag_stage_inputs = frag_ast.get_shader_resources().unwrap().stage_inputs;
    frag_ast
        .rename_interface_variable(&frag_stage_inputs, 0, "renamed")
        .unwrap();
    let frag_output = frag_ast.compile().unwrap();

    assert_eq!(
        vert_output,
        "\
#version 100

struct SPIRV_Cross_Interface_Location0
{
    vec4 InterfaceMember0;
    vec4 InterfaceMember1;
    vec4 InterfaceMember2;
    vec4 InterfaceMember3;
};

varying vec4 renamed_InterfaceMember0;
varying vec4 renamed_InterfaceMember1;
varying vec4 renamed_InterfaceMember2;
varying vec4 renamed_InterfaceMember3;
attribute vec4 a;
attribute vec4 b;
attribute vec4 c;
attribute vec4 d;

void main()
{
    {
        SPIRV_Cross_Interface_Location0 renamed = SPIRV_Cross_Interface_Location0(a, b, c, d);
        renamed_InterfaceMember0 = renamed.InterfaceMember0;
        renamed_InterfaceMember1 = renamed.InterfaceMember1;
        renamed_InterfaceMember2 = renamed.InterfaceMember2;
        renamed_InterfaceMember3 = renamed.InterfaceMember3;
    }
}

"
    );

    assert_eq!(
        frag_output,
        "\
#version 100
precision mediump float;
precision highp int;

struct SPIRV_Cross_Interface_Location0
{
    vec4 InterfaceMember0;
    vec4 InterfaceMember1;
    vec4 InterfaceMember2;
    vec4 InterfaceMember3;
};

varying vec4 renamed_InterfaceMember0;
varying vec4 renamed_InterfaceMember1;
varying vec4 renamed_InterfaceMember2;
varying vec4 renamed_InterfaceMember3;

void main()
{
    gl_FragData[0] = vec4(renamed_InterfaceMember0.x, renamed_InterfaceMember1.y, renamed_InterfaceMember2.z, renamed_InterfaceMember3.w);
}

"
    );
}

#[test]
fn ast_can_rename_combined_image_samplers() {
    let mut ast = spirv::Ast::<glsl::Target>::parse(&spirv::Module::from_words(words_from_bytes(
        include_bytes!("shaders/sampler.frag.spv"),
    )))
    .unwrap();
    ast.set_compiler_options(&glsl::CompilerOptions {
        version: glsl::Version::V4_10,
        enable_420_pack_extension: true,
        vertex: glsl::CompilerVertexOptions::default(),
    })
    .unwrap();
    for cis in ast.get_combined_image_samplers().unwrap() {
        let new_name = "combined_sampler".to_string()
            + "_"
            + &cis.sampler_id.to_string()
            + "_"
            + &cis.image_id.to_string()
            + "_"
            + &cis.combined_id.to_string();
        ast.set_name(cis.combined_id, &new_name).unwrap();
        assert_eq!(new_name, ast.get_name(cis.combined_id).unwrap());
    }

    assert_eq!(
        ast.compile().unwrap(),
        "\
#version 410
#ifdef GL_ARB_shading_language_420pack
#extension GL_ARB_shading_language_420pack : require
#endif

uniform sampler2D combined_sampler_16_12_26;

layout(location = 0) out vec4 target0;
layout(location = 0) in vec2 v_uv;

void main()
{
    target0 = texture(combined_sampler_16_12_26, v_uv);
}

"
    );
}
<|endoftext|>


<commit_before>// Copyright 2020 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use crate::MappedRegion;
use std::fmt::{self, Display};

#[derive(Debug, Eq, PartialEq)]
pub enum Error {
    // A null address is typically bad.  mmap allows it, but not external libraries
    NullAddress,
    // For external mappings that have weird sizes
    InvalidSize,
    // External library failed to map
    LibraryError(i32),
    // If external mapping is unsupported.
    Unsupported,
}

impl Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        use self::Error::*;

        match self {
            NullAddress => write!(f, "null address returned"),
            InvalidSize => write!(f, "invalid size returned"),
            LibraryError(ret) => write!(f, "library failed to map with {}", ret),
            Unsupported => write!(f, "external mapping unsupported"),
        }
    }
}

pub type Result<T> = std::result::Result<T, Error>;

// Maps a external library resource given an id, returning address and size upon success
pub type Map = fn(u32) -> Result<(u64, usize)>;
// Unmaps the resource given a resource id.
pub type Unmap = fn(u32) -> ();

/// ExternalMapping wraps an external library mapping.  This is useful in cases where where the
/// device memory is not compatible with the mmap interface, such as Vulkan VkDeviceMemory in the
/// non-exportable case or when exported as an opaque fd.
#[derive(Debug, PartialEq)]
pub struct ExternalMapping {
    resource_id: u32,
    ptr: u64,
    size: usize,
    unmap: Unmap,
}

unsafe impl Send for ExternalMapping {}
unsafe impl Sync for ExternalMapping {}
impl ExternalMapping {
    /// Creates an ExternalMapping given a library-specific resource id and map/unmap functions.
    /// The map function must return a valid host memory region.  In addition, callers of the
    /// function must guarantee that the map and unmap functions are thread-safe, never return a
    /// region overlapping already Rust referenced-data, and the backing store of the resource
    /// doesn't disappear before the unmap function is called.
    pub unsafe fn new(resource_id: u32, map: Map, unmap: Unmap) -> Result<ExternalMapping> {
        let (ptr, size) = map(resource_id)?;

        if ptr as *mut u8 == std::ptr::null_mut() {
            return Err(Error::NullAddress);
        }
        if size == 0 {
            return Err(Error::InvalidSize);
        }

        Ok(ExternalMapping {
            resource_id,
            ptr,
            size,
            unmap,
        })
    }
}

unsafe impl MappedRegion for ExternalMapping {
    /// used for passing this region to ioctls for setting guest memory.
    fn as_ptr(&self) -> *mut u8 {
        self.ptr as *mut u8
    }

    /// Returns the size of the memory region in bytes.
    fn size(&self) -> usize {
        self.size
    }
}

impl Drop for ExternalMapping {
    fn drop(&mut self) {
        // This is safe because we own this memory range, and nobody else is holding a reference to
        // it.
        (self.unmap)(self.resource_id)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn check_valid_external_map() {
        let map1: Map = |_resource_id| Ok((0xAAAABBBB, 500));
        let map2: Map = |_resource_id| Ok((0xBBBBAAAA, 1000));
        let unmap: Unmap = |_resource_id| {};
        let external_map1 = unsafe { ExternalMapping::new(0, map1, unmap).unwrap() };
        let external_map2 = unsafe { ExternalMapping::new(0, map2, unmap).unwrap() };

        assert_eq!(external_map1.as_ptr(), 0xAAAABBBB as *mut u8);
        assert_eq!(external_map1.size(), 500);
        assert_eq!(external_map2.as_ptr(), 0xBBBBAAAA as *mut u8);
        assert_eq!(external_map2.size(), 1000);
    }

    #[test]
    fn check_invalid_external_map() {
        let map1: Map = |_resource_id| Ok((0xAAAABBBB, 0));
        let map2: Map = |_resource_id| Ok((0, 500));
        let unmap: Unmap = |_resource_id| {};

        assert_eq!(
            unsafe { ExternalMapping::new(0, map1, unmap) },
            Err(Error::InvalidSize)
        );

        assert_eq!(
            unsafe { ExternalMapping::new(0, map2, unmap) },
            Err(Error::NullAddress)
        );
    }

    #[test]
    #[should_panic]
    fn check_external_map_drop() {
        let map = |_resource_id| Ok((0xAAAABBBB, 500));
        let unmap = |_resource_id| panic!();
        let _external_map = unsafe { ExternalMapping::new(0, map, unmap) };
    }
}
<commit_msg>sys_util: use is_null rather than == null_mut<commit_after>// Copyright 2020 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use crate::MappedRegion;
use std::fmt::{self, Display};

#[derive(Debug, Eq, PartialEq)]
pub enum Error {
    // A null address is typically bad.  mmap allows it, but not external libraries
    NullAddress,
    // For external mappings that have weird sizes
    InvalidSize,
    // External library failed to map
    LibraryError(i32),
    // If external mapping is unsupported.
    Unsupported,
}

impl Display for Error {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        use self::Error::*;

        match self {
            NullAddress => write!(f, "null address returned"),
            InvalidSize => write!(f, "invalid size returned"),
            LibraryError(ret) => write!(f, "library failed to map with {}", ret),
            Unsupported => write!(f, "external mapping unsupported"),
        }
    }
}

pub type Result<T> = std::result::Result<T, Error>;

// Maps a external library resource given an id, returning address and size upon success
pub type Map = fn(u32) -> Result<(u64, usize)>;
// Unmaps the resource given a resource id.
pub type Unmap = fn(u32) -> ();

/// ExternalMapping wraps an external library mapping.  This is useful in cases where where the
/// device memory is not compatible with the mmap interface, such as Vulkan VkDeviceMemory in the
/// non-exportable case or when exported as an opaque fd.
#[derive(Debug, PartialEq)]
pub struct ExternalMapping {
    resource_id: u32,
    ptr: u64,
    size: usize,
    unmap: Unmap,
}

unsafe impl Send for ExternalMapping {}
unsafe impl Sync for ExternalMapping {}
impl ExternalMapping {
    /// Creates an ExternalMapping given a library-specific resource id and map/unmap functions.
    /// The map function must return a valid host memory region.  In addition, callers of the
    /// function must guarantee that the map and unmap functions are thread-safe, never return a
    /// region overlapping already Rust referenced-data, and the backing store of the resource
    /// doesn't disappear before the unmap function is called.
    pub unsafe fn new(resource_id: u32, map: Map, unmap: Unmap) -> Result<ExternalMapping> {
        let (ptr, size) = map(resource_id)?;

        if (ptr as *mut u8).is_null() {
            return Err(Error::NullAddress);
        }
        if size == 0 {
            return Err(Error::InvalidSize);
        }

        Ok(ExternalMapping {
            resource_id,
            ptr,
            size,
            unmap,
        })
    }
}

unsafe impl MappedRegion for ExternalMapping {
    /// used for passing this region to ioctls for setting guest memory.
    fn as_ptr(&self) -> *mut u8 {
        self.ptr as *mut u8
    }

    /// Returns the size of the memory region in bytes.
    fn size(&self) -> usize {
        self.size
    }
}

impl Drop for ExternalMapping {
    fn drop(&mut self) {
        // This is safe because we own this memory range, and nobody else is holding a reference to
        // it.
        (self.unmap)(self.resource_id)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn check_valid_external_map() {
        let map1: Map = |_resource_id| Ok((0xAAAABBBB, 500));
        let map2: Map = |_resource_id| Ok((0xBBBBAAAA, 1000));
        let unmap: Unmap = |_resource_id| {};
        let external_map1 = unsafe { ExternalMapping::new(0, map1, unmap).unwrap() };
        let external_map2 = unsafe { ExternalMapping::new(0, map2, unmap).unwrap() };

        assert_eq!(external_map1.as_ptr(), 0xAAAABBBB as *mut u8);
        assert_eq!(external_map1.size(), 500);
        assert_eq!(external_map2.as_ptr(), 0xBBBBAAAA as *mut u8);
        assert_eq!(external_map2.size(), 1000);
    }

    #[test]
    fn check_invalid_external_map() {
        let map1: Map = |_resource_id| Ok((0xAAAABBBB, 0));
        let map2: Map = |_resource_id| Ok((0, 500));
        let unmap: Unmap = |_resource_id| {};

        assert_eq!(
            unsafe { ExternalMapping::new(0, map1, unmap) },
            Err(Error::InvalidSize)
        );

        assert_eq!(
            unsafe { ExternalMapping::new(0, map2, unmap) },
            Err(Error::NullAddress)
        );
    }

    #[test]
    #[should_panic]
    fn check_external_map_drop() {
        let map = |_resource_id| Ok((0xAAAABBBB, 500));
        let unmap = |_resource_id| panic!();
        let _external_map = unsafe { ExternalMapping::new(0, map, unmap) };
    }
}
<|endoftext|>


<commit_before>//! Provide access to locally cached memdb SDKs
//!
//! The `MemDbStash` pulls in remote SDKs from an S3 bucket and provides
//! access to it.  This is used by the symbol server to manage the local
//! cache and also to refer to memdb files that are mmap'ed in.
use std::fs;
use std::io;
use std::iter::FromIterator;
use std::path::{Path, PathBuf};
use std::collections::{HashMap, HashSet};
use std::collections::hash_map::Values as HashMapValuesIter;
use std::sync::{Arc, RwLock};

use serde_json;
use xz2::write::XzDecoder;
use chrono::UTC;

use super::config::Config;
use super::sdk::SdkInfo;
use super::s3::S3;
use super::memdb::MemDb;
use super::utils::{ProgressIndicator, copy_with_progress, HumanDuration};
use super::{Result, ResultExt, ErrorKind};

/// Helper for synching
pub struct SyncOptions {
    pub user_facing: bool,
}

/// The main memdb stash type
pub struct MemDbStash {
    path: PathBuf,
    s3: S3,
    local_state: RwLock<Option<Arc<SdkSyncState>>>,
    memdbs: RwLock<HashMap<SdkInfo, Arc<MemDb<'static>>>>,
}

/// Information about a remotely available SDK
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct RemoteSdk {
    filename: String,
    info: SdkInfo,
    size: u64,
    etag: String,
}

#[derive(Serialize, Deserialize, Default, Debug, Clone)]
struct SdkSyncState {
    sdks: HashMap<String, RemoteSdk>,
    revision: Option<u64>,
}

/// Information about the health of the stash sync
#[derive(Debug)]
pub struct SyncStatus {
    remote_total: u32,
    missing: u32,
    different: u32,
    revision: u64,
}

impl RemoteSdk {
    /// Creates a remote SDK object from some information
    pub fn new(filename: String, info: SdkInfo, etag: String, size: u64) -> RemoteSdk {
        RemoteSdk {
            filename: filename,
            info: info,
            etag: etag,
            size: size,
        }
    }

    /// The remotely visible filename for the SDK
    pub fn filename(&self) -> &str {
        &self.filename
    }

    /// The local filename the SDK has in the stash folder
    pub fn local_filename(&self) -> &str {
        self.filename.trim_right_matches('z')
    }

    /// The size of the SDK in bytes
    pub fn size(&self) -> u64 {
        self.size
    }

    /// Returns the SDK info
    pub fn info(&self) -> &SdkInfo {
        &self.info
    }
}

/// Iterator over the SDKs
pub type RemoteSdkIter<'a> = HashMapValuesIter<'a, String, RemoteSdk>;

impl SdkSyncState {
    pub fn get_sdk(&self, filename: &str) -> Option<&RemoteSdk> {
        self.sdks.get(filename)
    }

    pub fn update_sdk(&mut self, sdk: &RemoteSdk) {
        self.sdks.insert(sdk.local_filename().to_string(), sdk.clone());
    }

    pub fn sdks<'a>(&'a self) -> RemoteSdkIter<'a> {
        self.sdks.values()
    }
}

impl SyncStatus {

    /// Returns the lag (number of SDKs behind upstream)
    pub fn lag(&self) -> u32 {
        self.missing + self.different
    }

    /// Returns true if the local sync is still considered healthy
    pub fn is_healthy(&self) -> bool {
        let total = self.remote_total as f32;
        let lag = self.lag() as f32;
        lag / total < 0.10
    }

    /// Returns the revision of the stash
    pub fn revision(&self) -> u64 {
        self.revision
    }
}

impl Default for SyncOptions {
    fn default() -> SyncOptions {
        SyncOptions {
            user_facing: false,
        }
    }
}

impl MemDbStash {
    /// Opens a stash for a given config.
    pub fn new(config: &Config) -> Result<MemDbStash> {
        Ok(MemDbStash {
            path: config.get_symbol_dir()?.to_path_buf(),
            s3: S3::from_config(config)?,
            local_state: RwLock::new(None),
            memdbs: RwLock::new(HashMap::new()),
        })
    }

    fn get_local_sync_state_filename(&self) -> PathBuf {
        self.path.join("sync.state")
    }

    fn save_state(&self, new_state: &SdkSyncState, filename: &Path) -> Result<()> {
        let mut tmp_filename = filename.to_path_buf();
        tmp_filename.set_extension("tempstate");
        {
            let mut f = fs::File::create(&tmp_filename)?;
            serde_json::to_writer(&mut f, new_state)
                .chain_err(|| "Could not update sync state")?;
        }
        fs::rename(&tmp_filename, &filename)?;
        Ok(())
    }

    fn read_local_state(&self) -> Result<SdkSyncState> {
        let rv : SdkSyncState = match fs::File::open(&self.get_local_sync_state_filename()) {
            Ok(f) => serde_json::from_reader(f)
                .chain_err(|| "Parsing error on loading sync state")?,
            Err(err) => {
                if err.kind() == io::ErrorKind::NotFound {
                    Default::default()
                } else {
                    return Err(err).chain_err(|| "Error loading sync state");
                }
            }
        };
        let mut opt = self.local_state.write().unwrap();
        *opt = Some(Arc::new(rv.clone()));
        Ok(rv)
    }

    fn get_local_state(&self) -> Result<Arc<SdkSyncState>> {
        if let Some(ref arc) = *self.local_state.read().unwrap() {
            return Ok(arc.clone());
        }
        self.read_local_state()?;
        Ok(self.local_state.read().unwrap().as_ref().unwrap().clone())
    }

    fn save_local_state(&self, new_state: &SdkSyncState) -> Result<()> {
        self.save_state(new_state, &self.get_local_sync_state_filename())?;
        let mut opt = self.local_state.write().unwrap();
        *opt = Some(Arc::new(new_state.clone()));
        Ok(())
    }

    fn fetch_remote_state(&self) -> Result<SdkSyncState> {
        let mut sdks = HashMap::new();
        for remote_sdk in self.s3.list_upstream_sdks()? {
            sdks.insert(remote_sdk.local_filename().into(), remote_sdk);
        }
        Ok(SdkSyncState { sdks: sdks, revision: None })
    }

    fn update_sdk(&self, sdk: &RemoteSdk, options: &SyncOptions) -> Result<()> {
        // XXX: the progress bar here can stall out because we currently
        // need to buffer the download into memory in the s3 code :(
        let progress = if options.user_facing {
            ProgressIndicator::new(sdk.size() as usize)
        } else {
            info!("updating {}", sdk.info());
            ProgressIndicator::disabled()
        };
        let started = UTC::now();
        progress.set_message(&format!("Synchronizing {}", sdk.info()));
        let mut src = self.s3.download_sdk(sdk)?;
        let dst = fs::File::create(self.path.join(sdk.local_filename()))?;
        let mut dst = XzDecoder::new(dst);
        copy_with_progress(&progress, &mut src, &mut dst)?;
        progress.finish(&format!("Synchronized {}", sdk.info()));

        let duration = UTC::now() - started;
        if !options.user_facing {
            info!("updated {} in {}", sdk.info(), HumanDuration(duration));
        }
        Ok(())
    }

    fn remove_sdk(&self, sdk: &RemoteSdk, options: &SyncOptions) -> Result<()> {
        if options.user_facing {
            info!("  Deleting {}", sdk.info());
        } else {
            info!("removing {}", sdk.info());
        }
        if let Err(err) = fs::remove_file(self.path.join(sdk.local_filename())) {
            if err.kind() != io::ErrorKind::NotFound {
                return Err(err.into());
            }
        }
        Ok(())
    }

    /// Returns the current revision
    pub fn get_revision(&self) -> Result<u64> {
        Ok(self.read_local_state()?.revision.unwrap_or(0))
    }

    /// Checks the local stash against the server
    pub fn get_sync_status(&self) -> Result<SyncStatus> {
        let local_state = self.read_local_state()?;
        let remote_state = self.fetch_remote_state()?;

        let mut remote_total = 0;
        let mut missing = 0;
        let mut different = 0;

        for sdk in remote_state.sdks() {
            if let Some(local_sdk) = local_state.get_sdk(sdk.local_filename()) {
                if local_sdk != sdk {
                    different += 1;
                }
            } else {
                missing += 1;
            }
            remote_total += 1;
        }

        Ok(SyncStatus {
            remote_total: remote_total as u32,
            missing: missing as u32,
            different: different as u32,
            revision: local_state.revision.unwrap_or(0),
        })
    }

    /// Synchronize the local stash with the server
    pub fn sync(&self, options: SyncOptions) -> Result<()> {
        let mut local_state = self.read_local_state()?;
        let mut remote_state = self.fetch_remote_state()?;
        let started = UTC::now();
        let mut changed = false;

        let mut to_delete : HashSet<_> = HashSet::from_iter(
            local_state.sdks().map(|x| x.local_filename().to_string()));

        for sdk in remote_state.sdks() {
            if let Some(local_sdk) = local_state.get_sdk(sdk.local_filename()) {
                if local_sdk != sdk {
                    self.update_sdk(&sdk, &options)?;
                    changed = true;
                } else if options.user_facing {
                    println!("  ⸰ Unchanged {}", sdk.info());
                } else {
                    debug!("unchanged sdk {}", sdk.info());
                }
            } else {
                self.update_sdk(&sdk, &options)?;
                changed = true;
            }
            to_delete.remove(sdk.local_filename());
            local_state.update_sdk(&sdk);
            local_state.revision = Some(local_state.revision.unwrap_or(0) + 1);
            self.save_local_state(&local_state)?;
        }

        for local_filename in to_delete.iter() {
            if let Some(sdk) = local_state.get_sdk(local_filename) {
                self.remove_sdk(sdk, &options)?;
            }
        }

        let duration = UTC::now() - started;
        if options.user_facing {
            println!("Sync done in {}", HumanDuration(duration));
        } else if changed {
            info!("finished sync in {}", HumanDuration(duration));
        }

        // if we get this far, the remote state is indeed the local state.
        remote_state.revision = Some(local_state.revision.unwrap_or(0) + 1);
        self.save_local_state(&remote_state)?;

        Ok(())
    }

    /// Looks up an memdb by an SDK info if it's available.
    ///
    /// This returns a memdb wrapped in an arc as internally the system
    /// might try to unload the memdb if no longer needed.  If the MemDb
    /// does not exist, a `UnknownSdk` error is returned.
    pub fn get_memdb(&self, info: &SdkInfo) -> Result<Arc<MemDb<'static>>> {
        let local_state = self.get_local_state()?;

        // make sure we check in the local state first if the SDK exists.
        // if we go directly to the memdbs array or look at the file system
        // we might start to consider things that are not available yet or
        // not available any longer.
        if local_state.get_sdk(&info.memdb_filename()).is_some() {
            if let Some(arc) = self.memdbs.read().unwrap().get(info) {
                return Ok(arc.clone());
            }
            let memdb = MemDb::from_path(self.path.join(info.memdb_filename()))?;
            self.memdbs.write().unwrap().insert(info.clone(), Arc::new(memdb));
            if let Some(arc) = self.memdbs.read().unwrap().get(info) {
                return Ok(arc.clone());
            }
        }

        Err(ErrorKind::UnknownSdk.into())
    }

    /// Looks up an memdb by an SDK info as string if available.
    pub fn get_memdb_from_sdk_id(&self, sdk_id: &str) -> Result<Arc<MemDb<'static>>> {
        if let Some(sdk_info) = SdkInfo::from_filename(sdk_id) {
            self.get_memdb(&sdk_info)
        } else {
            Err(ErrorKind::UnknownSdk.into())
        }
    }
}
<commit_msg>Do not bump revision if there are no changes<commit_after>//! Provide access to locally cached memdb SDKs
//!
//! The `MemDbStash` pulls in remote SDKs from an S3 bucket and provides
//! access to it.  This is used by the symbol server to manage the local
//! cache and also to refer to memdb files that are mmap'ed in.
use std::fs;
use std::io;
use std::iter::FromIterator;
use std::path::{Path, PathBuf};
use std::collections::{HashMap, HashSet};
use std::collections::hash_map::Values as HashMapValuesIter;
use std::sync::{Arc, RwLock};

use serde_json;
use xz2::write::XzDecoder;
use chrono::UTC;

use super::config::Config;
use super::sdk::SdkInfo;
use super::s3::S3;
use super::memdb::MemDb;
use super::utils::{ProgressIndicator, copy_with_progress, HumanDuration};
use super::{Result, ResultExt, ErrorKind};

/// Helper for synching
pub struct SyncOptions {
    pub user_facing: bool,
}

/// The main memdb stash type
pub struct MemDbStash {
    path: PathBuf,
    s3: S3,
    local_state: RwLock<Option<Arc<SdkSyncState>>>,
    memdbs: RwLock<HashMap<SdkInfo, Arc<MemDb<'static>>>>,
}

/// Information about a remotely available SDK
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct RemoteSdk {
    filename: String,
    info: SdkInfo,
    size: u64,
    etag: String,
}

#[derive(Serialize, Deserialize, Default, Debug, Clone)]
struct SdkSyncState {
    sdks: HashMap<String, RemoteSdk>,
    revision: Option<u64>,
}

/// Information about the health of the stash sync
#[derive(Debug)]
pub struct SyncStatus {
    remote_total: u32,
    missing: u32,
    different: u32,
    revision: u64,
}

impl RemoteSdk {
    /// Creates a remote SDK object from some information
    pub fn new(filename: String, info: SdkInfo, etag: String, size: u64) -> RemoteSdk {
        RemoteSdk {
            filename: filename,
            info: info,
            etag: etag,
            size: size,
        }
    }

    /// The remotely visible filename for the SDK
    pub fn filename(&self) -> &str {
        &self.filename
    }

    /// The local filename the SDK has in the stash folder
    pub fn local_filename(&self) -> &str {
        self.filename.trim_right_matches('z')
    }

    /// The size of the SDK in bytes
    pub fn size(&self) -> u64 {
        self.size
    }

    /// Returns the SDK info
    pub fn info(&self) -> &SdkInfo {
        &self.info
    }
}

/// Iterator over the SDKs
pub type RemoteSdkIter<'a> = HashMapValuesIter<'a, String, RemoteSdk>;

impl SdkSyncState {
    pub fn get_sdk(&self, filename: &str) -> Option<&RemoteSdk> {
        self.sdks.get(filename)
    }

    pub fn update_sdk(&mut self, sdk: &RemoteSdk) {
        self.sdks.insert(sdk.local_filename().to_string(), sdk.clone());
    }

    pub fn sdks<'a>(&'a self) -> RemoteSdkIter<'a> {
        self.sdks.values()
    }
}

impl SyncStatus {

    /// Returns the lag (number of SDKs behind upstream)
    pub fn lag(&self) -> u32 {
        self.missing + self.different
    }

    /// Returns true if the local sync is still considered healthy
    pub fn is_healthy(&self) -> bool {
        let total = self.remote_total as f32;
        let lag = self.lag() as f32;
        lag / total < 0.10
    }

    /// Returns the revision of the stash
    pub fn revision(&self) -> u64 {
        self.revision
    }
}

impl Default for SyncOptions {
    fn default() -> SyncOptions {
        SyncOptions {
            user_facing: false,
        }
    }
}

impl MemDbStash {
    /// Opens a stash for a given config.
    pub fn new(config: &Config) -> Result<MemDbStash> {
        Ok(MemDbStash {
            path: config.get_symbol_dir()?.to_path_buf(),
            s3: S3::from_config(config)?,
            local_state: RwLock::new(None),
            memdbs: RwLock::new(HashMap::new()),
        })
    }

    fn get_local_sync_state_filename(&self) -> PathBuf {
        self.path.join("sync.state")
    }

    fn save_state(&self, new_state: &SdkSyncState, filename: &Path) -> Result<()> {
        let mut tmp_filename = filename.to_path_buf();
        tmp_filename.set_extension("tempstate");
        {
            let mut f = fs::File::create(&tmp_filename)?;
            serde_json::to_writer(&mut f, new_state)
                .chain_err(|| "Could not update sync state")?;
        }
        fs::rename(&tmp_filename, &filename)?;
        Ok(())
    }

    fn read_local_state(&self) -> Result<SdkSyncState> {
        let rv : SdkSyncState = match fs::File::open(&self.get_local_sync_state_filename()) {
            Ok(f) => serde_json::from_reader(f)
                .chain_err(|| "Parsing error on loading sync state")?,
            Err(err) => {
                if err.kind() == io::ErrorKind::NotFound {
                    Default::default()
                } else {
                    return Err(err).chain_err(|| "Error loading sync state");
                }
            }
        };
        let mut opt = self.local_state.write().unwrap();
        *opt = Some(Arc::new(rv.clone()));
        Ok(rv)
    }

    fn get_local_state(&self) -> Result<Arc<SdkSyncState>> {
        if let Some(ref arc) = *self.local_state.read().unwrap() {
            return Ok(arc.clone());
        }
        self.read_local_state()?;
        Ok(self.local_state.read().unwrap().as_ref().unwrap().clone())
    }

    fn save_local_state(&self, new_state: &SdkSyncState) -> Result<()> {
        self.save_state(new_state, &self.get_local_sync_state_filename())?;
        let mut opt = self.local_state.write().unwrap();
        *opt = Some(Arc::new(new_state.clone()));
        Ok(())
    }

    fn fetch_remote_state(&self) -> Result<SdkSyncState> {
        let mut sdks = HashMap::new();
        for remote_sdk in self.s3.list_upstream_sdks()? {
            sdks.insert(remote_sdk.local_filename().into(), remote_sdk);
        }
        Ok(SdkSyncState { sdks: sdks, revision: None })
    }

    fn update_sdk(&self, sdk: &RemoteSdk, options: &SyncOptions) -> Result<()> {
        // XXX: the progress bar here can stall out because we currently
        // need to buffer the download into memory in the s3 code :(
        let progress = if options.user_facing {
            ProgressIndicator::new(sdk.size() as usize)
        } else {
            info!("updating {}", sdk.info());
            ProgressIndicator::disabled()
        };
        let started = UTC::now();
        progress.set_message(&format!("Synchronizing {}", sdk.info()));
        let mut src = self.s3.download_sdk(sdk)?;
        let dst = fs::File::create(self.path.join(sdk.local_filename()))?;
        let mut dst = XzDecoder::new(dst);
        copy_with_progress(&progress, &mut src, &mut dst)?;
        progress.finish(&format!("Synchronized {}", sdk.info()));

        let duration = UTC::now() - started;
        if !options.user_facing {
            info!("updated {} in {}", sdk.info(), HumanDuration(duration));
        }
        Ok(())
    }

    fn remove_sdk(&self, sdk: &RemoteSdk, options: &SyncOptions) -> Result<()> {
        if options.user_facing {
            info!("  Deleting {}", sdk.info());
        } else {
            info!("removing {}", sdk.info());
        }
        if let Err(err) = fs::remove_file(self.path.join(sdk.local_filename())) {
            if err.kind() != io::ErrorKind::NotFound {
                return Err(err.into());
            }
        }
        Ok(())
    }

    /// Returns the current revision
    pub fn get_revision(&self) -> Result<u64> {
        Ok(self.read_local_state()?.revision.unwrap_or(0))
    }

    /// Checks the local stash against the server
    pub fn get_sync_status(&self) -> Result<SyncStatus> {
        let local_state = self.read_local_state()?;
        let remote_state = self.fetch_remote_state()?;

        let mut remote_total = 0;
        let mut missing = 0;
        let mut different = 0;

        for sdk in remote_state.sdks() {
            if let Some(local_sdk) = local_state.get_sdk(sdk.local_filename()) {
                if local_sdk != sdk {
                    different += 1;
                }
            } else {
                missing += 1;
            }
            remote_total += 1;
        }

        Ok(SyncStatus {
            remote_total: remote_total as u32,
            missing: missing as u32,
            different: different as u32,
            revision: local_state.revision.unwrap_or(0),
        })
    }

    /// Synchronize the local stash with the server
    pub fn sync(&self, options: SyncOptions) -> Result<()> {
        let mut local_state = self.read_local_state()?;
        let mut remote_state = self.fetch_remote_state()?;
        let started = UTC::now();
        let mut changed = false;

        let mut to_delete : HashSet<_> = HashSet::from_iter(
            local_state.sdks().map(|x| x.local_filename().to_string()));

        for sdk in remote_state.sdks() {
            let mut changed_something = false;
            if let Some(local_sdk) = local_state.get_sdk(sdk.local_filename()) {
                if local_sdk != sdk {
                    self.update_sdk(&sdk, &options)?;
                    changed_something = true;
                } else if options.user_facing {
                    println!("  ⸰ Unchanged {}", sdk.info());
                } else {
                    debug!("unchanged sdk {}", sdk.info());
                }
            } else {
                self.update_sdk(&sdk, &options)?;
                changed_something = true;
            }

            to_delete.remove(sdk.local_filename());
            if changed_something {
                changed = true;
                local_state.update_sdk(&sdk);
                local_state.revision = Some(local_state.revision.unwrap_or(0) + 1);
                self.save_local_state(&local_state)?;
            }
        }

        for local_filename in to_delete.iter() {
            if let Some(sdk) = local_state.get_sdk(local_filename) {
                self.remove_sdk(sdk, &options)?;
            }
        }

        let duration = UTC::now() - started;
        if options.user_facing {
            println!("Sync done in {}", HumanDuration(duration));
        } else if changed {
            info!("finished sync in {}", HumanDuration(duration));
        }

        // if we get this far, the remote state is indeed the local state.
        remote_state.revision = Some(local_state.revision.unwrap_or(0) + 1);
        self.save_local_state(&remote_state)?;

        Ok(())
    }

    /// Looks up an memdb by an SDK info if it's available.
    ///
    /// This returns a memdb wrapped in an arc as internally the system
    /// might try to unload the memdb if no longer needed.  If the MemDb
    /// does not exist, a `UnknownSdk` error is returned.
    pub fn get_memdb(&self, info: &SdkInfo) -> Result<Arc<MemDb<'static>>> {
        let local_state = self.get_local_state()?;

        // make sure we check in the local state first if the SDK exists.
        // if we go directly to the memdbs array or look at the file system
        // we might start to consider things that are not available yet or
        // not available any longer.
        if local_state.get_sdk(&info.memdb_filename()).is_some() {
            if let Some(arc) = self.memdbs.read().unwrap().get(info) {
                return Ok(arc.clone());
            }
            let memdb = MemDb::from_path(self.path.join(info.memdb_filename()))?;
            self.memdbs.write().unwrap().insert(info.clone(), Arc::new(memdb));
            if let Some(arc) = self.memdbs.read().unwrap().get(info) {
                return Ok(arc.clone());
            }
        }

        Err(ErrorKind::UnknownSdk.into())
    }

    /// Looks up an memdb by an SDK info as string if available.
    pub fn get_memdb_from_sdk_id(&self, sdk_id: &str) -> Result<Arc<MemDb<'static>>> {
        if let Some(sdk_info) = SdkInfo::from_filename(sdk_id) {
            self.get_memdb(&sdk_info)
        } else {
            Err(ErrorKind::UnknownSdk.into())
        }
    }
}
<|endoftext|>


<commit_before>use tag::TagType;

#[repr(C)]
struct MemoryMapTag {
	tag_type: u32,
	size: u32,
	entry_size: u32,
	/// This is currently set to `0`.
	entry_version: u32,
	first_entry: MemoryMapEntry
}

#[derive(Debug)]
#[repr(C, packed)]
struct MemoryMapEntry {
	/// The starting physical address
	base_addr: u64,
	/// The size of the memory region, in bytes.
	length: u64,
	/// The type of memory region contained.
	entry_type: u32,
	/// This should be set to `0` and ignored by the bootloader.
	reserved: u32
}

#[derive(Debug, PartialEq, Eq)]
enum MemoryType {
	Usable      = 1,
	UsableWithACPI = 3,
	Unusable       = 4,
}

impl MemoryMapEntry {
	fn base_address(&self) -> u64 {
		self.base_addr
	}

	fn length(&self) -> u64 {
		self.length
	}

	fn memory_type(&self) -> MemoryType {
		match self.entry_type {
			1 => MemoryType::Usable,
			3 => MemoryType::UsableWithACPI,
			_ => MemoryType::Unusable
		}
	}
}

struct MemoryMapIter {
	current_entry: *const MemoryMapEntry,
	// We need to know the entry size so we can calculate the
	// address of the next memory map entry.
	entry_size: u32,
}<commit_msg>Add iterator for memory maps.<commit_after>#![allow(dead_code)]

#[repr(C)]
struct MemoryMapTag {
	tag_type: u32,
	size: u32,
	entry_size: u32,
	/// This is currently set to `0`.
	entry_version: u32,
	first_entry: MemoryMapEntry
}

#[derive(Debug)]
#[repr(C, packed)]
struct MemoryMapEntry {
	/// The starting physical address
	base_addr: u64,
	/// The size of the memory region, in bytes.
	length: u64,
	/// The type of memory region contained.
	entry_type: u32,
	/// This should be set to `0` and ignored by the bootloader.
	reserved: u32
}

#[derive(Debug, PartialEq, Eq)]
enum MemoryType {
	Usable         = 1,
	UsableWithACPI = 3,
	Unusable       = 4,
}

impl MemoryMapEntry {
	fn base_address(&self) -> u64 {
		self.base_addr
	}

	fn length(&self) -> u64 {
		self.length
	}

	fn memory_type(&self) -> MemoryType {
		match self.entry_type {
			1 => MemoryType::Usable,
			3 => MemoryType::UsableWithACPI,
			_ => MemoryType::Unusable
		}
	}

	fn is_usable_region(&self) -> bool {
		self.memory_type() == MemoryType::Usable
	}
}

struct MemoryMapIter {
	current_entry: *const MemoryMapEntry,
	final_entry: *const MemoryMapEntry,
	// We need to know the entry size so we can calculate the
	// address of the next memory map entry.
	entry_size: u32,
}

impl MemoryMapIter {
	fn new(current_entry: *const MemoryMapEntry,
		   final_entry: *const MemoryMapEntry,
		   entry_size: u32) -> MemoryMapIter
	{
		MemoryMapIter {
			current_entry: current_entry,
			final_entry: final_entry,
			entry_size: entry_size,
		}
	}
}

impl Iterator for MemoryMapIter {
	type Item = &'static MemoryMapEntry;

	fn next(&mut self) -> Option<Self::Item> {
		// Loop until the next usable memory region. We ignore the unusable regions
		// per the Multiboot2 standard requirements.
		loop {
			if self.current_entry <= self.final_entry {
				let current_entry = unsafe { &*self.current_entry };
				let next_entry = (self.current_entry as u64) + (self.entry_size as u64);

				self.current_entry = next_entry as *const MemoryMapEntry;

				if current_entry.is_usable_region() {
					return Some(current_entry);
				}

			} else {
				// We have fallen off the end of the memory map.
				return None;
			}
		}
	}
}<|endoftext|>


<commit_before>use runtime::Runtime;
use std::error::Error;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::fmt::Display;
use std::path::Path;
use std::result::Result;

use module::todo::TodoModule;

mod todo;

#[derive(Debug)]
pub struct ModuleError {
    desc: String,
}

impl ModuleError {
    fn mk(desc: &'static str) -> ModuleError {
        ModuleError {
            desc: desc.to_owned().to_string(),
        }
    }
}

impl Error for ModuleError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        None
    }

}

impl Display for ModuleError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "ModuleError: {}", self.description())
    }
}

pub type ModuleResult = Result<(), ModuleError>;

pub trait Module {

    fn new(rt : &Runtime) -> Self;
    fn callnames() -> &'static [&'static str];
    fn name(&self) -> &'static str;

    fn execute(&self, rt : &Runtime) -> ModuleResult;
    fn shutdown(&self, rt : &Runtime) -> ModuleResult;

}

pub mod file {

    use regex::Regex;

    pub struct ParserError {
        summary: String,
        parsertext: String,
        index: i32,
        explanation: Option<String>,
    }

    impl ParserError {
        fn new(sum: &'static str, text: String, idx: i32, expl: &'static str) -> ParserError {
            ParserError {
                summary: String::from(sum),
                parsertext: text,
                index: idx,
                explanation: Some(String::from(expl)),
            }
        }

        fn short(sum: &'static str, text: String, idx: i32) -> ParserError {
            ParserError {
                summary: String::from(sum),
                parsertext: text,
                index: idx,
                explanation: None
            }
        }
    }

    pub mod header {

        pub enum FileHeaderSpec {
            Null,
            Bool,
            Integer,
            UInteger,
            Float,
            Text,
            Key { name: String, value_type: Box<FileHeaderSpec> },
            Array { allowed_types: Box<Vec<FileHeaderSpec>> },
        }

        pub enum FileHeaderData {
            Null,
            Bool(bool),
            Integer(i64),
            UInteger(u64),
            Float(f64),
            Text(String),
            Key { name: String, value: Box<FileHeaderData> },
            Array { values: Box<Vec<FileHeaderData>> },
        }

        pub trait FileHeaderParser {
            fn new(spec: &FileHeaderSpec) -> Self;
            fn read(&self, string: &String) -> Result<FileHeaderData, super::ParserError>;
            fn write(&self, data: &FileHeaderData) -> Result<String, super::ParserError>;
        }

    }

    pub trait FileData {
        fn get_fulltext(&self) -> String;
        fn get_abbrev(&self) -> String;
    }

    pub trait FileParser {
        fn new(header_parser: &header::FileHeaderParser) -> FileParser;
        fn read(&self, string: String) -> (header::FileHeaderData, FileData);
        fn write(&self, hdr: &header::FileHeaderData, data: &FileData) -> Result<String, ParserError>;
    }

    pub type HeaderDataTpl = (Option<String>, Option<String>);

    pub fn divide_text(text: String) -> Result<HeaderDataTpl, ParserError> {
        let re = Regex::new(r"(?m)^\-\-\-$\n(.*)^\-\-\-$\n(.*)").unwrap();

        let captures = re.captures(&text[..]).unwrap_or(
            return Err(ParserError::new("Cannot run regex on text",
                                        text.clone(), 0,
                                        "Cannot run regex on text to divide it into header and content."))
        );

        if captures.len() != 2 {
            return Err(ParserError::new("Unexpected Regex output",
                                        text.clone(), 0,
                                        "The regex to divide text into header and content had an unexpected output."))
        }

        let header  = captures.at(0).map(|s| String::from(s));
        let content = captures.at(1).map(|s| String::from(s));
        Ok((header, content))
    }

}
<commit_msg>FileParser -> FileDataParser<commit_after>use runtime::Runtime;
use std::error::Error;
use std::fmt::Formatter;
use std::fmt::Result as FMTResult;
use std::fmt::Display;
use std::path::Path;
use std::result::Result;

use module::todo::TodoModule;

mod todo;

#[derive(Debug)]
pub struct ModuleError {
    desc: String,
}

impl ModuleError {
    fn mk(desc: &'static str) -> ModuleError {
        ModuleError {
            desc: desc.to_owned().to_string(),
        }
    }
}

impl Error for ModuleError {

    fn description(&self) -> &str {
        &self.desc[..]
    }

    fn cause(&self) -> Option<&Error> {
        None
    }

}

impl Display for ModuleError {
    fn fmt(&self, f: &mut Formatter) -> FMTResult {
        write!(f, "ModuleError: {}", self.description())
    }
}

pub type ModuleResult = Result<(), ModuleError>;

pub trait Module {

    fn new(rt : &Runtime) -> Self;
    fn callnames() -> &'static [&'static str];
    fn name(&self) -> &'static str;

    fn execute(&self, rt : &Runtime) -> ModuleResult;
    fn shutdown(&self, rt : &Runtime) -> ModuleResult;

}

pub mod file {

    use regex::Regex;

    pub struct ParserError {
        summary: String,
        parsertext: String,
        index: i32,
        explanation: Option<String>,
    }

    impl ParserError {
        fn new(sum: &'static str, text: String, idx: i32, expl: &'static str) -> ParserError {
            ParserError {
                summary: String::from(sum),
                parsertext: text,
                index: idx,
                explanation: Some(String::from(expl)),
            }
        }

        fn short(sum: &'static str, text: String, idx: i32) -> ParserError {
            ParserError {
                summary: String::from(sum),
                parsertext: text,
                index: idx,
                explanation: None
            }
        }
    }

    pub mod header {

        pub enum FileHeaderSpec {
            Null,
            Bool,
            Integer,
            UInteger,
            Float,
            Text,
            Key { name: String, value_type: Box<FileHeaderSpec> },
            Array { allowed_types: Box<Vec<FileHeaderSpec>> },
        }

        pub enum FileHeaderData {
            Null,
            Bool(bool),
            Integer(i64),
            UInteger(u64),
            Float(f64),
            Text(String),
            Key { name: String, value: Box<FileHeaderData> },
            Array { values: Box<Vec<FileHeaderData>> },
        }

        pub trait FileHeaderParser {
            fn new(spec: &FileHeaderSpec) -> Self;
            fn read(&self, string: &String) -> Result<FileHeaderData, super::ParserError>;
            fn write(&self, data: &FileHeaderData) -> Result<String, super::ParserError>;
        }

    }

    pub trait FileData {
        fn get_fulltext(&self) -> String;
        fn get_abbrev(&self) -> String;
    }

    pub trait FileDataParser {
        fn new() -> FileDataParser;
        fn read(&self, string: String) -> FileData;
        fn write(&self, data: &FileData) -> Result<String, ParserError>;
    }

    pub type HeaderDataTpl = (Option<String>, Option<String>);

    pub fn divide_text(text: String) -> Result<HeaderDataTpl, ParserError> {
        let re = Regex::new(r"(?m)^\-\-\-$\n(.*)^\-\-\-$\n(.*)").unwrap();

        let captures = re.captures(&text[..]).unwrap_or(
            return Err(ParserError::new("Cannot run regex on text",
                                        text.clone(), 0,
                                        "Cannot run regex on text to divide it into header and content."))
        );

        if captures.len() != 2 {
            return Err(ParserError::new("Unexpected Regex output",
                                        text.clone(), 0,
                                        "The regex to divide text into header and content had an unexpected output."))
        }

        let header  = captures.at(0).map(|s| String::from(s));
        let content = captures.at(1).map(|s| String::from(s));
        Ok((header, content))
    }

}
<|endoftext|>


<commit_before>
use scope::*;

// pub enum MapWalkReturn<'a, T> {
//     NoMatch,
//     InterimMatch(&'a T),
//     FinalMatch(&'a T)
// }

pub enum MapWalkState<'a, K: 'a, T: 'a> {
    NoMatch,
    InterimMatch(&'a K, Option<&'a T>),
    FinalMatch(&'a T)
}

#[allow(dead_code)]
pub struct MapWalkIter<'a, K: 'a, T: 'a>
{
    ctx: &'a mut Context,
    scope_id: String,
    key: K,
    next_key: Option<K>,
    func: FnMut(&Symbols, &K) -> MapWalkState<'a, K, T>
}

impl<'a, K, T> Iterator for MapWalkIter<'a, K, T>
{
    type Item = MapWalkState<'a, K, T>;

    fn next(&mut self) -> Option<Self::Item> {
        let (map_id, parent_id) = {
            let scope = self.ctx.get_scope(&self.scope_id);
            if scope.is_none() { return None; }
            let scope = scope.unwrap();
            (scope.map_id().to_owned(), scope.parent_id().map(|parent_id| parent_id.to_owned()))
        };

        let map_ref = self.ctx.get_map(&map_id).expect("Map must exist for scope");
        let state = (self.func)(map_ref, &self.key);

        // match state {
        //     Some(MapWalkState::InterimMatch(ref next_key, ref v)) => {
        //         self.next_key = Some(next_key);
        //         return state;
        //     }

        //     Some(MapWalkState::FinalMatch(ref v)) => { return state; }

        //     _ => {}
        // };

        // // Prepare next iteration
        // if parent_id.is_none() { return None; }
        // self.next_scope_id = parent_id.map(|s| s.to_owned());

        // Some(MapWalkState::NoMatch)
        None
    }
}
<commit_msg>Working `WalkMapIter`.<commit_after>
use std::hash::Hash;
use scope::*;

// pub enum MapWalkReturn<'a, T> {
//     NoMatch,
//     InterimMatch(&'a T),
//     FinalMatch(&'a T)
// }

pub enum MapWalkState<'a, K: 'a, T: 'a> {
    NoMatch,
    InterimMatch(&'a K, Option<&'a T>),
    FinalMatch(&'a T)
}

#[allow(dead_code)]
pub struct MapWalkIter<'a, K: 'a, T: 'a>
{
    ctx: &'a mut Context,
    scope_id: String,
    key: K,
    func: Box<FnMut(&Symbols, &K) -> MapWalkState<'a, K, T>>
}

impl<'a, K: Hash + Clone, T> MapWalkIter<'a, K, T> {
    pub fn new(ctx: &'a mut Context, key: K, func: Box<FnMut(&Symbols, &K) -> MapWalkState<'a, K, T>>) -> Self {
        let scope_id = ctx.scope_ref().expect("Context must have valid scope").id().to_owned();
        // let scope_id = scope.id().to_owned();

        MapWalkIter {
            ctx: ctx,
            scope_id: scope_id,
            key: key,
            func: func
        }
    }
}

impl<'a, K: Hash + Clone, T> Iterator for MapWalkIter<'a, K, T>
{
    type Item = MapWalkState<'a, K, T>;

    fn next(&mut self) -> Option<Self::Item> {
        let (map_id, parent_id) = {
            let scope = self.ctx.get_scope(&self.scope_id);
            if scope.is_none() { return None; }
            let scope = scope.unwrap();
            (scope.map_id().to_owned(), scope.parent_id().map(|parent_id| parent_id.to_owned()))
        };

        let map_ref = self.ctx.get_map(&map_id).expect("Map must exist for scope");
        // let box ref func = func;
        let state = (self.func.as_mut())(map_ref, &self.key);

        match state {
            MapWalkState::InterimMatch(next_key, v) => {
                if parent_id.is_none() { return None; }
                self.scope_id = parent_id.unwrap();
                self.key = next_key.to_owned();
                Some(state)
            }

            MapWalkState::FinalMatch(..) => Some(state),
            _ if parent_id.is_some() => Some(MapWalkState::NoMatch),
            _ => None
        }
    }
}
<|endoftext|>


<commit_before>use std::cmp;
use std::fmt;

use std::io::prelude::*;
use std::io;

const DEFAULT_BUF_SIZE: usize = 64 * 1024;

/// A copy of `std::io::BufReader` with additional methods needed for multipart support.
pub struct CustomBufReader<R> {
    inner: R,
    buf: Vec<u8>,
    pos: usize,
    cap: usize,
}

impl<R: Read> CustomBufReader<R> {
    #[doc(hidden)]
    pub fn new(inner: R) -> Self {
        CustomBufReader::with_capacity(DEFAULT_BUF_SIZE, inner)
    }

    #[doc(hidden)]
    pub fn with_capacity(cap: usize, inner: R) -> Self {
        CustomBufReader {
            inner: inner,
            buf: vec![0; cap],
            pos: 0,
            cap: 0,
        }
    }

    #[doc(hidden)]
    pub fn fill_buf_min(&mut self, min: usize) -> io::Result<&[u8]> {
        if self.pos == self.cap {
            self.cap = try!(self.inner.read(&mut self.buf));
            self.pos = 0;
        } else if min > self.cap - self.pos {
            self.cap += try!(self.inner.read(&mut self.buf[self.cap..]));
        }

        Ok(&self.buf[self.pos..self.cap])
    }

    #[doc(hidden)]
    pub fn get_ref(&self) -> &R { &self.inner }
}

impl<R: Read> Read for CustomBufReader<R> {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        // If we don't have any buffered data and we're doing a massive read
        // (larger than our internal buffer), bypass our internal buffer
        // entirely.
        if self.pos == self.cap && buf.len() >= self.buf.len() {
            return self.inner.read(buf);
        }
        let nread = {
            let mut rem = try!(self.fill_buf());
            try!(rem.read(buf))
        };
        self.consume(nread);
        Ok(nread)
    }
}

impl<R: Read> BufRead for CustomBufReader<R> {
    fn fill_buf(&mut self) -> io::Result<&[u8]> {
        // If we've reached the end of our internal buffer then we need to fetch
        // some more data from the underlying reader.
        if self.pos == self.cap {
            self.cap = try!(self.inner.read(&mut self.buf));
            self.pos = 0;
        }
        Ok(&self.buf[self.pos..self.cap])
    }

    fn consume(&mut self, amt: usize) {
        self.pos = cmp::min(self.pos + amt, self.cap);
    }
}

impl<R> fmt::Debug for CustomBufReader<R> where R: fmt::Debug {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.debug_struct("BufReader")
            .field("reader", &self.inner)
            .field("buffer", &format_args!("{}/{}", self.cap - self.pos, self.buf.len()))
            .finish()
    }
}
<commit_msg>Fix building on Stable (until next release)<commit_after>use std::cmp;
use std::fmt;

use std::io::prelude::*;
use std::io;

const DEFAULT_BUF_SIZE: usize = 64 * 1024;

/// A copy of `std::io::BufReader` with additional methods needed for multipart support.
pub struct CustomBufReader<R> {
    inner: R,
    buf: Vec<u8>,
    pos: usize,
    cap: usize,
}

impl<R: Read> CustomBufReader<R> {
    #[doc(hidden)]
    pub fn new(inner: R) -> Self {
        CustomBufReader::with_capacity(DEFAULT_BUF_SIZE, inner)
    }

    #[doc(hidden)]
    pub fn with_capacity(cap: usize, inner: R) -> Self {
        CustomBufReader {
            inner: inner,
            buf: vec![0; cap],
            pos: 0,
            cap: 0,
        }
    }

    #[doc(hidden)]
    pub fn fill_buf_min(&mut self, min: usize) -> io::Result<&[u8]> {
        if self.pos == self.cap {
            self.cap = try!(self.inner.read(&mut self.buf));
            self.pos = 0;
        } else if min > self.cap - self.pos {
            self.cap += try!(self.inner.read(&mut self.buf[self.cap..]));
        }

        Ok(&self.buf[self.pos..self.cap])
    }

    #[doc(hidden)]
    pub fn get_ref(&self) -> &R { &self.inner }
}

impl<R: Read> Read for CustomBufReader<R> {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        // If we don't have any buffered data and we're doing a massive read
        // (larger than our internal buffer), bypass our internal buffer
        // entirely.
        if self.pos == self.cap && buf.len() >= self.buf.len() {
            return self.inner.read(buf);
        }
        let nread = {
            let mut rem = try!(self.fill_buf());
            try!(rem.read(buf))
        };
        self.consume(nread);
        Ok(nread)
    }
}

impl<R: Read> BufRead for CustomBufReader<R> {
    fn fill_buf(&mut self) -> io::Result<&[u8]> {
        // If we've reached the end of our internal buffer then we need to fetch
        // some more data from the underlying reader.
        if self.pos == self.cap {
            self.cap = try!(self.inner.read(&mut self.buf));
            self.pos = 0;
        }
        Ok(&self.buf[self.pos..self.cap])
    }

    fn consume(&mut self, amt: usize) {
        self.pos = cmp::min(self.pos + amt, self.cap);
    }
}

impl<R> fmt::Debug for CustomBufReader<R> where R: fmt::Debug {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        fmt.write_fmt(format_args!(
            "CustomBufReader {{ reader: {:?}, buffer: {}/{}}}",
            &self.inner, self.cap - self.pos, self.buf.len()
        ))

        /* FIXME (07/26/15): Switch to this impl after the next Stable release.
        fmt.debug_struct("CustomBufReader")
            .field("reader", &self.inner)
            .field("buffer", &format_args!("{}/{}", ))
            .finish()
        */
    }
}
<|endoftext|>