<commit_before>extern crate libc;

#[repr(C)]
#[derive(Copy, Clone)]
pub struct roaring_bitmap_s {
    pub high_low_container: roaring_array_s,
    pub copy_on_write: bool
}

#[repr(C)]
#[derive(Copy, Clone)]
pub struct roaring_array_s {
    pub size: ::libc::int32_t,
    pub allocation_size: ::libc::int32_t,
    pub containers: *mut *mut ::std::os::raw::c_void,
    pub keys: *mut ::libc::uint16_t,
    pub typecodes: *mut ::libc::uint8_t
}

#[link(name = "roaring", kind = "static")]
extern "C" {
    pub fn roaring_bitmap_create() -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_create_with_capacity(cap: ::libc::uint32_t) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_of_ptr(n_args: ::libc::size_t, vals: *const ::libc::uint32_t) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_of(n: ::libc::size_t, ...) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_copy(r: *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_printf(ra: *const roaring_bitmap_s);
    pub fn roaring_bitmap_and(x1: *const roaring_bitmap_s, x2: *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_and_inplace(x1: *mut roaring_bitmap_s, x2: *const roaring_bitmap_s);
    pub fn roaring_bitmap_or(x1: *const roaring_bitmap_s, x2: *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_or_inplace(x1: *mut roaring_bitmap_s, x2: *const roaring_bitmap_s);
    pub fn roaring_bitmap_or_many(number: ::libc::size_t, x: *mut *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_or_many_heap(number: ::libc::uint32_t, x: *mut *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_xor(x1: *const roaring_bitmap_s, x2: *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_xor_inplace(x1: *mut roaring_bitmap_s, x2: *const roaring_bitmap_s);
    pub fn roaring_bitmap_flip(x1: *const roaring_bitmap_s,
                               range_start: ::libc::uint64_t, range_end: ::libc::uint64_t)
                               -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_flip_inplace(x1: *mut roaring_bitmap_s,
                                       range_start: ::libc::uint64_t,
                                       range_end: ::libc::uint64_t);
    pub fn roaring_bitmap_xor_many(number: ::libc::size_t, x: *mut *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_andnot(x1: *const roaring_bitmap_s, x2: *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_andnot_inplace(x1: *mut roaring_bitmap_s, x2: *const roaring_bitmap_s);
    pub fn roaring_bitmap_free(r: *mut roaring_bitmap_s);
    pub fn roaring_bitmap_add(r: *mut roaring_bitmap_s, x: ::libc::uint32_t);
    pub fn roaring_bitmap_add_many(r: *mut roaring_bitmap_s, n_args: ::libc::size_t, vals: *const ::libc::uint32_t);
    pub fn roaring_bitmap_remove(r: *mut roaring_bitmap_s, x: ::libc::uint32_t);
    pub fn roaring_bitmap_contains(r: *const roaring_bitmap_s, x: ::libc::uint32_t) -> bool;
    pub fn roaring_bitmap_get_cardinality(ra: *const roaring_bitmap_s) -> ::libc::uint64_t;
    pub fn roaring_bitmap_to_uint32_array(ra: *const roaring_bitmap_s, cardinality: *mut ::libc::uint32_t);
    pub fn roaring_bitmap_remove_run_compression(r: *mut roaring_bitmap_s) -> bool;
    pub fn roaring_bitmap_run_optimize(r: *mut roaring_bitmap_s) -> bool;
    pub fn roaring_bitmap_serialize(ra: *const roaring_bitmap_s, buf: *mut ::libc::c_char) -> ::libc::size_t;
    pub fn roaring_bitmap_deserialize(buf: *const ::libc::c_void) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_portable_deserialize(buf: *const ::libc::c_char) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_portable_size_in_bytes(ra: *const roaring_bitmap_s) -> ::libc::size_t;
    pub fn roaring_bitmap_portable_serialize(ra: *const roaring_bitmap_s, buf: *mut ::libc::c_char) -> ::libc::size_t;
    pub fn roaring_bitmap_is_empty(ra: *const roaring_bitmap_s) -> bool;
    /* TODO  pub fn roaring_iterate(ra: *mut roaring_bitmap_s,
                                    iterator: roaring_iterator,
                                    ptr: *mut ::libc::c_void) -> bool; */
    pub fn roaring_bitmap_equals(ra1: *mut roaring_bitmap_s, ra2: *mut roaring_bitmap_s) -> bool;
}
<commit_msg>Add roaring_bitmap_statistics to croaring-sys<commit_after>extern crate libc;

#[repr(C)]
#[derive(Copy, Clone)]
pub struct roaring_bitmap_s {
    pub high_low_container: roaring_array_s,
    pub copy_on_write: bool
}

#[repr(C)]
#[derive(Copy, Clone)]
pub struct roaring_array_s {
    pub size: ::libc::int32_t,
    pub allocation_size: ::libc::int32_t,
    pub containers: *mut *mut ::std::os::raw::c_void,
    pub keys: *mut ::libc::uint16_t,
    pub typecodes: *mut ::libc::uint8_t
}

#[repr(C)]
#[derive(Copy, Clone)]
pub struct roaring_statistics_s {
    pub n_containers: ::libc::uint32_t,
    pub n_array_containers: ::libc::uint32_t,
    pub n_run_containers: ::libc::uint32_t,
    pub n_bitset_containers: ::libc::uint32_t,
    pub n_values_array_containers: ::libc::uint32_t,
    pub n_values_run_containers: ::libc::uint32_t,
    pub n_values_bitset_containers: ::libc::uint32_t,
    pub n_bytes_array_containers: ::libc::uint32_t,
    pub n_bytes_run_containers: ::libc::uint32_t,
    pub n_bytes_bitset_containers: ::libc::uint32_t,
    pub max_value: ::libc::uint32_t,
    pub min_value: ::libc::uint32_t,
    pub sum_value: ::libc::uint64_t,
    pub cardinality: ::libc::uint64_t
}

impl Default for roaring_statistics_s {
    fn default() -> roaring_statistics_s { unsafe { std::mem::zeroed() } }
}

#[link(name = "roaring", kind = "static")]
extern "C" {
    pub fn roaring_bitmap_create() -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_create_with_capacity(cap: ::libc::uint32_t) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_of_ptr(n_args: ::libc::size_t, vals: *const ::libc::uint32_t) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_of(n: ::libc::size_t, ...) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_copy(r: *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_printf(ra: *const roaring_bitmap_s);
    pub fn roaring_bitmap_and(x1: *const roaring_bitmap_s, x2: *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_and_inplace(x1: *mut roaring_bitmap_s, x2: *const roaring_bitmap_s);
    pub fn roaring_bitmap_or(x1: *const roaring_bitmap_s, x2: *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_or_inplace(x1: *mut roaring_bitmap_s, x2: *const roaring_bitmap_s);
    pub fn roaring_bitmap_or_many(number: ::libc::size_t, x: *mut *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_or_many_heap(number: ::libc::uint32_t, x: *mut *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_xor(x1: *const roaring_bitmap_s, x2: *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_xor_inplace(x1: *mut roaring_bitmap_s, x2: *const roaring_bitmap_s);
    pub fn roaring_bitmap_flip(x1: *const roaring_bitmap_s,
                               range_start: ::libc::uint64_t, range_end: ::libc::uint64_t)
                               -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_flip_inplace(x1: *mut roaring_bitmap_s,
                                       range_start: ::libc::uint64_t,
                                       range_end: ::libc::uint64_t);
    pub fn roaring_bitmap_xor_many(number: ::libc::size_t, x: *mut *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_andnot(x1: *const roaring_bitmap_s, x2: *const roaring_bitmap_s) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_andnot_inplace(x1: *mut roaring_bitmap_s, x2: *const roaring_bitmap_s);
    pub fn roaring_bitmap_free(r: *mut roaring_bitmap_s);
    pub fn roaring_bitmap_add(r: *mut roaring_bitmap_s, x: ::libc::uint32_t);
    pub fn roaring_bitmap_add_many(r: *mut roaring_bitmap_s, n_args: ::libc::size_t, vals: *const ::libc::uint32_t);
    pub fn roaring_bitmap_remove(r: *mut roaring_bitmap_s, x: ::libc::uint32_t);
    pub fn roaring_bitmap_contains(r: *const roaring_bitmap_s, x: ::libc::uint32_t) -> bool;
    pub fn roaring_bitmap_get_cardinality(ra: *const roaring_bitmap_s) -> ::libc::uint64_t;
    pub fn roaring_bitmap_to_uint32_array(ra: *const roaring_bitmap_s, cardinality: *mut ::libc::uint32_t);
    pub fn roaring_bitmap_remove_run_compression(r: *mut roaring_bitmap_s) -> bool;
    pub fn roaring_bitmap_run_optimize(r: *mut roaring_bitmap_s) -> bool;
    pub fn roaring_bitmap_serialize(ra: *const roaring_bitmap_s, buf: *mut ::libc::c_char) -> ::libc::size_t;
    pub fn roaring_bitmap_deserialize(buf: *const ::libc::c_void) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_portable_deserialize(buf: *const ::libc::c_char) -> *mut roaring_bitmap_s;
    pub fn roaring_bitmap_portable_size_in_bytes(ra: *const roaring_bitmap_s) -> ::libc::size_t;
    pub fn roaring_bitmap_portable_serialize(ra: *const roaring_bitmap_s, buf: *mut ::libc::c_char) -> ::libc::size_t;
    pub fn roaring_bitmap_is_empty(ra: *const roaring_bitmap_s) -> bool;
    /* TODO  pub fn roaring_iterate(ra: *mut roaring_bitmap_s,
                                    iterator: roaring_iterator,
                                    ptr: *mut ::libc::c_void) -> bool; */
    pub fn roaring_bitmap_equals(ra1: *mut roaring_bitmap_s, ra2: *mut roaring_bitmap_s) -> bool;
    pub fn roaring_bitmap_statistics(ra: *const roaring_bitmap_s, stat: *mut roaring_statistics_s);
}
<|endoftext|>


<commit_before>//
// Copyright 2021 The Project Oak Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

use std::{io::Read, path::PathBuf};

pub fn read_file(path: &PathBuf) -> String {
    let mut file = std::fs::File::open(path).expect("could not open file");
    let mut contents = String::new();
    file.read_to_string(&mut contents)
        .expect("could not read file contents");
    contents
}

/// Return an iterator of all the first-party and non-ignored files in the repository, which can be
/// then additionally filtered by the caller.
pub fn source_files() -> impl Iterator<Item = PathBuf> {
    let walker = walkdir::WalkDir::new(".").into_iter();
    walker
        .filter_entry(|e| !is_ignored_entry(e))
        .filter_map(Result::ok)
        .map(|e| e.into_path())
}

pub fn file_contains(path: &PathBuf, pattern: &str) -> bool {
    if path.is_file() {
        let mut file = std::fs::File::open(path).expect("could not open file");
        let mut contents = String::new();
        // Content may be non-UTF-8, in which case we just return false.
        if file.read_to_string(&mut contents).is_ok() {
            contents.contains(pattern)
        } else {
            false
        }
    } else {
        false
    }
}

pub fn example_toml_files() -> impl Iterator<Item = PathBuf> {
    source_files().filter(is_example_toml_file)
}

pub fn fuzz_config_toml_files() -> impl Iterator<Item = PathBuf> {
    source_files().filter(is_fuzz_config_toml_file)
}

/// Return an iterator of all known Cargo Manifest files that define crates.
pub fn crate_manifest_files() -> impl Iterator<Item = PathBuf> {
    source_files()
        .filter(is_cargo_toml_file)
        .filter(|p| is_cargo_package_file(p))
}

/// Return an iterator of all known Cargo Manifest files that define workspaces.
pub fn workspace_manifest_files() -> impl Iterator<Item = PathBuf> {
    source_files()
        .filter(is_cargo_toml_file)
        .filter(|p| is_cargo_workspace_file(p))
}

/// Return whether the provided path refers to a source file in a programming language.
pub fn is_source_code_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".cc")
        || filename.ends_with(".h")
        || filename.ends_with(".rs")
        || filename.ends_with(".proto")
        || filename.ends_with(".js")
        || filename.ends_with(".go")
}

/// Return whether the provided path refers to a source file that can be formatted by clang-tidy.
pub fn is_clang_format_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".cc") || filename.ends_with(".h") || filename.ends_with(".proto")
}

/// Return whether the provided path refers to a Bazel file (`BUILD`, `WORKSPACE`, or `*.bzl`)
pub fn is_bazel_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename == "BUILD" || filename == "WORKSPACE" || filename.ends_with(".bzl")
}

pub fn is_build_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename == "BUILD"
}

/// Return whether the provided path refers to a markdown file (`*.md`)
pub fn is_markdown_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".md")
}

pub fn is_dockerfile(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with("Dockerfile")
}

pub fn is_toml_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".toml")
}

pub fn is_yaml_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".yaml") || filename.ends_with(".yml")
}

pub fn is_html_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".htm") || filename.ends_with(".html")
}

pub fn is_javascript_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".js") || filename.ends_with(".mjs")
}

pub fn is_typescript_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".ts") || filename.ends_with(".tsx")
}

pub fn is_shell_script(path: &PathBuf) -> bool {
    if path.is_file() {
        let mut file = std::fs::File::open(path).expect("could not open file");
        let mut contents = String::new();
        match file.read_to_string(&mut contents) {
            Ok(_size) => contents.starts_with("#!"),
            Err(_err) => false,
        }
    } else {
        false
    }
}

/// Return whether the provided path refers to a `fuzz` crate for fuzz testing with `cargo-fuzz`.
pub fn is_fuzzing_toml_file(path: &PathBuf) -> bool {
    format!("{:?}", path).contains("/fuzz/")
}

/// Return whether the provided path refers to a `Cargo.toml` file. Note that it does not
/// differentiate between workspace-level and crate-level files.
fn is_cargo_toml_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename == "Cargo.toml"
}

/// Return whether the provided path refers to a workspace-level `Cargo.toml` file, by looking at
/// the contents of the file.
fn is_cargo_workspace_file(path: &PathBuf) -> bool {
    // We naively look for the `[workspace]` string to appear in the contents of the file. A better
    // alternative would be to actually parse the file as `toml` and figure out whether it has a
    // `workspace` section, but it seems overkill for now.
    file_contains(path, "[workspace]")
}

/// Return whether the provided path refers to a `Cargo.toml` file that defines a crate, by looking
/// at the contents of the file.
fn is_cargo_package_file(path: &PathBuf) -> bool {
    // We naively look for the `[package]` string to appear in the contents of the file. A better
    // alternative would be to actually parse the file as `toml` and figure out whether it has a
    // `package` section, but it seems overkill for now.
    file_contains(path, "[package]")
}

fn is_example_toml_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename == "example.toml"
}

fn is_fuzz_config_toml_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename == "fuzz.toml"
}

fn is_ignored_entry(entry: &walkdir::DirEntry) -> bool {
    let path = entry.clone().into_path();
    // Simple heuristic to try and exclude generated files.
    is_ignored_path(&path) || file_contains(&path, "DO NOT EDIT")
}

/// Return whether to ignore the specified path. This is used by the `walker` package to efficiently
/// avoid descending into blacklisted directories.
fn is_ignored_path(path: &PathBuf) -> bool {
    let components = path.components().collect::<std::collections::HashSet<_>>();
    components.contains(&std::path::Component::Normal(".git".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-cache".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-bin".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-out".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-oak".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-clang-bin".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-clang-out".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-clang-oak".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-client-bin".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-client-out".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-client-oak".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-emscripten-bin".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-emscripten-out".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-emscripten-oak".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-wasm-bin".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-wasm-out".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-wasm-oak".as_ref()))
        || components.contains(&std::path::Component::Normal("cargo-cache".as_ref()))
        || components.contains(&std::path::Component::Normal("node_modules".as_ref()))
        || components.contains(&std::path::Component::Normal("protoc_out".as_ref())) // Code generated by protoc
        || components.contains(&std::path::Component::Normal("dist".as_ref())) // Code generated by webpack
        || components.contains(&std::path::Component::Normal("target".as_ref())) // Rust artifacts.
        || components.contains(&std::path::Component::Normal("third_party".as_ref()))
}

pub fn to_string(path: PathBuf) -> String {
    path.to_str().unwrap().to_string()
}
<commit_msg>Add cache to ignored paths for runner (#2175)<commit_after>//
// Copyright 2021 The Project Oak Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

use std::{io::Read, path::PathBuf};

pub fn read_file(path: &PathBuf) -> String {
    let mut file = std::fs::File::open(path).expect("could not open file");
    let mut contents = String::new();
    file.read_to_string(&mut contents)
        .expect("could not read file contents");
    contents
}

/// Return an iterator of all the first-party and non-ignored files in the repository, which can be
/// then additionally filtered by the caller.
pub fn source_files() -> impl Iterator<Item = PathBuf> {
    let walker = walkdir::WalkDir::new(".").into_iter();
    walker
        .filter_entry(|e| !is_ignored_entry(e))
        .filter_map(Result::ok)
        .map(|e| e.into_path())
}

pub fn file_contains(path: &PathBuf, pattern: &str) -> bool {
    if path.is_file() {
        let mut file = std::fs::File::open(path).expect("could not open file");
        let mut contents = String::new();
        // Content may be non-UTF-8, in which case we just return false.
        if file.read_to_string(&mut contents).is_ok() {
            contents.contains(pattern)
        } else {
            false
        }
    } else {
        false
    }
}

pub fn example_toml_files() -> impl Iterator<Item = PathBuf> {
    source_files().filter(is_example_toml_file)
}

pub fn fuzz_config_toml_files() -> impl Iterator<Item = PathBuf> {
    source_files().filter(is_fuzz_config_toml_file)
}

/// Return an iterator of all known Cargo Manifest files that define crates.
pub fn crate_manifest_files() -> impl Iterator<Item = PathBuf> {
    source_files()
        .filter(is_cargo_toml_file)
        .filter(|p| is_cargo_package_file(p))
}

/// Return an iterator of all known Cargo Manifest files that define workspaces.
pub fn workspace_manifest_files() -> impl Iterator<Item = PathBuf> {
    source_files()
        .filter(is_cargo_toml_file)
        .filter(|p| is_cargo_workspace_file(p))
}

/// Return whether the provided path refers to a source file in a programming language.
pub fn is_source_code_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".cc")
        || filename.ends_with(".h")
        || filename.ends_with(".rs")
        || filename.ends_with(".proto")
        || filename.ends_with(".js")
        || filename.ends_with(".go")
}

/// Return whether the provided path refers to a source file that can be formatted by clang-tidy.
pub fn is_clang_format_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".cc") || filename.ends_with(".h") || filename.ends_with(".proto")
}

/// Return whether the provided path refers to a Bazel file (`BUILD`, `WORKSPACE`, or `*.bzl`)
pub fn is_bazel_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename == "BUILD" || filename == "WORKSPACE" || filename.ends_with(".bzl")
}

pub fn is_build_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename == "BUILD"
}

/// Return whether the provided path refers to a markdown file (`*.md`)
pub fn is_markdown_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".md")
}

pub fn is_dockerfile(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with("Dockerfile")
}

pub fn is_toml_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".toml")
}

pub fn is_yaml_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".yaml") || filename.ends_with(".yml")
}

pub fn is_html_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".htm") || filename.ends_with(".html")
}

pub fn is_javascript_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".js") || filename.ends_with(".mjs")
}

pub fn is_typescript_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename.ends_with(".ts") || filename.ends_with(".tsx")
}

pub fn is_shell_script(path: &PathBuf) -> bool {
    if path.is_file() {
        let mut file = std::fs::File::open(path).expect("could not open file");
        let mut contents = String::new();
        match file.read_to_string(&mut contents) {
            Ok(_size) => contents.starts_with("#!"),
            Err(_err) => false,
        }
    } else {
        false
    }
}

/// Return whether the provided path refers to a `fuzz` crate for fuzz testing with `cargo-fuzz`.
pub fn is_fuzzing_toml_file(path: &PathBuf) -> bool {
    format!("{:?}", path).contains("/fuzz/")
}

/// Return whether the provided path refers to a `Cargo.toml` file. Note that it does not
/// differentiate between workspace-level and crate-level files.
fn is_cargo_toml_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename == "Cargo.toml"
}

/// Return whether the provided path refers to a workspace-level `Cargo.toml` file, by looking at
/// the contents of the file.
fn is_cargo_workspace_file(path: &PathBuf) -> bool {
    // We naively look for the `[workspace]` string to appear in the contents of the file. A better
    // alternative would be to actually parse the file as `toml` and figure out whether it has a
    // `workspace` section, but it seems overkill for now.
    file_contains(path, "[workspace]")
}

/// Return whether the provided path refers to a `Cargo.toml` file that defines a crate, by looking
/// at the contents of the file.
fn is_cargo_package_file(path: &PathBuf) -> bool {
    // We naively look for the `[package]` string to appear in the contents of the file. A better
    // alternative would be to actually parse the file as `toml` and figure out whether it has a
    // `package` section, but it seems overkill for now.
    file_contains(path, "[package]")
}

fn is_example_toml_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename == "example.toml"
}

fn is_fuzz_config_toml_file(path: &PathBuf) -> bool {
    let filename = path.file_name().and_then(|s| s.to_str()).unwrap_or("");
    filename == "fuzz.toml"
}

fn is_ignored_entry(entry: &walkdir::DirEntry) -> bool {
    let path = entry.clone().into_path();
    // Simple heuristic to try and exclude generated files.
    is_ignored_path(&path) || file_contains(&path, "DO NOT EDIT")
}

/// Return whether to ignore the specified path. This is used by the `walker` package to efficiently
/// avoid descending into blacklisted directories.
fn is_ignored_path(path: &PathBuf) -> bool {
    let components = path.components().collect::<std::collections::HashSet<_>>();
    components.contains(&std::path::Component::Normal(".git".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-cache".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-bin".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-out".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-oak".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-clang-bin".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-clang-out".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-clang-oak".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-client-bin".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-client-out".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-client-oak".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-emscripten-bin".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-emscripten-out".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-emscripten-oak".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-wasm-bin".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-wasm-out".as_ref()))
        || components.contains(&std::path::Component::Normal("bazel-wasm-oak".as_ref()))
        || components.contains(&std::path::Component::Normal("cache".as_ref()))
        || components.contains(&std::path::Component::Normal("cargo-cache".as_ref()))
        || components.contains(&std::path::Component::Normal("node_modules".as_ref()))
        || components.contains(&std::path::Component::Normal("protoc_out".as_ref())) // Code generated by protoc
        || components.contains(&std::path::Component::Normal("dist".as_ref())) // Code generated by webpack
        || components.contains(&std::path::Component::Normal("target".as_ref())) // Rust artifacts.
        || components.contains(&std::path::Component::Normal("third_party".as_ref()))
}

pub fn to_string(path: PathBuf) -> String {
    path.to_str().unwrap().to_string()
}
<|endoftext|>


<commit_before>//
// zhtta.rs
//
// Running on Rust 0.8
//
// Starting code for PS3
//
// Note: it would be very unwise to run this server on a machine that is
// on the Internet and contains any sensitive files!
//
// University of Virginia - cs4414 Fall 2013
// Weilin Xu and David Evans
// Version 0.2

extern mod extra;

use std::rt::io::*;
use std::rt::io::net::ip::SocketAddr;
use std::io::println;
use std::cell::Cell;
use std::{os, str, io};
use extra::arc;
use std::comm::*;

static PORT: int = 4414;
static IP: &'static str = "0.0.0.0";
static mut visitor_count: uint = 0;

struct sched_msg {
    stream: Option<std::rt::io::net::tcp::TcpStream>,
    filepath: ~std::path::PosixPath
}

fn main() {
    let req_vec: ~[sched_msg] = ~[];
    let shared_req_vec = arc::RWArc::new(req_vec);
    let add_vec = shared_req_vec.clone();
    let take_vec = shared_req_vec.clone();

    let (port, chan) = stream();
    let chan = SharedChan::new(chan);

    // add file requests into queue.
    do spawn {
        loop {
            do add_vec.write |vec| {
                // port.recv() will block the code and keep locking the RWArc, so we simply use peek() to check if there's message to recv.
                // But a asynchronous solution will be much better.
                if (port.peek()) {
                    let tf:sched_msg = port.recv();
                    (*vec).push(tf);
                    println(fmt!("add to queue, size: %ud", (*vec).len()));
                }
            }
        }
    }

    // take file requests from queue, and send a response.
    // FIFO
    do spawn {
        loop {
            do take_vec.write |vec| {
                if ((*vec).len() > 0) {
                    // FILO didn't make sense in service scheduling, so we modify it as FIFO by using shift_opt() rather than pop().
                    let tf_opt: Option<sched_msg> = (*vec).shift_opt();
                    let mut tf = tf_opt.unwrap();
                    println(fmt!("shift from queue, size: %ud", (*vec).len()));

                    match io::read_whole_file(tf.filepath) { // killed if file size is larger than memory size.
                        Ok(file_data) => {
                            println(fmt!("begin serving file [%?]", tf.filepath));
                            // A web server should always reply a HTTP header for any legal HTTP request.
                            tf.stream.write("HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream; charset=UTF-8\r\n\r\n".as_bytes());
                            tf.stream.write(file_data);
                            println(fmt!("finish file [%?]", tf.filepath));
                        }
                        Err(err) => {
                            println(err);
                        }
                    }
                }
            }
        }
    }

    let ip = match FromStr::from_str(IP) { Some(ip) => ip,
                                           None => { println(fmt!("Error: Invalid IP address <%s>", IP));
                                                     return;},
                                         };

    let socket = net::tcp::TcpListener::bind(SocketAddr {ip: ip, port: PORT as u16});

    println(fmt!("Listening on tcp port %d ...", PORT));
    let mut acceptor = socket.listen().unwrap();

    // we can limit the incoming connection count.
    //for stream in acceptor.incoming().take(10 as uint) {
    for stream in acceptor.incoming() {
        let stream = Cell::new(stream);

        // Start a new task to handle the connection
        let child_chan = chan.clone();
        do spawn {
            unsafe {
                visitor_count += 1;
            }

            let mut stream = stream.take();
            let mut buf = [0, ..500];
            stream.read(buf);
            let request_str = str::from_utf8(buf);

            let req_group : ~[&str]= request_str.splitn_iter(' ', 3).collect();
            if req_group.len() > 2 {
                let path = req_group[1];
                println(fmt!("Request for path: \n%?", path));

                let file_path = ~os::getcwd().push(path.replace("/../", ""));
                if !os::path_exists(file_path) || os::path_is_dir(file_path) {
                    println(fmt!("Request received:\n%s", request_str));
                    let response: ~str = fmt!(
                        "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=UTF-8\r\n\r\n
			<doctype !html>
			<title>Hello, Rust!</title>

Greetings, Krusty!

Visitor count: %u

			\r\n", unsafe{visitor_count});

                    stream.write(response.as_bytes());
                }
                else {
                    // may do scheduling here
                    let msg: sched_msg = sched_msg{stream: stream, filepath: file_path.clone()};
                    child_chan.send(msg);

                    println(fmt!("get file request: %?", file_path));
                }
            }
            println!("connection terminates")
        }
    }
}

<commit_msg>upgraded the starting code from Weilin to v0.3<commit_after>//
// zhtta.rs
//
// Running on Rust 0.8
//
// Starting code for PS3
//
// Note: it would be very unwise to run this server on a machine that is
// on the Internet and contains any sensitive files!
//
// University of Virginia - cs4414 Fall 2013
// Weilin Xu and David Evans
// Version 0.3

extern mod extra;

use std::rt::io::*;
use std::rt::io::net::ip::SocketAddr;
use std::io::println;
use std::cell::Cell;
use std::{os, str, io};
use extra::arc;
use std::comm::*;

static PORT: int = 4414;
static IP: &'static str = "127.0.0.1";
static mut visitor_count: uint = 0;

struct sched_msg {
    stream: Option<std::rt::io::net::tcp::TcpStream>,
    filepath: ~std::path::PosixPath
}

fn main() {
    let req_vec: ~[sched_msg] = ~[];
    let shared_req_vec = arc::RWArc::new(req_vec);
    let add_vec = shared_req_vec.clone();
    let take_vec = shared_req_vec.clone();

    let (port, chan) = stream();
    let chan = SharedChan::new(chan);

    // dequeue file requests, and send responses.
    // FIFO
    do spawn {
        let (sm_port, sm_chan) = stream();

        // a task for sending responses.
        do spawn {
            loop {
                let mut tf: sched_msg = sm_port.recv(); // wait for the dequeued request to handle
                match io::read_whole_file(tf.filepath) { // killed if file size is larger than memory size.
                    Ok(file_data) => {
                        println(fmt!("begin serving file [%?]", tf.filepath));
                        // A web server should always reply a HTTP header for any legal HTTP request.
                        tf.stream.write("HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream; charset=UTF-8\r\n\r\n".as_bytes());
                        tf.stream.write(file_data);
                        println(fmt!("finish file [%?]", tf.filepath));
                    }
                    Err(err) => {
                        println(err);
                    }
                }
            }
        }

        loop {
            port.recv(); // wait for arrving notification
            do take_vec.write |vec| {
                if ((*vec).len() > 0) {
                    // LIFO didn't make sense in service scheduling, so we modify it as FIFO by using shift_opt() rather than pop().
                    let tf_opt: Option<sched_msg> = (*vec).shift_opt();
                    let tf = tf_opt.unwrap();
                    println(fmt!("shift from queue, size: %ud", (*vec).len()));
                    sm_chan.send(tf); // send the request to send-response-task to serve.
                }
            }
        }
    }

    let ip = match FromStr::from_str(IP) { Some(ip) => ip,
                                           None => { println(fmt!("Error: Invalid IP address <%s>", IP));
                                                     return;},
                                         };

    let socket = net::tcp::TcpListener::bind(SocketAddr {ip: ip, port: PORT as u16});

    println(fmt!("Listening on %s:%d ...", ip.to_str(), PORT));
    let mut acceptor = socket.listen().unwrap();

    for stream in acceptor.incoming() {
        let stream = Cell::new(stream);

        // Start a new task to handle the each connection
        let child_chan = chan.clone();
        let child_add_vec = add_vec.clone();
        do spawn {
            unsafe {
                visitor_count += 1;
            }

            let mut stream = stream.take();
            let mut buf = [0, ..500];
            stream.read(buf);
            let request_str = str::from_utf8(buf);

            let req_group : ~[&str]= request_str.splitn_iter(' ', 3).collect();
            if req_group.len() > 2 {
                let path = req_group[1];
                println(fmt!("Request for path: \n%?", path));

                let file_path = ~os::getcwd().push(path.replace("/../", ""));
                if !os::path_exists(file_path) || os::path_is_dir(file_path) {
                    println(fmt!("Request received:\n%s", request_str));
                    let response: ~str = fmt!(
                        "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=UTF-8\r\n\r\n
			<doctype !html><title>Hello, Rust!</title>

Greetings, Krusty!

Visitor count: %u

			\r\n", unsafe{visitor_count});

                    stream.write(response.as_bytes());
                }
                else {
                    // Requests scheduling
                    let msg: sched_msg = sched_msg{stream: stream, filepath: file_path.clone()};
                    let (sm_port, sm_chan) = std::comm::stream();
                    sm_chan.send(msg);

                    do child_add_vec.write |vec| {
                        let msg = sm_port.recv();
                        (*vec).push(msg); // enqueue new request.
                        println("add to queue");
                    }
                    child_chan.send(""); //notify the new arriving request.
                    println(fmt!("get file request: %?", file_path));
                }
            }
            println!("connection terminates")
        }
    }
}
<|endoftext|>


<commit_before>///! Universally unique identifiers.
///!
///! The identifiers generated by this module:
///!
///! - are URL safe
///! - contain information on the type of object that they
///!   are identifying
///! - are roughly sortable by time of generation
///! - have an extremely low probability of collision
///!
///! Generated identifiers have a fixed length of 32 characters made up
///! of three parts separated by dots:
///!
///! - 2 characters in the range `[a-z]` that identifying the "family" of
///!   identifiers, usually the type of object the identifier is for
///!   e.g. `fi` = file, `re` = request
///!
///! - 10 characters in the range `[0-9a-f]` that are the hexadecimal encoding of the
///!   32bit number of seconds since Unix Timestamp Epoch 1500000000000 left padded
///!   with zeros
///!
///! - 18 characters in the range `[0-9A-Za-z]` that are randomly generated
///!
///! See
///!  - https://segment.com/blog/a-brief-history-of-the-uuid/
///!  - https://zelark.github.io/nano-id-cc/
///!  - https://gist.github.com/fnky/76f533366f75cf75802c8052b577e2a5
use nanoid::nanoid;
use std::convert::TryFrom;
use strum::ToString;

/// The available families of identifiers
#[derive(ToString)]
pub enum Family {
    #[strum(serialize = "no")]
    Node,

    #[strum(serialize = "do")]
    Document,

    #[strum(serialize = "fi")]
    File,
}

/// The epoch used for calculating the time stamp in the second part of
/// an identifier. Chosen as a recent time that was easily remembered
/// (happens to correspond to 2017-07-14T02:40:00.000Z).
const EPOCH: i64 = 1500000000;

/// The characters used in the third part of the identifier
const CHARACTERS: [char; 62] = [
    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I',
    'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b',
    'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u',
    'v', 'w', 'x', 'y', 'z',
];

// Generate a universally unique identifier
pub fn generate(family: Family) -> String {
    let diff = chrono::Utc::now().timestamp() - EPOCH;
    let seconds =
        u32::try_from(diff).expect("Unable to convert to u32 (must be waaaay in the future");
    let rand = nanoid!(18, &CHARACTERS);
    format!("{}.{:010x}.{}", family.to_string(), seconds, rand)
}

#[cfg(test)]
mod tests {
    use eyre::Result;
    use regex::Regex;

    use super::*;

    #[test]
    fn test_generate() -> Result<()> {
        let id = generate(Family::File);

        assert_eq!(id.len(), 32);

        let re = Regex::new(r"[a-z]{2}\.[0-9a-f]{10}\.[0-9A-Za-z]{18}")?;
        assert!(re.is_match(&id));

        Ok(())
    }
}
<commit_msg>refactor(UUIDs): Make generate_chars available<commit_after>///! Universally unique identifiers.
///!
///! The identifiers generated by this module:
///!
///! - are URL safe
///! - contain information on the type of object that they
///!   are identifying
///! - are roughly sortable by time of generation
///! - have an extremely low probability of collision
///!
///! Generated identifiers have a fixed length of 32 characters made up
///! of three parts separated by dots:
///!
///! - 2 characters in the range `[a-z]` that identifying the "family" of
///!   identifiers, usually the type of object the identifier is for
///!   e.g. `fi` = file, `re` = request
///!
///! - 10 characters in the range `[0-9a-f]` that are the hexadecimal encoding of the
///!   32bit number of seconds since Unix Timestamp Epoch 1500000000000 left padded
///!   with zeros
///!
///! - 18 characters in the range `[0-9A-Za-z]` that are randomly generated
///!
///! See
///!  - https://segment.com/blog/a-brief-history-of-the-uuid/
///!  - https://zelark.github.io/nano-id-cc/
///!  - https://gist.github.com/fnky/76f533366f75cf75802c8052b577e2a5
use nanoid::nanoid;
use std::convert::TryFrom;
use strum::ToString;

/// The available families of identifiers
#[derive(ToString)]
pub enum Family {
    #[strum(serialize = "no")]
    Node,

    #[strum(serialize = "do")]
    Document,

    #[strum(serialize = "fi")]
    File,
}

/// The epoch used for calculating the time stamp in the second part of
/// an identifier. Chosen as a recent time that was easily remembered
/// (happens to correspond to 2017-07-14T02:40:00.000Z).
const EPOCH: i64 = 1500000000;

/// The characters used in the third part of the identifier
const CHARACTERS: [char; 62] = [
    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I',
    'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b',
    'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u',
    'v', 'w', 'x', 'y', 'z',
];

// Generate random characters
pub fn generate_chars(size: usize) -> String {
    nanoid!(size, &CHARACTERS)
}

// Generate a universally unique identifier
pub fn generate(family: Family) -> String {
    let diff = chrono::Utc::now().timestamp() - EPOCH;
    let seconds =
        u32::try_from(diff).expect("Unable to convert to u32 (must be waaaay in the future");
    let rand = generate_chars(18);
    format!("{}.{:010x}.{}", family.to_string(), seconds, rand)
}
#[cfg(test)]
mod tests {
    use eyre::Result;
    use regex::Regex;

    use super::*;

    #[test]
    fn test_generate() -> Result<()> {
        let id = generate(Family::File);

        assert_eq!(id.len(), 32);

        let re = Regex::new(r"[a-z]{2}\.[0-9a-f]{10}\.[0-9A-Za-z]{18}")?;
        assert!(re.is_match(&id));

        Ok(())
    }
}
<|endoftext|>


<commit_before>pub fn is_letter(c: char) -> bool {
    c == '_' || c.is_alphabetic()
}

pub fn is_digit(c: char) -> bool {
    c.is_digit(10)
}

pub fn is_letter_or_digit(c: char) -> bool {
    is_letter(c) || is_digit(c)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn is_letter_test() {
        assert_eq!(is_letter('a'), true);
        assert_eq!(is_letter('c'), true);
        assert_eq!(is_letter('k'), true);
        assert_eq!(is_letter('l'), true);
        assert_eq!(is_letter('_'), true);
        assert_eq!(is_letter('1'), false);
        assert_eq!(is_letter('3'), false);
        assert_eq!(is_letter(' '), false);
        assert_eq!(is_letter('}'), false);
    }

    #[test]
    fn is_digit_test() {
        assert_eq!(is_digit('a'), false);
        assert_eq!(is_digit('c'), false);
        assert_eq!(is_digit('k'), false);
        assert_eq!(is_digit('l'), false);
        assert_eq!(is_digit('_'), false);
        assert_eq!(is_digit('1'), true);
        assert_eq!(is_digit('3'), true);
        assert_eq!(is_digit(' '), false);
        assert_eq!(is_digit('}'), false);
    }

    #[test]
    fn is_letter_or_digit_test() {
        assert_eq!(is_letter_or_digit('a'), true);
        assert_eq!(is_letter_or_digit('c'), true);
        assert_eq!(is_letter_or_digit('k'), true);
        assert_eq!(is_letter_or_digit('l'), true);
        assert_eq!(is_letter_or_digit('_'), true);
        assert_eq!(is_letter_or_digit('1'), true);
        assert_eq!(is_letter_or_digit('3'), true);
        assert_eq!(is_letter_or_digit(' '), false);
        assert_eq!(is_letter_or_digit('}'), false);
    }
}
<commit_msg>Make is_x functions for char handle reference<commit_after>pub fn is_letter(c: &char) -> bool {
    *c == '_' || c.is_alphabetic()
}

pub fn is_digit(c: &char) -> bool {
    c.is_digit(10)
}

pub fn is_letter_or_digit(c: &char) -> bool {
    is_letter(c) || is_digit(c)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn is_letter_test() {
        assert_eq!(is_letter(&'a'), true);
        assert_eq!(is_letter(&'c'), true);
        assert_eq!(is_letter(&'k'), true);
        assert_eq!(is_letter(&'l'), true);
        assert_eq!(is_letter(&'_'), true);
        assert_eq!(is_letter(&'1'), false);
        assert_eq!(is_letter(&'3'), false);
        assert_eq!(is_letter(&' '), false);
        assert_eq!(is_letter(&'}'), false);
    }

    #[test]
    fn is_digit_test() {
        assert_eq!(is_digit(&'a'), false);
        assert_eq!(is_digit(&'c'), false);
        assert_eq!(is_digit(&'k'), false);
        assert_eq!(is_digit(&'l'), false);
        assert_eq!(is_digit(&'_'), false);
        assert_eq!(is_digit(&'1'), true);
        assert_eq!(is_digit(&'3'), true);
        assert_eq!(is_digit(&' '), false);
        assert_eq!(is_digit(&'}'), false);
    }

    #[test]
    fn is_letter_or_digit_test() {
        assert_eq!(is_letter_or_digit(&'a'), true);
        assert_eq!(is_letter_or_digit(&'c'), true);
        assert_eq!(is_letter_or_digit(&'k'), true);
        assert_eq!(is_letter_or_digit(&'l'), true);
        assert_eq!(is_letter_or_digit(&'_'), true);
        assert_eq!(is_letter_or_digit(&'1'), true);
        assert_eq!(is_letter_or_digit(&'3'), true);
        assert_eq!(is_letter_or_digit(&' '), false);
        assert_eq!(is_letter_or_digit(&'}'), false);
    }
}
<|endoftext|>


<commit_before>use crate::server::ServerData;
use crate::Database;

use actix_web::{error::ResponseError, get, http::StatusCode, web, HttpRequest, HttpResponse};
use cemu_smm::proto::SMMCourse::{
    SMMCourse_AutoScroll, SMMCourse_CourseTheme, SMMCourse_GameStyle,
};
use mongodb::{oid::ObjectId, ordered::OrderedDocument, Bson};
use protobuf::ProtobufEnum;
use serde::Deserialize;
use serde_qs::actix::QsQuery;
use std::sync::MutexGuard;

#[get("/getcourses")]
pub fn get_courses(
    data: web::Data<ServerData>,
    query: QsQuery<GetCourses>,
    req: HttpRequest,
) -> Result<String, GetCoursesError> {
    data.get_courses(query.into_inner())
}

#[derive(Deserialize, Debug)]
pub struct GetCourses {
    #[serde(default)]
    limit: Limit,
    skip: Option<u32>,
    id: Option<String>,
    ids: Option<Vec<String>>,
    title: Option<String>,
    maker: Option<String>,
    owner: Option<String>,
    uploader: Option<String>,
    game_style: Option<Vec<SMMCourse_GameStyle>>,
    course_theme: Option<Vec<SMMCourse_CourseTheme>>,
    course_theme_sub: Option<Vec<SMMCourse_CourseTheme>>,
    auto_scroll: Option<Vec<SMMCourse_AutoScroll>>,
    auto_scroll_sub: Option<Vec<SMMCourse_AutoScroll>>,
    width_from: Option<i32>,
    width_to: Option<i32>,
    width_sub_from: Option<i32>,
    width_sub_to: Option<i32>,
    nintendoid: Option<String>,
    difficulty_from: Option<u32>,
    difficulty_to: Option<u32>,
    videoid: Option<String>,
    lastmodified_from: Option<u32>,
    lastmodified_to: Option<u32>,
    uploaded_from: Option<u32>,
    uploaded_to: Option<u32>,
    stars_from: Option<u32>,
    stars_to: Option<u32>,
}

impl GetCourses {
    pub fn into_ordered_document(
        self,
        database: &MutexGuard<Database>,
    ) -> Result<Vec<OrderedDocument>, GetCoursesError> {
        let mut pipeline = vec![];

        if let Some(pipeline_match) = self.get_match(database)? {
            pipeline.push(doc! { "$match" => pipeline_match });
        }

        let limit = self.get_limit()?;
        pipeline.push(doc! {
            "$limit" => limit
        });

        if let Some(skip) = self.skip {
            pipeline.push(doc! {
                "$skip" => skip
            });
        }

        Ok(pipeline)
    }

    fn get_match(
        &self,
        database: &MutexGuard<Database>,
    ) -> Result<Option<OrderedDocument>, GetCoursesError> {
        let mut res = doc! {};
        if let Some(id) = &self.id {
            GetCourses::insert_objectid(&mut res, "_id".to_string(), id)?;
        }

        if let Some(ids) = self.ids.clone() {
            let ids: Vec<Bson> = ids
                .iter()
                .map(|id| -> Result<Bson, GetCoursesError> {
                    let object_id = ObjectId::with_string(id)
                        .map_err(|_| GetCoursesError::DeserializeError("ids".to_string()))?;
                    Ok(Bson::ObjectId(object_id))
                })
                .filter_map(Result::ok)
                .collect();
            res.insert_bson(
                "_id".to_string(),
                Bson::Document(doc! {
                    "$in" => ids
                }),
            );
        }

        if let Some(title) = self.title.clone() {
            GetCourses::insert_regexp(&mut res, "title".to_string(), title);
        }

        if let Some(maker) = self.maker.clone() {
            GetCourses::insert_regexp(&mut res, "maker".to_string(), maker);
        }

        if let Some(owner) = &self.owner {
            GetCourses::insert_objectid(&mut res, "owner".to_string(), owner)?;
        }

        if let Some(uploader) = &self.uploader {
            let filter = doc! {
                "username" => Bson::RegExp(format!("^{}$", uploader), "i".to_string())
            };
            match database.find_account(filter) {
                Some(account) => {
                    res.insert_bson("owner".to_string(), Bson::ObjectId(account.get_id()));
                }
                None => return Err(GetCoursesError::UploaderUnknown(uploader.clone())),
            };
        }

        if let Some(game_styles) = &self.game_style {
            GetCourses::insert_enum(&mut res, "gameStyle".to_string(), game_styles);
        }

        if let Some(course_themes) = &self.course_theme {
            GetCourses::insert_enum(&mut res, "courseTheme".to_string(), course_themes);
        }

        if let Some(course_themes) = &self.course_theme_sub {
            GetCourses::insert_enum(&mut res, "courseThemeSub".to_string(), course_themes);
        }

        if let Some(auto_scrolls) = &self.auto_scroll {
            GetCourses::insert_enum(&mut res, "autoScroll".to_string(), auto_scrolls);
        }

        if let Some(auto_scrolls) = &self.auto_scroll_sub {
            GetCourses::insert_enum(&mut res, "autoScrollSub".to_string(), auto_scrolls);
        }

        match res.is_empty() {
            true => Ok(None),
            false => Ok(Some(res)),
        }
    }

    fn get_limit(&self) -> Result<u32, GetCoursesError> {
        let limit = self.limit.0;
        if limit <= 0 {
            return Err(GetCoursesError::LimitTooLow);
        }
        if limit > 120 {
            return Err(GetCoursesError::LimitTooHigh);
        }
        Ok(limit + self.skip.unwrap_or_default())
    }

    fn insert_regexp(doc: &mut OrderedDocument, key: String, regexp: String) {
        doc.insert_bson(
            key,
            Bson::RegExp(format!(".*{}.*", regexp), "i".to_string()),
        );
    }

    fn insert_objectid(
        doc: &mut OrderedDocument,
        key: String,
        oid: &str,
    ) -> Result<(), GetCoursesError> {
        doc.insert_bson(
            key.clone(),
            Bson::ObjectId(
                ObjectId::with_string(oid).map_err(|_| GetCoursesError::DeserializeError(key))?,
            ),
        );
        Ok(())
    }

    fn insert_enum<T>(doc: &mut OrderedDocument, key: String, enums: &Vec<T>)
    where
        T: ProtobufEnum,
    {
        let enums: Vec<Bson> = enums.iter().map(|val| Bson::I32(val.value())).collect();
        doc.insert_bson(
            key.to_string(),
            Bson::Document(doc! {
                "$in" => enums
            }),
        );
    }
}

#[derive(Deserialize, Debug)]
struct Limit(u32);

impl Default for Limit {
    fn default() -> Limit {
        Limit(120)
    }
}

#[derive(Fail, Debug)]
pub enum GetCoursesError {
    #[fail(display = "limit must be at least 1")]
    LimitTooLow,
    #[fail(display = "limit must be at most 120")]
    LimitTooHigh,
    #[fail(display = "could not deserialize {} fom hex string", _0)]
    DeserializeError(String),
    #[fail(display = "uploader with name {} unknown", _0)]
    UploaderUnknown(String),
}

impl ResponseError for GetCoursesError {
    fn error_response(&self) -> HttpResponse {
        match *self {
            GetCoursesError::LimitTooLow => HttpResponse::new(StatusCode::BAD_REQUEST),
            GetCoursesError::LimitTooHigh => HttpResponse::new(StatusCode::BAD_REQUEST),
            GetCoursesError::DeserializeError(_) => HttpResponse::new(StatusCode::BAD_REQUEST),
            GetCoursesError::UploaderUnknown(_) => HttpResponse::new(StatusCode::BAD_REQUEST),
        }
    }
}
<commit_msg>more filter options<commit_after>use crate::server::ServerData;
use crate::Database;

use actix_web::{error::ResponseError, get, http::StatusCode, web, HttpRequest, HttpResponse};
use cemu_smm::proto::SMMCourse::{
    SMMCourse_AutoScroll, SMMCourse_CourseTheme, SMMCourse_GameStyle,
};
use mongodb::{oid::ObjectId, ordered::OrderedDocument, Bson};
use protobuf::ProtobufEnum;
use serde::Deserialize;
use serde_qs::actix::QsQuery;
use std::sync::MutexGuard;

#[get("/getcourses")]
pub fn get_courses(
    data: web::Data<ServerData>,
    query: QsQuery<GetCourses>,
    req: HttpRequest,
) -> Result<String, GetCoursesError> {
    data.get_courses(query.into_inner())
}

#[derive(Deserialize, Debug)]
pub struct GetCourses {
    #[serde(default)]
    limit: Limit,
    skip: Option<u32>,
    id: Option<String>,
    ids: Option<Vec<String>>,
    title: Option<String>,
    maker: Option<String>,
    owner: Option<String>,
    uploader: Option<String>,
    game_style: Option<Vec<SMMCourse_GameStyle>>,
    course_theme: Option<Vec<SMMCourse_CourseTheme>>,
    course_theme_sub: Option<Vec<SMMCourse_CourseTheme>>,
    auto_scroll: Option<Vec<SMMCourse_AutoScroll>>,
    auto_scroll_sub: Option<Vec<SMMCourse_AutoScroll>>,
    width_gte: Option<i32>,
    width_lte: Option<i32>,
    width_sub_gte: Option<i32>,
    width_sub_lte: Option<i32>,
    nintendo_id: Option<String>,
    difficulty_gte: Option<i32>,
    difficulty_lte: Option<i32>,
    video_id: Option<String>,
    lastmodified_gte: Option<i64>,
    lastmodified_lte: Option<i64>,
    uploaded_gte: Option<i64>,
    uploaded_lte: Option<i64>,
    stars_gte: Option<i32>,
    stars_lte: Option<i32>,
}

impl GetCourses {
    pub fn into_ordered_document(
        self,
        database: &MutexGuard<Database>,
    ) -> Result<Vec<OrderedDocument>, GetCoursesError> {
        let mut pipeline = vec![];

        if let Some(pipeline_match) = self.get_match(database)? {
            pipeline.push(doc! { "$match" => pipeline_match });
        }

        let limit = self.get_limit()?;
        pipeline.push(doc! {
            "$limit" => limit
        });

        if let Some(skip) = self.skip {
            pipeline.push(doc! {
                "$skip" => skip
            });
        }

        Ok(pipeline)
    }

    fn get_match(
        &self,
        database: &MutexGuard<Database>,
    ) -> Result<Option<OrderedDocument>, GetCoursesError> {
        let mut res = doc! {};
        if let Some(id) = &self.id {
            GetCourses::insert_objectid(&mut res, "_id".to_string(), id)?;
        }

        if let Some(ids) = self.ids.clone() {
            let ids: Vec<Bson> = ids
                .iter()
                .map(|id| -> Result<Bson, GetCoursesError> {
                    let object_id = ObjectId::with_string(id)
                        .map_err(|_| GetCoursesError::DeserializeError("ids".to_string()))?;
                    Ok(Bson::ObjectId(object_id))
                })
                .filter_map(Result::ok)
                .collect();
            res.insert_bson(
                "_id".to_string(),
                Bson::Document(doc! {
                    "$in" => ids
                }),
            );
        }

        if let Some(title) = self.title.clone() {
            GetCourses::insert_regexp(&mut res, "title".to_string(), title);
        }

        if let Some(maker) = self.maker.clone() {
            GetCourses::insert_regexp(&mut res, "maker".to_string(), maker);
        }

        if let Some(owner) = &self.owner {
            GetCourses::insert_objectid(&mut res, "owner".to_string(), owner)?;
        }

        if let Some(uploader) = &self.uploader {
            let filter = doc! {
                "username" => Bson::RegExp(format!("^{}$", uploader), "i".to_string())
            };
            match database.find_account(filter) {
                Some(account) => {
                    res.insert_bson("owner".to_string(), Bson::ObjectId(account.get_id()));
                }
                None => return Err(GetCoursesError::UploaderUnknown(uploader.clone())),
            };
        }

        if let Some(game_styles) = &self.game_style {
            GetCourses::insert_enum(&mut res, "gameStyle".to_string(), game_styles);
        }

        if let Some(course_themes) = &self.course_theme {
            GetCourses::insert_enum(&mut res, "courseTheme".to_string(), course_themes);
        }

        if let Some(course_themes) = &self.course_theme_sub {
            GetCourses::insert_enum(&mut res, "courseThemeSub".to_string(), course_themes);
        }

        if let Some(auto_scrolls) = &self.auto_scroll {
            GetCourses::insert_enum(&mut res, "autoScroll".to_string(), auto_scrolls);
        }

        if let Some(auto_scrolls) = &self.auto_scroll_sub {
            GetCourses::insert_enum(&mut res, "autoScrollSub".to_string(), auto_scrolls);
        }

        GetCourses::insert_boundaries(
            &mut res,
            "width".to_string(),
            self.width_gte,
            self.width_lte,
        );

        GetCourses::insert_boundaries(
            &mut res,
            "widthSub".to_string(),
            self.width_sub_gte,
            self.width_sub_lte,
        );

        if let Some(nintendo_id) = &self.nintendo_id {
            res.insert_bson("nintendoid".to_string(), Bson::String(nintendo_id.clone()));
        }

        GetCourses::insert_boundaries(
            &mut res,
            "difficulty".to_string(),
            self.difficulty_gte,
            self.difficulty_lte,
        );

        GetCourses::insert_boundaries(
            &mut res,
            "stars".to_string(),
            self.stars_gte,
            self.stars_lte,
        );

        match res.is_empty() {
            true => Ok(None),
            false => Ok(Some(res)),
        }
    }

    fn get_limit(&self) -> Result<u32, GetCoursesError> {
        let limit = self.limit.0;
        if limit <= 0 {
            return Err(GetCoursesError::LimitTooLow);
        }
        if limit > 120 {
            return Err(GetCoursesError::LimitTooHigh);
        }
        Ok(limit + self.skip.unwrap_or_default())
    }

    fn insert_regexp(doc: &mut OrderedDocument, key: String, regexp: String) {
        doc.insert_bson(
            key,
            Bson::RegExp(format!(".*{}.*", regexp), "i".to_string()),
        );
    }

    fn insert_objectid(
        doc: &mut OrderedDocument,
        key: String,
        oid: &str,
    ) -> Result<(), GetCoursesError> {
        doc.insert_bson(
            key.clone(),
            Bson::ObjectId(
                ObjectId::with_string(oid).map_err(|_| GetCoursesError::DeserializeError(key))?,
            ),
        );
        Ok(())
    }

    fn insert_enum<T>(doc: &mut OrderedDocument, key: String, enums: &Vec<T>)
    where
        T: ProtobufEnum,
    {
        let enums: Vec<Bson> = enums.iter().map(|val| Bson::I32(val.value())).collect();
        doc.insert_bson(
            key.to_string(),
            Bson::Document(doc! {
                "$in" => enums
            }),
        );
    }

    fn insert_boundaries(
        doc: &mut OrderedDocument,
        key: String,
        gte: Option<i32>,
        lte: Option<i32>,
    ) {
        let mut boundaries = None;
        if let Some(gte) = gte {
            boundaries = Some(doc! {
                "$gte" => gte
            });
        }
        if let Some(lte) = lte {
            match &mut boundaries {
                Some(boundaries) => {
                    boundaries.insert("$lte", lte);
                }
                None => {
                    boundaries = Some(doc! {
                        "$lte" => lte
                    })
                }
            }
        }
        if let Some(boundaries) = boundaries {
            doc.insert_bson(key, Bson::Document(boundaries));
        }
    }
}

#[derive(Deserialize, Debug)]
struct Limit(u32);

impl Default for Limit {
    fn default() -> Limit {
        Limit(120)
    }
}

#[derive(Fail, Debug)]
pub enum GetCoursesError {
    #[fail(display = "limit must be at least 1")]
    LimitTooLow,
    #[fail(display = "limit must be at most 120")]
    LimitTooHigh,
    #[fail(display = "could not deserialize {} fom hex string", _0)]
    DeserializeError(String),
    #[fail(display = "uploader with name {} unknown", _0)]
    UploaderUnknown(String),
}

impl ResponseError for GetCoursesError {
    fn error_response(&self) -> HttpResponse {
        match *self {
            GetCoursesError::LimitTooLow => HttpResponse::new(StatusCode::BAD_REQUEST),
            GetCoursesError::LimitTooHigh => HttpResponse::new(StatusCode::BAD_REQUEST),
            GetCoursesError::DeserializeError(_) => HttpResponse::new(StatusCode::BAD_REQUEST),
            GetCoursesError::UploaderUnknown(_) => HttpResponse::new(StatusCode::BAD_REQUEST),
        }
    }
}
<|endoftext|>


<commit_before>extern crate glob;
extern crate prost_build;

fn main() {
    use glob::glob;

    prost_build::compile_protos(&["../common_protos/scaii.proto"], &["../common_protos"]).unwrap();

    // Make our protobuf file a reason to recompile
    // (See http://doc.crates.io/build-script.html for more info
    // on outputs)
    println!("cargo:rerun-if-changed=../common_protos/scaii.proto");

    // Unfortunately, rerun-if-changed overrides Cargo's default handling, which
    // is to rerun if any of our files change, so we have to re-add all our source
    // files
    println!("cargo:rerun-if-changed=Cargo.toml");

    for entry in glob("src/*.rs").expect("Could not parse glob pattern") {
        if let Ok(path) = entry {
            println!("cargo:rerun-if-changed={}", path.display())
        }
    }
}
<commit_msg>Add new protos to build script<commit_after>extern crate glob;
extern crate prost_build;

fn main() {
    use glob::glob;

    prost_build::compile_protos(&["../common_protos/scaii.proto"], &["../common_protos"]).unwrap();

    // Make our protobuf file a reason to recompile
    // (See http://doc.crates.io/build-script.html for more info
    // on outputs)
    for entry in glob("../common_protos/*.rs") {
        println!("cargo:rerun-if-changed={}", path.display());
    }

    // Unfortunately, rerun-if-changed overrides Cargo's default handling, which
    // is to rerun if any of our files change, so we have to re-add all our source
    // files
    println!("cargo:rerun-if-changed=Cargo.toml");

    for entry in glob("src/*.rs").expect("Could not parse glob pattern") {
        if let Ok(path) = entry {
            println!("cargo:rerun-if-changed={}", path.display())
        }
    }
}
<|endoftext|>


<commit_before>//! Lexing `&str` into a sequence of Rust tokens.
//!
//! Note that strictly speaking the parser in this crate is not required to work
//! on tokens which originated from text. Macros, eg, can synthesize tokens out
//! of thin air. So, ideally, lexer should be an orthogonal crate. It is however
//! convenient to include a text-based lexer here!
//!
//! Note that these tokens, unlike the tokens we feed into the parser, do
//! include info about comments and whitespace.

use std::ops;

use crate::{
    SyntaxKind::{self, *},
    T,
};

pub struct LexedStr<'a> {
    text: &'a str,
    kind: Vec<SyntaxKind>,
    start: Vec<u32>,
    error: Vec<LexError>,
}

struct LexError {
    msg: String,
    token: u32,
}

impl<'a> LexedStr<'a> {
    pub fn new(text: &'a str) -> LexedStr<'a> {
        let mut res = LexedStr { text, kind: Vec::new(), start: Vec::new(), error: Vec::new() };

        let mut offset = 0;
        if let Some(shebang_len) = rustc_lexer::strip_shebang(text) {
            res.push(SHEBANG, offset);
            offset = shebang_len
        };
        for token in rustc_lexer::tokenize(&text[offset..]) {
            let token_text = &text[offset..][..token.len];

            let (kind, err) = from_rustc(&token.kind, token_text);
            res.push(kind, offset);
            offset += token.len;

            if let Some(err) = err {
                let token = res.len() as u32;
                let msg = err.to_string();
                res.error.push(LexError { msg, token });
            }
        }
        res.push(EOF, offset);

        res
    }

    pub fn single_token(text: &'a str) -> Option<(SyntaxKind, Option<String>)> {
        if text.is_empty() {
            return None;
        }

        let token = rustc_lexer::first_token(text);
        if token.len != text.len() {
            return None;
        }

        let (kind, err) = from_rustc(&token.kind, text);
        Some((kind, err.map(|it| it.to_owned())))
    }

    pub fn as_str(&self) -> &str {
        self.text
    }

    pub fn len(&self) -> usize {
        self.kind.len() - 1
    }

    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    pub fn kind(&self, i: usize) -> SyntaxKind {
        assert!(i < self.len());
        self.kind[i]
    }

    pub fn text(&self, i: usize) -> &str {
        self.range_text(i..i + 1)
    }
    pub fn range_text(&self, r: ops::Range<usize>) -> &str {
        assert!(r.start < r.end && r.end <= self.len());
        let lo = self.start[r.start] as usize;
        let hi = self.start[r.end] as usize;
        &self.text[lo..hi]
    }

    // Naming is hard.
    pub fn text_range(&self, i: usize) -> ops::Range<usize> {
        assert!(i < self.len());
        let lo = self.start[i] as usize;
        let hi = self.start[i + 1] as usize;
        lo..hi
    }
    pub fn text_start(&self, i: usize) -> usize {
        assert!(i <= self.len());
        self.start[i] as usize
    }
    pub fn text_len(&self, i: usize) -> usize {
        assert!(i < self.len());
        let r = self.text_range(i);
        r.end - r.start
    }

    pub fn error(&self, i: usize) -> Option<&str> {
        assert!(i < self.len());
        let err = self.error.binary_search_by_key(&(i as u32), |i| i.token).ok()?;
        Some(self.error[err].msg.as_str())
    }

    pub fn errors(&self) -> impl Iterator<Item = (usize, &str)> + '_ {
        self.error.iter().map(|it| (it.token as usize, it.msg.as_str()))
    }

    fn push(&mut self, kind: SyntaxKind, offset: usize) {
        self.kind.push(kind);
        self.start.push(offset as u32);
    }
}

/// Returns `SyntaxKind` and an optional tokenize error message.
fn from_rustc(
    kind: &rustc_lexer::TokenKind,
    token_text: &str,
) -> (SyntaxKind, Option<&'static str>) {
    // A note on an intended tradeoff:
    // We drop some useful information here (see patterns with double dots `..`)
    // Storing that info in `SyntaxKind` is not possible due to its layout requirements of
    // being `u16` that come from `rowan::SyntaxKind`.
    let mut err = "";

    let syntax_kind = {
        match kind {
            rustc_lexer::TokenKind::LineComment { doc_style: _ } => COMMENT,
            rustc_lexer::TokenKind::BlockComment { doc_style: _, terminated } => {
                if !terminated {
                    err = "Missing trailing `*/` symbols to terminate the block comment";
                }
                COMMENT
            }

            rustc_lexer::TokenKind::Whitespace => WHITESPACE,

            rustc_lexer::TokenKind::Ident if token_text == "_" => UNDERSCORE,
            rustc_lexer::TokenKind::Ident => SyntaxKind::from_keyword(token_text).unwrap_or(IDENT),

            rustc_lexer::TokenKind::RawIdent => IDENT,
            rustc_lexer::TokenKind::Literal { kind, .. } => return from_rustc_literal(kind),

            rustc_lexer::TokenKind::Lifetime { starts_with_number } => {
                if *starts_with_number {
                    err = "Lifetime name cannot start with a number";
                }
                LIFETIME_IDENT
            }

            rustc_lexer::TokenKind::Semi => T![;],
            rustc_lexer::TokenKind::Comma => T![,],
            rustc_lexer::TokenKind::Dot => T![.],
            rustc_lexer::TokenKind::OpenParen => T!['('],
            rustc_lexer::TokenKind::CloseParen => T![')'],
            rustc_lexer::TokenKind::OpenBrace => T!['{'],
            rustc_lexer::TokenKind::CloseBrace => T!['}'],
            rustc_lexer::TokenKind::OpenBracket => T!['['],
            rustc_lexer::TokenKind::CloseBracket => T![']'],
            rustc_lexer::TokenKind::At => T![@],
            rustc_lexer::TokenKind::Pound => T![#],
            rustc_lexer::TokenKind::Tilde => T![~],
            rustc_lexer::TokenKind::Question => T![?],
            rustc_lexer::TokenKind::Colon => T![:],
            rustc_lexer::TokenKind::Dollar => T![$],
            rustc_lexer::TokenKind::Eq => T![=],
            rustc_lexer::TokenKind::Bang => T![!],
            rustc_lexer::TokenKind::Lt => T![<],
            rustc_lexer::TokenKind::Gt => T![>],
            rustc_lexer::TokenKind::Minus => T![-],
            rustc_lexer::TokenKind::And => T![&],
            rustc_lexer::TokenKind::Or => T![|],
            rustc_lexer::TokenKind::Plus => T![+],
            rustc_lexer::TokenKind::Star => T![*],
            rustc_lexer::TokenKind::Slash => T![/],
            rustc_lexer::TokenKind::Caret => T![^],
            rustc_lexer::TokenKind::Percent => T![%],
            rustc_lexer::TokenKind::Unknown => ERROR,
        }
    };

    let err = if err.is_empty() { None } else { Some(err) };
    (syntax_kind, err)
}

fn from_rustc_literal(kind: &rustc_lexer::LiteralKind) -> (SyntaxKind, Option<&'static str>) {
    let mut err = "";

    let syntax_kind = match *kind {
        rustc_lexer::LiteralKind::Int { empty_int, base: _ } => {
            if empty_int {
                err = "Missing digits after the integer base prefix";
            }
            INT_NUMBER
        }
        rustc_lexer::LiteralKind::Float { empty_exponent, base: _ } => {
            if empty_exponent {
                err = "Missing digits after the exponent symbol";
            }
            FLOAT_NUMBER
        }
        rustc_lexer::LiteralKind::Char { terminated } => {
            if !terminated {
                err = "Missing trailing `'` symbol to terminate the character literal";
            }
            CHAR
        }
        rustc_lexer::LiteralKind::Byte { terminated } => {
            if !terminated {
                err = "Missing trailing `'` symbol to terminate the byte literal";
            }
            BYTE
        }
        rustc_lexer::LiteralKind::Str { terminated } => {
            if !terminated {
                err = "Missing trailing `\"` symbol to terminate the string literal";
            }
            STRING
        }
        rustc_lexer::LiteralKind::ByteStr { terminated } => {
            if !terminated {
                err = "Missing trailing `\"` symbol to terminate the byte string literal";
            }
            BYTE_STRING
        }
        rustc_lexer::LiteralKind::RawStr { err: raw_str_err, .. } => {
            if let Some(raw_str_err) = raw_str_err {
                err = match raw_str_err {
                    rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw string literal",
                    rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
                        "Missing trailing `\"` to terminate the raw string literal"
                    } else {
                        "Missing trailing `\"` with `#` symbols to terminate the raw string literal"
                    },
                    rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols",
                };
            };
            STRING
        }
        rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => {
            if let Some(raw_str_err) = raw_str_err {
                err = match raw_str_err {
                    rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw byte string literal",
                    rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
                        "Missing trailing `\"` to terminate the raw byte string literal"
                    } else {
                        "Missing trailing `\"` with `#` symbols to terminate the raw byte string literal"
                    },
                    rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols",
                };
            };

            BYTE_STRING
        }
    };

    let err = if err.is_empty() { None } else { Some(err) };
    (syntax_kind, err)
}
<commit_msg>Auto merge of #12136 - jonas-schievink:lexedstr-converter, r=jonas-schievink<commit_after>//! Lexing `&str` into a sequence of Rust tokens.
//!
//! Note that strictly speaking the parser in this crate is not required to work
//! on tokens which originated from text. Macros, eg, can synthesize tokens out
//! of thin air. So, ideally, lexer should be an orthogonal crate. It is however
//! convenient to include a text-based lexer here!
//!
//! Note that these tokens, unlike the tokens we feed into the parser, do
//! include info about comments and whitespace.

use std::ops;

use crate::{
    SyntaxKind::{self, *},
    T,
};

pub struct LexedStr<'a> {
    text: &'a str,
    kind: Vec<SyntaxKind>,
    start: Vec<u32>,
    error: Vec<LexError>,
}

struct LexError {
    msg: String,
    token: u32,
}

impl<'a> LexedStr<'a> {
    pub fn new(text: &'a str) -> LexedStr<'a> {
        let mut conv = Converter::new(text);
        if let Some(shebang_len) = rustc_lexer::strip_shebang(text) {
            conv.res.push(SHEBANG, conv.offset);
            conv.offset = shebang_len;
        };

        for token in rustc_lexer::tokenize(&text[conv.offset..]) {
            let token_text = &text[conv.offset..][..token.len];

            conv.extend_token(&token.kind, token_text);
        }

        conv.finalize_with_eof()
    }

    pub fn single_token(text: &'a str) -> Option<(SyntaxKind, Option<String>)> {
        if text.is_empty() {
            return None;
        }

        let token = rustc_lexer::first_token(text);
        if token.len != text.len() {
            return None;
        }

        let mut conv = Converter::new(text);
        conv.extend_token(&token.kind, text);
        match &*conv.res.kind {
            [kind] => Some((*kind, conv.res.error.pop().map(|it| it.msg.clone()))),
            _ => None,
        }
    }

    pub fn as_str(&self) -> &str {
        self.text
    }

    pub fn len(&self) -> usize {
        self.kind.len() - 1
    }

    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    pub fn kind(&self, i: usize) -> SyntaxKind {
        assert!(i < self.len());
        self.kind[i]
    }

    pub fn text(&self, i: usize) -> &str {
        self.range_text(i..i + 1)
    }
    pub fn range_text(&self, r: ops::Range<usize>) -> &str {
        assert!(r.start < r.end && r.end <= self.len());
        let lo = self.start[r.start] as usize;
        let hi = self.start[r.end] as usize;
        &self.text[lo..hi]
    }

    // Naming is hard.
    pub fn text_range(&self, i: usize) -> ops::Range<usize> {
        assert!(i < self.len());
        let lo = self.start[i] as usize;
        let hi = self.start[i + 1] as usize;
        lo..hi
    }
    pub fn text_start(&self, i: usize) -> usize {
        assert!(i <= self.len());
        self.start[i] as usize
    }
    pub fn text_len(&self, i: usize) -> usize {
        assert!(i < self.len());
        let r = self.text_range(i);
        r.end - r.start
    }

    pub fn error(&self, i: usize) -> Option<&str> {
        assert!(i < self.len());
        let err = self.error.binary_search_by_key(&(i as u32), |i| i.token).ok()?;
        Some(self.error[err].msg.as_str())
    }

    pub fn errors(&self) -> impl Iterator<Item = (usize, &str)> + '_ {
        self.error.iter().map(|it| (it.token as usize, it.msg.as_str()))
    }

    fn push(&mut self, kind: SyntaxKind, offset: usize) {
        self.kind.push(kind);
        self.start.push(offset as u32);
    }
}

struct Converter<'a> {
    res: LexedStr<'a>,
    offset: usize,
}

impl<'a> Converter<'a> {
    fn new(text: &'a str) -> Self {
        Self {
            res: LexedStr { text, kind: Vec::new(), start: Vec::new(), error: Vec::new() },
            offset: 0,
        }
    }

    fn finalize_with_eof(mut self) -> LexedStr<'a> {
        self.res.push(EOF, self.offset);
        self.res
    }

    fn push(&mut self, kind: SyntaxKind, len: usize, err: Option<&str>) {
        self.res.push(kind, self.offset);
        self.offset += len;

        if let Some(err) = err {
            let token = self.res.len() as u32;
            let msg = err.to_string();
            self.res.error.push(LexError { msg, token });
        }
    }

    fn extend_token(&mut self, kind: &rustc_lexer::TokenKind, token_text: &str) {
        // A note on an intended tradeoff:
        // We drop some useful information here (see patterns with double dots `..`)
        // Storing that info in `SyntaxKind` is not possible due to its layout requirements of
        // being `u16` that come from `rowan::SyntaxKind`.
        let mut err = "";

        let syntax_kind = {
            match kind {
                rustc_lexer::TokenKind::LineComment { doc_style: _ } => COMMENT,
                rustc_lexer::TokenKind::BlockComment { doc_style: _, terminated } => {
                    if !terminated {
                        err = "Missing trailing `*/` symbols to terminate the block comment";
                    }
                    COMMENT
                }

                rustc_lexer::TokenKind::Whitespace => WHITESPACE,

                rustc_lexer::TokenKind::Ident if token_text == "_" => UNDERSCORE,
                rustc_lexer::TokenKind::Ident => {
                    SyntaxKind::from_keyword(token_text).unwrap_or(IDENT)
                }

                rustc_lexer::TokenKind::RawIdent => IDENT,
                rustc_lexer::TokenKind::Literal { kind, .. } => {
                    self.extend_literal(token_text.len(), kind);
                    return;
                }

                rustc_lexer::TokenKind::Lifetime { starts_with_number } => {
                    if *starts_with_number {
                        err = "Lifetime name cannot start with a number";
                    }
                    LIFETIME_IDENT
                }

                rustc_lexer::TokenKind::Semi => T![;],
                rustc_lexer::TokenKind::Comma => T![,],
                rustc_lexer::TokenKind::Dot => T![.],
                rustc_lexer::TokenKind::OpenParen => T!['('],
                rustc_lexer::TokenKind::CloseParen => T![')'],
                rustc_lexer::TokenKind::OpenBrace => T!['{'],
                rustc_lexer::TokenKind::CloseBrace => T!['}'],
                rustc_lexer::TokenKind::OpenBracket => T!['['],
                rustc_lexer::TokenKind::CloseBracket => T![']'],
                rustc_lexer::TokenKind::At => T![@],
                rustc_lexer::TokenKind::Pound => T![#],
                rustc_lexer::TokenKind::Tilde => T![~],
                rustc_lexer::TokenKind::Question => T![?],
                rustc_lexer::TokenKind::Colon => T![:],
                rustc_lexer::TokenKind::Dollar => T![$],
                rustc_lexer::TokenKind::Eq => T![=],
                rustc_lexer::TokenKind::Bang => T![!],
                rustc_lexer::TokenKind::Lt => T![<],
                rustc_lexer::TokenKind::Gt => T![>],
                rustc_lexer::TokenKind::Minus => T![-],
                rustc_lexer::TokenKind::And => T![&],
                rustc_lexer::TokenKind::Or => T![|],
                rustc_lexer::TokenKind::Plus => T![+],
                rustc_lexer::TokenKind::Star => T![*],
                rustc_lexer::TokenKind::Slash => T![/],
                rustc_lexer::TokenKind::Caret => T![^],
                rustc_lexer::TokenKind::Percent => T![%],
                rustc_lexer::TokenKind::Unknown => ERROR,
            }
        };

        let err = if err.is_empty() { None } else { Some(err) };
        self.push(syntax_kind, token_text.len(), err);
    }

    fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) {
        let mut err = "";

        let syntax_kind = match *kind {
            rustc_lexer::LiteralKind::Int { empty_int, base: _ } => {
                if empty_int {
                    err = "Missing digits after the integer base prefix";
                }
                INT_NUMBER
            }
            rustc_lexer::LiteralKind::Float { empty_exponent, base: _ } => {
                if empty_exponent {
                    err = "Missing digits after the exponent symbol";
                }
                FLOAT_NUMBER
            }
            rustc_lexer::LiteralKind::Char { terminated } => {
                if !terminated {
                    err = "Missing trailing `'` symbol to terminate the character literal";
                }
                CHAR
            }
            rustc_lexer::LiteralKind::Byte { terminated } => {
                if !terminated {
                    err = "Missing trailing `'` symbol to terminate the byte literal";
                }
                BYTE
            }
            rustc_lexer::LiteralKind::Str { terminated } => {
                if !terminated {
                    err = "Missing trailing `\"` symbol to terminate the string literal";
                }
                STRING
            }
            rustc_lexer::LiteralKind::ByteStr { terminated } => {
                if !terminated {
                    err = "Missing trailing `\"` symbol to terminate the byte string literal";
                }
                BYTE_STRING
            }
            rustc_lexer::LiteralKind::RawStr { err: raw_str_err, .. } => {
                if let Some(raw_str_err) = raw_str_err {
                    err = match raw_str_err {
                        rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw string literal",
                        rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
                            "Missing trailing `\"` to terminate the raw string literal"
                        } else {
                            "Missing trailing `\"` with `#` symbols to terminate the raw string literal"
                        },
                        rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols",
                    };
                };
                STRING
            }
            rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => {
                if let Some(raw_str_err) = raw_str_err {
                    err = match raw_str_err {
                        rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw byte string literal",
                        rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
                            "Missing trailing `\"` to terminate the raw byte string literal"
                        } else {
                            "Missing trailing `\"` with `#` symbols to terminate the raw byte string literal"
                        },
                        rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols",
                    };
                };

                BYTE_STRING
            }
        };

        let err = if err.is_empty() { None } else { Some(err) };
        self.push(syntax_kind, len, err);
    }
}
<|endoftext|>