<commit_before>use std::env;
use std::path::Path;

use lazy_static::lazy_static;
use raw_cpuid::{CpuId, ExtendedFeatures, FeatureInfo};

lazy_static! {
    static ref CPU_ID: CpuId = CpuId::new();
    static ref CPU_FEATURES: Option<FeatureInfo> = CPU_ID.get_feature_info();
    static ref CPU_EXTENDED_FEATURES: Option<ExtendedFeatures> = CPU_ID.get_extended_feature_info();
    static ref TARGET_FEATURES: Vec<String> = env::var("CARGO_CFG_TARGET_FEATURE")
        .map_or_else(|_| vec![], |s| s.split(',').map(|s| s.to_owned()).collect());
    static ref TARGET_ARCH: String = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
    static ref TARGET_ENV: String = env::var("CARGO_CFG_TARGET_ENV").unwrap();
}

fn has_target_feature(feature: &str) -> bool {
    TARGET_FEATURES.iter().any(|name| name == feature)
}

fn has_aesni() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_aesni())
}

fn has_sse41() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_sse41())
}

fn has_sse42() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_sse42())
}

fn has_avx() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_avx())
}

fn has_avx2() -> bool {
    cfg!(feature = "native")
        && CPU_EXTENDED_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_avx2())
}

fn support_aesni() -> bool {
    cfg!(feature = "aes") || has_target_feature("aes") || has_aesni()
}

#[allow(dead_code)]
fn support_sse41() -> bool {
    cfg!(feature = "sse41") || has_target_feature("sse41") || has_sse41()
}

fn support_sse42() -> bool {
    cfg!(feature = "sse42") || has_target_feature("sse42") || has_sse42()
}

fn support_avx() -> bool {
    cfg!(feature = "avx") || has_target_feature("avx") || has_avx()
}

fn support_avx2() -> bool {
    cfg!(feature = "avx2") || has_target_feature("avx2") || has_avx2()
}

#[cfg(all(not(feature = "gen"), any(target_os = "macos", target_os = "linux")))]
fn generate_binding(_out_file: &Path) {
    cargo_emit::warning!("pregenerated binding file.");
}

#[cfg(any(feature = "gen", not(any(target_os = "macos", target_os = "linux"))))]
fn generate_binding(out_file: &Path) {
    cargo_emit::warning!("generate binding file @ {:?}.", out_file);

    let _ = bindgen::builder()
        .clang_args(&["-x", "c++", "-std=c++11"])
        .clang_args(&[
            "-Dt1ha_EXPORTS",
            "-DXXH_STATIC_LINKING_ONLY",
            "-Isrc/highwayhash",
        ])
        .clang_args(if support_aesni() {
            &[
                "-maes",
                "-DT1HA0_RUNTIME_SELECT=1",
                "-DT1HA0_AESNI_AVAILABLE=1",
            ][..]
        } else {
            &[][..]
        })
        .clang_args(
            vec![
                if cfg!(feature = "native") {
                    Some("-march=native")
                } else {
                    None
                },
                if support_sse41() {
                    Some("-msse4.1")
                } else {
                    None
                },
                if support_sse42() {
                    Some("-msse4.2")
                } else {
                    None
                },
                if support_avx() { Some("-mavx") } else { None },
                if support_avx2() { Some("-mavx2") } else { None },
                if cfg!(feature = "city") {
                    Some("-DCITY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "farm") {
                    Some("-DFARM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "highway") {
                    Some("-DHIGHWAY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "komi") {
                    Some("-DKOMI_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "lookup3") {
                    Some("-DLOOKUP3=1")
                } else {
                    None
                },
                if cfg!(feature = "meow") && matches!(TARGET_ARCH.as_str(), "x86" | "x86_64") {
                    Some("-DMEOW_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "metro") {
                    Some("-DMETRO_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "mum") {
                    Some("-DMUM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "murmur") {
                    Some("-DMURMUR_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "mx3") {
                    Some("-DMX3_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "nm") {
                    Some("-DNM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "pengy") {
                    Some("-DPENGY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "prv") {
                    Some("-DPRV_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "spooky") {
                    Some("-DSPOOKY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "t1ha") {
                    Some("-DT1_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "wy") {
                    Some("-DWY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "xx") {
                    Some("-DXX_HASH=1")
                } else {
                    None
                },
            ]
            .into_iter()
            .flatten(),
        )
        .header("src/fasthash.hpp")
        .size_t_is_usize(true)
        .generate_inline_functions(true)
        .disable_name_namespacing()
        .allowlist_function("^CityHash.*")
        .allowlist_function("^farmhash.*")
        .allowlist_function("^HighwayHash.*")
        .allowlist_function("^komi.*")
        .allowlist_function("^lookup3.*")
        .allowlist_function("^metrohash.*")
        .allowlist_function("^mum_hash.*")
        .allowlist_function("^MurmurHash.*")
        .allowlist_function("^mx3hash.*")
        .allowlist_function("^NMHASH.*")
        .allowlist_function("^pengy.*")
        .allowlist_function("^prvhash.*")
        .allowlist_function("^SpookyHasher.*")
        .allowlist_function("^t1ha.*")
        .allowlist_function("^wyhash.*")
        .allowlist_function("^XXH.*")
        .allowlist_function("^Meow.*")
        .blocklist_function("^t1ha_selfcheck__.*")
        .allowlist_var("^Meow.*")
        .allowlist_var("^PRH64S_.*")
        .generate()
        .unwrap()
        .write_to_file(out_file)
        .expect("fail to write bindings");

    cargo_emit::warning!("generate binding file @ {:?}.", out_file);
}

fn build_fasthash() {
    let mut build = cc::Build::new();

    build
        .cpp(true)
        .include("src/highwayhash")
        .flag("-std=c++11")
        .flag_if_supported("-Wno-implicit-fallthrough")
        .flag_if_supported("-Wno-unknown-attributes")
        .flag_if_supported("-Wno-sign-compare")
        .file("src/fasthash.cpp");

    if cfg!(feature = "city") {
        build.flag("-DCITY_HASH=1").file("src/smhasher/City.cpp");
    }

    if cfg!(feature = "farm") {
        build
            .flag("-DFARM_HASH=1")
            .file("src/smhasher/farmhash-c.c");
    }

    if cfg!(feature = "komi") {
        build.flag("-DKOMI_HASH=1");
    }

    if cfg!(feature = "highway") {
        build.flag("-DHIGHWAY_HASH=1");
    }

    if cfg!(feature = "lookup3") {
        build.flag("-DLOOKUP3=1").file("src/smhasher/lookup3.cpp");
    }

    if cfg!(feature = "meow") && matches!(TARGET_ARCH.as_str(), "x86" | "x86_64") {
        build.flag("-DMEOW_HASH=1");
    }

    if cfg!(feature = "mx3") {
        build.flag("-DMX3_HASH=1");
    }

    if cfg!(feature = "nm") {
        build.flag("-DNM_HASH=1");
    }

    if cfg!(feature = "pengy") {
        build
            .flag("-DPENGY_HASH=1")
            .file("src/pengyhash/pengyhash.c");
    }

    if cfg!(feature = "prv") {
        build.flag("-DPRV_HASH=1");
    }

    if cfg!(feature = "t1ha") {
        build.flag("-DT1_HASH=1");
    }

    if cfg!(feature = "metro") {
        build
            .flag("-DMETRO_HASH=1")
            .file("src/smhasher/metrohash/metrohash64.cpp")
            .file("src/smhasher/metrohash/metrohash128.cpp");

        if support_sse42() {
            build
                .file("src/smhasher/metrohash/metrohash64crc.cpp")
                .file("src/smhasher/metrohash/metrohash128crc.cpp");
        }
    }

    if cfg!(feature = "mum") {
        build.flag("-DMUM_HASH=1").file("src/smhasher/mum.cc");
    }

    if cfg!(feature = "murmur") {
        build
            .flag("-DMURMUR_HASH=1")
            .file("src/smhasher/MurmurHash1.cpp")
            .file("src/smhasher/MurmurHash2.cpp")
            .file("src/smhasher/MurmurHash3.cpp");
    }

    if cfg!(feature = "spooky") {
        build
            .flag("-DSPOOKY_HASH=1")
            .file("src/smhasher/Spooky.cpp");
    }

    if cfg!(feature = "wy") {
        build.flag("-DWY_HASH=1");
    }

    if cfg!(feature = "xx") {
        build.flag("-DXX_HASH=1").file("src/xxHash/xxhash.c");
    }

    if cfg!(feature = "native") {
        build.flag("-march=native");
    } else {
        if has_target_feature("aes") {
            build.flag("-maes");
        }
        if has_target_feature("sse41") {
            build.flag("-msse41");
        }
        if has_target_feature("sse42") {
            build.flag("-msse4.2");
        }
        if has_target_feature("avx") {
            build.flag("-mavx");
        }
        if has_target_feature("avx2") {
            build.flag("-mavx2");
        }
    }

    build.static_flag(true).compile("fasthash");
}

fn build_t1() {
    let mut build = cc::Build::new();

    build
        .file("src/t1ha/src/t1ha0.c")
        .file("src/t1ha/src/t1ha1.c")
        .file("src/t1ha/src/t1ha2.c");

    // indirect functions are not supported on all targets (e.g. x86_64-unknown-linux-musl)
    if TARGET_ENV.as_str() == "musl" {
        build.define("T1HA_USE_INDIRECT_FUNCTIONS", Some("0"));
    }

    if support_aesni() {
        build
            .define("T1HA0_RUNTIME_SELECT", Some("1"))
            .define("T1HA0_AESNI_AVAILABLE", Some("1"))
            .flag("-maes")
            .file("src/t1ha/src/t1ha0_ia32aes_noavx.c")
            .file("src/t1ha/src/t1ha0_ia32aes_avx.c")
            .file("src/t1ha/src/t1ha0_ia32aes_avx2.c");

        if support_avx() {
            build.flag("-mavx");
        }

        if support_avx2() {
            build.flag("-mavx2");
        }
    }

    build.static_flag(true).compile("t1ha");
}

fn build_highway() {
    let mut build = cc::Build::new();

    build
        .cpp(true)
        .flag("-std=c++11")
        .flag_if_supported("-Wno-sign-compare")
        .include("src/highwayhash")
        .file("src/highwayhash/highwayhash/arch_specific.cc")
        .file("src/highwayhash/highwayhash/instruction_sets.cc")
        .file("src/highwayhash/highwayhash/os_specific.cc")
        .file("src/highwayhash/highwayhash/hh_portable.cc")
        .file("src/highwayhash/highwayhash/c_bindings.cc");

    match TARGET_ARCH.as_str() {
        "x86" | "x86_64" => {
            build
                .flag("-msse4.1")
                .flag("-mavx2")
                .file("src/highwayhash/highwayhash/hh_sse41.cc")
                .file("src/highwayhash/highwayhash/hh_avx2.cc");
        }

        "aarch64" => {
            build.file("src/highwayhash/highwayhash/hh_neon.cc");
        }

        "powerpc64" => {
            build
                .flag("-mvsx")
                .file("src/highwayhash/highwayhash/hh_vsx.cc");
        }
        _ => {}
    }

    build.static_flag(true).compile("highwayhash");
}

fn main() {
    if has_aesni() {
        cargo_emit::rustc_cfg!(r#"feature="aes""#);
    }
    if has_sse41() {
        cargo_emit::rustc_cfg!(r#"feature="sse41""#);
    }
    if has_sse42() {
        cargo_emit::rustc_cfg!(r#"feature="sse42""#);
    }
    if has_avx() {
        cargo_emit::rustc_cfg!(r#"feature="avx""#);
    }
    if has_avx2() {
        cargo_emit::rustc_cfg!(r#"feature="avx2""#);
    }

    build_fasthash();
    if cfg!(feature = "t1ha") {
        build_t1();
    }
    if cfg!(feature = "highway") {
        build_highway();
    }

    let out_dir = env::var("OUT_DIR").unwrap();
    let out_file = Path::new(&out_dir).join("fasthash.rs");

    cargo_emit::rerun_if_changed!("src/fasthash.hpp");
    cargo_emit::rerun_if_changed!("src/fasthash.cpp");

    generate_binding(&out_file);
}
<commit_msg>add -mpower8-vector for ppc64<commit_after>use std::env;
use std::path::Path;

use lazy_static::lazy_static;
use raw_cpuid::{CpuId, ExtendedFeatures, FeatureInfo};

lazy_static! {
    static ref CPU_ID: CpuId = CpuId::new();
    static ref CPU_FEATURES: Option<FeatureInfo> = CPU_ID.get_feature_info();
    static ref CPU_EXTENDED_FEATURES: Option<ExtendedFeatures> = CPU_ID.get_extended_feature_info();
    static ref TARGET_FEATURES: Vec<String> = env::var("CARGO_CFG_TARGET_FEATURE")
        .map_or_else(|_| vec![], |s| s.split(',').map(|s| s.to_owned()).collect());
    static ref TARGET_ARCH: String = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
    static ref TARGET_ENV: String = env::var("CARGO_CFG_TARGET_ENV").unwrap();
}

fn has_target_feature(feature: &str) -> bool {
    TARGET_FEATURES.iter().any(|name| name == feature)
}

fn has_aesni() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_aesni())
}

fn has_sse41() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_sse41())
}

fn has_sse42() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_sse42())
}

fn has_avx() -> bool {
    cfg!(feature = "native")
        && CPU_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_avx())
}

fn has_avx2() -> bool {
    cfg!(feature = "native")
        && CPU_EXTENDED_FEATURES
            .as_ref()
            .map_or(false, |features| features.has_avx2())
}

fn support_aesni() -> bool {
    cfg!(feature = "aes") || has_target_feature("aes") || has_aesni()
}

#[allow(dead_code)]
fn support_sse41() -> bool {
    cfg!(feature = "sse41") || has_target_feature("sse41") || has_sse41()
}

fn support_sse42() -> bool {
    cfg!(feature = "sse42") || has_target_feature("sse42") || has_sse42()
}

fn support_avx() -> bool {
    cfg!(feature = "avx") || has_target_feature("avx") || has_avx()
}

fn support_avx2() -> bool {
    cfg!(feature = "avx2") || has_target_feature("avx2") || has_avx2()
}

#[cfg(all(not(feature = "gen"), any(target_os = "macos", target_os = "linux")))]
fn generate_binding(_out_file: &Path) {
    cargo_emit::warning!("pregenerated binding file.");
}

#[cfg(any(feature = "gen", not(any(target_os = "macos", target_os = "linux"))))]
fn generate_binding(out_file: &Path) {
    cargo_emit::warning!("generate binding file @ {:?}.", out_file);

    let _ = bindgen::builder()
        .clang_args(&["-x", "c++", "-std=c++11"])
        .clang_args(&[
            "-Dt1ha_EXPORTS",
            "-DXXH_STATIC_LINKING_ONLY",
            "-Isrc/highwayhash",
        ])
        .clang_args(if support_aesni() {
            &[
                "-maes",
                "-DT1HA0_RUNTIME_SELECT=1",
                "-DT1HA0_AESNI_AVAILABLE=1",
            ][..]
        } else {
            &[][..]
        })
        .clang_args(
            vec![
                if cfg!(feature = "native") {
                    Some("-march=native")
                } else {
                    None
                },
                if support_sse41() {
                    Some("-msse4.1")
                } else {
                    None
                },
                if support_sse42() {
                    Some("-msse4.2")
                } else {
                    None
                },
                if support_avx() { Some("-mavx") } else { None },
                if support_avx2() { Some("-mavx2") } else { None },
                if cfg!(feature = "city") {
                    Some("-DCITY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "farm") {
                    Some("-DFARM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "highway") {
                    Some("-DHIGHWAY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "komi") {
                    Some("-DKOMI_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "lookup3") {
                    Some("-DLOOKUP3=1")
                } else {
                    None
                },
                if cfg!(feature = "meow") && matches!(TARGET_ARCH.as_str(), "x86" | "x86_64") {
                    Some("-DMEOW_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "metro") {
                    Some("-DMETRO_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "mum") {
                    Some("-DMUM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "murmur") {
                    Some("-DMURMUR_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "mx3") {
                    Some("-DMX3_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "nm") {
                    Some("-DNM_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "pengy") {
                    Some("-DPENGY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "prv") {
                    Some("-DPRV_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "spooky") {
                    Some("-DSPOOKY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "t1ha") {
                    Some("-DT1_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "wy") {
                    Some("-DWY_HASH=1")
                } else {
                    None
                },
                if cfg!(feature = "xx") {
                    Some("-DXX_HASH=1")
                } else {
                    None
                },
            ]
            .into_iter()
            .flatten(),
        )
        .header("src/fasthash.hpp")
        .size_t_is_usize(true)
        .generate_inline_functions(true)
        .disable_name_namespacing()
        .allowlist_function("^CityHash.*")
        .allowlist_function("^farmhash.*")
        .allowlist_function("^HighwayHash.*")
        .allowlist_function("^komi.*")
        .allowlist_function("^lookup3.*")
        .allowlist_function("^metrohash.*")
        .allowlist_function("^mum_hash.*")
        .allowlist_function("^MurmurHash.*")
        .allowlist_function("^mx3hash.*")
        .allowlist_function("^NMHASH.*")
        .allowlist_function("^pengy.*")
        .allowlist_function("^prvhash.*")
        .allowlist_function("^SpookyHasher.*")
        .allowlist_function("^t1ha.*")
        .allowlist_function("^wyhash.*")
        .allowlist_function("^XXH.*")
        .allowlist_function("^Meow.*")
        .blocklist_function("^t1ha_selfcheck__.*")
        .allowlist_var("^Meow.*")
        .allowlist_var("^PRH64S_.*")
        .generate()
        .unwrap()
        .write_to_file(out_file)
        .expect("fail to write bindings");

    cargo_emit::warning!("generate binding file @ {:?}.", out_file);
}

fn build_fasthash() {
    let mut build = cc::Build::new();

    build
        .cpp(true)
        .include("src/highwayhash")
        .flag("-std=c++11")
        .flag_if_supported("-Wno-implicit-fallthrough")
        .flag_if_supported("-Wno-unknown-attributes")
        .flag_if_supported("-Wno-sign-compare")
        .file("src/fasthash.cpp");

    if cfg!(feature = "city") {
        build.flag("-DCITY_HASH=1").file("src/smhasher/City.cpp");
    }

    if cfg!(feature = "farm") {
        build
            .flag("-DFARM_HASH=1")
            .file("src/smhasher/farmhash-c.c");
    }

    if cfg!(feature = "komi") {
        build.flag("-DKOMI_HASH=1");
    }

    if cfg!(feature = "highway") {
        build.flag("-DHIGHWAY_HASH=1");
    }

    if cfg!(feature = "lookup3") {
        build.flag("-DLOOKUP3=1").file("src/smhasher/lookup3.cpp");
    }

    if cfg!(feature = "meow") && matches!(TARGET_ARCH.as_str(), "x86" | "x86_64") {
        build.flag("-DMEOW_HASH=1");
    }

    if cfg!(feature = "mx3") {
        build.flag("-DMX3_HASH=1");
    }

    if cfg!(feature = "nm") {
        build.flag("-DNM_HASH=1");
    }

    if cfg!(feature = "pengy") {
        build
            .flag("-DPENGY_HASH=1")
            .file("src/pengyhash/pengyhash.c");
    }

    if cfg!(feature = "prv") {
        build.flag("-DPRV_HASH=1");
    }

    if cfg!(feature = "t1ha") {
        build.flag("-DT1_HASH=1");
    }

    if cfg!(feature = "metro") {
        build
            .flag("-DMETRO_HASH=1")
            .file("src/smhasher/metrohash/metrohash64.cpp")
            .file("src/smhasher/metrohash/metrohash128.cpp");

        if support_sse42() {
            build
                .file("src/smhasher/metrohash/metrohash64crc.cpp")
                .file("src/smhasher/metrohash/metrohash128crc.cpp");
        }
    }

    if cfg!(feature = "mum") {
        build.flag("-DMUM_HASH=1").file("src/smhasher/mum.cc");
    }

    if cfg!(feature = "murmur") {
        build
            .flag("-DMURMUR_HASH=1")
            .file("src/smhasher/MurmurHash1.cpp")
            .file("src/smhasher/MurmurHash2.cpp")
            .file("src/smhasher/MurmurHash3.cpp");
    }

    if cfg!(feature = "spooky") {
        build
            .flag("-DSPOOKY_HASH=1")
            .file("src/smhasher/Spooky.cpp");
    }

    if cfg!(feature = "wy") {
        build.flag("-DWY_HASH=1");
    }

    if cfg!(feature = "xx") {
        build.flag("-DXX_HASH=1").file("src/xxHash/xxhash.c");
    }

    if cfg!(feature = "native") {
        build.flag("-march=native");
    } else {
        if has_target_feature("aes") {
            build.flag("-maes");
        }
        if has_target_feature("sse41") {
            build.flag("-msse41");
        }
        if has_target_feature("sse42") {
            build.flag("-msse4.2");
        }
        if has_target_feature("avx") {
            build.flag("-mavx");
        }
        if has_target_feature("avx2") {
            build.flag("-mavx2");
        }
    }

    build.static_flag(true).compile("fasthash");
}

fn build_t1() {
    let mut build = cc::Build::new();

    build
        .file("src/t1ha/src/t1ha0.c")
        .file("src/t1ha/src/t1ha1.c")
        .file("src/t1ha/src/t1ha2.c");

    // indirect functions are not supported on all targets (e.g. x86_64-unknown-linux-musl)
    if TARGET_ENV.as_str() == "musl" {
        build.define("T1HA_USE_INDIRECT_FUNCTIONS", Some("0"));
    }

    if support_aesni() {
        build
            .define("T1HA0_RUNTIME_SELECT", Some("1"))
            .define("T1HA0_AESNI_AVAILABLE", Some("1"))
            .flag("-maes")
            .file("src/t1ha/src/t1ha0_ia32aes_noavx.c")
            .file("src/t1ha/src/t1ha0_ia32aes_avx.c")
            .file("src/t1ha/src/t1ha0_ia32aes_avx2.c");

        if support_avx() {
            build.flag("-mavx");
        }

        if support_avx2() {
            build.flag("-mavx2");
        }
    }

    build.static_flag(true).compile("t1ha");
}

fn build_highway() {
    let mut build = cc::Build::new();

    build
        .cpp(true)
        .flag("-std=c++11")
        .flag_if_supported("-Wno-sign-compare")
        .include("src/highwayhash")
        .file("src/highwayhash/highwayhash/arch_specific.cc")
        .file("src/highwayhash/highwayhash/instruction_sets.cc")
        .file("src/highwayhash/highwayhash/os_specific.cc")
        .file("src/highwayhash/highwayhash/hh_portable.cc")
        .file("src/highwayhash/highwayhash/c_bindings.cc");

    match TARGET_ARCH.as_str() {
        "x86" | "x86_64" => {
            build
                .flag("-msse4.1")
                .flag("-mavx2")
                .file("src/highwayhash/highwayhash/hh_sse41.cc")
                .file("src/highwayhash/highwayhash/hh_avx2.cc");
        }

        "aarch64" => {
            build.file("src/highwayhash/highwayhash/hh_neon.cc");
        }

        "powerpc64" => {
            build
                .flag("-mvsx")
                .flag("-mpower8-vector")
                .file("src/highwayhash/highwayhash/hh_vsx.cc");
        }
        _ => {}
    }

    build.static_flag(true).compile("highwayhash");
}

fn main() {
    if has_aesni() {
        cargo_emit::rustc_cfg!(r#"feature="aes""#);
    }
    if has_sse41() {
        cargo_emit::rustc_cfg!(r#"feature="sse41""#);
    }
    if has_sse42() {
        cargo_emit::rustc_cfg!(r#"feature="sse42""#);
    }
    if has_avx() {
        cargo_emit::rustc_cfg!(r#"feature="avx""#);
    }
    if has_avx2() {
        cargo_emit::rustc_cfg!(r#"feature="avx2""#);
    }

    build_fasthash();
    if cfg!(feature = "t1ha") {
        build_t1();
    }
    if cfg!(feature = "highway") {
        build_highway();
    }

    let out_dir = env::var("OUT_DIR").unwrap();
    let out_file = Path::new(&out_dir).join("fasthash.rs");

    cargo_emit::rerun_if_changed!("src/fasthash.hpp");
    cargo_emit::rerun_if_changed!("src/fasthash.cpp");

    generate_binding(&out_file);
}
<|endoftext|>


<commit_before>use futures::{Future, Stream, Sink, IntoFuture};
use futures::stream::MergedItem;
use futures::future::Either;
use futures::sync::mpsc;
use tokio_core::net::{TcpListener, TcpStream};
use tokio_core::reactor::{Core, Timeout};
use tokio_io::AsyncRead;

use std::io;
use std::net::SocketAddr;
use std::thread;
use std::time::Duration;
use std::collections::hash_map::{HashMap, Entry};

use messages::{Any, RawMessage};
use events::{EventHandler, NetworkEvent};

use super::error::{other_error, result_ok, forget_result, into_other, log_error};
use super::codec::MessagesCodec;
use super::Node;
use super::handler::NetworkRequest;

pub fn run_node_handler(node: Node) -> io::Result<()> {
    let (sender, receiver) = (node.channel.0, node.channel.1);
    let listen_addr = sender.listen_addr;
    // Channels
    let (events_tx, events_rx) = mpsc::channel(64);
    let (requests_tx, requests_rx) = (sender.network, receiver.network);
    let timeouts_rx = receiver.timeout;

    let events_tx = events_tx.clone();
    let requests_tx = requests_tx.clone();
    let network_thread = thread::spawn(move || {
        let mut core = Core::new().unwrap();

        // Outgoing connections handler
        let mut outgoing_connections: HashMap<SocketAddr, mpsc::Sender<RawMessage>> =
            HashMap::new();

        // Requests handler
        let handle = core.handle();
        let events_tx = events_tx.clone();
        let requests_handle = requests_rx.for_each(|request| {
            debug!("Tokio: Handle request {:?}", request);

            match request {
                NetworkRequest::SendMessage(peer, msg) => {
                    let conn_tx = match outgoing_connections.entry(peer) {
                        Entry::Occupied(entry) => entry.get().clone(),
                        Entry::Vacant(entry) => {
                            let (conn_tx, conn_rx) = mpsc::channel(10);
                            let socket = TcpStream::connect(&peer, &handle);

                            let requests_tx = requests_tx.clone();
                            let connect_handle = socket
                                .and_then(move |sock| {
                                    info!("Tokio: Established connection with peer={}", peer);

                                    let stream = sock.framed(MessagesCodec);
                                    let (sink, stream) = stream.split();

                                    let writer = conn_rx
                                        .map_err(|_| other_error("Can't send data into socket"))
                                        .forward(sink);
                                    let reader = stream.for_each(result_ok).map_err(into_other);

                                    reader
                                        .select2(writer)
                                        .map_err(|_| other_error("Socket error"))
                                        .and_then(|res| match res {
                                            Either::A((_, _reader)) => Ok(()).into_future(),
                                            Either::B((_, _writer)) => Ok(()).into_future(),
                                        })
                                })
                                .map_err(log_error)
                                .then(move |_| {
                                    info!("Tokio: Connection with peer={} closed", peer);
                                    let request = NetworkRequest::DisconnectWithPeer(peer);
                                    requests_tx
                                        .clone()
                                        .send(request)
                                        .map(forget_result)
                                        .map_err(into_other)
                                })
                                .map_err(log_error);
                            handle.spawn(connect_handle);

                            entry.insert(conn_tx.clone());
                            conn_tx
                        }
                    };

                    let duration = Duration::from_secs(5);
                    let send_timeout = Timeout::new(duration, &handle)
                        .unwrap()
                        .and_then(result_ok)
                        .map_err(|_| other_error("Can't timeout"));

                    let send_handle = conn_tx.send(msg).map(forget_result).map_err(log_error);

                    let requests_tx = requests_tx.clone();
                    let timeouted_connect = send_handle
                        .select2(send_timeout)
                        .map_err(|_| other_error("Unable to send message"))
                        .and_then(move |either| match either {
                            Either::A((send, _timeout_fut)) => Ok(send),
                            Either::B((_, _connect_fut)) => Err(other_error("Send timeout")),
                        })
                        .or_else(move |_| {
                            let request = NetworkRequest::DisconnectWithPeer(peer);
                            requests_tx.clone().send(request).map(forget_result)
                        })
                        .map_err(log_error);

                    handle.spawn(timeouted_connect);
                }
                NetworkRequest::DisconnectWithPeer(peer) => {
                    outgoing_connections.remove(&peer);

                    let event = NetworkEvent::PeerDisconnected(peer);
                    let event_handle = events_tx.clone().send(event).map(forget_result).map_err(
                        log_error,
                    );
                    handle.spawn(event_handle);
                }
            }

            Ok(())
        });

        // Incoming connections handler
        let listener = TcpListener::bind(&listen_addr, &core.handle()).unwrap();
        let events_tx = events_tx.clone();
        let server = listener
            .incoming()
            .fold(events_tx, move |events_tx, (sock, addr)| {
                info!("Tokio: Accepted incoming connection with peer={}", addr);

                let stream = sock.framed(MessagesCodec);
                let (_, stream) = stream.split();
                let events_tx = events_tx.clone();
                stream
                    .into_future()
                    .map_err(|e| e.0)
                    .and_then(move |(raw, stream)| {
                        let msg = raw.map(Any::from_raw);
                        if let Some(Ok(Any::Connect(msg))) = msg {
                            Ok((msg, stream))
                        } else {
                            Err(other_error("First message is not Connect"))
                        }
                    })
                    .and_then(move |(connect, stream)| {
                        info!("Tokio: Received handshake message={:?}", connect);

                        let addr = connect.addr();
                        let event = NetworkEvent::PeerConnected(addr, connect);
                        let connect_event = events_tx.clone().send(event).map_err(into_other);

                        let events_tx = events_tx.clone();
                        let messages_stream = stream.for_each(move |raw| {
                            let event = NetworkEvent::MessageReceived(addr, raw);
                            events_tx.clone().send(event).map(forget_result).map_err(
                                into_other,
                            )
                        });

                        messages_stream
                            .join(connect_event)
                            .map(move |(_, stream)| stream)
                            .map_err(into_other)
                    })
                    .map_err(into_other)
            })
            .map(forget_result)
            .map_err(log_error);
        core.handle().spawn(server);

        core.run(requests_handle).unwrap();
    });

    let mut handler = node.handler;
    let mut core = node.core;
    handler.initialize();

    let events_handle = events_rx.merge(timeouts_rx).for_each(move |item| {
        match item {
            MergedItem::First(event) => {
                handler.handle_network_event(event);
            }
            MergedItem::Second(timeout) => {
                handler.handle_timeout(timeout);
            }
            MergedItem::Both(event, timeout) => {
                handler.handle_timeout(timeout);
                handler.handle_network_event(event);
            }
        }
        Ok(())
    });
    core.run(events_handle).unwrap();
    debug!("Tokio: Handler thread is gone");

    network_thread.join().unwrap();
    Ok(())
}
<commit_msg>Use events aggregator<commit_after>use futures::{Future, Stream, Sink, IntoFuture, Poll, Async};
use futures::stream::Fuse;
use futures::future::Either;
use futures::sync::mpsc;
use tokio_core::net::{TcpListener, TcpStream};
use tokio_core::reactor::{Core, Timeout};
use tokio_io::AsyncRead;

use std::io;
use std::net::SocketAddr;
use std::thread;
use std::time::Duration;
use std::collections::hash_map::{HashMap, Entry};

use messages::{Any, RawMessage};
use events::{EventHandler, NetworkEvent};
use node::{NodeTimeout, ExternalMessage};

use super::error::{other_error, result_ok, forget_result, into_other, log_error};
use super::codec::MessagesCodec;
use super::Node;
use super::handler::NetworkRequest;

#[derive(Debug)]
pub enum Event {
    Network(NetworkEvent),
    Timeout(NodeTimeout),
    Api(ExternalMessage),
}

#[derive(Debug)]
pub struct EventsAggregator<S1, S2, S3>
where
    S1: Stream,
    S2: Stream,
    S3: Stream,
{
    timeout: Fuse<S1>,
    network: Fuse<S2>,
    api: Fuse<S3>,
}

impl<S1, S2, S3> EventsAggregator<S1, S2, S3>
where
    S1: Stream,
    S2: Stream,
    S3: Stream,
{
    pub fn new(timeout: S1, network: S2, api: S3) -> EventsAggregator<S1, S2, S3> {
        EventsAggregator {
            network: network.fuse(),
            timeout: timeout.fuse(),
            api: api.fuse(),
        }
    }
}

impl<S1, S2, S3> Stream for EventsAggregator<S1, S2, S3>
where
    S1: Stream<Item = NodeTimeout>,
    S2: Stream<
        Item = NetworkEvent,
        Error = S1::Error,
    >,
    S3: Stream<
        Item = ExternalMessage,
        Error = S1::Error,
    >,
{
    type Item = Event;
    type Error = S1::Error;

    fn poll(&mut self) -> Poll<Option<Event>, Self::Error> {
        let mut stream_finished = false;
        // Check timeout events
        match self.timeout.poll()? {
            Async::Ready(Some(item)) => return Ok(Async::Ready(Some(Event::Timeout(item)))),
            // Just finish stream
            Async::Ready(None) => stream_finished = true,
            Async::NotReady => {}
        };
        // Check network events
        match self.network.poll()? {
            Async::Ready(Some(item)) => return Ok(Async::Ready(Some(Event::Network(item)))),
            // Just finish stream
            Async::Ready(None) => stream_finished = true,
            Async::NotReady => {}
        };
        // Check api events
        match self.api.poll()? {
            Async::Ready(Some(item)) => return Ok(Async::Ready(Some(Event::Api(item)))),
            // Just finish stream
            Async::Ready(None) => stream_finished = true,
            Async::NotReady => {}
        };

        Ok(if stream_finished {
            Async::Ready(None)
        } else {
            Async::NotReady
        })
    }
}

pub fn run_node_handler(node: Node) -> io::Result<()> {
    let (sender, receiver) = (node.channel.0, node.channel.1);
    let listen_addr = sender.listen_addr;
    // Channels
    let (events_tx, events_rx) = mpsc::channel(64);
    let (requests_tx, requests_rx) = (sender.network, receiver.network);
    let timeouts_rx = receiver.timeout;
    let api_rx = receiver.external;

    let events_tx = events_tx.clone();
    let requests_tx = requests_tx.clone();
    let network_thread = thread::spawn(move || {
        let mut core = Core::new().unwrap();

        // Outgoing connections handler
        let mut outgoing_connections: HashMap<SocketAddr, mpsc::Sender<RawMessage>> =
            HashMap::new();

        // Requests handler
        let handle = core.handle();
        let events_tx = events_tx.clone();
        let requests_handle = requests_rx.for_each(|request| {
            debug!("Tokio: Handle request {:?}", request);

            match request {
                NetworkRequest::SendMessage(peer, msg) => {
                    let conn_tx = match outgoing_connections.entry(peer) {
                        Entry::Occupied(entry) => entry.get().clone(),
                        Entry::Vacant(entry) => {
                            let (conn_tx, conn_rx) = mpsc::channel(10);
                            let socket = TcpStream::connect(&peer, &handle);

                            let requests_tx = requests_tx.clone();
                            let connect_handle = socket
                                .and_then(move |sock| {
                                    info!("Tokio: Established connection with peer={}", peer);

                                    let stream = sock.framed(MessagesCodec);
                                    let (sink, stream) = stream.split();

                                    let writer = conn_rx
                                        .map_err(|_| other_error("Can't send data into socket"))
                                        .forward(sink);
                                    let reader = stream.for_each(result_ok).map_err(into_other);

                                    reader
                                        .select2(writer)
                                        .map_err(|_| other_error("Socket error"))
                                        .and_then(|res| match res {
                                            Either::A((_, _reader)) => Ok(()).into_future(),
                                            Either::B((_, _writer)) => Ok(()).into_future(),
                                        })
                                })
                                .map_err(log_error)
                                .then(move |_| {
                                    info!("Tokio: Connection with peer={} closed", peer);
                                    let request = NetworkRequest::DisconnectWithPeer(peer);
                                    requests_tx
                                        .clone()
                                        .send(request)
                                        .map(forget_result)
                                        .map_err(into_other)
                                })
                                .map_err(log_error);
                            handle.spawn(connect_handle);

                            entry.insert(conn_tx.clone());
                            conn_tx
                        }
                    };

                    let duration = Duration::from_secs(5);
                    let send_timeout = Timeout::new(duration, &handle)
                        .unwrap()
                        .and_then(result_ok)
                        .map_err(|_| other_error("Can't timeout"));

                    let send_handle = conn_tx.send(msg).map(forget_result).map_err(log_error);

                    let requests_tx = requests_tx.clone();
                    let timeouted_connect = send_handle
                        .select2(send_timeout)
                        .map_err(|_| other_error("Unable to send message"))
                        .and_then(move |either| match either {
                            Either::A((send, _timeout_fut)) => Ok(send),
                            Either::B((_, _connect_fut)) => Err(other_error("Send timeout")),
                        })
                        .or_else(move |_| {
                            let request = NetworkRequest::DisconnectWithPeer(peer);
                            requests_tx.clone().send(request).map(forget_result)
                        })
                        .map_err(log_error);

                    handle.spawn(timeouted_connect);
                }
                NetworkRequest::DisconnectWithPeer(peer) => {
                    outgoing_connections.remove(&peer);

                    let event = NetworkEvent::PeerDisconnected(peer);
                    let event_handle = events_tx.clone().send(event).map(forget_result).map_err(
                        log_error,
                    );
                    handle.spawn(event_handle);
                }
            }

            Ok(())
        });

        // Incoming connections handler
        let listener = TcpListener::bind(&listen_addr, &core.handle()).unwrap();
        let events_tx = events_tx.clone();
        let server = listener
            .incoming()
            .fold(events_tx, move |events_tx, (sock, addr)| {
                info!("Tokio: Accepted incoming connection with peer={}", addr);

                let stream = sock.framed(MessagesCodec);
                let (_, stream) = stream.split();
                let events_tx = events_tx.clone();
                stream
                    .into_future()
                    .map_err(|e| e.0)
                    .and_then(move |(raw, stream)| {
                        let msg = raw.map(Any::from_raw);
                        if let Some(Ok(Any::Connect(msg))) = msg {
                            Ok((msg, stream))
                        } else {
                            Err(other_error("First message is not Connect"))
                        }
                    })
                    .and_then(move |(connect, stream)| {
                        info!("Tokio: Received handshake message={:?}", connect);

                        let addr = connect.addr();
                        let event = NetworkEvent::PeerConnected(addr, connect);
                        let connect_event = events_tx.clone().send(event).map_err(into_other);

                        let events_tx = events_tx.clone();
                        let messages_stream = stream.for_each(move |raw| {
                            let event = NetworkEvent::MessageReceived(addr, raw);
                            events_tx.clone().send(event).map(forget_result).map_err(
                                into_other,
                            )
                        });

                        messages_stream
                            .join(connect_event)
                            .map(move |(_, stream)| stream)
                            .map_err(into_other)
                    })
                    .map_err(into_other)
            })
            .map(forget_result)
            .map_err(log_error);
        core.handle().spawn(server);

        core.run(requests_handle).unwrap();
    });

    let mut handler = node.handler;
    let mut core = node.core;
    handler.initialize();

    let events_handle = EventsAggregator::new(timeouts_rx, events_rx, api_rx)
        .for_each(move |event| {
            match event {
                Event::Timeout(timeout) => handler.handle_timeout(timeout),
                Event::Network(network) => handler.handle_network_event(network),
                Event::Api(api) => handler.handle_application_event(api),
            }
            Ok(())
        });
    core.run(events_handle).unwrap();
    debug!("Tokio: Handler thread is gone");

    network_thread.join().unwrap();
    Ok(())
}
<|endoftext|>


<commit_before>use crate::schema::*;

use rustc::hir::def_id::{DefId, DefIndex};
use rustc_serialize::opaque::Encoder;
use std::marker::PhantomData;
use std::num::NonZeroUsize;
use std::u32;
use log::debug;

/// Helper trait, for encoding to, and decoding from, a fixed number of bytes.
trait FixedSizeEncoding {
    const BYTE_LEN: usize;

    // FIXME(eddyb) convert to and from `[u8; Self::BYTE_LEN]` instead,
    // once that starts being allowed by the compiler (i.e. lazy normalization).
    fn from_bytes(b: &[u8]) -> Self;
    fn write_to_bytes(self, b: &mut [u8]);

    // FIXME(eddyb) make these generic functions, or at least defaults here.
    // (same problem as above, needs `[u8; Self::BYTE_LEN]`)
    // For now, a macro (`fixed_size_encoding_byte_len_and_defaults`) is used.
    fn read_from_bytes_at(b: &[u8], i: usize) -> Self;
    fn write_to_bytes_at(self, b: &mut [u8], i: usize);
}

// HACK(eddyb) this shouldn't be needed (see comments on the methods above).
macro_rules! fixed_size_encoding_byte_len_and_defaults {
    ($byte_len:expr) => {
        const BYTE_LEN: usize = $byte_len;
        fn read_from_bytes_at(b: &[u8], i: usize) -> Self {
            const BYTE_LEN: usize = $byte_len;
            // HACK(eddyb) ideally this would be done with fully safe code,
            // but slicing `[u8]` with `i * N..` is optimized worse, due to the
            // possibility of `i * N` overflowing, than indexing `[[u8; N]]`.
            let b = unsafe {
                std::slice::from_raw_parts(
                    b.as_ptr() as *const [u8; BYTE_LEN],
                    b.len() / BYTE_LEN,
                )
            };
            Self::from_bytes(&b[i])
        }
        fn write_to_bytes_at(self, b: &mut [u8], i: usize) {
            const BYTE_LEN: usize = $byte_len;
            // HACK(eddyb) ideally this would be done with fully safe code,
            // see similar comment in `read_from_bytes_at` for why it can't yet.
            let b = unsafe {
                std::slice::from_raw_parts_mut(
                    b.as_mut_ptr() as *mut [u8; BYTE_LEN],
                    b.len() / BYTE_LEN,
                )
            };
            self.write_to_bytes(&mut b[i]);
        }
    }
}

impl FixedSizeEncoding for u32 {
    fixed_size_encoding_byte_len_and_defaults!(4);

    fn from_bytes(b: &[u8]) -> Self {
        let mut bytes = [0; Self::BYTE_LEN];
        bytes.copy_from_slice(&b[..Self::BYTE_LEN]);
        Self::from_le_bytes(bytes)
    }

    fn write_to_bytes(self, b: &mut [u8]) {
        b[..Self::BYTE_LEN].copy_from_slice(&self.to_le_bytes());
    }
}

/// While we are generating the metadata, we also track the position
/// of each DefIndex. It is not required that all definitions appear
/// in the metadata, nor that they are serialized in order, and
/// therefore we first allocate the vector here and fill it with
/// `u32::MAX`. Whenever an index is visited, we fill in the
/// appropriate spot by calling `record_position`. We should never
/// visit the same index twice.
crate struct Index<'tcx> {
    positions: Vec<u8>,
    _marker: PhantomData<&'tcx ()>,
}

impl Index<'tcx> {
    crate fn new(max_index: usize) -> Self {
        Index {
            positions: vec![0xff; max_index * 4],
            _marker: PhantomData,
        }
    }

    crate fn record(&mut self, def_id: DefId, entry: Lazy<Entry<'tcx>>) {
        assert!(def_id.is_local());
        self.record_index(def_id.index, entry);
    }

    fn record_index(&mut self, item: DefIndex, entry: Lazy<Entry<'tcx>>) {
        assert!(entry.position.get() < (u32::MAX as usize));
        let position = entry.position.get() as u32;
        let array_index = item.index();

        let positions = &mut self.positions;
        assert!(u32::read_from_bytes_at(positions, array_index) == u32::MAX,
                "recorded position for item {:?} twice, first at {:?} and now at {:?}",
                item,
                u32::read_from_bytes_at(positions, array_index),
                position);

        position.write_to_bytes_at(positions, array_index)
    }

    crate fn write_index(&self, buf: &mut Encoder) -> Lazy<[Self]> {
        let pos = buf.position();
        buf.emit_raw_bytes(&self.positions);
        Lazy::from_position_and_meta(
            NonZeroUsize::new(pos as usize).unwrap(),
            self.positions.len() / 4,
        )
    }
}

impl Lazy<[Index<'tcx>]> {
    /// Given the metadata, extract out the offset of a particular
    /// DefIndex (if any).
    #[inline(never)]
    crate fn lookup(&self, bytes: &[u8], def_index: DefIndex) -> Option<Lazy<Entry<'tcx>>> {
        debug!("Index::lookup: index={:?} len={:?}",
               def_index,
               self.meta);

        let bytes = &bytes[self.position.get()..][..self.meta * 4];
        let position = u32::read_from_bytes_at(bytes, def_index.index());
        if position == u32::MAX {
            debug!("Index::lookup: position=u32::MAX");
            None
        } else {
            debug!("Index::lookup: position={:?}", position);
            Some(Lazy::from_position(NonZeroUsize::new(position as usize).unwrap()))
        }
    }
}
<commit_msg>rustc_metadata: use 0 in index::Index to indicate missing entries.<commit_after>use crate::schema::*;

use rustc::hir::def_id::{DefId, DefIndex};
use rustc_serialize::opaque::Encoder;
use std::convert::TryInto;
use std::marker::PhantomData;
use std::num::NonZeroUsize;
use log::debug;

/// Helper trait, for encoding to, and decoding from, a fixed number of bytes.
trait FixedSizeEncoding {
    const BYTE_LEN: usize;

    // FIXME(eddyb) convert to and from `[u8; Self::BYTE_LEN]` instead,
    // once that starts being allowed by the compiler (i.e. lazy normalization).
    fn from_bytes(b: &[u8]) -> Self;
    fn write_to_bytes(self, b: &mut [u8]);

    // FIXME(eddyb) make these generic functions, or at least defaults here.
    // (same problem as above, needs `[u8; Self::BYTE_LEN]`)
    // For now, a macro (`fixed_size_encoding_byte_len_and_defaults`) is used.
    fn read_from_bytes_at(b: &[u8], i: usize) -> Self;
    fn write_to_bytes_at(self, b: &mut [u8], i: usize);
}

// HACK(eddyb) this shouldn't be needed (see comments on the methods above).
macro_rules! fixed_size_encoding_byte_len_and_defaults {
    ($byte_len:expr) => {
        const BYTE_LEN: usize = $byte_len;
        fn read_from_bytes_at(b: &[u8], i: usize) -> Self {
            const BYTE_LEN: usize = $byte_len;
            // HACK(eddyb) ideally this would be done with fully safe code,
            // but slicing `[u8]` with `i * N..` is optimized worse, due to the
            // possibility of `i * N` overflowing, than indexing `[[u8; N]]`.
            let b = unsafe {
                std::slice::from_raw_parts(
                    b.as_ptr() as *const [u8; BYTE_LEN],
                    b.len() / BYTE_LEN,
                )
            };
            Self::from_bytes(&b[i])
        }
        fn write_to_bytes_at(self, b: &mut [u8], i: usize) {
            const BYTE_LEN: usize = $byte_len;
            // HACK(eddyb) ideally this would be done with fully safe code,
            // see similar comment in `read_from_bytes_at` for why it can't yet.
            let b = unsafe {
                std::slice::from_raw_parts_mut(
                    b.as_mut_ptr() as *mut [u8; BYTE_LEN],
                    b.len() / BYTE_LEN,
                )
            };
            self.write_to_bytes(&mut b[i]);
        }
    }
}

impl FixedSizeEncoding for u32 {
    fixed_size_encoding_byte_len_and_defaults!(4);

    fn from_bytes(b: &[u8]) -> Self {
        let mut bytes = [0; Self::BYTE_LEN];
        bytes.copy_from_slice(&b[..Self::BYTE_LEN]);
        Self::from_le_bytes(bytes)
    }

    fn write_to_bytes(self, b: &mut [u8]) {
        b[..Self::BYTE_LEN].copy_from_slice(&self.to_le_bytes());
    }
}

/// While we are generating the metadata, we also track the position
/// of each DefIndex. It is not required that all definitions appear
/// in the metadata, nor that they are serialized in order, and
/// therefore we first allocate the vector here and fill it with
/// `0`. Whenever an index is visited, we fill in the
/// appropriate spot by calling `record_position`. We should never
/// visit the same index twice.
crate struct Index<'tcx> {
    positions: Vec<u8>,
    _marker: PhantomData<&'tcx ()>,
}

impl Index<'tcx> {
    crate fn new(max_index: usize) -> Self {
        Index {
            positions: vec![0; max_index * 4],
            _marker: PhantomData,
        }
    }

    crate fn record(&mut self, def_id: DefId, entry: Lazy<Entry<'tcx>>) {
        assert!(def_id.is_local());
        self.record_index(def_id.index, entry);
    }

    fn record_index(&mut self, item: DefIndex, entry: Lazy<Entry<'tcx>>) {
        let position: u32 = entry.position.get().try_into().unwrap();
        let array_index = item.index();

        let positions = &mut self.positions;
        assert!(u32::read_from_bytes_at(positions, array_index) == 0,
                "recorded position for item {:?} twice, first at {:?} and now at {:?}",
                item,
                u32::read_from_bytes_at(positions, array_index),
                position);

        position.write_to_bytes_at(positions, array_index)
    }

    crate fn write_index(&self, buf: &mut Encoder) -> Lazy<[Self]> {
        let pos = buf.position();
        buf.emit_raw_bytes(&self.positions);
        Lazy::from_position_and_meta(
            NonZeroUsize::new(pos as usize).unwrap(),
            self.positions.len() / 4,
        )
    }
}

impl Lazy<[Index<'tcx>]> {
    /// Given the metadata, extract out the offset of a particular
    /// DefIndex (if any).
    #[inline(never)]
    crate fn lookup(&self, bytes: &[u8], def_index: DefIndex) -> Option<Lazy<Entry<'tcx>>> {
        debug!("Index::lookup: index={:?} len={:?}",
               def_index,
               self.meta);

        let bytes = &bytes[self.position.get()..][..self.meta * 4];
        let position = u32::read_from_bytes_at(bytes, def_index.index());
        debug!("Index::lookup: position={:?}", position);
        NonZeroUsize::new(position as usize).map(Lazy::from_position)
    }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use abi::{ArgAttribute, FnType, LayoutExt, Reg, RegKind};
use common::CrateContext;

#[derive(PartialEq)]
pub enum Flavor {
    General,
    Fastcall
}

pub fn compute_abi_info<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>,
                                  fty: &mut FnType<'tcx>,
                                  flavor: Flavor) {
    if !fty.ret.is_ignore() {
        if fty.ret.layout.is_aggregate() {
            // Returning a structure. Most often, this will use
            // a hidden first argument. On some platforms, though,
            // small structs are returned as integers.
            //
            // Some links:
            // http://www.angelcode.com/dev/callconv/callconv.html
            // Clang's ABI handling is in lib/CodeGen/TargetInfo.cpp
            let t = &ccx.sess().target.target;
            if t.options.is_like_osx || t.options.is_like_windows
                || t.options.is_like_openbsd {
                let size = fty.ret.layout.size(ccx);
                match size.bytes() {
                    1 => fty.ret.cast_to(ccx, Reg::i8()),
                    2 => fty.ret.cast_to(ccx, Reg::i16()),
                    4 => fty.ret.cast_to(ccx, Reg::i32()),
                    8 => fty.ret.cast_to(ccx, Reg::i64()),
                    _ => fty.ret.make_indirect(ccx)
                }
            } else {
                fty.ret.make_indirect(ccx);
            }
        } else {
            fty.ret.extend_integer_width_to(32);
        }
    }

    for arg in &mut fty.args {
        if arg.is_ignore() { continue; }
        if arg.layout.is_aggregate() {
            arg.make_indirect(ccx);
            arg.attrs.set(ArgAttribute::ByVal);
        } else {
            arg.extend_integer_width_to(32);
        }
    }

    if flavor == Flavor::Fastcall {
        // Mark arguments as InReg like clang does it,
        // so our fastcall is compatible with C/C++ fastcall.

        // Clang reference: lib/CodeGen/TargetInfo.cpp
        // See X86_32ABIInfo::shouldPrimitiveUseInReg(), X86_32ABIInfo::updateFreeRegs()

        // IsSoftFloatABI is only set to true on ARM platforms,
        // which in turn can't be x86?

        let mut free_regs = 2;

        for arg in &mut fty.args {
            if arg.is_ignore() || arg.is_indirect() { continue; }

            // At this point we know this must be a primitive of sorts.
            let unit = arg.layout.homogeneous_aggregate(ccx).unwrap();
            let size = arg.layout.size(ccx);
            assert_eq!(unit.size, size);
            if unit.kind == RegKind::Float {
                continue;
            }

            let size_in_regs = (size.bits() + 31) / 32;

            if size_in_regs == 0 {
                continue;
            }

            if size_in_regs > free_regs {
                break;
            }

            free_regs -= size_in_regs;

            if size.bits() <= 32 && unit.kind == RegKind::Integer {
                arg.attrs.set(ArgAttribute::InReg);
            }

            if free_regs == 0 {
                break;
            }
        }
    }
}
<commit_msg>x86: return single-float aggregates in a float register<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use abi::{ArgAttribute, FnType, LayoutExt, Reg, RegKind};
use common::CrateContext;

use rustc::ty::layout::{self, Layout, TyLayout};

#[derive(PartialEq)]
pub enum Flavor {
    General,
    Fastcall
}

fn is_single_fp_element<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>,
                                  layout: TyLayout<'tcx>) -> bool {
    match *layout {
        Layout::Scalar { value: layout::F32, .. } |
        Layout::Scalar { value: layout::F64, .. } => true,
        Layout::Univariant { .. } => {
            if layout.field_count() == 1 {
                is_single_fp_element(ccx, layout.field(ccx, 0))
            } else {
                false
            }
        }
        _ => false
    }
}

pub fn compute_abi_info<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>,
                                  fty: &mut FnType<'tcx>,
                                  flavor: Flavor) {
    if !fty.ret.is_ignore() {
        if fty.ret.layout.is_aggregate() {
            // Returning a structure. Most often, this will use
            // a hidden first argument. On some platforms, though,
            // small structs are returned as integers.
            //
            // Some links:
            // http://www.angelcode.com/dev/callconv/callconv.html
            // Clang's ABI handling is in lib/CodeGen/TargetInfo.cpp
            let t = &ccx.sess().target.target;
            if t.options.is_like_osx || t.options.is_like_windows
                || t.options.is_like_openbsd {
                let size = fty.ret.layout.size(ccx);

                // According to Clang, everyone but MSVC returns single-element
                // float aggregates directly in a floating-point register.
                if !t.options.is_like_msvc && is_single_fp_element(ccx, fty.ret.layout) {
                    match size.bytes() {
                        4 => fty.ret.cast_to(ccx, Reg::f32()),
                        8 => fty.ret.cast_to(ccx, Reg::f64()),
                        _ => fty.ret.make_indirect(ccx)
                    }
                } else {
                    match size.bytes() {
                        1 => fty.ret.cast_to(ccx, Reg::i8()),
                        2 => fty.ret.cast_to(ccx, Reg::i16()),
                        4 => fty.ret.cast_to(ccx, Reg::i32()),
                        8 => fty.ret.cast_to(ccx, Reg::i64()),
                        _ => fty.ret.make_indirect(ccx)
                    }
                }
            } else {
                fty.ret.make_indirect(ccx);
            }
        } else {
            fty.ret.extend_integer_width_to(32);
        }
    }

    for arg in &mut fty.args {
        if arg.is_ignore() { continue; }
        if arg.layout.is_aggregate() {
            arg.make_indirect(ccx);
            arg.attrs.set(ArgAttribute::ByVal);
        } else {
            arg.extend_integer_width_to(32);
        }
    }

    if flavor == Flavor::Fastcall {
        // Mark arguments as InReg like clang does it,
        // so our fastcall is compatible with C/C++ fastcall.

        // Clang reference: lib/CodeGen/TargetInfo.cpp
        // See X86_32ABIInfo::shouldPrimitiveUseInReg(), X86_32ABIInfo::updateFreeRegs()

        // IsSoftFloatABI is only set to true on ARM platforms,
        // which in turn can't be x86?

        let mut free_regs = 2;

        for arg in &mut fty.args {
            if arg.is_ignore() || arg.is_indirect() { continue; }

            // At this point we know this must be a primitive of sorts.
            let unit = arg.layout.homogeneous_aggregate(ccx).unwrap();
            let size = arg.layout.size(ccx);
            assert_eq!(unit.size, size);
            if unit.kind == RegKind::Float {
                continue;
            }

            let size_in_regs = (size.bits() + 31) / 32;

            if size_in_regs == 0 {
                continue;
            }

            if size_in_regs > free_regs {
                break;
            }

            free_regs -= size_in_regs;

            if size.bits() <= 32 && unit.kind == RegKind::Integer {
                arg.attrs.set(ArgAttribute::InReg);
            }

            if free_regs == 0 {
                break;
            }
        }
    }
}
<|endoftext|>


<commit_before>use itertools::Itertools;
use regex::Regex;
use reqwest;
use rust_htslib::bcf;
use rust_htslib::bcf::Read;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::error::Error;
use std::str;

#[derive(Serialize, Deserialize, Debug)]
struct Dgidb {
    #[serde(rename = "matchedTerms")]
    matched_terms: Vec<MatchedTerm>,
}

#[derive(Serialize, Deserialize, Debug)]
struct MatchedTerm {
    #[serde(rename = "geneName")]
    gene_name: String,
    interactions: Vec<Interaction>,
}

#[derive(Serialize, Deserialize, Debug)]
struct Interaction {
    #[serde(rename = "drugName")]
    drug_name: String,
    #[serde(rename = "interactionTypes")]
    interaction_types: Vec<String>,
}

pub fn annotate_dgidb(
    vcf_path: &str,
    api_path: String,
    field_name: &str,
    genes_per_request: usize,
) -> Result<(), Box<dyn Error>> {
    let gene_drug_interactions = request_interaction_drugs(vcf_path, api_path, genes_per_request)?;
    modify_vcf_entries(vcf_path, gene_drug_interactions, field_name)
}

fn request_interaction_drugs(
    vcf_path: &str,
    api_path: String,
    genes_per_request: usize,
) -> Result<Option<HashMap<String, Vec<(String, Vec<String>)>>>, Box<dyn Error>> {
    let mut genes = collect_genes(vcf_path)?;
    if genes.is_empty() {
        return Ok(None);
    }
    let mut gene_drug_interactions: HashMap<String, Vec<(String, Vec<String>)>> = HashMap::new();
    for gene_slice in genes
        .drain()
        .map(|gene| gene)
        .collect_vec()
        .chunks(genes_per_request)
    {
        let mut slice_api_path = api_path.clone();
        slice_api_path.push_str(gene_slice.join(",").as_str());
        let res: Dgidb = reqwest::get(&slice_api_path)?.json()?;

        for term in res.matched_terms {
            if !term.interactions.is_empty() {
                gene_drug_interactions.insert(
                    term.gene_name,
                    term.interactions
                        .iter()
                        .map(|interaction| {
                            (
                                interaction.drug_name.clone(),
                                interaction.interaction_types.clone(),
                            )
                        })
                        .collect(),
                );
            }
        }
    }
    Ok(Some(gene_drug_interactions))
}

fn collect_genes(vcf_path: &str) -> Result<HashSet<String>, Box<dyn Error>> {
    let mut total_genes = HashSet::new();
    let mut reader = bcf::Reader::from_path(vcf_path)?;
    for result in reader.records() {
        let mut rec = result?;
        let genes_opt = extract_genes(&mut rec)?;
        match genes_opt {
            Some(genes) => {
                for gene in genes {
                    total_genes.insert(gene);
                }
            }
            None => {}
        }
    }
    Ok(total_genes)
}

//TODO Remove split by ',' when updating to latest htslib release
fn extract_genes<'a>(
    rec: &'a mut bcf::Record,
) -> Result<Option<impl Iterator<Item = String> + 'a>, Box<dyn Error>> {
    let annotation = rec.info("ANN".as_bytes()).string()?;
    match annotation {
        Some(transcripts) => Ok(Some(transcripts[0].split(|c| *c == b',').map(
            |transcript| {
                str::from_utf8(transcript.split(|c| *c == b'|').nth(3).unwrap())
                    .unwrap()
                    .to_owned()
            },
        ))),
        None => Ok(None),
    }
}

fn modify_vcf_entries(
    vcf_path: &str,
    gene_drug_interactions_opt: Option<HashMap<String, Vec<(String, Vec<String>)>>>,
    field_name: &str,
) -> Result<(), Box<dyn Error>> {
    let mut reader = bcf::Reader::from_path(vcf_path)?;
    let mut header = bcf::header::Header::from_template(reader.header());
    header.push_record(format!("##INFO=<ID={},Number=.,Type=String,Description=\"Combination of gene, drug, interaction types extracted from dgiDB. Each combination is pipe-seperated annotated as GENE|DRUG|TYPE\">", field_name).as_bytes());
    let mut writer = bcf::Writer::from_stdout(&header, true, true)?;
    match gene_drug_interactions_opt {
        None => {
            for result in reader.records() {
                let mut rec = result?;
                writer.translate(&mut rec);
                writer.write(&rec)?;
            }
        }
        Some(gene_drug_interactions) => {
            for result in reader.records() {
                let mut rec = result?;
                writer.translate(&mut rec);
                let genes = extract_genes(&mut rec)?.map(|genes| genes.collect_vec());
                if let Some(mut genes) = genes {
                    genes.dedup();
                    let field_entries = build_dgidb_field(&gene_drug_interactions, genes)?;
                    let field_entries: Vec<&[u8]> =
                        field_entries.iter().map(|v| v.as_slice()).collect();
                    rec.push_info_string(field_name.as_bytes(), &field_entries[..])?;
                }
                writer.write(&rec)?;
            }
        }
    }
    Ok(())
}

fn build_dgidb_field(
    gene_drug_interactions: &HashMap<String, Vec<(String, Vec<String>)>>,
    genes: Vec<String>,
) -> Result<Vec<Vec<u8>>, Box<dyn Error>> {
    let mut field_entries: Vec<Vec<u8>> = Vec::new();
    let re = Regex::new(r"\s\(\w+\)").unwrap();
    for gene in genes.iter() {
        match gene_drug_interactions.get(gene) {
            Some(drug_interactions) => {
                drug_interactions
                    .iter()
                    .for_each(|(drug, interaction_types)| {
                        if !interaction_types.is_empty() {
                            interaction_types.iter().for_each(|interaction_type| {
                                field_entries.push(
                                    format!(
                                        "{g}|{d}|{t}",
                                        g = gene,
                                        d = re.replace(drug, ""),
                                        t = interaction_type
                                    )
                                    .as_bytes()
                                    .to_vec(),
                                )
                            })
                        } else {
                            field_entries.push(
                                format!("{g}|{d}|.", g = gene, d = re.replace(drug, ""))
                                    .as_bytes()
                                    .to_vec(),
                            )
                        }
                    });
            }
            None => field_entries.push(format!("{g}|.|.", g = gene).as_bytes().to_vec()),
        }
    }
    Ok(field_entries)
}
<commit_msg>Changes for rust_htslib<commit_after>use itertools::Itertools;
use regex::Regex;
use reqwest;
use rust_htslib::bcf;
use rust_htslib::bcf::{Read, Format};
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::error::Error;
use std::str;

#[derive(Serialize, Deserialize, Debug)]
struct Dgidb {
    #[serde(rename = "matchedTerms")]
    matched_terms: Vec<MatchedTerm>,
}

#[derive(Serialize, Deserialize, Debug)]
struct MatchedTerm {
    #[serde(rename = "geneName")]
    gene_name: String,
    interactions: Vec<Interaction>,
}

#[derive(Serialize, Deserialize, Debug)]
struct Interaction {
    #[serde(rename = "drugName")]
    drug_name: String,
    #[serde(rename = "interactionTypes")]
    interaction_types: Vec<String>,
}

pub fn annotate_dgidb(
    vcf_path: &str,
    api_path: String,
    field_name: &str,
    genes_per_request: usize,
) -> Result<(), Box<dyn Error>> {
    let gene_drug_interactions = request_interaction_drugs(vcf_path, api_path, genes_per_request)?;
    modify_vcf_entries(vcf_path, gene_drug_interactions, field_name)
}

fn request_interaction_drugs(
    vcf_path: &str,
    api_path: String,
    genes_per_request: usize,
) -> Result<Option<HashMap<String, Vec<(String, Vec<String>)>>>, Box<dyn Error>> {
    let mut genes = collect_genes(vcf_path)?;
    if genes.is_empty() {
        return Ok(None);
    }
    let mut gene_drug_interactions: HashMap<String, Vec<(String, Vec<String>)>> = HashMap::new();
    for gene_slice in genes
        .drain()
        .map(|gene| gene)
        .collect_vec()
        .chunks(genes_per_request)
    {
        let mut slice_api_path = api_path.clone();
        slice_api_path.push_str(gene_slice.join(",").as_str());
        let res: Dgidb = reqwest::get(&slice_api_path)?.json()?;

        for term in res.matched_terms {
            if !term.interactions.is_empty() {
                gene_drug_interactions.insert(
                    term.gene_name,
                    term.interactions
                        .iter()
                        .map(|interaction| {
                            (
                                interaction.drug_name.clone(),
                                interaction.interaction_types.clone(),
                            )
                        })
                        .collect(),
                );
            }
        }
    }
    Ok(Some(gene_drug_interactions))
}

fn collect_genes(vcf_path: &str) -> Result<HashSet<String>, Box<dyn Error>> {
    let mut total_genes = HashSet::new();
    let mut reader = bcf::Reader::from_path(vcf_path)?;
    for result in reader.records() {
        let mut rec = result?;
        let genes_opt = extract_genes(&mut rec)?;
        match genes_opt {
            Some(genes) => {
                for gene in genes {
                    total_genes.insert(gene);
                }
            }
            None => {}
        }
    }
    Ok(total_genes)
}

//TODO Remove split by ',' when updating to latest htslib release
fn extract_genes<'a>(
    rec: &'a mut bcf::Record,
) -> Result<Option<impl Iterator<Item = String> + 'a>, Box<dyn Error>> {
    let annotation = rec.info("ANN".as_bytes()).string()?;
    match annotation {
        Some(transcripts) => Ok(Some(transcripts[0].split(|c| *c == b',').map(
            |transcript| {
                str::from_utf8(transcript.split(|c| *c == b'|').nth(3).unwrap())
                    .unwrap()
                    .to_owned()
            },
        ))),
        None => Ok(None),
    }
}

fn modify_vcf_entries(
    vcf_path: &str,
    gene_drug_interactions_opt: Option<HashMap<String, Vec<(String, Vec<String>)>>>,
    field_name: &str,
) -> Result<(), Box<dyn Error>> {
    let mut reader = bcf::Reader::from_path(vcf_path)?;
    let mut header = bcf::header::Header::from_template(reader.header());
    header.push_record(format!("##INFO=<ID={},Number=.,Type=String,Description=\"Combination of gene, drug, interaction types extracted from dgiDB. Each combination is pipe-seperated annotated as GENE|DRUG|TYPE\">", field_name).as_bytes());
    let mut writer = bcf::Writer::from_stdout(&header, true, Format::VCF)?;
    match gene_drug_interactions_opt {
        None => {
            for result in reader.records() {
                let mut rec = result?;
                writer.translate(&mut rec);
                writer.write(&rec)?;
            }
        }
        Some(gene_drug_interactions) => {
            for result in reader.records() {
                let mut rec = result?;
                writer.translate(&mut rec);
                let genes = extract_genes(&mut rec)?.map(|genes| genes.collect_vec());
                if let Some(mut genes) = genes {
                    genes.dedup();
                    let field_entries = build_dgidb_field(&gene_drug_interactions, genes)?;
                    let field_entries: Vec<&[u8]> =
                        field_entries.iter().map(|v| v.as_slice()).collect();
                    rec.push_info_string(field_name.as_bytes(), &field_entries[..])?;
                }
                writer.write(&rec)?;
            }
        }
    }
    Ok(())
}

fn build_dgidb_field(
    gene_drug_interactions: &HashMap<String, Vec<(String, Vec<String>)>>,
    genes: Vec<String>,
) -> Result<Vec<Vec<u8>>, Box<dyn Error>> {
    let mut field_entries: Vec<Vec<u8>> = Vec::new();
    let re = Regex::new(r"\s\(\w+\)").unwrap();
    for gene in genes.iter() {
        match gene_drug_interactions.get(gene) {
            Some(drug_interactions) => {
                drug_interactions
                    .iter()
                    .for_each(|(drug, interaction_types)| {
                        if !interaction_types.is_empty() {
                            interaction_types.iter().for_each(|interaction_type| {
                                field_entries.push(
                                    format!(
                                        "{g}|{d}|{t}",
                                        g = gene,
                                        d = re.replace(drug, ""),
                                        t = interaction_type
                                    )
                                    .as_bytes()
                                    .to_vec(),
                                )
                            })
                        } else {
                            field_entries.push(
                                format!("{g}|{d}|.", g = gene, d = re.replace(drug, ""))
                                    .as_bytes()
                                    .to_vec(),
                            )
                        }
                    });
            }
            None => field_entries.push(format!("{g}|.|.", g = gene).as_bytes().to_vec()),
        }
    }
    Ok(field_entries)
}
<|endoftext|>


<commit_before>extern crate hyper;
extern crate hyper_rustls;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate serde_json;
extern crate scraper;
extern crate chrono;
extern crate dotenv;

use hyper::Client;

use scraper::{Html, Selector};

use chrono::{UTC, TimeZone};

use std::collections::HashMap;
use std::fs::{File, OpenOptions};
use std::io::Read;
use std::env;

const NEWS_URL: &'static str = "http://na.finalfantasyxiv.com/lodestone/news/";

struct NewsScraper {
  client: Client,
  database: NewsDatabase
}

#[derive(Debug, Default, Serialize, Deserialize)]
struct NewsDatabase {
  items: HashMap<String, NewsItem>
}

#[derive(Debug, Serialize, Deserialize)]
struct NewsItem {
  title: String,
  url: String,
  kind: NewsKind,
  time: i64,
  tag: Option<String>
}

#[derive(Debug, Serialize, Deserialize)]
enum NewsKind {
  SpecialNotice,
  News,
  Topic
}

impl ToString for NewsKind {
  fn to_string(&self) -> String {
    match *self {
      NewsKind::News => "News",
      NewsKind::Topic => "Topic",
      NewsKind::SpecialNotice => "Special notice"
    }.to_string()
  }
}

fn main() {
  dotenv::dotenv().ok();

  let webhook_url = match env::var("LB_NEWS_WEBHOOK") {
    Ok(u) => u,
    Err(_) => {
      println!("No webhook specified");
      return;
    }
  };

  let database = match NewsDatabase::load() {
    Some(db) => db,
    None => NewsDatabase::default()
  };
  let mut scraper = NewsScraper::new(database);

  let news = match scraper.download_news() {
    Some(n) => n,
    None => {
      println!("could not download news");
      return;
    }
  };

  let downloaded_news = scraper.parse_news(&news);
  for (id, item) in downloaded_news {
    if !scraper.database.items.contains_key(&id) {
      let mut embed = json!({
        "type": "rich",
        "timestamp": UTC.timestamp(item.time, 0).to_rfc3339(),
        "fields": [
          {
            "name": "Title",
            "value": item.title,
            "inline": false
          },
          {
            "name": "Link",
            "value": item.url,
            "inline": false
          },
          {
            "name": "Kind",
            "value": item.kind.to_string(),
            "inline": true
          }
        ]
      });
      if let Some(ref tag) = item.tag {
        embed["fields"].as_array_mut().unwrap().push(json!({
          "name": "Tag",
          "value": tag,
          "inline": true
        }));
      }
      let data = json!({
        "embeds": [embed]
      });
      let res = scraper.client.post(&webhook_url)
        .header(hyper::header::ContentType::json())
        .body(&data.to_string())
        .send();
      let mut data = match res {
        Ok(r) => r,
        Err(e) => {
          println!("error sending webhook: {}", e);
          continue;
        }
      };
      let mut content = String::new();
      if let Err(e) = data.read_to_string(&mut content) {
        println!("could not read webhook response: {}", e);
        continue;
      }
      if data.status.class() != hyper::status::StatusClass::Success {
        println!("discord says no to {}", item.title);
        println!("{}", content);
      } else {
        println!("webhook sent: {}", item.title);
        scraper.database.items.insert(id, item);
      }
    }
  }
  if scraper.database.save().is_none() {
    println!("could not save database");
  }
}

impl NewsDatabase {
  fn load() -> Option<Self> {
    let news_file = match File::open("./lodestone_news.json") {
      Ok(n) => n,
      Err(_) => return None
    };
    serde_json::from_reader(news_file).ok()
  }

  fn save(&self) -> Option<()> {
    let news_file = match OpenOptions::new().create(true).write(true).open("./lodestone_news.json") {
      Ok(n) => n,
      Err(_) => return None
    };
    serde_json::to_writer(news_file, self).ok()
  }
}

impl NewsScraper {
  fn new(database: NewsDatabase) -> Self {
    NewsScraper {
      client: Client::with_connector(hyper::net::HttpsConnector::new(hyper_rustls::TlsClient::new())),
      database: database
    }
  }

  fn download_news(&self) -> Option<String> {
    let mut response = match self.client.get(NEWS_URL).send() {
      Ok(r) => r,
      Err(_) => return None
    };

    let mut content = String::new();
    if response.read_to_string(&mut content).is_err() {
      return None;
    }
    Some(content)
  }

  fn parse_news(&self, news: &str) -> HashMap<String, NewsItem> {
    let html = Html::parse_document(news);
    let news_selector = Selector::parse("div.news__content.parts__space--add > ul:nth-of-type(2) > li").unwrap();
    let topics_selector = Selector::parse("div.news__content.parts__space--add > ul:nth-of-type(3) > li").unwrap();
    let title_selector = Selector::parse("p.news__list--title").unwrap();
    let time_script_selector = Selector::parse("time.news__list--time > script").unwrap();
    let mut lis: Vec<_> = html.select(&news_selector).map(|x| (NewsKind::News, x)).collect();
    lis.append(&mut html.select(&topics_selector).map(|x| (NewsKind::Topic, x)).collect());
    let mut items = HashMap::with_capacity(lis.len());
    for (kind, li) in lis {
      let child = match kind {
        NewsKind::News => li.first_child().and_then(|v| v.value().as_element()),
        NewsKind::Topic => li.select(&title_selector).next().and_then(|v| v.first_child().and_then(|x| x.value().as_element())),
        _ => {
          println!("unsupported news kind");
          continue;
        }
      };

      let child = match child {
        Some(c) => c,
        None => {
          println!("could not get news item child");
          continue
        }
      };

      let href = match child.attr("href") {
        Some(h) => h,
        None => {
          println!("invalid link in news item");
          continue;
        }
      };

      let (title, tag) = match kind {
        NewsKind::News => {
          let title = match li.select(&title_selector).next() {
            Some(t) => t,
            None => {
              println!("missing title in news item");
              continue;
            }
          };

          let tag: Option<String> = title.first_child()
            .and_then(scraper::ElementRef::wrap)
            .map(|c| c.text().collect::<String>())
            .map(|tag| tag[1..tag.len() - 1].to_string());

          let text_iter = title.text();
          let title: String = if tag.is_some() {
            text_iter.skip(1).collect()
          } else {
            text_iter.collect()
          };

          (title, tag)
        },
        NewsKind::Topic => {
          let text = li.select(&title_selector).next()
            .and_then(|v| v.first_child())
            .and_then(scraper::ElementRef::wrap)
            .map(|v| v.text().collect());
          match text {
            Some(t) => (t, None),
            None => {
              println!("invalid topic: no title");
              continue;
            }
          }
        },
        _ => unreachable!()
      };

      let time_script = match li.select(&time_script_selector).next() {
        Some(ts) => ts,
        None => {
          println!("news item missing time script");
          continue
        }
      };

      let time_script: String = time_script.text().collect();
      let time_string = match time_script.split("strftime(").nth(1).and_then(|v| v.split(',').next()) {
        Some(time) => time,
        None => {
          println!("invalid script in news item");
          continue
        }
      };
      let time: i64 = match time_string.parse() {
        Ok(t) => t,
        Err(_) => {
          println!("invalid time in time script");
          continue
        }
      };

      let id = match href.split('/').last() {
        Some(i) => i,
        None => {
          println!("invalid href in news item");
          continue;
        }
      };

      let news_item = NewsItem {
        title: title.trim().to_string(),
        url: format!("http://na.finalfantasyxiv.com{}", href),
        kind: kind,
        time: time,
        tag: tag.map(|x| x.trim().to_string())
      };
      items.insert(id.to_string(), news_item);
    }

    items
  }
}
<commit_msg>feat: add special notices<commit_after>extern crate hyper;
extern crate hyper_rustls;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate serde_json;
extern crate scraper;
extern crate chrono;
extern crate dotenv;

use hyper::Client;

use scraper::{Html, Selector};

use chrono::{UTC, TimeZone};

use std::collections::HashMap;
use std::fs::{File, OpenOptions};
use std::io::Read;
use std::env;

const NEWS_URL: &'static str = "http://na.finalfantasyxiv.com/lodestone/news/";

struct NewsScraper {
  client: Client,
  database: NewsDatabase
}

#[derive(Debug, Default, Serialize, Deserialize)]
struct NewsDatabase {
  items: HashMap<String, NewsItem>
}

#[derive(Debug, Serialize, Deserialize)]
struct NewsItem {
  title: String,
  url: String,
  kind: NewsKind,
  time: i64,
  tag: Option<String>
}

#[derive(Debug, Serialize, Deserialize)]
enum NewsKind {
  SpecialNotice,
  News,
  Topic
}

impl ToString for NewsKind {
  fn to_string(&self) -> String {
    match *self {
      NewsKind::News => "News",
      NewsKind::Topic => "Topic",
      NewsKind::SpecialNotice => "Special notice"
    }.to_string()
  }
}

fn main() {
  dotenv::dotenv().ok();

  let webhook_url = match env::var("LB_NEWS_WEBHOOK") {
    Ok(u) => u,
    Err(_) => {
      println!("No webhook specified");
      return;
    }
  };

  let database = match NewsDatabase::load() {
    Some(db) => db,
    None => NewsDatabase::default()
  };
  let mut scraper = NewsScraper::new(database);

  let news = match scraper.download_news() {
    Some(n) => n,
    None => {
      println!("could not download news");
      return;
    }
  };

  let downloaded_news = scraper.parse_news(&news);
  for (id, item) in downloaded_news {
    if !scraper.database.items.contains_key(&id) {
      let mut embed = json!({
        "type": "rich",
        "timestamp": UTC.timestamp(item.time, 0).to_rfc3339(),
        "fields": [
          {
            "name": "Title",
            "value": item.title,
            "inline": false
          },
          {
            "name": "Link",
            "value": item.url,
            "inline": false
          },
          {
            "name": "Kind",
            "value": item.kind.to_string(),
            "inline": true
          }
        ]
      });
      if let Some(ref tag) = item.tag {
        embed["fields"].as_array_mut().unwrap().push(json!({
          "name": "Tag",
          "value": tag,
          "inline": true
        }));
      }
      let data = json!({
        "embeds": [embed]
      });
      let res = scraper.client.post(&webhook_url)
        .header(hyper::header::ContentType::json())
        .body(&data.to_string())
        .send();
      let mut data = match res {
        Ok(r) => r,
        Err(e) => {
          println!("error sending webhook: {}", e);
          continue;
        }
      };
      let mut content = String::new();
      if let Err(e) = data.read_to_string(&mut content) {
        println!("could not read webhook response: {}", e);
        continue;
      }
      if data.status.class() != hyper::status::StatusClass::Success {
        println!("discord says no to {}", item.title);
        println!("{}", content);
      } else {
        println!("webhook sent: {}", item.title);
        scraper.database.items.insert(id, item);
      }
    }
  }
  if scraper.database.save().is_none() {
    println!("could not save database");
  }
}

impl NewsDatabase {
  fn load() -> Option<Self> {
    let news_file = match File::open("./lodestone_news.json") {
      Ok(n) => n,
      Err(_) => return None
    };
    serde_json::from_reader(news_file).ok()
  }

  fn save(&self) -> Option<()> {
    let news_file = match OpenOptions::new().create(true).write(true).open("./lodestone_news.json") {
      Ok(n) => n,
      Err(_) => return None
    };
    serde_json::to_writer(news_file, self).ok()
  }
}

impl NewsScraper {
  fn new(database: NewsDatabase) -> Self {
    NewsScraper {
      client: Client::with_connector(hyper::net::HttpsConnector::new(hyper_rustls::TlsClient::new())),
      database: database
    }
  }

  fn download_news(&self) -> Option<String> {
    let mut response = match self.client.get(NEWS_URL).send() {
      Ok(r) => r,
      Err(_) => return None
    };

    let mut content = String::new();
    if response.read_to_string(&mut content).is_err() {
      return None;
    }
    Some(content)
  }

  fn parse_news(&self, news: &str) -> HashMap<String, NewsItem> {
    let html = Html::parse_document(news);
    let special_notices_selector = Selector::parse("div.news__content.parts__space--add > ul:nth-of-type(1) > li").unwrap();
    let news_selector = Selector::parse("div.news__content.parts__space--add > ul:nth-of-type(2) > li").unwrap();
    let topics_selector = Selector::parse("div.news__content.parts__space--add > ul:nth-of-type(3) > li").unwrap();
    let title_selector = Selector::parse("p.news__list--title").unwrap();
    let time_script_selector = Selector::parse("time.news__list--time > script").unwrap();
    let mut lis: Vec<_> = html.select(&news_selector).map(|x| (NewsKind::News, x)).collect();
    lis.append(&mut html.select(&topics_selector).map(|x| (NewsKind::Topic, x)).collect());
    lis.append(&mut html.select(&special_notices_selector).map(|x| (NewsKind::SpecialNotice, x)).collect());
    let mut items = HashMap::with_capacity(lis.len());
    for (kind, li) in lis {
      let child = match kind {
        NewsKind::News | NewsKind::SpecialNotice => li.first_child().and_then(|v| v.value().as_element()),
        NewsKind::Topic => li.select(&title_selector).next().and_then(|v| v.first_child().and_then(|x| x.value().as_element())),
      };

      let child = match child {
        Some(c) => c,
        None => {
          println!("could not get news item child");
          continue
        }
      };

      let href = match child.attr("href") {
        Some(h) => h,
        None => {
          println!("invalid link in news item");
          continue;
        }
      };

      let (title, tag) = match kind {
        NewsKind::News | NewsKind::SpecialNotice => {
          let title = match li.select(&title_selector).next() {
            Some(t) => t,
            None => {
              println!("missing title in news item");
              continue;
            }
          };

          let tag: Option<String> = title.first_child()
            .and_then(scraper::ElementRef::wrap)
            .map(|c| c.text().collect::<String>())
            .map(|tag| tag[1..tag.len() - 1].to_string());

          let text_iter = title.text();
          let title: String = if tag.is_some() {
            text_iter.skip(1).collect()
          } else {
            text_iter.collect()
          };

          (title, tag)
        },
        NewsKind::Topic => {
          let text = li.select(&title_selector).next()
            .and_then(|v| v.first_child())
            .and_then(scraper::ElementRef::wrap)
            .map(|v| v.text().collect());
          match text {
            Some(t) => (t, None),
            None => {
              println!("invalid topic/special notice: no title");
              continue;
            }
          }
        }
      };

      let time_script = match li.select(&time_script_selector).next() {
        Some(ts) => ts,
        None => {
          println!("news item missing time script");
          continue
        }
      };

      let time_script: String = time_script.text().collect();
      let time_string = match time_script.split("strftime(").nth(1).and_then(|v| v.split(',').next()) {
        Some(time) => time,
        None => {
          println!("invalid script in news item");
          continue
        }
      };
      let time: i64 = match time_string.parse() {
        Ok(t) => t,
        Err(_) => {
          println!("invalid time in time script");
          continue
        }
      };

      let id = match href.split('/').last() {
        Some(i) => i,
        None => {
          println!("invalid href in news item");
          continue;
        }
      };

      let news_item = NewsItem {
        title: title.trim().to_string(),
        url: format!("http://na.finalfantasyxiv.com{}", href),
        kind: kind,
        time: time,
        tag: tag.map(|x| x.trim().to_string())
      };
      items.insert(id.to_string(), news_item);
    }

    items
  }
}
<|endoftext|>