<commit_before>use byteorder::{ReadBytesExt, LittleEndian};
use std::io::{Read, Seek, SeekFrom};

use color::ColorType;
use image::{DecodingResult, ImageResult, ImageDecoder, ImageError};

use bmp::BMPDecoder;
use png::PNGDecoder;
use self::InnerDecoder::*;

// http://www.w3.org/TR/PNG-Structure.html
// The first eight bytes of a PNG file always contain the following (decimal) values:
const PNG_SIGNATURE: [u8; 8] = [137, 80, 78, 71, 13, 10, 26, 10];

/// An ico decoder
pub struct ICODecoder<R: Read> {
    selected_entry: DirEntry,
    inner_decoder: InnerDecoder<R>,
}

enum InnerDecoder<R: Read> {
    BMP(BMPDecoder<R>),
    PNG(PNGDecoder<R>)
}

#[derive(Clone, Copy, Default)]
struct DirEntry {
    width: u8,
    height: u8,
    color_count: u8,
    reserved: u8,

    num_color_planes: u16,
    bits_per_pixel: u16,

    image_length: u32,
    image_offset: u32,
}

impl<R: Read + Seek> ICODecoder<R> {
    /// Create a new decoder that decodes from the stream ```r```
    pub fn new(mut r: R) -> ImageResult<ICODecoder<R>> {
        let entries = try!(read_entries(&mut r));
        let entry = try!(best_entry(entries));
        let decoder = try!(entry.decoder(r));

        Ok(ICODecoder {
            selected_entry: entry,
            inner_decoder: decoder,
        })
    }
}

fn read_entries<R: Read>(r: &mut R) -> ImageResult<Vec<DirEntry>> {
    let _reserved = try!(r.read_u16::<LittleEndian>());
    let _type = try!(r.read_u16::<LittleEndian>());
    let count = try!(r.read_u16::<LittleEndian>());
    (0..count).map(|_| read_entry(r)).collect()
}

fn read_entry<R: Read>(r: &mut R) -> ImageResult<DirEntry> {
    let mut entry = DirEntry::default();

    entry.width = try!(r.read_u8());
    entry.height = try!(r.read_u8());
    entry.color_count = try!(r.read_u8());
    entry.reserved = try!(r.read_u8());

    entry.num_color_planes = try!(r.read_u16::<LittleEndian>());
    entry.bits_per_pixel = try!(r.read_u16::<LittleEndian>());

    entry.image_length = try!(r.read_u32::<LittleEndian>());
    entry.image_offset = try!(r.read_u32::<LittleEndian>());

    Ok(entry)
}

/// Find the entry with the highest (color depth, size).
fn best_entry(mut entries: Vec<DirEntry>) -> ImageResult<DirEntry> {
    let mut best = try!(entries.pop().ok_or(ImageError::ImageEnd));
    let mut best_score = (best.bits_per_pixel, best.real_width() * best.real_height());

    for entry in entries.into_iter() {
        let score = (entry.bits_per_pixel, entry.real_width() * entry.real_height());
        if score > best_score {
            best = entry;
            best_score = score;
        }
    }
    Ok(best)
}

impl DirEntry {
    fn real_width(&self) -> u16 {
        match self.width {
            0 => 256,
            w => w as u16
        }
    }

    fn real_height(&self) -> u16 {
        match self.height {
            0 => 256,
            h => h as u16
        }
    }

    fn seek_to_start<R: Read + Seek>(&self, r: &mut R) -> ImageResult<()> {
        try!(r.seek(SeekFrom::Start(self.image_offset as u64)));
        Ok(())
    }

    fn is_png<R: Read + Seek>(&self, r: &mut R) -> ImageResult<bool> {
        try!(self.seek_to_start(r));

        // Read the first 8 bytes to sniff the image.
        // TODO: Use Read::read_exact when it is stabilized.
        let mut signature = [0u8; 8];
        if try!(r.read(&mut signature)) != signature.len() {
            return Err(ImageError::ImageEnd)
        }

        Ok(signature == PNG_SIGNATURE)
    }

    fn decoder<R: Read + Seek>(&self, mut r: R) -> ImageResult<InnerDecoder<R>> {
        let is_png = try!(self.is_png(&mut r));
        try!(self.seek_to_start(&mut r));

        if is_png {
            Ok(PNG(PNGDecoder::new(r)))
        } else {
            let mut decoder = BMPDecoder::new(r);
            try!(decoder.read_metadata_in_ico_format(self.image_offset));
            Ok(BMP(decoder))
        }
    }
}

impl<R: Read + Seek> ImageDecoder for ICODecoder<R> {
    fn dimensions(&mut self) -> ImageResult<(u32, u32)> {
        match self.inner_decoder {
            BMP(ref mut decoder) => decoder.dimensions(),
            PNG(ref mut decoder) => decoder.dimensions()
        }
    }

    fn colortype(&mut self) -> ImageResult<ColorType> {
        match self.inner_decoder {
            BMP(ref mut decoder) => decoder.colortype(),
            PNG(ref mut decoder) => decoder.colortype()
        }
    }

    fn row_len(&mut self) -> ImageResult<usize> {
        match self.inner_decoder {
            BMP(ref mut decoder) => decoder.row_len(),
            PNG(ref mut decoder) => decoder.row_len()
        }
    }

    fn read_scanline(&mut self, buf: &mut [u8]) -> ImageResult<u32> {
        match self.inner_decoder {
            BMP(ref mut decoder) => decoder.read_scanline(buf),
            PNG(ref mut decoder) => decoder.read_scanline(buf)
        }
    }

    fn read_image(&mut self) -> ImageResult<DecodingResult> {
        match self.inner_decoder {
            PNG(ref mut decoder) => {
                decoder.read_image()
            }
            BMP(ref mut decoder) => {
                let (width, height) = try!(decoder.dimensions());

                // The ICO decoder needs an alpha chanel to apply the AND mask.
                if try!(decoder.colortype()) != ColorType::RGBA(8) {
                    return Err(ImageError::UnsupportedError("Unsupported color type".to_string()))
                }
                let mut pixel_data = match try!(decoder.read_image()) {
                    DecodingResult::U8(v) => v,
                    _ => unreachable!()
                };

                // If there's an AND mask following the image, read and apply it.
                let r = decoder.reader();
                let mask_start = try!(r.seek(SeekFrom::Current(0)));
                let mask_end = (self.selected_entry.image_offset + self.selected_entry.image_length) as u64;
                let mask_length = mask_end - mask_start;

                if mask_length > 0 {
                    // A mask row contains 1 bit per pixel, padded to 4 bytes.
                    let mask_row_bytes = ((width + 31) / 32) * 4;
                    let expected_length = (mask_row_bytes * height) as u64;
                    if mask_length < expected_length {
                        return Err(ImageError::ImageEnd)
                    }

                    for y in 0..height {
                        let mut x = 0;
                        for _ in 0..mask_row_bytes {
                            // Apply the bits of each byte until we reach the end of the row.
                            let mask_byte = try!(r.read_u8());
                            for bit in (0..8).rev() {
                                if x >= width { break }
                                if mask_byte & (1 << bit) != 0 {
                                    // Set alpha channel to transparent.
                                    pixel_data[((height - y - 1) * width + x) as usize * 4 + 3] = 0;
                                }
                                x += 1;
                            }
                        }
                    }
                }
                Ok(DecodingResult::U8(pixel_data))
            }
        }
    }
}
<commit_msg>avoid arithmetic operation overflowed<commit_after>use byteorder::{ReadBytesExt, LittleEndian};
use std::io::{Read, Seek, SeekFrom};

use color::ColorType;
use image::{DecodingResult, ImageResult, ImageDecoder, ImageError};

use bmp::BMPDecoder;
use png::PNGDecoder;
use self::InnerDecoder::*;

// http://www.w3.org/TR/PNG-Structure.html
// The first eight bytes of a PNG file always contain the following (decimal) values:
const PNG_SIGNATURE: [u8; 8] = [137, 80, 78, 71, 13, 10, 26, 10];

/// An ico decoder
pub struct ICODecoder<R: Read> {
    selected_entry: DirEntry,
    inner_decoder: InnerDecoder<R>,
}

enum InnerDecoder<R: Read> {
    BMP(BMPDecoder<R>),
    PNG(PNGDecoder<R>)
}

#[derive(Clone, Copy, Default)]
struct DirEntry {
    width: u8,
    height: u8,
    color_count: u8,
    reserved: u8,

    num_color_planes: u16,
    bits_per_pixel: u16,

    image_length: u32,
    image_offset: u32,
}

impl<R: Read + Seek> ICODecoder<R> {
    /// Create a new decoder that decodes from the stream ```r```
    pub fn new(mut r: R) -> ImageResult<ICODecoder<R>> {
        let entries = try!(read_entries(&mut r));
        let entry = try!(best_entry(entries));
        let decoder = try!(entry.decoder(r));

        Ok(ICODecoder {
            selected_entry: entry,
            inner_decoder: decoder,
        })
    }
}

fn read_entries<R: Read>(r: &mut R) -> ImageResult<Vec<DirEntry>> {
    let _reserved = try!(r.read_u16::<LittleEndian>());
    let _type = try!(r.read_u16::<LittleEndian>());
    let count = try!(r.read_u16::<LittleEndian>());
    (0..count).map(|_| read_entry(r)).collect()
}

fn read_entry<R: Read>(r: &mut R) -> ImageResult<DirEntry> {
    let mut entry = DirEntry::default();

    entry.width = try!(r.read_u8());
    entry.height = try!(r.read_u8());
    entry.color_count = try!(r.read_u8());
    entry.reserved = try!(r.read_u8());

    entry.num_color_planes = try!(r.read_u16::<LittleEndian>());
    entry.bits_per_pixel = try!(r.read_u16::<LittleEndian>());

    entry.image_length = try!(r.read_u32::<LittleEndian>());
    entry.image_offset = try!(r.read_u32::<LittleEndian>());

    Ok(entry)
}

/// Find the entry with the highest (color depth, size).
fn best_entry(mut entries: Vec<DirEntry>) -> ImageResult<DirEntry> {
    let mut best = try!(entries.pop().ok_or(ImageError::ImageEnd));
    let mut best_score = (best.bits_per_pixel, best.real_width() as u32 * best.real_height() as u32);

    for entry in entries.into_iter() {
        let score = (entry.bits_per_pixel, entry.real_width() as u32 * entry.real_height() as u32);
        if score > best_score {
            best = entry;
            best_score = score;
        }
    }
    Ok(best)
}

impl DirEntry {
    fn real_width(&self) -> u16 {
        match self.width {
            0 => 256,
            w => w as u16
        }
    }

    fn real_height(&self) -> u16 {
        match self.height {
            0 => 256,
            h => h as u16
        }
    }

    fn seek_to_start<R: Read + Seek>(&self, r: &mut R) -> ImageResult<()> {
        try!(r.seek(SeekFrom::Start(self.image_offset as u64)));
        Ok(())
    }

    fn is_png<R: Read + Seek>(&self, r: &mut R) -> ImageResult<bool> {
        try!(self.seek_to_start(r));

        // Read the first 8 bytes to sniff the image.
        // TODO: Use Read::read_exact when it is stabilized.
        let mut signature = [0u8; 8];
        if try!(r.read(&mut signature)) != signature.len() {
            return Err(ImageError::ImageEnd)
        }

        Ok(signature == PNG_SIGNATURE)
    }

    fn decoder<R: Read + Seek>(&self, mut r: R) -> ImageResult<InnerDecoder<R>> {
        let is_png = try!(self.is_png(&mut r));
        try!(self.seek_to_start(&mut r));

        if is_png {
            Ok(PNG(PNGDecoder::new(r)))
        } else {
            let mut decoder = BMPDecoder::new(r);
            try!(decoder.read_metadata_in_ico_format(self.image_offset));
            Ok(BMP(decoder))
        }
    }
}

impl<R: Read + Seek> ImageDecoder for ICODecoder<R> {
    fn dimensions(&mut self) -> ImageResult<(u32, u32)> {
        match self.inner_decoder {
            BMP(ref mut decoder) => decoder.dimensions(),
            PNG(ref mut decoder) => decoder.dimensions()
        }
    }

    fn colortype(&mut self) -> ImageResult<ColorType> {
        match self.inner_decoder {
            BMP(ref mut decoder) => decoder.colortype(),
            PNG(ref mut decoder) => decoder.colortype()
        }
    }

    fn row_len(&mut self) -> ImageResult<usize> {
        match self.inner_decoder {
            BMP(ref mut decoder) => decoder.row_len(),
            PNG(ref mut decoder) => decoder.row_len()
        }
    }

    fn read_scanline(&mut self, buf: &mut [u8]) -> ImageResult<u32> {
        match self.inner_decoder {
            BMP(ref mut decoder) => decoder.read_scanline(buf),
            PNG(ref mut decoder) => decoder.read_scanline(buf)
        }
    }

    fn read_image(&mut self) -> ImageResult<DecodingResult> {
        match self.inner_decoder {
            PNG(ref mut decoder) => {
                decoder.read_image()
            }
            BMP(ref mut decoder) => {
                let (width, height) = try!(decoder.dimensions());

                // The ICO decoder needs an alpha chanel to apply the AND mask.
                if try!(decoder.colortype()) != ColorType::RGBA(8) {
                    return Err(ImageError::UnsupportedError("Unsupported color type".to_string()))
                }
                let mut pixel_data = match try!(decoder.read_image()) {
                    DecodingResult::U8(v) => v,
                    _ => unreachable!()
                };

                // If there's an AND mask following the image, read and apply it.
                let r = decoder.reader();
                let mask_start = try!(r.seek(SeekFrom::Current(0)));
                let mask_end = (self.selected_entry.image_offset + self.selected_entry.image_length) as u64;
                let mask_length = mask_end - mask_start;

                if mask_length > 0 {
                    // A mask row contains 1 bit per pixel, padded to 4 bytes.
                    let mask_row_bytes = ((width + 31) / 32) * 4;
                    let expected_length = (mask_row_bytes * height) as u64;
                    if mask_length < expected_length {
                        return Err(ImageError::ImageEnd)
                    }

                    for y in 0..height {
                        let mut x = 0;
                        for _ in 0..mask_row_bytes {
                            // Apply the bits of each byte until we reach the end of the row.
                            let mask_byte = try!(r.read_u8());
                            for bit in (0..8).rev() {
                                if x >= width { break }
                                if mask_byte & (1 << bit) != 0 {
                                    // Set alpha channel to transparent.
                                    pixel_data[((height - y - 1) * width + x) as usize * 4 + 3] = 0;
                                }
                                x += 1;
                            }
                        }
                    }
                }
                Ok(DecodingResult::U8(pixel_data))
            }
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.

use std::collections::HashSet;
use std::env::VarError;
use std::io::prelude::*;
use std::io::BufReader;
use std::path::{Path, PathBuf};
use std::{env, fs, io};

use cmake::Config as CmakeConfig;
use pkg_config::{Config as PkgConfig, Library};
use walkdir::WalkDir;

const GRPC_VERSION: &str = "1.33.1";

fn probe_library(library: &str, cargo_metadata: bool) -> Library {
    match PkgConfig::new()
        .atleast_version(GRPC_VERSION)
        .cargo_metadata(cargo_metadata)
        .probe(library)
    {
        Ok(lib) => lib,
        Err(e) => panic!("can't find library {} via pkg-config: {:?}", library, e),
    }
}

fn prepare_grpc() {
    let modules = vec![
        "grpc",
        "grpc/third_party/cares/cares",
        "grpc/third_party/address_sorting",
        "grpc/third_party/abseil-cpp",
        "grpc/third_party/re2",
    ];

    for module in modules {
        if is_directory_empty(module).unwrap_or(true) {
            panic!(
                "Can't find module {}. You need to run `git submodule \
                 update --init --recursive` first to build the project.",
                module
            );
        }
    }
}

fn is_directory_empty<P: AsRef<Path>>(p: P) -> Result<bool, io::Error> {
    let mut entries = fs::read_dir(p)?;
    Ok(entries.next().is_none())
}

fn trim_start<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
    if s.starts_with(prefix) {
        Some(s.trim_start_matches(prefix))
    } else {
        None
    }
}

fn build_grpc(cc: &mut cc::Build, library: &str) {
    prepare_grpc();

    let target = env::var("TARGET").unwrap();
    let dst = {
        let mut config = CmakeConfig::new("grpc");

        if get_env("CARGO_CFG_TARGET_OS").map_or(false, |s| s == "macos") {
            config.cxxflag("-stdlib=libc++");
        }

        // Ensure CoreFoundation be found in macos or ios
        if get_env("CARGO_CFG_TARGET_OS").map_or(false, |s| s == "macos")
            || get_env("CARGO_CFG_TARGET_OS").map_or(false, |s| s == "ios")
        {
            println!("cargo:rustc-link-lib=framework=CoreFoundation");
        }

        if env::var("CARGO_CFG_TARGET_ENV").unwrap() == "musl" {
            config.define("CMAKE_CXX_COMPILER", "g++");
        }

        // Cross-compile support for iOS
        match target.as_str() {
            "aarch64-apple-ios" => {
                config
                    .define("CMAKE_OSX_SYSROOT", "iphoneos")
                    .define("CMAKE_OSX_ARCHITECTURES", "arm64");
            }
            "armv7-apple-ios" => {
                config
                    .define("CMAKE_OSX_SYSROOT", "iphoneos")
                    .define("CMAKE_OSX_ARCHITECTURES", "armv7");
            }
            "armv7s-apple-ios" => {
                config
                    .define("CMAKE_OSX_SYSROOT", "iphoneos")
                    .define("CMAKE_OSX_ARCHITECTURES", "armv7s");
            }
            "i386-apple-ios" => {
                config
                    .define("CMAKE_OSX_SYSROOT", "iphonesimulator")
                    .define("CMAKE_OSX_ARCHITECTURES", "i386");
            }
            "x86_64-apple-ios" => {
                config
                    .define("CMAKE_OSX_SYSROOT", "iphonesimulator")
                    .define("CMAKE_OSX_ARCHITECTURES", "x86_64");
            }
            _ => {}
        };

        // Allow overriding of the target passed to cmake
        // (needed for Android crosscompile)
        if let Ok(val) = env::var("CMAKE_TARGET_OVERRIDE") {
            config.target(&val);
        }

        // We don't need to generate install targets.
        config.define("gRPC_INSTALL", "false");
        // We don't need to build csharp target.
        config.define("gRPC_BUILD_CSHARP_EXT", "false");
        // We don't need to build codegen target.
        config.define("gRPC_BUILD_CODEGEN", "false");
        // We don't need to build benchmarks.
        config.define("gRPC_BENCHMARK_PROVIDER", "none");
        config.define("gRPC_SSL_PROVIDER", "package");
        if cfg!(feature = "openssl") {
            if cfg!(feature = "openssl-vendored") {
                config.register_dep("openssl");
            }
        } else {
            build_boringssl(&mut config);
        }
        if cfg!(feature = "no-omit-frame-pointer") {
            config
                .cflag("-fno-omit-frame-pointer")
                .cxxflag("-fno-omit-frame-pointer");
        }
        // Uses zlib from libz-sys.
        setup_libz(&mut config);
        config.build_target(library).uses_cxx11().build()
    };

    let lib_suffix = if target.contains("msvc") {
        ".lib"
    } else {
        ".a"
    };
    let build_dir = format!("{}/build", dst.display());
    for e in WalkDir::new(&build_dir) {
        let e = e.unwrap();
        if e.file_name().to_string_lossy().ends_with(lib_suffix) {
            println!(
                "cargo:rustc-link-search=native={}",
                e.path().parent().unwrap().display()
            );
        }
    }

    let collect = |path, to: &mut HashSet<_>| {
        let f = fs::File::open(format!("{}/libs/opt/pkgconfig/{}.pc", build_dir, path)).unwrap();
        for l in io::BufReader::new(f).lines() {
            let l = l.unwrap();
            if l.starts_with("Libs: ") {
                for lib in l.split_whitespace() {
                    if let Some(s) = lib.strip_prefix("-l") {
                        to.insert(s.to_string());
                    }
                }
            }
        }
    };
    let mut libs = HashSet::new();
    collect("gpr", &mut libs);
    collect(library, &mut libs);
    for l in libs {
        println!("cargo:rustc-link-lib=static={}", l);
    }

    if cfg!(feature = "secure") {
        if cfg!(feature = "openssl") && !cfg!(feature = "openssl-vendored") {
            figure_ssl_path(&build_dir);
        } else {
            println!("cargo:rustc-link-lib=static=ssl");
            println!("cargo:rustc-link-lib=static=crypto");
        }
    } else {
        // grpc_unsecure.pc is not accurate, see also grpc/grpc#24512.
        println!("cargo:rustc-link-lib=static=upb");
        println!("cargo:rustc-link-lib=static=cares");
        println!("cargo:rustc-link-lib=static=z");
        println!("cargo:rustc-link-lib=static=address_sorting");
    }

    cc.include("grpc/include");
}

fn figure_ssl_path(build_dir: &str) {
    let path = format!("{}/CMakeCache.txt", build_dir);
    let f = BufReader::new(std::fs::File::open(&path).unwrap());
    let mut cnt = 0;
    for l in f.lines() {
        let l = l.unwrap();
        let t = trim_start(&l, "OPENSSL_CRYPTO_LIBRARY:FILEPATH=")
            .or_else(|| trim_start(&l, "OPENSSL_SSL_LIBRARY:FILEPATH="));
        if let Some(s) = t {
            let path = Path::new(s);
            println!(
                "cargo:rustc-link-search=native={}",
                path.parent().unwrap().display()
            );
            cnt += 1;
        }
    }
    if cnt != 2 {
        panic!(
            "CMake cache invalid, file {} contains {} ssl keys!",
            path, cnt
        );
    }
    println!("cargo:rustc-link-lib=ssl");
    println!("cargo:rustc-link-lib=crypto");
}

fn build_boringssl(config: &mut CmakeConfig) {
    let boringssl_artifact = boringssl_src::Build::new().build();
    config.define(
        "OPENSSL_ROOT_DIR",
        format!("{}", boringssl_artifact.root_dir().display()),
    );
    // To avoid linking system library, set lib path explicitly.
    println!(
        "cargo:rustc-link-search=native={}",
        boringssl_artifact.lib_dir().display()
    );
}

fn setup_libz(config: &mut CmakeConfig) {
    config.define("gRPC_ZLIB_PROVIDER", "package");
    config.register_dep("Z");
    // cmake script expect libz.a being under ${DEP_Z_ROOT}/lib, but libz-sys crate put it
    // under ${DEP_Z_ROOT}/build. Append the path to CMAKE_PREFIX_PATH to get around it.
    let zlib_root = env::var("DEP_Z_ROOT").unwrap();
    let prefix_path = if let Ok(prefix_path) = env::var("CMAKE_PREFIX_PATH") {
        format!("{};{}/build", prefix_path, zlib_root)
    } else {
        format!("{}/build", zlib_root)
    };
    // To avoid linking system library, set lib path explicitly.
    println!("cargo:rustc-link-search=native={}/build", zlib_root);
    println!("cargo:rustc-link-search=native={}/lib", zlib_root);
    env::set_var("CMAKE_PREFIX_PATH", prefix_path);
}

fn get_env(name: &str) -> Option<String> {
    println!("cargo:rerun-if-env-changed={}", name);
    match env::var(name) {
        Ok(s) => Some(s),
        Err(VarError::NotPresent) => None,
        Err(VarError::NotUnicode(s)) => {
            panic!("unrecognize env var of {}: {:?}", name, s.to_string_lossy());
        }
    }
}

// Generate the bindings to grpc C-core.
// Try to disable the generation of platform-related bindings.
fn bindgen_grpc(mut config: bindgen::Builder, file_path: &PathBuf) {
    // Search header files with API interface
    let mut headers = Vec::new();
    for result in WalkDir::new(Path::new("./grpc/include")) {
        let dent = result.expect("Error happened when search headers");
        if !dent.file_type().is_file() {
            continue;
        }
        let mut file = fs::File::open(dent.path()).expect("couldn't open headers");
        let mut buf = String::new();
        file.read_to_string(&mut buf)
            .expect("Coundn't read header content");
        if buf.contains("GRPCAPI") || buf.contains("GPRAPI") {
            headers.push(String::from(dent.path().to_str().unwrap()));
        }
    }

    // To control the order of bindings
    headers.sort();
    for path in headers {
        config = config.header(path);
    }

    println!("cargo:rerun-if-env-changed=TEST_BIND");
    let gen_tests = env::var("TEST_BIND").map_or(false, |s| s == "1");

    let cfg = config
        .header("grpc_wrap.cc")
        .clang_arg("-xc++")
        .clang_arg("-I./grpc/include")
        .clang_arg("-std=c++11")
        .rustfmt_bindings(true)
        .impl_debug(true)
        .whitelist_function(r"\bgrpc_.*")
        .whitelist_function(r"\bgpr_.*")
        .whitelist_function(r"\bgrpcwrap_.*")
        .whitelist_var(r"\bGRPC_.*")
        .whitelist_type(r"\bgrpc_.*")
        .whitelist_type(r"\bgpr_.*")
        .whitelist_type(r"\bgrpcwrap_.*")
        .whitelist_type(r"\bcensus_context.*")
        .whitelist_type(r"\bverify_peer_options.*")
        .blacklist_type(r"(__)?pthread.*")
        .blacklist_function(r"\bgpr_mu_.*")
        .blacklist_function(r"\bgpr_cv_.*")
        .blacklist_function(r"\bgpr_once_.*")
        .blacklist_type(r"gpr_mu")
        .blacklist_type(r"gpr_cv")
        .blacklist_type(r"gpr_once")
        .constified_enum_module(r"grpc_status_code")
        .layout_tests(gen_tests)
        .default_enum_style(bindgen::EnumVariation::Rust {
            non_exhaustive: false,
        });
    println!("running {}", cfg.command_line_flags().join(" "));
    cfg.generate()
        .expect("Unable to generate grpc bindings")
        .write_to_file(file_path)
        .expect("Couldn't write bindings!");
}

// Determine if need to update bindings. Supported platforms do not
// need to be updated by default unless the UPDATE_BIND is specified.
// Other platforms use bindgen to generate the bindings every time.
fn config_binding_path(config: bindgen::Builder) {
    let file_path: PathBuf;
    let target = env::var("TARGET").unwrap();
    match target.as_str() {
        "x86_64-unknown-linux-gnu" | "aarch64-unknown-linux-gnu" => {
            // Cargo treats nonexistent files changed, so we only emit the rerun-if-changed
            // directive when we expect the target-specific pre-generated binding file to be
            // present.
            println!("cargo:rerun-if-changed=bindings/{}-bindings.rs", &target);

            file_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap())
                .join("bindings")
                .join(format!("{}-bindings.rs", &target));
            if env::var("UPDATE_BIND").map(|s| s == "1").unwrap_or(false) {
                bindgen_grpc(config, &file_path);
            }
        }
        _ => {
            file_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("grpc-bindings.rs");
            bindgen_grpc(config, &file_path);
        }
    };
    println!(
        "cargo:rustc-env=BINDING_PATH={}",
        file_path.to_str().unwrap()
    );
}

fn main() {
    println!("cargo:rerun-if-changed=grpc_wrap.cc");
    println!("cargo:rerun-if-changed=grpc");
    println!("cargo:rerun-if-env-changed=UPDATE_BIND");

    // create a builder to compile grpc_wrap.cc
    let mut cc = cc::Build::new();
    // create a config to generate binding file
    let mut bind_config = bindgen::Builder::default();

    let library = if cfg!(feature = "secure") {
        cc.define("GRPC_SYS_SECURE", None);
        bind_config = bind_config.clang_arg("-DGRPC_SYS_SECURE");
        "grpc"
    } else {
        "grpc_unsecure"
    };

    if get_env("CARGO_CFG_TARGET_OS").map_or(false, |s| s == "windows") {
        // At lease vista
        cc.define("_WIN32_WINNT", Some("0x600"));
        bind_config = bind_config.clang_arg("-D _WIN32_WINNT=0x600");
    }

    if get_env("GRPCIO_SYS_USE_PKG_CONFIG").map_or(false, |s| s == "1") {
        // Print cargo metadata.
        let lib_core = probe_library(library, true);
        for inc_path in lib_core.include_paths {
            cc.include(inc_path);
        }
    } else {
        build_grpc(&mut cc, library);
    }

    cc.cpp(true);
    if !cfg!(target_env = "msvc") {
        cc.flag("-std=c++11");
    }
    cc.file("grpc_wrap.cc");
    cc.warnings_into_errors(true);
    cc.compile("libgrpc_wrap.a");

    config_binding_path(bind_config);
}
<commit_msg>Allow CXX environment variable to override g++ for musl build (#500)<commit_after>// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0.

use std::collections::HashSet;
use std::env::VarError;
use std::io::prelude::*;
use std::io::BufReader;
use std::path::{Path, PathBuf};
use std::{env, fs, io};

use cmake::Config as CmakeConfig;
use pkg_config::{Config as PkgConfig, Library};
use walkdir::WalkDir;

const GRPC_VERSION: &str = "1.33.1";

fn probe_library(library: &str, cargo_metadata: bool) -> Library {
    match PkgConfig::new()
        .atleast_version(GRPC_VERSION)
        .cargo_metadata(cargo_metadata)
        .probe(library)
    {
        Ok(lib) => lib,
        Err(e) => panic!("can't find library {} via pkg-config: {:?}", library, e),
    }
}

fn prepare_grpc() {
    let modules = vec![
        "grpc",
        "grpc/third_party/cares/cares",
        "grpc/third_party/address_sorting",
        "grpc/third_party/abseil-cpp",
        "grpc/third_party/re2",
    ];

    for module in modules {
        if is_directory_empty(module).unwrap_or(true) {
            panic!(
                "Can't find module {}. You need to run `git submodule \
                 update --init --recursive` first to build the project.",
                module
            );
        }
    }
}

fn is_directory_empty<P: AsRef<Path>>(p: P) -> Result<bool, io::Error> {
    let mut entries = fs::read_dir(p)?;
    Ok(entries.next().is_none())
}

fn trim_start<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
    if s.starts_with(prefix) {
        Some(s.trim_start_matches(prefix))
    } else {
        None
    }
}

fn build_grpc(cc: &mut cc::Build, library: &str) {
    prepare_grpc();

    let target = env::var("TARGET").unwrap();
    let dst = {
        let mut config = CmakeConfig::new("grpc");

        if get_env("CARGO_CFG_TARGET_OS").map_or(false, |s| s == "macos") {
            config.cxxflag("-stdlib=libc++");
        }

        // Ensure CoreFoundation be found in macos or ios
        if get_env("CARGO_CFG_TARGET_OS").map_or(false, |s| s == "macos")
            || get_env("CARGO_CFG_TARGET_OS").map_or(false, |s| s == "ios")
        {
            println!("cargo:rustc-link-lib=framework=CoreFoundation");
        }

        if let Some(val) = get_env("CXX") {
            config.define("CMAKE_CXX_COMPILER", val);
        } else if env::var("CARGO_CFG_TARGET_ENV").unwrap() == "musl" {
            config.define("CMAKE_CXX_COMPILER", "g++");
        }

        // Cross-compile support for iOS
        match target.as_str() {
            "aarch64-apple-ios" => {
                config
                    .define("CMAKE_OSX_SYSROOT", "iphoneos")
                    .define("CMAKE_OSX_ARCHITECTURES", "arm64");
            }
            "armv7-apple-ios" => {
                config
                    .define("CMAKE_OSX_SYSROOT", "iphoneos")
                    .define("CMAKE_OSX_ARCHITECTURES", "armv7");
            }
            "armv7s-apple-ios" => {
                config
                    .define("CMAKE_OSX_SYSROOT", "iphoneos")
                    .define("CMAKE_OSX_ARCHITECTURES", "armv7s");
            }
            "i386-apple-ios" => {
                config
                    .define("CMAKE_OSX_SYSROOT", "iphonesimulator")
                    .define("CMAKE_OSX_ARCHITECTURES", "i386");
            }
            "x86_64-apple-ios" => {
                config
                    .define("CMAKE_OSX_SYSROOT", "iphonesimulator")
                    .define("CMAKE_OSX_ARCHITECTURES", "x86_64");
            }
            _ => {}
        };

        // Allow overriding of the target passed to cmake
        // (needed for Android crosscompile)
        if let Ok(val) = env::var("CMAKE_TARGET_OVERRIDE") {
            config.target(&val);
        }

        // We don't need to generate install targets.
        config.define("gRPC_INSTALL", "false");
        // We don't need to build csharp target.
        config.define("gRPC_BUILD_CSHARP_EXT", "false");
        // We don't need to build codegen target.
        config.define("gRPC_BUILD_CODEGEN", "false");
        // We don't need to build benchmarks.
        config.define("gRPC_BENCHMARK_PROVIDER", "none");
        config.define("gRPC_SSL_PROVIDER", "package");
        if cfg!(feature = "openssl") {
            if cfg!(feature = "openssl-vendored") {
                config.register_dep("openssl");
            }
        } else {
            build_boringssl(&mut config);
        }
        if cfg!(feature = "no-omit-frame-pointer") {
            config
                .cflag("-fno-omit-frame-pointer")
                .cxxflag("-fno-omit-frame-pointer");
        }
        // Uses zlib from libz-sys.
        setup_libz(&mut config);
        config.build_target(library).uses_cxx11().build()
    };

    let lib_suffix = if target.contains("msvc") {
        ".lib"
    } else {
        ".a"
    };
    let build_dir = format!("{}/build", dst.display());
    for e in WalkDir::new(&build_dir) {
        let e = e.unwrap();
        if e.file_name().to_string_lossy().ends_with(lib_suffix) {
            println!(
                "cargo:rustc-link-search=native={}",
                e.path().parent().unwrap().display()
            );
        }
    }

    let collect = |path, to: &mut HashSet<_>| {
        let f = fs::File::open(format!("{}/libs/opt/pkgconfig/{}.pc", build_dir, path)).unwrap();
        for l in io::BufReader::new(f).lines() {
            let l = l.unwrap();
            if l.starts_with("Libs: ") {
                for lib in l.split_whitespace() {
                    if let Some(s) = lib.strip_prefix("-l") {
                        to.insert(s.to_string());
                    }
                }
            }
        }
    };
    let mut libs = HashSet::new();
    collect("gpr", &mut libs);
    collect(library, &mut libs);
    for l in libs {
        println!("cargo:rustc-link-lib=static={}", l);
    }

    if cfg!(feature = "secure") {
        if cfg!(feature = "openssl") && !cfg!(feature = "openssl-vendored") {
            figure_ssl_path(&build_dir);
        } else {
            println!("cargo:rustc-link-lib=static=ssl");
            println!("cargo:rustc-link-lib=static=crypto");
        }
    } else {
        // grpc_unsecure.pc is not accurate, see also grpc/grpc#24512.
        println!("cargo:rustc-link-lib=static=upb");
        println!("cargo:rustc-link-lib=static=cares");
        println!("cargo:rustc-link-lib=static=z");
        println!("cargo:rustc-link-lib=static=address_sorting");
    }

    cc.include("grpc/include");
}

fn figure_ssl_path(build_dir: &str) {
    let path = format!("{}/CMakeCache.txt", build_dir);
    let f = BufReader::new(std::fs::File::open(&path).unwrap());
    let mut cnt = 0;
    for l in f.lines() {
        let l = l.unwrap();
        let t = trim_start(&l, "OPENSSL_CRYPTO_LIBRARY:FILEPATH=")
            .or_else(|| trim_start(&l, "OPENSSL_SSL_LIBRARY:FILEPATH="));
        if let Some(s) = t {
            let path = Path::new(s);
            println!(
                "cargo:rustc-link-search=native={}",
                path.parent().unwrap().display()
            );
            cnt += 1;
        }
    }
    if cnt != 2 {
        panic!(
            "CMake cache invalid, file {} contains {} ssl keys!",
            path, cnt
        );
    }
    println!("cargo:rustc-link-lib=ssl");
    println!("cargo:rustc-link-lib=crypto");
}

fn build_boringssl(config: &mut CmakeConfig) {
    let boringssl_artifact = boringssl_src::Build::new().build();
    config.define(
        "OPENSSL_ROOT_DIR",
        format!("{}", boringssl_artifact.root_dir().display()),
    );
    // To avoid linking system library, set lib path explicitly.
    println!(
        "cargo:rustc-link-search=native={}",
        boringssl_artifact.lib_dir().display()
    );
}

fn setup_libz(config: &mut CmakeConfig) {
    config.define("gRPC_ZLIB_PROVIDER", "package");
    config.register_dep("Z");
    // cmake script expect libz.a being under ${DEP_Z_ROOT}/lib, but libz-sys crate put it
    // under ${DEP_Z_ROOT}/build. Append the path to CMAKE_PREFIX_PATH to get around it.
    let zlib_root = env::var("DEP_Z_ROOT").unwrap();
    let prefix_path = if let Ok(prefix_path) = env::var("CMAKE_PREFIX_PATH") {
        format!("{};{}/build", prefix_path, zlib_root)
    } else {
        format!("{}/build", zlib_root)
    };
    // To avoid linking system library, set lib path explicitly.
    println!("cargo:rustc-link-search=native={}/build", zlib_root);
    println!("cargo:rustc-link-search=native={}/lib", zlib_root);
    env::set_var("CMAKE_PREFIX_PATH", prefix_path);
}

fn get_env(name: &str) -> Option<String> {
    println!("cargo:rerun-if-env-changed={}", name);
    match env::var(name) {
        Ok(s) => Some(s),
        Err(VarError::NotPresent) => None,
        Err(VarError::NotUnicode(s)) => {
            panic!("unrecognize env var of {}: {:?}", name, s.to_string_lossy());
        }
    }
}

// Generate the bindings to grpc C-core.
// Try to disable the generation of platform-related bindings.
fn bindgen_grpc(mut config: bindgen::Builder, file_path: &PathBuf) {
    // Search header files with API interface
    let mut headers = Vec::new();
    for result in WalkDir::new(Path::new("./grpc/include")) {
        let dent = result.expect("Error happened when search headers");
        if !dent.file_type().is_file() {
            continue;
        }
        let mut file = fs::File::open(dent.path()).expect("couldn't open headers");
        let mut buf = String::new();
        file.read_to_string(&mut buf)
            .expect("Coundn't read header content");
        if buf.contains("GRPCAPI") || buf.contains("GPRAPI") {
            headers.push(String::from(dent.path().to_str().unwrap()));
        }
    }

    // To control the order of bindings
    headers.sort();
    for path in headers {
        config = config.header(path);
    }

    println!("cargo:rerun-if-env-changed=TEST_BIND");
    let gen_tests = env::var("TEST_BIND").map_or(false, |s| s == "1");

    let cfg = config
        .header("grpc_wrap.cc")
        .clang_arg("-xc++")
        .clang_arg("-I./grpc/include")
        .clang_arg("-std=c++11")
        .rustfmt_bindings(true)
        .impl_debug(true)
        .whitelist_function(r"\bgrpc_.*")
        .whitelist_function(r"\bgpr_.*")
        .whitelist_function(r"\bgrpcwrap_.*")
        .whitelist_var(r"\bGRPC_.*")
        .whitelist_type(r"\bgrpc_.*")
        .whitelist_type(r"\bgpr_.*")
        .whitelist_type(r"\bgrpcwrap_.*")
        .whitelist_type(r"\bcensus_context.*")
        .whitelist_type(r"\bverify_peer_options.*")
        .blacklist_type(r"(__)?pthread.*")
        .blacklist_function(r"\bgpr_mu_.*")
        .blacklist_function(r"\bgpr_cv_.*")
        .blacklist_function(r"\bgpr_once_.*")
        .blacklist_type(r"gpr_mu")
        .blacklist_type(r"gpr_cv")
        .blacklist_type(r"gpr_once")
        .constified_enum_module(r"grpc_status_code")
        .layout_tests(gen_tests)
        .default_enum_style(bindgen::EnumVariation::Rust {
            non_exhaustive: false,
        });
    println!("running {}", cfg.command_line_flags().join(" "));
    cfg.generate()
        .expect("Unable to generate grpc bindings")
        .write_to_file(file_path)
        .expect("Couldn't write bindings!");
}

// Determine if need to update bindings. Supported platforms do not
// need to be updated by default unless the UPDATE_BIND is specified.
// Other platforms use bindgen to generate the bindings every time.
fn config_binding_path(config: bindgen::Builder) {
    let file_path: PathBuf;
    let target = env::var("TARGET").unwrap();
    match target.as_str() {
        "x86_64-unknown-linux-gnu" | "aarch64-unknown-linux-gnu" => {
            // Cargo treats nonexistent files changed, so we only emit the rerun-if-changed
            // directive when we expect the target-specific pre-generated binding file to be
            // present.
            println!("cargo:rerun-if-changed=bindings/{}-bindings.rs", &target);

            file_path = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap())
                .join("bindings")
                .join(format!("{}-bindings.rs", &target));
            if env::var("UPDATE_BIND").map(|s| s == "1").unwrap_or(false) {
                bindgen_grpc(config, &file_path);
            }
        }
        _ => {
            file_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("grpc-bindings.rs");
            bindgen_grpc(config, &file_path);
        }
    };
    println!(
        "cargo:rustc-env=BINDING_PATH={}",
        file_path.to_str().unwrap()
    );
}

fn main() {
    println!("cargo:rerun-if-changed=grpc_wrap.cc");
    println!("cargo:rerun-if-changed=grpc");
    println!("cargo:rerun-if-env-changed=UPDATE_BIND");

    // create a builder to compile grpc_wrap.cc
    let mut cc = cc::Build::new();
    // create a config to generate binding file
    let mut bind_config = bindgen::Builder::default();

    let library = if cfg!(feature = "secure") {
        cc.define("GRPC_SYS_SECURE", None);
        bind_config = bind_config.clang_arg("-DGRPC_SYS_SECURE");
        "grpc"
    } else {
        "grpc_unsecure"
    };

    if get_env("CARGO_CFG_TARGET_OS").map_or(false, |s| s == "windows") {
        // At lease vista
        cc.define("_WIN32_WINNT", Some("0x600"));
        bind_config = bind_config.clang_arg("-D _WIN32_WINNT=0x600");
    }

    if get_env("GRPCIO_SYS_USE_PKG_CONFIG").map_or(false, |s| s == "1") {
        // Print cargo metadata.
        let lib_core = probe_library(library, true);
        for inc_path in lib_core.include_paths {
            cc.include(inc_path);
        }
    } else {
        build_grpc(&mut cc, library);
    }

    cc.cpp(true);
    if !cfg!(target_env = "msvc") {
        cc.flag("-std=c++11");
    }
    cc.file("grpc_wrap.cc");
    cc.warnings_into_errors(true);
    cc.compile("libgrpc_wrap.a");

    config_binding_path(bind_config);
}
<|endoftext|>


<commit_before>use std::{
    collections::{hash_map::Entry, HashMap},
    sync::Arc,
    time::{Duration, Instant, SystemTime},
};

use async_channel::TrySendError;
use axum::http::StatusCode;
use clap::Clap;
use futures_util::StreamExt;
use rustc_hash::FxHashMap;
use shakmaty::{
    uci::Uci, variant::VariantPosition, zobrist::Zobrist, ByColor, CastlingMode, Outcome, Position,
};
use tokio::{
    sync::{watch, RwLock},
    task::JoinHandle,
    time::{sleep, timeout},
};

use crate::{
    db::Database,
    model::{
        GameInfo, GameInfoPlayer, Mode, Month, PersonalEntry, PersonalKeyBuilder, PersonalStatus,
        UserId,
    },
};

mod lila;

use lila::{Game, Lila};

const MAX_PLIES: usize = 50;

#[derive(Clap, Clone)]
pub struct IndexerOpt {
    #[clap(long = "lila", default_value = "https://lichess.org")]
    lila: String,
    #[clap(long = "bearer")]
    bearer: Option<String>,
    #[clap(long = "indexers", default_value = "16")]
    indexers: usize,
}

#[derive(Clone)]
pub struct IndexerStub {
    db: Arc<Database>,
    indexing: Arc<RwLock<HashMap<UserId, watch::Sender<()>>>>,
    tx: async_channel::Sender<IndexerMessage>,
}

impl IndexerStub {
    pub fn spawn(db: Arc<Database>, opt: IndexerOpt) -> (IndexerStub, Vec<JoinHandle<()>>) {
        let indexing = Arc::new(RwLock::new(HashMap::new()));

        let (tx, rx) = async_channel::bounded(1000);
        let mut join_handles = Vec::with_capacity(opt.indexers);
        for idx in 0..opt.indexers {
            join_handles.push(tokio::spawn(
                IndexerActor {
                    idx,
                    rx: rx.clone(),
                    indexing: Arc::clone(&indexing),
                    db: Arc::clone(&db),
                    lila: Lila::new(opt.clone()),
                }
                .run(),
            ));
        }

        (IndexerStub { db, indexing, tx }, join_handles)
    }

    pub async fn num_indexing(&self) -> usize {
        let guard = self.indexing.read().await;
        guard.len()
    }

    pub async fn index_player(&self, player: &UserId) -> Option<watch::Receiver<()>> {
        // Optimization: First try subscribing to an existing indexing run,
        // without acquiring a write lock.
        {
            let guard = self.indexing.read().await;
            if let Some(sender) = guard.get(player) {
                return Some(sender.subscribe());
            }
        }

        // Check player indexing status.
        let mut status = self
            .db
            .queryable()
            .get_player_status(player)
            .expect("get player status")
            .unwrap_or_default();

        let since_created_at = match status
            .maybe_revisit_ongoing()
            .or_else(|| status.maybe_index())
        {
            Some(since) => since,
            None => return None, // Do not reindex so soon!
        };

        // Queue indexing request.
        let mut guard = self.indexing.write().await;
        let entry = match guard.entry(player.to_owned()) {
            Entry::Occupied(entry) => return Some(entry.get().subscribe()),
            Entry::Vacant(entry) => entry,
        };

        match self.tx.try_send(IndexerMessage::IndexPlayer {
            player: player.to_owned(),
            status,
            since_created_at,
        }) {
            Ok(_) => {
                let (sender, receiver) = watch::channel(());
                entry.insert(sender);
                Some(receiver)
            }
            Err(TrySendError::Full(_)) => {
                log::error!(
                    "not queuing {} because indexer queue is full",
                    player.as_str()
                );
                None
            }
            Err(TrySendError::Closed(_)) => panic!("all indexers died"),
        }
    }
}

struct IndexerActor {
    idx: usize,
    indexing: Arc<RwLock<HashMap<UserId, watch::Sender<()>>>>,
    rx: async_channel::Receiver<IndexerMessage>,
    db: Arc<Database>,
    lila: Lila,
}

impl IndexerActor {
    async fn run(self) {
        while let Ok(msg) = self.rx.recv().await {
            match msg {
                IndexerMessage::IndexPlayer {
                    player,
                    status,
                    since_created_at,
                } => {
                    self.index_player(&player, status, since_created_at).await;

                    let mut guard = self.indexing.write().await;
                    guard.remove(&player);
                }
            }
        }
    }

    async fn index_player(
        &self,
        player: &UserId,
        mut status: PersonalStatus,
        since_created_at: u64,
    ) {
        let started_at = Instant::now();
        log::info!(
            "indexer {:02}: starting {} (created_at >= {})",
            self.idx,
            player.as_str(),
            since_created_at
        );

        let mut games = match timeout(
            Duration::from_secs(10),
            self.lila.user_games(player, since_created_at),
        )
        .await
        {
            Ok(Ok(games)) => games,
            Ok(Err(err)) if err.status() == Some(StatusCode::NOT_FOUND) => {
                log::warn!(
                    "indexer {:02}: did not find player {}",
                    self.idx,
                    player.as_str()
                );
                return;
            }
            Ok(Err(err)) => {
                log::error!("indexer {:02}: request failed: {}", self.idx, err);
                sleep(Duration::from_secs(5)).await;
                return;
            }
            Err(timed_out) => {
                log::error!("indexer {:02}: request to lila: {}", self.idx, timed_out);
                return;
            }
        };

        let hash = ByColor::new_with(|color| PersonalKeyBuilder::with_user_pov(player, color));

        let mut num_games = 0;
        loop {
            let game = match timeout(Duration::from_secs(10), games.next()).await {
                Ok(Some(Ok(game))) => game,
                Ok(Some(Err(err))) => {
                    log::error!("indexer {:02}: {}", self.idx, err);
                    continue;
                }
                Ok(None) => break,
                Err(timed_out) => {
                    log::error!("indexer {:02}: stream from lila: {}", self.idx, timed_out);
                    return;
                }
            };

            self.index_game(player, &hash, game, &mut status);

            num_games += 1;
            if num_games % 1024 == 0 {
                self.db
                    .queryable()
                    .put_player_status(player, &status)
                    .expect("put player status");

                log::info!(
                    "indexer {:02}: indexed {} games for {} ...",
                    self.idx,
                    num_games,
                    player.as_str()
                );
            }
        }

        status.indexed_at = SystemTime::now();
        self.db
            .queryable()
            .put_player_status(player, &status)
            .expect("put player status");

        let elapsed = started_at.elapsed();
        log::info!(
            "indexer {:02}: finished {} games for {} in {:.3?} ({:.3?}/game, {:.1} games/s)",
            self.idx,
            num_games,
            player.as_str(),
            elapsed,
            (elapsed.as_nanos() as u64)
                .checked_div(num_games)
                .map(Duration::from_nanos)
                .unwrap_or_default(),
            (num_games as f64) / elapsed.as_secs_f64()
        );
    }

    fn index_game(
        &self,
        player: &UserId,
        hash: &ByColor<PersonalKeyBuilder>,
        game: Game,
        status: &mut PersonalStatus,
    ) {
        status.latest_created_at = game.created_at;

        if game.status.is_ongoing() {
            if status.revisit_ongoing_created_at.is_none() {
                log::debug!(
                    "indexer {:02}: will revisit ongoing game {} eventually",
                    self.idx,
                    game.id
                );
                status.revisit_ongoing_created_at = Some(game.created_at);
            }
            return;
        }

        if game.status.is_unindexable() {
            return;
        }

        if game.players.any(|p| p.user.is_none()) {
            return;
        }

        let color = match game
            .players
            .find(|p| p.user.as_ref().map_or(false, |user| user.name == *player))
        {
            Some(color) => color,
            None => {
                log::error!(
                    "indexer {:02}: {} did not play in {}",
                    self.idx,
                    player.as_str(),
                    game.id
                );
                return;
            }
        };

        // Skip game if already indexed from this side. This cannot race with
        // writes, because all writes for the same player are sequenced by
        // this actor. So making a transaction is not required.
        let queryable = self.db.queryable();
        if queryable
            .get_game_info(game.id)
            .expect("get game info")
            .map_or(false, |info| info.indexed.into_color(color))
        {
            log::debug!(
                "indexer {:02}: {}/{} already indexed",
                self.idx,
                game.id,
                color
            );
            return;
        }

        // Prepare basic information and setup initial position.
        let month = Month::from_time_saturating(game.last_move_at);
        let outcome = Outcome::from_winner(game.winner);
        let variant = game.variant.into();
        let pos = match game.initial_fen {
            Some(fen) => VariantPosition::from_setup(variant, &fen, CastlingMode::Chess960),
            None => Ok(VariantPosition::new(variant)),
        };
        let opponent_rating = match game.players.by_color(!color).rating {
            Some(rating) => rating,
            None => {
                log::warn!(
                    "indexer {:02}: skipping {} without opponent rating",
                    self.idx,
                    game.id
                );
                return;
            }
        };

        let mut pos: Zobrist<_, u128> = match pos {
            Ok(pos) => Zobrist::new(pos),
            Err(err) => {
                log::warn!("indexer {:02}: not indexing {}: {}", self.idx, game.id, err);
                return;
            }
        };

        // Build an intermediate table to remove loops (due to repetitions).
        let mut table: FxHashMap<u128, Uci> =
            FxHashMap::with_capacity_and_hasher(game.moves.len(), Default::default());

        for (ply, san) in game.moves.into_iter().enumerate() {
            if ply >= MAX_PLIES {
                break;
            }

            let m = match san.to_move(&pos) {
                Ok(m) => m,
                Err(err) => {
                    log::warn!(
                        "indexer {:02}: cutting off {} at ply {}: {}: {}",
                        self.idx,
                        game.id,
                        ply,
                        err,
                        san
                    );
                    break;
                }
            };

            let uci = m.to_uci(CastlingMode::Chess960);
            table.insert(pos.zobrist_hash(), uci);

            pos.play_unchecked(&m);
        }

        // Write to database. All writes regarding this game are batched and
        // atomically committed, so the database will always be in a consistent
        // state.
        let mut batch = queryable.batch();

        batch.merge_game_info(
            game.id,
            GameInfo {
                winner: outcome.winner(),
                speed: game.speed,
                mode: Mode::from_rated(game.rated),
                month,
                players: game.players.map(|p| GameInfoPlayer {
                    name: p.user.map(|p| p.name.to_string()),
                    rating: p.rating,
                }),
                indexed: ByColor::new_with(|c| color == c),
            },
        );

        for (zobrist, uci) in table {
            batch.merge_personal(
                hash.by_color(color)
                    .with_zobrist(variant, zobrist)
                    .with_month(month),
                PersonalEntry::new_single(
                    uci.clone(),
                    game.speed,
                    Mode::from_rated(game.rated),
                    game.id,
                    outcome,
                    opponent_rating,
                ),
            );
        }

        batch.write().expect("atomically commit game and moves");
    }
}

enum IndexerMessage {
    IndexPlayer {
        player: UserId,
        status: PersonalStatus,
        since_created_at: u64,
    },
}
<commit_msg>longer request and stream timeout<commit_after>use std::{
    collections::{hash_map::Entry, HashMap},
    sync::Arc,
    time::{Duration, Instant, SystemTime},
};

use async_channel::TrySendError;
use axum::http::StatusCode;
use clap::Clap;
use futures_util::StreamExt;
use rustc_hash::FxHashMap;
use shakmaty::{
    uci::Uci, variant::VariantPosition, zobrist::Zobrist, ByColor, CastlingMode, Outcome, Position,
};
use tokio::{
    sync::{watch, RwLock},
    task::JoinHandle,
    time::{sleep, timeout},
};

use crate::{
    db::Database,
    model::{
        GameInfo, GameInfoPlayer, Mode, Month, PersonalEntry, PersonalKeyBuilder, PersonalStatus,
        UserId,
    },
};

mod lila;

use lila::{Game, Lila};

const MAX_PLIES: usize = 50;

#[derive(Clap, Clone)]
pub struct IndexerOpt {
    #[clap(long = "lila", default_value = "https://lichess.org")]
    lila: String,
    #[clap(long = "bearer")]
    bearer: Option<String>,
    #[clap(long = "indexers", default_value = "16")]
    indexers: usize,
}

#[derive(Clone)]
pub struct IndexerStub {
    db: Arc<Database>,
    indexing: Arc<RwLock<HashMap<UserId, watch::Sender<()>>>>,
    tx: async_channel::Sender<IndexerMessage>,
}

impl IndexerStub {
    pub fn spawn(db: Arc<Database>, opt: IndexerOpt) -> (IndexerStub, Vec<JoinHandle<()>>) {
        let indexing = Arc::new(RwLock::new(HashMap::new()));

        let (tx, rx) = async_channel::bounded(1000);
        let mut join_handles = Vec::with_capacity(opt.indexers);
        for idx in 0..opt.indexers {
            join_handles.push(tokio::spawn(
                IndexerActor {
                    idx,
                    rx: rx.clone(),
                    indexing: Arc::clone(&indexing),
                    db: Arc::clone(&db),
                    lila: Lila::new(opt.clone()),
                }
                .run(),
            ));
        }

        (IndexerStub { db, indexing, tx }, join_handles)
    }

    pub async fn num_indexing(&self) -> usize {
        let guard = self.indexing.read().await;
        guard.len()
    }

    pub async fn index_player(&self, player: &UserId) -> Option<watch::Receiver<()>> {
        // Optimization: First try subscribing to an existing indexing run,
        // without acquiring a write lock.
        {
            let guard = self.indexing.read().await;
            if let Some(sender) = guard.get(player) {
                return Some(sender.subscribe());
            }
        }

        // Check player indexing status.
        let mut status = self
            .db
            .queryable()
            .get_player_status(player)
            .expect("get player status")
            .unwrap_or_default();

        let since_created_at = match status
            .maybe_revisit_ongoing()
            .or_else(|| status.maybe_index())
        {
            Some(since) => since,
            None => return None, // Do not reindex so soon!
        };

        // Queue indexing request.
        let mut guard = self.indexing.write().await;
        let entry = match guard.entry(player.to_owned()) {
            Entry::Occupied(entry) => return Some(entry.get().subscribe()),
            Entry::Vacant(entry) => entry,
        };

        match self.tx.try_send(IndexerMessage::IndexPlayer {
            player: player.to_owned(),
            status,
            since_created_at,
        }) {
            Ok(_) => {
                let (sender, receiver) = watch::channel(());
                entry.insert(sender);
                Some(receiver)
            }
            Err(TrySendError::Full(_)) => {
                log::error!(
                    "not queuing {} because indexer queue is full",
                    player.as_str()
                );
                None
            }
            Err(TrySendError::Closed(_)) => panic!("all indexers died"),
        }
    }
}

struct IndexerActor {
    idx: usize,
    indexing: Arc<RwLock<HashMap<UserId, watch::Sender<()>>>>,
    rx: async_channel::Receiver<IndexerMessage>,
    db: Arc<Database>,
    lila: Lila,
}

impl IndexerActor {
    async fn run(self) {
        while let Ok(msg) = self.rx.recv().await {
            match msg {
                IndexerMessage::IndexPlayer {
                    player,
                    status,
                    since_created_at,
                } => {
                    self.index_player(&player, status, since_created_at).await;

                    let mut guard = self.indexing.write().await;
                    guard.remove(&player);
                }
            }
        }
    }

    async fn index_player(
        &self,
        player: &UserId,
        mut status: PersonalStatus,
        since_created_at: u64,
    ) {
        let started_at = Instant::now();
        log::info!(
            "indexer {:02}: starting {} (created_at >= {})",
            self.idx,
            player.as_str(),
            since_created_at
        );

        let mut games = match timeout(
            Duration::from_secs(60),
            self.lila.user_games(player, since_created_at),
        )
        .await
        {
            Ok(Ok(games)) => games,
            Ok(Err(err)) if err.status() == Some(StatusCode::NOT_FOUND) => {
                log::warn!(
                    "indexer {:02}: did not find player {}",
                    self.idx,
                    player.as_str()
                );
                return;
            }
            Ok(Err(err)) => {
                log::error!("indexer {:02}: request failed: {}", self.idx, err);
                sleep(Duration::from_secs(5)).await;
                return;
            }
            Err(timed_out) => {
                log::error!("indexer {:02}: request to lila: {}", self.idx, timed_out);
                return;
            }
        };

        let hash = ByColor::new_with(|color| PersonalKeyBuilder::with_user_pov(player, color));

        let mut num_games = 0;
        loop {
            let game = match timeout(Duration::from_secs(60), games.next()).await {
                Ok(Some(Ok(game))) => game,
                Ok(Some(Err(err))) => {
                    log::error!("indexer {:02}: {}", self.idx, err);
                    continue;
                }
                Ok(None) => break,
                Err(timed_out) => {
                    log::error!("indexer {:02}: stream from lila: {}", self.idx, timed_out);
                    return;
                }
            };

            self.index_game(player, &hash, game, &mut status);

            num_games += 1;
            if num_games % 1024 == 0 {
                self.db
                    .queryable()
                    .put_player_status(player, &status)
                    .expect("put player status");

                log::info!(
                    "indexer {:02}: indexed {} games for {} ...",
                    self.idx,
                    num_games,
                    player.as_str()
                );
            }
        }

        status.indexed_at = SystemTime::now();
        self.db
            .queryable()
            .put_player_status(player, &status)
            .expect("put player status");

        let elapsed = started_at.elapsed();
        log::info!(
            "indexer {:02}: finished {} games for {} in {:.3?} ({:.3?}/game, {:.1} games/s)",
            self.idx,
            num_games,
            player.as_str(),
            elapsed,
            (elapsed.as_nanos() as u64)
                .checked_div(num_games)
                .map(Duration::from_nanos)
                .unwrap_or_default(),
            (num_games as f64) / elapsed.as_secs_f64()
        );
    }

    fn index_game(
        &self,
        player: &UserId,
        hash: &ByColor<PersonalKeyBuilder>,
        game: Game,
        status: &mut PersonalStatus,
    ) {
        status.latest_created_at = game.created_at;

        if game.status.is_ongoing() {
            if status.revisit_ongoing_created_at.is_none() {
                log::debug!(
                    "indexer {:02}: will revisit ongoing game {} eventually",
                    self.idx,
                    game.id
                );
                status.revisit_ongoing_created_at = Some(game.created_at);
            }
            return;
        }

        if game.status.is_unindexable() {
            return;
        }

        if game.players.any(|p| p.user.is_none()) {
            return;
        }

        let color = match game
            .players
            .find(|p| p.user.as_ref().map_or(false, |user| user.name == *player))
        {
            Some(color) => color,
            None => {
                log::error!(
                    "indexer {:02}: {} did not play in {}",
                    self.idx,
                    player.as_str(),
                    game.id
                );
                return;
            }
        };

        // Skip game if already indexed from this side. This cannot race with
        // writes, because all writes for the same player are sequenced by
        // this actor. So making a transaction is not required.
        let queryable = self.db.queryable();
        if queryable
            .get_game_info(game.id)
            .expect("get game info")
            .map_or(false, |info| info.indexed.into_color(color))
        {
            log::debug!(
                "indexer {:02}: {}/{} already indexed",
                self.idx,
                game.id,
                color
            );
            return;
        }

        // Prepare basic information and setup initial position.
        let month = Month::from_time_saturating(game.last_move_at);
        let outcome = Outcome::from_winner(game.winner);
        let variant = game.variant.into();
        let pos = match game.initial_fen {
            Some(fen) => VariantPosition::from_setup(variant, &fen, CastlingMode::Chess960),
            None => Ok(VariantPosition::new(variant)),
        };
        let opponent_rating = match game.players.by_color(!color).rating {
            Some(rating) => rating,
            None => {
                log::warn!(
                    "indexer {:02}: skipping {} without opponent rating",
                    self.idx,
                    game.id
                );
                return;
            }
        };

        let mut pos: Zobrist<_, u128> = match pos {
            Ok(pos) => Zobrist::new(pos),
            Err(err) => {
                log::warn!("indexer {:02}: not indexing {}: {}", self.idx, game.id, err);
                return;
            }
        };

        // Build an intermediate table to remove loops (due to repetitions).
        let mut table: FxHashMap<u128, Uci> =
            FxHashMap::with_capacity_and_hasher(game.moves.len(), Default::default());

        for (ply, san) in game.moves.into_iter().enumerate() {
            if ply >= MAX_PLIES {
                break;
            }

            let m = match san.to_move(&pos) {
                Ok(m) => m,
                Err(err) => {
                    log::warn!(
                        "indexer {:02}: cutting off {} at ply {}: {}: {}",
                        self.idx,
                        game.id,
                        ply,
                        err,
                        san
                    );
                    break;
                }
            };

            let uci = m.to_uci(CastlingMode::Chess960);
            table.insert(pos.zobrist_hash(), uci);

            pos.play_unchecked(&m);
        }

        // Write to database. All writes regarding this game are batched and
        // atomically committed, so the database will always be in a consistent
        // state.
        let mut batch = queryable.batch();

        batch.merge_game_info(
            game.id,
            GameInfo {
                winner: outcome.winner(),
                speed: game.speed,
                mode: Mode::from_rated(game.rated),
                month,
                players: game.players.map(|p| GameInfoPlayer {
                    name: p.user.map(|p| p.name.to_string()),
                    rating: p.rating,
                }),
                indexed: ByColor::new_with(|c| color == c),
            },
        );

        for (zobrist, uci) in table {
            batch.merge_personal(
                hash.by_color(color)
                    .with_zobrist(variant, zobrist)
                    .with_month(month),
                PersonalEntry::new_single(
                    uci.clone(),
                    game.speed,
                    Mode::from_rated(game.rated),
                    game.id,
                    outcome,
                    opponent_rating,
                ),
            );
        }

        batch.write().expect("atomically commit game and moves");
    }
}

enum IndexerMessage {
    IndexPlayer {
        player: UserId,
        status: PersonalStatus,
        since_created_at: u64,
    },
}
<|endoftext|>


<commit_before>// (c) 2016 Productize SPRL <joost@productize.be>

// from parent
use Result;
use str_error;
use footprint;
use Sexp;

#[derive(Debug)]
pub struct Layout {
    pub version:i64,
    pub host:Host,
    pub general:General,
    pub page:String,
    pub setup:Setup,
    pub layers:Vec<Layer>,
    pub elements:Vec<Element>,
}

#[derive(Clone,Debug)]
pub enum Element {
    Module(footprint::Module),
    Net(Net),
    NetClass(NetClass),

    GrText(GrText),
    GrLine(GrLine),
    GrArc(GrArc),
    Dimension(Dimension),
    //Segment,
    //Via,
    //FilledPolygon,
    Zone(Zone),
    Other(Sexp),
}

#[derive(Clone,Debug)]
pub struct Zone {
    pub net:i64,
    pub net_name:String,
    pub other:Vec<Sexp>,
}

#[derive(Clone,Debug)]
pub struct Host {
    pub tool:String,
    pub build:String,
}

#[derive(Clone,Debug)]
pub struct General {
    pub links:i64,
    pub no_connects:i64,
    pub area:Area,
    pub thickness:f64,
    pub drawings:i64,
    pub tracks:i64,
    pub zones:i64,
    pub modules:i64,
    pub nets:i64,
}

#[derive(Clone,Debug)]
pub struct Area {
    pub x1:f64, pub y1:f64,
    pub x2:f64, pub y2:f64,
}

#[derive(Clone,Debug)]
pub struct Layer {
    pub num:i64,
    pub layer:footprint::Layer,
    pub layer_type:LayerType,
    pub hide:bool,
}

#[derive(Clone,Debug)]
pub enum LayerType {
    Signal,
    User,
}

#[derive(Clone,Debug)]
pub struct Setup {
    pub elements:Vec<SetupElement>,
    pub pcbplotparams:Vec<SetupElement>,
}

#[derive(Clone,Debug)]
pub struct SetupElement {
    pub name:String,
    pub value1:String,
    pub value2:Option<String>,
}

#[derive(Clone,PartialEq,Debug)]
pub struct Net {
    pub num:i64,
    pub name:String,
}

#[derive(Clone,PartialEq,Debug)]
pub struct NetClass {
    pub name:String,
    pub desc:String,
    pub clearance:f64,
    pub trace_width:f64,
    pub via_dia:f64,
    pub via_drill:f64,
    pub uvia_dia:f64,
    pub uvia_drill:f64,
    pub nets:Vec<String>,
}

#[derive(Clone,Debug)]
pub struct GrText {
    pub value:String,
    pub at:footprint::At,
    pub layer:footprint::Layer,
    pub effects:footprint::Effects,
    pub tstamp:String,
}

// only used internally
pub enum GrElement {
    Start(footprint::Xy),
    End(footprint::Xy),
    Angle(i64),
    Layer(footprint::Layer),
    Width(f64),
    TStamp(String),
    At(footprint::At),
    Effects(footprint::Effects),
}

#[derive(Clone,Debug)]
pub struct GrLine {
    pub start:footprint::Xy,
    pub end:footprint::Xy,
    pub angle:i64,
    pub layer:footprint::Layer,
    pub width:f64,
    pub tstamp:String,
}

#[derive(Clone,Debug)]
pub struct GrArc {
    pub start:footprint::Xy,
    pub end:footprint::Xy,
    pub angle:i64,
    pub layer:footprint::Layer,
    pub width:f64,
    pub tstamp:String,
}

#[derive(Clone,Debug)]
pub struct Dimension {
    pub name:String,
    pub width:f64,
    pub layer:footprint::Layer,
    pub tstamp:Option<String>,
    pub text:GrText,
    pub feature1:footprint::Pts,
    pub feature2:footprint::Pts,
    pub crossbar:footprint::Pts,
    pub arrow1a:footprint::Pts,
    pub arrow1b:footprint::Pts,
    pub arrow2a:footprint::Pts,
    pub arrow2b:footprint::Pts,
}

impl Default for Layout {
    fn default() -> Layout {
        Layout {
            version:4,
            host:Host::default(),
            elements:vec![],
            setup:Setup::default(),
            general:General::default(),
            page:String::from("A4"),
            layers:vec![],
        }
    }
}

impl Layout {
    pub fn nets(&self) -> Vec<&Net> {
        let mut v = vec![];
        for element in &self.elements {
            if let Element::Net(ref net) = *element {
                v.push(net)
            }
        }
        v
    }

    pub fn netclasses(&self) -> Vec<&NetClass> {
        let mut v = vec![];
        for element in &self.elements {
            if let Element::NetClass(ref net_class) = *element {
                v.push(net_class)
            }
        }
        v
    }

    pub fn get_module(&self, reference:&str) -> Option<&footprint::Module> {
        for ref x in &self.elements[..] {
            match **x {
                Element::Module(ref m) => {
                    if m.is_reference(reference) {
                        return Some(m)
                    }
                },
                Element::Net(_) |
                Element::NetClass(_) |
                Element::GrText(_) |
                Element::GrLine(_) |
                Element::GrArc(_) |
                Element::Dimension(_) |
                Element::Zone(_) |
                Element::Other(_) => (),
            }
        }
        None
    }

    pub fn modify_module<F>(&mut self, reference:&str, fun:F) -> Result<()>
        where F:Fn(&mut footprint::Module) -> ()
    {
        for ref mut x in &mut self.elements[..] {
            match **x {
                Element::Module(ref mut m) => {
                    if m.is_reference(reference) {
                        return Ok(fun(m))
                    }
                },
                Element::Net(_) |
                Element::NetClass(_) |
                Element::GrText(_) |
                Element::GrLine(_) |
                Element::GrArc(_) |
                Element::Dimension(_) |
                Element::Zone(_) |
                Element::Other(_) => (),
            }
        }
        str_error(format!("did not find module with reference {}", reference))
    }

    pub fn add_net(&mut self, num:i64, name:&'static str) {
        self.elements.push(Element::Net(Net { num:num, name:String::from(name) }));
    }
    pub fn add_netclass(&mut self, name:&'static str, desc:&'static str, clearance:f64, trace_width:f64, via_dia:f64, via_drill:f64, uvia_dia:f64, uvia_drill:f64, nets:Vec<String>) {
        self.elements.push(Element::NetClass(NetClass {
            name:String::from(name),
            desc:String::from(desc),
            clearance:clearance,
            trace_width:trace_width,
            via_dia:via_dia,
            via_drill:via_drill,
            uvia_dia:uvia_dia,
            uvia_drill:uvia_drill,
            nets:nets
            }));
    }
}

impl Default for Host {
    fn default() -> Host {
        Host { tool:String::from("pcbnew"), build:String::from("custom") }
    }
}

impl Default for General {
    fn default() -> General {
        General {
            links:0,
            no_connects:0,
            area:Area::default(),
            thickness:1.6,
            drawings:0,
            tracks:0,
            zones:0,
            modules:0,
            nets:0,
        }
    }
}

impl Default for Area {
    fn default() -> Area {
        Area {
            x1:0.0, y1:0.0,
            x2:0.0, y2:0.0,
        }
    }
}

impl NetClass {
    pub fn equal_no_net(&self, other:&NetClass) -> bool {
        let mut s1 = self.clone();
        s1.nets = vec![];
        let mut s2 = other.clone();
        s2.nets = vec![];
        s1 == s2
    }
    pub fn has_net(&self, name:&'static str) -> bool {
        for net in &self.nets {
            if &net[..] == name {
                return true
            }
        }
        false
    }
}

impl Default for Setup {
    fn default() -> Setup {
        Setup { elements:vec![], pcbplotparams:vec![] }
    }
}

impl Setup {
    pub fn get(&self, s:&str) -> Option<&String> {
        for element in &self.elements {
            if &element.name[..] == &s[..] {
                return Some(&element.value1)
            }
        }
        None
    }

    pub fn update_element(&mut self, name:&'static str, value:String) {
        for element in &mut self.elements {
            if &element.name[..] == name {
                element.value1 = value;
                return;

            }
        }
        self.elements.push(SetupElement { name:String::from(name), value1:value, value2:None })
    }
}
<commit_msg>add support for renaming net<commit_after>// (c) 2016 Productize SPRL <joost@productize.be>

// from parent
use Result;
use str_error;
use footprint;
use Sexp;

#[derive(Debug)]
pub struct Layout {
    pub version:i64,
    pub host:Host,
    pub general:General,
    pub page:String,
    pub setup:Setup,
    pub layers:Vec<Layer>,
    pub elements:Vec<Element>,
}

#[derive(Clone,Debug)]
pub enum Element {
    Module(footprint::Module),
    Net(Net),
    NetClass(NetClass),

    GrText(GrText),
    GrLine(GrLine),
    GrArc(GrArc),
    Dimension(Dimension),
    //Segment,
    //Via,
    //FilledPolygon,
    Zone(Zone),
    Other(Sexp),
}

#[derive(Clone,Debug)]
pub struct Zone {
    pub net:i64,
    pub net_name:String,
    pub other:Vec<Sexp>,
}

#[derive(Clone,Debug)]
pub struct Host {
    pub tool:String,
    pub build:String,
}

#[derive(Clone,Debug)]
pub struct General {
    pub links:i64,
    pub no_connects:i64,
    pub area:Area,
    pub thickness:f64,
    pub drawings:i64,
    pub tracks:i64,
    pub zones:i64,
    pub modules:i64,
    pub nets:i64,
}

#[derive(Clone,Debug)]
pub struct Area {
    pub x1:f64, pub y1:f64,
    pub x2:f64, pub y2:f64,
}

#[derive(Clone,Debug)]
pub struct Layer {
    pub num:i64,
    pub layer:footprint::Layer,
    pub layer_type:LayerType,
    pub hide:bool,
}

#[derive(Clone,Debug)]
pub enum LayerType {
    Signal,
    User,
}

#[derive(Clone,Debug)]
pub struct Setup {
    pub elements:Vec<SetupElement>,
    pub pcbplotparams:Vec<SetupElement>,
}

#[derive(Clone,Debug)]
pub struct SetupElement {
    pub name:String,
    pub value1:String,
    pub value2:Option<String>,
}

#[derive(Clone,PartialEq,Debug)]
pub struct Net {
    pub num:i64,
    pub name:String,
}

#[derive(Clone,PartialEq,Debug)]
pub struct NetClass {
    pub name:String,
    pub desc:String,
    pub clearance:f64,
    pub trace_width:f64,
    pub via_dia:f64,
    pub via_drill:f64,
    pub uvia_dia:f64,
    pub uvia_drill:f64,
    pub nets:Vec<String>,
}

#[derive(Clone,Debug)]
pub struct GrText {
    pub value:String,
    pub at:footprint::At,
    pub layer:footprint::Layer,
    pub effects:footprint::Effects,
    pub tstamp:String,
}

// only used internally
pub enum GrElement {
    Start(footprint::Xy),
    End(footprint::Xy),
    Angle(i64),
    Layer(footprint::Layer),
    Width(f64),
    TStamp(String),
    At(footprint::At),
    Effects(footprint::Effects),
}

#[derive(Clone,Debug)]
pub struct GrLine {
    pub start:footprint::Xy,
    pub end:footprint::Xy,
    pub angle:i64,
    pub layer:footprint::Layer,
    pub width:f64,
    pub tstamp:String,
}

#[derive(Clone,Debug)]
pub struct GrArc {
    pub start:footprint::Xy,
    pub end:footprint::Xy,
    pub angle:i64,
    pub layer:footprint::Layer,
    pub width:f64,
    pub tstamp:String,
}

#[derive(Clone,Debug)]
pub struct Dimension {
    pub name:String,
    pub width:f64,
    pub layer:footprint::Layer,
    pub tstamp:Option<String>,
    pub text:GrText,
    pub feature1:footprint::Pts,
    pub feature2:footprint::Pts,
    pub crossbar:footprint::Pts,
    pub arrow1a:footprint::Pts,
    pub arrow1b:footprint::Pts,
    pub arrow2a:footprint::Pts,
    pub arrow2b:footprint::Pts,
}

impl Default for Layout {
    fn default() -> Layout {
        Layout {
            version:4,
            host:Host::default(),
            elements:vec![],
            setup:Setup::default(),
            general:General::default(),
            page:String::from("A4"),
            layers:vec![],
        }
    }
}

impl Layout {
    pub fn nets(&self) -> Vec<&Net> {
        let mut v = vec![];
        for element in &self.elements {
            if let Element::Net(ref net) = *element {
                v.push(net)
            }
        }
        v
    }

    pub fn change_net_name(&mut self, i:i64, new_name:&str) {
        for element in &mut self.elements {
            if let Element::Net(ref mut net) = *element {
                if net.num == i {
                    net.name.clear();
                    net.name.push_str(new_name);
                }
            }
        }
    }

    pub fn netclasses(&self) -> Vec<&NetClass> {
        let mut v = vec![];
        for element in &self.elements {
            if let Element::NetClass(ref net_class) = *element {
                v.push(net_class)
            }
        }
        v
    }

    pub fn get_module(&self, reference:&str) -> Option<&footprint::Module> {
        for ref x in &self.elements[..] {
            match **x {
                Element::Module(ref m) => {
                    if m.is_reference(reference) {
                        return Some(m)
                    }
                },
                Element::Net(_) |
                Element::NetClass(_) |
                Element::GrText(_) |
                Element::GrLine(_) |
                Element::GrArc(_) |
                Element::Dimension(_) |
                Element::Zone(_) |
                Element::Other(_) => (),
            }
        }
        None
    }

    pub fn modify_module<F>(&mut self, reference:&str, fun:F) -> Result<()>
        where F:Fn(&mut footprint::Module) -> ()
    {
        for ref mut x in &mut self.elements {
            match **x {
                Element::Module(ref mut m) => {
                    if m.is_reference(reference) {
                        return Ok(fun(m))
                    }
                },
                Element::Net(_) |
                Element::NetClass(_) |
                Element::GrText(_) |
                Element::GrLine(_) |
                Element::GrArc(_) |
                Element::Dimension(_) |
                Element::Zone(_) |
                Element::Other(_) => (),
            }
        }
        str_error(format!("did not find module with reference {}", reference))
    }

    pub fn add_net(&mut self, num:i64, name:&'static str) {
        self.elements.push(Element::Net(Net { num:num, name:String::from(name) }));
    }
    pub fn add_netclass(&mut self, name:&'static str, desc:&'static str, clearance:f64, trace_width:f64, via_dia:f64, via_drill:f64, uvia_dia:f64, uvia_drill:f64, nets:Vec<String>) {
        self.elements.push(Element::NetClass(NetClass {
            name:String::from(name),
            desc:String::from(desc),
            clearance:clearance,
            trace_width:trace_width,
            via_dia:via_dia,
            via_drill:via_drill,
            uvia_dia:uvia_dia,
            uvia_drill:uvia_drill,
            nets:nets
            }));
    }
}

impl Default for Host {
    fn default() -> Host {
        Host { tool:String::from("pcbnew"), build:String::from("custom") }
    }
}

impl Default for General {
    fn default() -> General {
        General {
            links:0,
            no_connects:0,
            area:Area::default(),
            thickness:1.6,
            drawings:0,
            tracks:0,
            zones:0,
            modules:0,
            nets:0,
        }
    }
}

impl Default for Area {
    fn default() -> Area {
        Area {
            x1:0.0, y1:0.0,
            x2:0.0, y2:0.0,
        }
    }
}

impl NetClass {
    pub fn equal_no_net(&self, other:&NetClass) -> bool {
        let mut s1 = self.clone();
        s1.nets = vec![];
        let mut s2 = other.clone();
        s2.nets = vec![];
        s1 == s2
    }
    pub fn has_net(&self, name:&'static str) -> bool {
        for net in &self.nets {
            if &net[..] == name {
                return true
            }
        }
        false
    }
}

impl Default for Setup {
    fn default() -> Setup {
        Setup { elements:vec![], pcbplotparams:vec![] }
    }
}

impl Setup {
    pub fn get(&self, s:&str) -> Option<&String> {
        for element in &self.elements {
            if &element.name[..] == &s[..] {
                return Some(&element.value1)
            }
        }
        None
    }

    pub fn update_element(&mut self, name:&'static str, value:String) {
        for element in &mut self.elements {
            if &element.name[..] == name {
                element.value1 = value;
                return;

            }
        }
        self.elements.push(SetupElement { name:String::from(name), value1:value, value2:None })
    }
}
<|endoftext|>


<commit_before>#[doc = "A standard linked list"];

import core::option;
import option::*;
import option::{some, none};

enum list<T> {
    cons(T, @list<T>),
    nil,
}

#[doc = "Create a list from a vector"]
fn from_vec<T: copy>(v: ~[T]) -> @list<T> {
    vec::foldr(v, @nil::<T>, |h, t| @cons(h, t))
}

#[doc = "
Left fold

Applies `f` to `u` and the first element in the list, then applies `f` to the
result of the previous call and the second element, and so on, returning the
accumulated result.

# Arguments

* ls - The list to fold
* z - The initial value
* f - The function to apply
"]
fn foldl<T: copy, U>(z: T, ls: @list<U>, f: fn(T, U) -> T) -> T {
    let mut accum: T = z;
    do iter(ls) |elt| { accum = f(accum, elt);}
    accum
}

#[doc = "
Search for an element that matches a given predicate

Apply function `f` to each element of `v`, starting from the first.
When function `f` returns true then an option containing the element
is returned. If `f` matches no elements then none is returned.
"]
fn find<T: copy>(ls: @list<T>, f: fn(T) -> bool) -> option<T> {
    let mut ls = ls;
    loop {
        ls = alt *ls {
          cons(hd, tl) {
            if f(hd) { ret some(hd); }
            tl
          }
          nil { ret none; }
        }
    };
}

#[doc = "Returns true if a list contains an element with the given value"]
fn has<T: copy>(ls: @list<T>, elt: T) -> bool {
    for each(ls) |e| {
        if e == elt { ret true; }
    }
    ret false;
}

#[doc = "Returns true if the list is empty"]
pure fn is_empty<T: copy>(ls: @list<T>) -> bool {
    alt *ls {
        nil { true }
        _ { false }
    }
}

#[doc = "Returns true if the list is not empty"]
pure fn is_not_empty<T: copy>(ls: @list<T>) -> bool {
    ret !is_empty(ls);
}

#[doc = "Returns the length of a list"]
fn len<T>(ls: @list<T>) -> uint {
    let mut count = 0u;
    iter(ls, |_e| count += 1u);
    count
}

#[doc = "Returns all but the first element of a list"]
pure fn tail<T: copy>(ls: @list<T>) -> @list<T> {
    alt *ls {
        cons(_, tl) { ret tl; }
        nil { fail "list empty" }
    }
}

#[doc = "Returns the first element of a list"]
pure fn head<T: copy>(ls: @list<T>) -> T {
    alt check *ls { cons(hd, _) { hd } }
}

#[doc = "Appends one list to another"]
pure fn append<T: copy>(l: @list<T>, m: @list<T>) -> @list<T> {
    alt *l {
      nil { ret m; }
      cons(x, xs) { let rest = append(xs, m); ret @cons(x, rest); }
    }
}

#[doc = "Iterate over a list"]
fn iter<T>(l: @list<T>, f: fn(T)) {
    let mut cur = l;
    loop {
        cur = alt *cur {
          cons(hd, tl) {
            f(hd);
            tl
          }
          nil { break; }
        }
    }
}

#[doc = "Iterate over a list"]
fn each<T>(l: @list<T>, f: fn(T) -> bool) {
    let mut cur = l;
    loop {
        cur = alt *cur {
          cons(hd, tl) {
            if !f(hd) { ret; }
            tl
          }
          nil { break; }
        }
    }
}

#[cfg(test)]
mod tests {

    #[test]
    fn test_is_empty() {
        let empty : @list::list<int> = from_vec(~[]);
        let full1 = from_vec(~[1]);
        let full2 = from_vec(~['r', 'u']);

        assert is_empty(empty);
        assert !is_empty(full1);
        assert !is_empty(full2);

        assert !is_not_empty(empty);
        assert is_not_empty(full1);
        assert is_not_empty(full2);
    }

    #[test]
    fn test_from_vec() {
        let l = from_vec(~[0, 1, 2]);

        assert (head(l) == 0);

        let tail_l = tail(l);
        assert (head(tail_l) == 1);

        let tail_tail_l = tail(tail_l);
        assert (head(tail_tail_l) == 2);
    }

    #[test]
    fn test_from_vec_empty() {
        let empty : @list::list<int> = from_vec(~[]);
        assert (empty == @list::nil::<int>);
    }

    #[test]
    fn test_foldl() {
        fn add(&&a: uint, &&b: int) -> uint { ret a + (b as uint); }
        let l = from_vec(~[0, 1, 2, 3, 4]);
        let empty = @list::nil::<int>;
        assert (list::foldl(0u, l, add) == 10u);
        assert (list::foldl(0u, empty, add) == 0u);
    }

    #[test]
    fn test_foldl2() {
        fn sub(&&a: int, &&b: int) -> int {
            a - b
        }
        let l = from_vec(~[1, 2, 3, 4]);
        assert (list::foldl(0, l, sub) == -10);
    }

    #[test]
    fn test_find_success() {
        fn match(&&i: int) -> bool { ret i == 2; }
        let l = from_vec(~[0, 1, 2]);
        assert (list::find(l, match) == option::some(2));
    }

    #[test]
    fn test_find_fail() {
        fn match(&&_i: int) -> bool { ret false; }
        let l = from_vec(~[0, 1, 2]);
        let empty = @list::nil::<int>;
        assert (list::find(l, match) == option::none::<int>);
        assert (list::find(empty, match) == option::none::<int>);
    }

    #[test]
    fn test_has() {
        let l = from_vec(~[5, 8, 6]);
        let empty = @list::nil::<int>;
        assert (list::has(l, 5));
        assert (!list::has(l, 7));
        assert (list::has(l, 8));
        assert (!list::has(empty, 5));
    }

    #[test]
    fn test_len() {
        let l = from_vec(~[0, 1, 2]);
        let empty = @list::nil::<int>;
        assert (list::len(l) == 3u);
        assert (list::len(empty) == 0u);
    }

}

// Local Variables:
// mode: rust;
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// End:
<commit_msg>Make list::from_vec take a slice, not a ~.<commit_after>#[doc = "A standard linked list"];

import core::option;
import option::*;
import option::{some, none};

enum list<T> {
    cons(T, @list<T>),
    nil,
}

#[doc = "Create a list from a vector"]
fn from_vec<T: copy>(v: &[T]) -> @list<T> {
    vec::foldr(v, @nil::<T>, |h, t| @cons(h, t))
}

#[doc = "
Left fold

Applies `f` to `u` and the first element in the list, then applies `f` to the
result of the previous call and the second element, and so on, returning the
accumulated result.

# Arguments

* ls - The list to fold
* z - The initial value
* f - The function to apply
"]
fn foldl<T: copy, U>(z: T, ls: @list<U>, f: fn(T, U) -> T) -> T {
    let mut accum: T = z;
    do iter(ls) |elt| { accum = f(accum, elt);}
    accum
}

#[doc = "
Search for an element that matches a given predicate

Apply function `f` to each element of `v`, starting from the first.
When function `f` returns true then an option containing the element
is returned. If `f` matches no elements then none is returned.
"]
fn find<T: copy>(ls: @list<T>, f: fn(T) -> bool) -> option<T> {
    let mut ls = ls;
    loop {
        ls = alt *ls {
          cons(hd, tl) {
            if f(hd) { ret some(hd); }
            tl
          }
          nil { ret none; }
        }
    };
}

#[doc = "Returns true if a list contains an element with the given value"]
fn has<T: copy>(ls: @list<T>, elt: T) -> bool {
    for each(ls) |e| {
        if e == elt { ret true; }
    }
    ret false;
}

#[doc = "Returns true if the list is empty"]
pure fn is_empty<T: copy>(ls: @list<T>) -> bool {
    alt *ls {
        nil { true }
        _ { false }
    }
}

#[doc = "Returns true if the list is not empty"]
pure fn is_not_empty<T: copy>(ls: @list<T>) -> bool {
    ret !is_empty(ls);
}

#[doc = "Returns the length of a list"]
fn len<T>(ls: @list<T>) -> uint {
    let mut count = 0u;
    iter(ls, |_e| count += 1u);
    count
}

#[doc = "Returns all but the first element of a list"]
pure fn tail<T: copy>(ls: @list<T>) -> @list<T> {
    alt *ls {
        cons(_, tl) { ret tl; }
        nil { fail "list empty" }
    }
}

#[doc = "Returns the first element of a list"]
pure fn head<T: copy>(ls: @list<T>) -> T {
    alt check *ls { cons(hd, _) { hd } }
}

#[doc = "Appends one list to another"]
pure fn append<T: copy>(l: @list<T>, m: @list<T>) -> @list<T> {
    alt *l {
      nil { ret m; }
      cons(x, xs) { let rest = append(xs, m); ret @cons(x, rest); }
    }
}

#[doc = "Iterate over a list"]
fn iter<T>(l: @list<T>, f: fn(T)) {
    let mut cur = l;
    loop {
        cur = alt *cur {
          cons(hd, tl) {
            f(hd);
            tl
          }
          nil { break; }
        }
    }
}

#[doc = "Iterate over a list"]
fn each<T>(l: @list<T>, f: fn(T) -> bool) {
    let mut cur = l;
    loop {
        cur = alt *cur {
          cons(hd, tl) {
            if !f(hd) { ret; }
            tl
          }
          nil { break; }
        }
    }
}

#[cfg(test)]
mod tests {

    #[test]
    fn test_is_empty() {
        let empty : @list::list<int> = from_vec(~[]);
        let full1 = from_vec(~[1]);
        let full2 = from_vec(~['r', 'u']);

        assert is_empty(empty);
        assert !is_empty(full1);
        assert !is_empty(full2);

        assert !is_not_empty(empty);
        assert is_not_empty(full1);
        assert is_not_empty(full2);
    }

    #[test]
    fn test_from_vec() {
        let l = from_vec(~[0, 1, 2]);

        assert (head(l) == 0);

        let tail_l = tail(l);
        assert (head(tail_l) == 1);

        let tail_tail_l = tail(tail_l);
        assert (head(tail_tail_l) == 2);
    }

    #[test]
    fn test_from_vec_empty() {
        let empty : @list::list<int> = from_vec(~[]);
        assert (empty == @list::nil::<int>);
    }

    #[test]
    fn test_foldl() {
        fn add(&&a: uint, &&b: int) -> uint { ret a + (b as uint); }
        let l = from_vec(~[0, 1, 2, 3, 4]);
        let empty = @list::nil::<int>;
        assert (list::foldl(0u, l, add) == 10u);
        assert (list::foldl(0u, empty, add) == 0u);
    }

    #[test]
    fn test_foldl2() {
        fn sub(&&a: int, &&b: int) -> int {
            a - b
        }
        let l = from_vec(~[1, 2, 3, 4]);
        assert (list::foldl(0, l, sub) == -10);
    }

    #[test]
    fn test_find_success() {
        fn match(&&i: int) -> bool { ret i == 2; }
        let l = from_vec(~[0, 1, 2]);
        assert (list::find(l, match) == option::some(2));
    }

    #[test]
    fn test_find_fail() {
        fn match(&&_i: int) -> bool { ret false; }
        let l = from_vec(~[0, 1, 2]);
        let empty = @list::nil::<int>;
        assert (list::find(l, match) == option::none::<int>);
        assert (list::find(empty, match) == option::none::<int>);
    }

    #[test]
    fn test_has() {
        let l = from_vec(~[5, 8, 6]);
        let empty = @list::nil::<int>;
        assert (list::has(l, 5));
        assert (!list::has(l, 7));
        assert (list::has(l, 8));
        assert (!list::has(empty, 5));
    }

    #[test]
    fn test_len() {
        let l = from_vec(~[0, 1, 2]);
        let empty = @list::nil::<int>;
        assert (list::len(l) == 3u);
        assert (list::len(empty) == 0u);
    }

}

// Local Variables:
// mode: rust;
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// End:
<|endoftext|>