<commit_before>extern crate bazel_protos;
extern crate boxfuture;
extern crate clap;
extern crate fs;
extern crate futures;
extern crate protobuf;

use boxfuture::{Boxable, BoxFuture};
use clap::{App, Arg, SubCommand};
use fs::{Fingerprint, GetFileDigest, ResettablePool, Snapshot, Store, VFS};
use futures::future::{self, Future, join_all};
use protobuf::Message;
use std::error::Error;
use std::fs::File;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process::exit;
use std::sync::Arc;

#[derive(Debug)]
enum ExitCode {
  UnknownError = 1,
  NotFound = 2,
}

#[derive(Debug)]
struct ExitError(pub String, pub ExitCode);

impl From<String> for ExitError {
  fn from(s: String) -> Self {
    ExitError(s, ExitCode::UnknownError)
  }
}

fn main() {
  match execute(
    App::new("fs_util")
      .subcommand(
        SubCommand::with_name("file")
          .subcommand(
            SubCommand::with_name("cat")
              .about("Output the contents of a file by fingerprint.")
              .arg(Arg::with_name("fingerprint").required(true).takes_value(
                true,
              )),
          )
          .subcommand(
            SubCommand::with_name("save")
              .about(
                "Ingest a file by path, which allows it to be used in Directories/Snapshots. \
Outputs a fingerprint of its contents and its size in bytes, separated by a space.",
              )
              .arg(Arg::with_name("path").required(true).takes_value(true)),
          ),
      )
      .subcommand(
        SubCommand::with_name("directory")
          .subcommand(
            SubCommand::with_name("materialize")
              .about(
                "Materialize a directory by fingerprint to the filesystem. \
Destination must not exist before this command is run.",
              )
              .arg(Arg::with_name("fingerprint").required(true).takes_value(
                true,
              ))
              .arg(Arg::with_name("destination").required(true).takes_value(
                true,
              )),
          )
          .subcommand(
            SubCommand::with_name("save")
              .about(
                "Ingest a directory recursively. Saves all files found therein and saves Directory \
protos for each directory found. Outputs a fingerprint of the canonical top-level Directory proto \
and the size of the serialized proto in bytes, separated by a space.",
              )
              .arg(
                Arg::with_name("globs")
                  .required(true)
                  .takes_value(true)
                  .multiple(true)
                  .help(
                    "globs matching the files and directories which should be included in the \
directory, relative to the root.",
                  ),
              )
                .arg(Arg::with_name("root").long("root").required(true).takes_value(true).help(
                  "Root under which the globs live. The Directory proto produced will be relative \
to this directory.",
                )),
          )
          .subcommand(
            SubCommand::with_name("cat-proto")
              .about(
                "Output the bytes of a serialized Directory proto addressed by fingerprint.",
              )
              .arg(
                Arg::with_name("output-format")
                  .long("output-format")
                  .takes_value(true)
                  .default_value("binary")
                  .possible_values(&["binary", "text"]),
              )
              .arg(Arg::with_name("fingerprint").required(true).takes_value(
                true,
              )),
          ),
      )
      .subcommand(
        SubCommand::with_name("cat")
          .about(
            "Output the contents of a file or Directory proto addressed by fingerprint.",
          )
          .arg(Arg::with_name("fingerprint").required(true).takes_value(
            true,
          )),
      )
      .arg(
        Arg::with_name("local-store-path")
          .takes_value(true)
          .long("local-store-path")
          .required(true),
      )
      .get_matches(),
  ) {
    Ok(_) => {}
    Err(err) => {
      eprintln!("{}", err.0);
      exit(err.1 as i32)
    }
  };
}

fn execute(top_match: clap::ArgMatches) -> Result<(), ExitError> {
  let store_dir = top_match.value_of("local-store-path").unwrap();
  let pool = Arc::new(ResettablePool::new("fsutil-pool-".to_string()));
  let store = Arc::new(Store::local_only(store_dir, pool.clone()).map_err(|e| {
    format!(
      "Failed to open/create store for directory {}: {}",
      store_dir,
      e
    )
  })?);

  match top_match.subcommand() {
    ("file", Some(sub_match)) => {
      match sub_match.subcommand() {
        ("cat", Some(args)) => {
          let fingerprint = Fingerprint::from_hex_string(args.value_of("fingerprint").unwrap())?;
          let write_result = store
            .load_file_bytes_with(fingerprint, |bytes| io::stdout().write_all(&bytes).unwrap())
            .wait()?;
          write_result.ok_or_else(|| {
            ExitError(
              format!("File with fingerprint {} not found", fingerprint),
              ExitCode::NotFound,
            )
          })
        }
        ("save", Some(args)) => {
          let path = PathBuf::from(args.value_of("path").unwrap());
          // Canonicalize path to guarantee that a relative path has a parent.
          let posix_fs = make_posix_fs(
            path
              .canonicalize()
              .map_err(|e| {
                format!("Error canonicalizing path {:?}: {}", path, e.description())
              })?
              .parent()
              .ok_or_else(|| {
                format!("File being saved must have parent but {:?} did not", path)
              })?,
            pool,
          );
          let file = posix_fs
            .stat(PathBuf::from(path.file_name().unwrap()))
            .unwrap();
          match file {
            fs::Stat::File(f) => {
              let digest = FileSaver {
                store: store,
                posix_fs: Arc::new(posix_fs),
              }.digest(&f)
                .wait()
                .unwrap();
              Ok(println!("{} {}", digest.0, digest.1))
            }
            o => Err(
              format!(
                "Tried to save file {:?} but it was not a file, was a {:?}",
                path,
                o
              ).into(),
            ),
          }

        }
        (_, _) => unimplemented!(),
      }
    }
    ("directory", Some(sub_match)) => {
      match sub_match.subcommand() {
        ("materialize", Some(args)) => {
          let destination = PathBuf::from(args.value_of("destination").unwrap());
          let fingerprint = Fingerprint::from_hex_string(args.value_of("fingerprint").unwrap())?;
          materialize_directory(store, destination, fingerprint).wait()
        }
        ("save", Some(args)) => {
          let posix_fs = Arc::new(make_posix_fs(args.value_of("root").unwrap(), pool));
          let digest = posix_fs
            .expand(fs::PathGlobs::create(
              &args
                .values_of("globs")
                .unwrap()
                .map(|s| s.to_string())
                .collect::<Vec<String>>(),
              &[],
            )?)
            .map_err(|e| format!("Error expanding globs: {}", e.description()))
            .and_then(move |paths| {
              Snapshot::from_path_stats(
                store.clone(),
                Arc::new(FileSaver {
                  store: store.clone(),
                  posix_fs: posix_fs,
                }),
                paths,
              )
            })
            .map(|snapshot| snapshot.digest.unwrap())
            .wait()?;
          Ok(println!("{} {}", digest.0, digest.1))
        }
        ("cat-proto", Some(args)) => {
          let fingerprint = Fingerprint::from_hex_string(args.value_of("fingerprint").unwrap())?;
          let proto_bytes = match args.value_of("output-format").unwrap() {
            "binary" => {
              store.load_directory(fingerprint).wait().map(|maybe_d| {
                maybe_d.map(|d| d.write_to_bytes().unwrap())
              })
            }
            "text" => {
              store.load_directory(fingerprint).wait().map(|maybe_p| {
                maybe_p.map(|p| format!("{:?}\n", p).as_bytes().to_vec())
              })
            }
            format => Err(format!(
              "Unexpected value of --output-format arg: {}",
              format
            )),
          }?;
          match proto_bytes {
            Some(bytes) => {
              io::stdout().write(&bytes).unwrap();
              Ok(())
            }
            None => Err(ExitError(
              format!(
                "Directory with fingerprint {} not found",
                fingerprint
              ),
              ExitCode::NotFound,
            )),
          }
        }
        (_, _) => unimplemented!(),
      }
    }
    ("cat", Some(args)) => {
      let fingerprint = Fingerprint::from_hex_string(args.value_of("fingerprint").unwrap())?;
      let v = match store
        .load_file_bytes_with(fingerprint, |bytes| Vec::from(bytes))
        .wait()? {
        None => {
          store
            .load_directory(fingerprint)
            .map(|maybe_dir| {
              maybe_dir.map(|dir| {
                dir.write_to_bytes().expect(
                  "Error serializing Directory proto",
                )
              })
            })
            .wait()?
        }
        some => some,
      };
      match v {
        Some(bytes) => {
          io::stdout().write(&bytes).unwrap();
          Ok(())
        }
        None => Err(ExitError(
          format!("Fingerprint {} not found", fingerprint),
          ExitCode::NotFound,
        )),
      }
    }

    (_, _) => unimplemented!(),
  }
}

fn make_posix_fs<P: AsRef<Path>>(root: P, pool: Arc<ResettablePool>) -> fs::PosixFS {
  fs::PosixFS::new(&root, pool, vec![]).unwrap()
}

struct FileSaver {
  store: Arc<Store>,
  posix_fs: Arc<fs::PosixFS>,
}

impl GetFileDigest<String> for FileSaver {
  fn digest(&self, file: &fs::File) -> BoxFuture<fs::Digest, String> {
    let file_copy = file.clone();
    let store = self.store.clone();
    self
      .posix_fs
      .read_file(&file)
      .map_err(move |err| {
        format!("Error reading file {:?}: {}", file_copy, err.description())
      })
      .and_then(move |content| store.store_file_bytes(content.content))
      .to_boxed()
  }
}

fn materialize_directory(
  store: Arc<Store>,
  destination: PathBuf,
  fingerprint: Fingerprint,
) -> BoxFuture<(), ExitError> {
  let mkdir = make_clean_dir(&destination).map_err(|e| {
    format!(
      "Error making directory {:?}: {}",
      destination,
      e.description()
    ).into()
  });
  match mkdir {
    Ok(()) => {}
    Err(e) => return future::err(e).to_boxed(),
  };
  store
    .load_directory(fingerprint)
    .map_err(|e| e.into())
    .and_then(move |directory_opt| {
      directory_opt.ok_or_else(|| {
        ExitError(
          format!("Directory with fingerprint {} not found", fingerprint),
          ExitCode::NotFound,
        )
      })
    })
    .and_then(move |directory| {
      let file_futures = directory
        .get_files()
        .iter()
        .map(|file_node| {
          let store = store.clone();
          let path = destination.join(file_node.get_name());
          future::result(Fingerprint::from_hex_string(
            file_node.get_digest().get_hash(),
          )).map_err(|e| e.into())
            .and_then(move |fingerprint| {
              materialize_file(store, path, fingerprint)
            })
        })
        .collect::<Vec<_>>();
      let directory_futures = directory
        .get_directories()
        .iter()
        .map(|directory_node| {
          let store = store.clone();
          let path = destination.join(directory_node.get_name());
          future::result(Fingerprint::from_hex_string(
            directory_node.get_digest().get_hash(),
          )).map_err(|e| e.into())
            .and_then(move |fingerprint| {
              materialize_directory(store, path, fingerprint)
            })
        })
        .collect::<Vec<_>>();
      join_all(file_futures)
        .join(join_all(directory_futures))
        .map(|_| ())
    })
    .to_boxed()
}

fn materialize_file(
  store: Arc<Store>,
  destination: PathBuf,
  fingerprint: Fingerprint,
) -> BoxFuture<(), ExitError> {
  store
    .load_file_bytes_with(fingerprint, move |bytes| {
      File::create(&destination)
        .and_then(|mut f| f.write_all(bytes))
        .map_err(|e| {
          format!("Error writing file {:?}: {}", destination, e.description())
        })
    })
    .map_err(|e| e.into())
    .and_then(move |write_result| match write_result {
      Some(Ok(())) => Ok(()),
      Some(Err(e)) => Err(e.into()),
      None => {
        Err(ExitError(
          format!("File with fingerprint {} not found", fingerprint),
          ExitCode::NotFound,
        ))
      }
    })
    .to_boxed()
}

fn make_clean_dir(path: &Path) -> Result<(), io::Error> {
  let parent = path.parent().ok_or_else(|| {
    io::Error::new(io::ErrorKind::NotFound, format!("{:?} had no parent", path))
  })?;
  std::fs::create_dir_all(parent)?;
  std::fs::create_dir(path)
}
<commit_msg>fs_util backfills from remote CAS if --server-address is set (#5179)<commit_after>extern crate bazel_protos;
extern crate boxfuture;
extern crate clap;
extern crate fs;
extern crate futures;
extern crate protobuf;

use boxfuture::{Boxable, BoxFuture};
use clap::{App, Arg, SubCommand};
use fs::{Fingerprint, GetFileDigest, ResettablePool, Snapshot, Store, VFS};
use futures::future::{self, Future, join_all};
use protobuf::Message;
use std::error::Error;
use std::fs::File;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process::exit;
use std::sync::Arc;

#[derive(Debug)]
enum ExitCode {
  UnknownError = 1,
  NotFound = 2,
}

#[derive(Debug)]
struct ExitError(pub String, pub ExitCode);

impl From<String> for ExitError {
  fn from(s: String) -> Self {
    ExitError(s, ExitCode::UnknownError)
  }
}

fn main() {
  match execute(
    App::new("fs_util")
      .subcommand(
        SubCommand::with_name("file")
          .subcommand(
            SubCommand::with_name("cat")
              .about("Output the contents of a file by fingerprint.")
              .arg(Arg::with_name("fingerprint").required(true).takes_value(
                true,
              )),
          )
          .subcommand(
            SubCommand::with_name("save")
              .about(
                "Ingest a file by path, which allows it to be used in Directories/Snapshots. \
Outputs a fingerprint of its contents and its size in bytes, separated by a space.",
              )
              .arg(Arg::with_name("path").required(true).takes_value(true)),
          ),
      )
      .subcommand(
        SubCommand::with_name("directory")
          .subcommand(
            SubCommand::with_name("materialize")
              .about(
                "Materialize a directory by fingerprint to the filesystem. \
Destination must not exist before this command is run.",
              )
              .arg(Arg::with_name("fingerprint").required(true).takes_value(
                true,
              ))
              .arg(Arg::with_name("destination").required(true).takes_value(
                true,
              )),
          )
          .subcommand(
            SubCommand::with_name("save")
              .about(
                "Ingest a directory recursively. Saves all files found therein and saves Directory \
protos for each directory found. Outputs a fingerprint of the canonical top-level Directory proto \
and the size of the serialized proto in bytes, separated by a space.",
              )
              .arg(
                Arg::with_name("globs")
                  .required(true)
                  .takes_value(true)
                  .multiple(true)
                  .help(
                    "globs matching the files and directories which should be included in the \
directory, relative to the root.",
                  ),
              )
                .arg(Arg::with_name("root").long("root").required(true).takes_value(true).help(
                  "Root under which the globs live. The Directory proto produced will be relative \
to this directory.",
                )),
          )
          .subcommand(
            SubCommand::with_name("cat-proto")
              .about(
                "Output the bytes of a serialized Directory proto addressed by fingerprint.",
              )
              .arg(
                Arg::with_name("output-format")
                  .long("output-format")
                  .takes_value(true)
                  .default_value("binary")
                  .possible_values(&["binary", "text"]),
              )
              .arg(Arg::with_name("fingerprint").required(true).takes_value(
                true,
              )),
          ),
      )
      .subcommand(
        SubCommand::with_name("cat")
          .about(
            "Output the contents of a file or Directory proto addressed by fingerprint.",
          )
          .arg(Arg::with_name("fingerprint").required(true).takes_value(
            true,
          )),
      )
      .arg(
        Arg::with_name("local-store-path")
          .takes_value(true)
          .long("local-store-path")
          .required(true),
      )
        .arg(
          Arg::with_name("server-address")
              .takes_value(true)
              .long("server-address")
              .required(false)
        )
      .get_matches(),
  ) {
    Ok(_) => {}
    Err(err) => {
      eprintln!("{}", err.0);
      exit(err.1 as i32)
    }
  };
}

fn execute(top_match: clap::ArgMatches) -> Result<(), ExitError> {
  let store_dir = top_match.value_of("local-store-path").unwrap();
  let pool = Arc::new(ResettablePool::new("fsutil-pool-".to_string()));
  let store = {
    let store_result = match top_match.value_of("server-address") {
      Some(cas_address) => {
        Store::backfills_from_remote(store_dir, pool.clone(), cas_address.to_owned())
      }
      None => Store::local_only(store_dir, pool.clone()),
    };
    let store = store_result.map_err(|e| {
      format!(
        "Failed to open/create store for directory {}: {}",
        store_dir,
        e
      )
    })?;
    Arc::new(store)
  };

  match top_match.subcommand() {
    ("file", Some(sub_match)) => {
      match sub_match.subcommand() {
        ("cat", Some(args)) => {
          let fingerprint = Fingerprint::from_hex_string(args.value_of("fingerprint").unwrap())?;
          let write_result = store
            .load_file_bytes_with(fingerprint, |bytes| io::stdout().write_all(&bytes).unwrap())
            .wait()?;
          write_result.ok_or_else(|| {
            ExitError(
              format!("File with fingerprint {} not found", fingerprint),
              ExitCode::NotFound,
            )
          })
        }
        ("save", Some(args)) => {
          let path = PathBuf::from(args.value_of("path").unwrap());
          // Canonicalize path to guarantee that a relative path has a parent.
          let posix_fs = make_posix_fs(
            path
              .canonicalize()
              .map_err(|e| {
                format!("Error canonicalizing path {:?}: {}", path, e.description())
              })?
              .parent()
              .ok_or_else(|| {
                format!("File being saved must have parent but {:?} did not", path)
              })?,
            pool,
          );
          let file = posix_fs
            .stat(PathBuf::from(path.file_name().unwrap()))
            .unwrap();
          match file {
            fs::Stat::File(f) => {
              let digest = FileSaver {
                store: store,
                posix_fs: Arc::new(posix_fs),
              }.digest(&f)
                .wait()
                .unwrap();
              Ok(println!("{} {}", digest.0, digest.1))
            }
            o => Err(
              format!(
                "Tried to save file {:?} but it was not a file, was a {:?}",
                path,
                o
              ).into(),
            ),
          }

        }
        (_, _) => unimplemented!(),
      }
    }
    ("directory", Some(sub_match)) => {
      match sub_match.subcommand() {
        ("materialize", Some(args)) => {
          let destination = PathBuf::from(args.value_of("destination").unwrap());
          let fingerprint = Fingerprint::from_hex_string(args.value_of("fingerprint").unwrap())?;
          materialize_directory(store, destination, fingerprint).wait()
        }
        ("save", Some(args)) => {
          let posix_fs = Arc::new(make_posix_fs(args.value_of("root").unwrap(), pool));
          let digest = posix_fs
            .expand(fs::PathGlobs::create(
              &args
                .values_of("globs")
                .unwrap()
                .map(|s| s.to_string())
                .collect::<Vec<String>>(),
              &[],
            )?)
            .map_err(|e| format!("Error expanding globs: {}", e.description()))
            .and_then(move |paths| {
              Snapshot::from_path_stats(
                store.clone(),
                Arc::new(FileSaver {
                  store: store.clone(),
                  posix_fs: posix_fs,
                }),
                paths,
              )
            })
            .map(|snapshot| snapshot.digest.unwrap())
            .wait()?;
          Ok(println!("{} {}", digest.0, digest.1))
        }
        ("cat-proto", Some(args)) => {
          let fingerprint = Fingerprint::from_hex_string(args.value_of("fingerprint").unwrap())?;
          let proto_bytes = match args.value_of("output-format").unwrap() {
            "binary" => {
              store.load_directory(fingerprint).wait().map(|maybe_d| {
                maybe_d.map(|d| d.write_to_bytes().unwrap())
              })
            }
            "text" => {
              store.load_directory(fingerprint).wait().map(|maybe_p| {
                maybe_p.map(|p| format!("{:?}\n", p).as_bytes().to_vec())
              })
            }
            format => Err(format!(
              "Unexpected value of --output-format arg: {}",
              format
            )),
          }?;
          match proto_bytes {
            Some(bytes) => {
              io::stdout().write(&bytes).unwrap();
              Ok(())
            }
            None => Err(ExitError(
              format!(
                "Directory with fingerprint {} not found",
                fingerprint
              ),
              ExitCode::NotFound,
            )),
          }
        }
        (_, _) => unimplemented!(),
      }
    }
    ("cat", Some(args)) => {
      let fingerprint = Fingerprint::from_hex_string(args.value_of("fingerprint").unwrap())?;
      let v = match store
        .load_file_bytes_with(fingerprint, |bytes| Vec::from(bytes))
        .wait()? {
        None => {
          store
            .load_directory(fingerprint)
            .map(|maybe_dir| {
              maybe_dir.map(|dir| {
                dir.write_to_bytes().expect(
                  "Error serializing Directory proto",
                )
              })
            })
            .wait()?
        }
        some => some,
      };
      match v {
        Some(bytes) => {
          io::stdout().write(&bytes).unwrap();
          Ok(())
        }
        None => Err(ExitError(
          format!("Fingerprint {} not found", fingerprint),
          ExitCode::NotFound,
        )),
      }
    }

    (_, _) => unimplemented!(),
  }
}

fn make_posix_fs<P: AsRef<Path>>(root: P, pool: Arc<ResettablePool>) -> fs::PosixFS {
  fs::PosixFS::new(&root, pool, vec![]).unwrap()
}

struct FileSaver {
  store: Arc<Store>,
  posix_fs: Arc<fs::PosixFS>,
}

impl GetFileDigest<String> for FileSaver {
  fn digest(&self, file: &fs::File) -> BoxFuture<fs::Digest, String> {
    let file_copy = file.clone();
    let store = self.store.clone();
    self
      .posix_fs
      .read_file(&file)
      .map_err(move |err| {
        format!("Error reading file {:?}: {}", file_copy, err.description())
      })
      .and_then(move |content| store.store_file_bytes(content.content))
      .to_boxed()
  }
}

fn materialize_directory(
  store: Arc<Store>,
  destination: PathBuf,
  fingerprint: Fingerprint,
) -> BoxFuture<(), ExitError> {
  let mkdir = make_clean_dir(&destination).map_err(|e| {
    format!(
      "Error making directory {:?}: {}",
      destination,
      e.description()
    ).into()
  });
  match mkdir {
    Ok(()) => {}
    Err(e) => return future::err(e).to_boxed(),
  };
  store
    .load_directory(fingerprint)
    .map_err(|e| e.into())
    .and_then(move |directory_opt| {
      directory_opt.ok_or_else(|| {
        ExitError(
          format!("Directory with fingerprint {} not found", fingerprint),
          ExitCode::NotFound,
        )
      })
    })
    .and_then(move |directory| {
      let file_futures = directory
        .get_files()
        .iter()
        .map(|file_node| {
          let store = store.clone();
          let path = destination.join(file_node.get_name());
          future::result(Fingerprint::from_hex_string(
            file_node.get_digest().get_hash(),
          )).map_err(|e| e.into())
            .and_then(move |fingerprint| {
              materialize_file(store, path, fingerprint)
            })
        })
        .collect::<Vec<_>>();
      let directory_futures = directory
        .get_directories()
        .iter()
        .map(|directory_node| {
          let store = store.clone();
          let path = destination.join(directory_node.get_name());
          future::result(Fingerprint::from_hex_string(
            directory_node.get_digest().get_hash(),
          )).map_err(|e| e.into())
            .and_then(move |fingerprint| {
              materialize_directory(store, path, fingerprint)
            })
        })
        .collect::<Vec<_>>();
      join_all(file_futures)
        .join(join_all(directory_futures))
        .map(|_| ())
    })
    .to_boxed()
}

fn materialize_file(
  store: Arc<Store>,
  destination: PathBuf,
  fingerprint: Fingerprint,
) -> BoxFuture<(), ExitError> {
  store
    .load_file_bytes_with(fingerprint, move |bytes| {
      File::create(&destination)
        .and_then(|mut f| f.write_all(bytes))
        .map_err(|e| {
          format!("Error writing file {:?}: {}", destination, e.description())
        })
    })
    .map_err(|e| e.into())
    .and_then(move |write_result| match write_result {
      Some(Ok(())) => Ok(()),
      Some(Err(e)) => Err(e.into()),
      None => {
        Err(ExitError(
          format!("File with fingerprint {} not found", fingerprint),
          ExitCode::NotFound,
        ))
      }
    })
    .to_boxed()
}

fn make_clean_dir(path: &Path) -> Result<(), io::Error> {
  let parent = path.parent().ok_or_else(|| {
    io::Error::new(io::ErrorKind::NotFound, format!("{:?} had no parent", path))
  })?;
  std::fs::create_dir_all(parent)?;
  std::fs::create_dir(path)
}
<|endoftext|>


<commit_before>use cube_shell::cube_shell;
use id_allocator::IdAllocator;
use terrain_vram_buffers::TerrainVRAMBuffers;
use terrain::BlockPosition;
use terrain::Terrain;
use physics::Physics;
use state::EntityId;
use std::collections::HashSet;
use std::collections::RingBuf;
use std::iter::range_inclusive;
use stopwatch::TimerSet;
use yaglw::gl_context::GLContext;

#[cfg(test)]
use cube_shell::cube_shell_area;
#[cfg(test)]
use nalgebra::Pnt3;
#[cfg(test)]
use std::cmp::max;
#[cfg(test)]
use std::num::SignedInt;

pub const LOAD_DISTANCE: int = 10;

pub const BLOCK_UPDATE_BUDGET: int = 16;
pub const BLOCK_LOAD_COST: int = 2;
pub const BLOCK_UNLOAD_COST: int = 1;

/// Keep surroundings loaded around a given world position.
pub struct SurroundingsLoader<'a> {
  pub terrain: Terrain<'a>,

  pub load_queue: RingBuf<BlockPosition>,
  pub unload_queue: RingBuf<BlockPosition>,

  // the set of blocks that are currently loaded
  pub loaded: HashSet<BlockPosition>,
  // The set of all blocks we want loaded.
  pub want_loaded: HashSet<BlockPosition>,

  pub last_position: Option<BlockPosition>,
}

impl<'a> SurroundingsLoader<'a> {
  pub fn new() -> SurroundingsLoader<'a> {
    SurroundingsLoader {
      terrain: Terrain::new(),

      load_queue: RingBuf::new(),
      unload_queue: RingBuf::new(),

      loaded: HashSet::new(),
      want_loaded: HashSet::new(),

      last_position: None,
    }
  }

  pub fn update(
    &mut self,
    timers: &TimerSet,
    gl: &mut GLContext,
    terrain_buffers: &mut TerrainVRAMBuffers<'a>,
    id_allocator: &mut IdAllocator<EntityId>,
    physics: &mut Physics,
    position: BlockPosition,
  ) {
    timers.time("update.update_queues", || {
      self.update_queues(timers, position);
    });
    timers.time("update.load_some", || {
      self.load_some(timers, gl, terrain_buffers, id_allocator, physics);
    });
  }

  #[inline]
  fn update_queues(&mut self, timers: &TimerSet, block_position: BlockPosition) {
    if Some(block_position) != self.last_position {
      self.last_position = Some(block_position);

      let mut want_loaded = Vec::new();
      let mut want_loaded_set = HashSet::new();

      timers.time("update.update_queues.want_loaded", || {
        want_loaded.push(block_position);
        want_loaded_set.insert(block_position);

        for radius in range_inclusive(1, LOAD_DISTANCE) {
          let blocks_at_radius = cube_shell(block_position, radius);
          want_loaded.push_all(blocks_at_radius.as_slice());
          for block_position in blocks_at_radius.into_iter() {
            want_loaded_set.insert(block_position);
          }
        }
      });

      self.load_queue.clear();

      timers.time("update.update_queues.unload_queue", || {
        for block_position in self.want_loaded.iter() {
          let is_loaded = self.loaded.contains(block_position);
          if is_loaded {
            let is_needed = want_loaded_set.contains(block_position);
            if !is_needed {
              self.unload_queue.push_back(*block_position);
            }
          }
        }
      });

      timers.time("update.update_queues.load_queue", || {
        for block_position in want_loaded.iter() {
          let is_loaded = self.loaded.contains(block_position);
          if !is_loaded {
            self.load_queue.push_back(*block_position);
          }
        }
      });

      self.want_loaded = want_loaded_set;
    }
  }

  #[inline]
  fn load_some(
    &mut self,
    timers: &TimerSet,
    gl: &mut GLContext,
    terrain_buffers: &mut TerrainVRAMBuffers<'a>,
    id_allocator: &mut IdAllocator<EntityId>,
    physics: &mut Physics,
  ) {
    let mut budget = BLOCK_UPDATE_BUDGET;
    while budget > 0 {
      match self.unload_queue.pop_front() {
        None =>
          match self.load_queue.pop_front() {
            None => {
              break;
            },
            Some(block_position) => {
              timers.time("update.load_some.load", || {
                let block = self.terrain.load(timers, id_allocator, &block_position);

                timers.time("update.load_some.load.physics", || {
                  for (&id, bounds) in block.bounds.iter() {
                    physics.insert_terrain(id, bounds);
                  }
                });

                timers.time("update.load_some.load.vram", || {
                  terrain_buffers.push(
                    gl,
                    block.vertices.as_slice(),
                    block.normals.as_slice(),
                    block.typs.as_slice(),
                    block.ids.as_slice(),
                  );
                });
              });

              self.loaded.insert(block_position);
              budget -= BLOCK_LOAD_COST;
            },
          },
        Some(block_position) => {
          timers.time("update.load_some.unload", || {
            let block = self.terrain.all_blocks.get(&block_position).unwrap();
            for id in block.ids.iter() {
              physics.remove_terrain(*id);
              terrain_buffers.swap_remove(gl, *id);
            }

            self.loaded.remove(&block_position);
            budget -= BLOCK_UNLOAD_COST;
          })
        }
      }
    }
  }
}

#[test]
fn shell_ordering() {
  fn radius_between(p1: &BlockPosition, p2: &BlockPosition) -> int {
    let dx = (p1.x - p2.x).abs();
    let dy = (p1.y - p2.y).abs();;
    let dz = (p1.z - p2.z).abs();;
    max(max(dx, dy), dz)
  }

  let mut loader = SurroundingsLoader::new();
  let timers = TimerSet::new();
  let position = Pnt3::new(1, -4, 7);
  loader.update_queues(&timers, position);
  let mut loader = loader.load_queue.into_iter();

  // The load queue should contain cube shells in increasing order of radius, up to LOAD_DISTANCE radius.
  for radius in range_inclusive(0, LOAD_DISTANCE) {
    for _ in range(0, cube_shell_area(radius)) {
      let load_position = loader.next();
      // The next load position should be in the cube shell of the given radius, relative to the center position.
      assert_eq!(radius_between(&position, &load_position.unwrap()), radius);
    }
  }

  // The load queue should be exactly the shells specified above, and nothing else.
  assert!(loader.next().is_none());
}
<commit_msg>tweak terrain load budget<commit_after>use cube_shell::cube_shell;
use id_allocator::IdAllocator;
use terrain_vram_buffers::TerrainVRAMBuffers;
use terrain::BlockPosition;
use terrain::Terrain;
use physics::Physics;
use state::EntityId;
use std::collections::HashSet;
use std::collections::RingBuf;
use std::iter::range_inclusive;
use stopwatch::TimerSet;
use yaglw::gl_context::GLContext;

#[cfg(test)]
use cube_shell::cube_shell_area;
#[cfg(test)]
use nalgebra::Pnt3;
#[cfg(test)]
use std::cmp::max;
#[cfg(test)]
use std::num::SignedInt;

pub const LOAD_DISTANCE: int = 10;

// values are approximately in microseconds, but they don't have to be.
pub const BLOCK_UPDATE_BUDGET: int = 5000;
pub const BLOCK_LOAD_COST: int = 500;
pub const BLOCK_UNLOAD_COST: int = 300;

/// Keep surroundings loaded around a given world position.
pub struct SurroundingsLoader<'a> {
  pub terrain: Terrain<'a>,

  pub load_queue: RingBuf<BlockPosition>,
  pub unload_queue: RingBuf<BlockPosition>,

  // the set of blocks that are currently loaded
  pub loaded: HashSet<BlockPosition>,
  // The set of all blocks we want loaded.
  pub want_loaded: HashSet<BlockPosition>,

  pub last_position: Option<BlockPosition>,
}

impl<'a> SurroundingsLoader<'a> {
  pub fn new() -> SurroundingsLoader<'a> {
    SurroundingsLoader {
      terrain: Terrain::new(),

      load_queue: RingBuf::new(),
      unload_queue: RingBuf::new(),

      loaded: HashSet::new(),
      want_loaded: HashSet::new(),

      last_position: None,
    }
  }

  pub fn update(
    &mut self,
    timers: &TimerSet,
    gl: &mut GLContext,
    terrain_buffers: &mut TerrainVRAMBuffers<'a>,
    id_allocator: &mut IdAllocator<EntityId>,
    physics: &mut Physics,
    position: BlockPosition,
  ) {
    timers.time("update.update_queues", || {
      self.update_queues(timers, position);
    });
    timers.time("update.load_some", || {
      self.load_some(timers, gl, terrain_buffers, id_allocator, physics);
    });
  }

  #[inline]
  fn update_queues(&mut self, timers: &TimerSet, block_position: BlockPosition) {
    if Some(block_position) != self.last_position {
      self.last_position = Some(block_position);

      let mut want_loaded = Vec::new();
      let mut want_loaded_set = HashSet::new();

      timers.time("update.update_queues.want_loaded", || {
        want_loaded.push(block_position);
        want_loaded_set.insert(block_position);

        for radius in range_inclusive(1, LOAD_DISTANCE) {
          let blocks_at_radius = cube_shell(block_position, radius);
          want_loaded.push_all(blocks_at_radius.as_slice());
          for block_position in blocks_at_radius.into_iter() {
            want_loaded_set.insert(block_position);
          }
        }
      });

      self.load_queue.clear();

      timers.time("update.update_queues.unload_queue", || {
        for block_position in self.want_loaded.iter() {
          let is_loaded = self.loaded.contains(block_position);
          if is_loaded {
            let is_needed = want_loaded_set.contains(block_position);
            if !is_needed {
              self.unload_queue.push_back(*block_position);
            }
          }
        }
      });

      timers.time("update.update_queues.load_queue", || {
        for block_position in want_loaded.iter() {
          let is_loaded = self.loaded.contains(block_position);
          if !is_loaded {
            self.load_queue.push_back(*block_position);
          }
        }
      });

      self.want_loaded = want_loaded_set;
    }
  }

  #[inline]
  fn load_some(
    &mut self,
    timers: &TimerSet,
    gl: &mut GLContext,
    terrain_buffers: &mut TerrainVRAMBuffers<'a>,
    id_allocator: &mut IdAllocator<EntityId>,
    physics: &mut Physics,
  ) {
    let mut budget = BLOCK_UPDATE_BUDGET;
    while budget > 0 {
      match self.unload_queue.pop_front() {
        None =>
          match self.load_queue.pop_front() {
            None => {
              break;
            },
            Some(block_position) => {
              timers.time("update.load_some.load", || {
                let block = self.terrain.load(timers, id_allocator, &block_position);

                timers.time("update.load_some.load.physics", || {
                  for (&id, bounds) in block.bounds.iter() {
                    physics.insert_terrain(id, bounds);
                  }
                });

                timers.time("update.load_some.load.vram", || {
                  terrain_buffers.push(
                    gl,
                    block.vertices.as_slice(),
                    block.normals.as_slice(),
                    block.typs.as_slice(),
                    block.ids.as_slice(),
                  );
                });
              });

              self.loaded.insert(block_position);
              budget -= BLOCK_LOAD_COST;
            },
          },
        Some(block_position) => {
          timers.time("update.load_some.unload", || {
            let block = self.terrain.all_blocks.get(&block_position).unwrap();
            for id in block.ids.iter() {
              physics.remove_terrain(*id);
              terrain_buffers.swap_remove(gl, *id);
            }

            self.loaded.remove(&block_position);
            budget -= BLOCK_UNLOAD_COST;
          })
        }
      }
    }
  }
}

#[test]
fn shell_ordering() {
  fn radius_between(p1: &BlockPosition, p2: &BlockPosition) -> int {
    let dx = (p1.x - p2.x).abs();
    let dy = (p1.y - p2.y).abs();;
    let dz = (p1.z - p2.z).abs();;
    max(max(dx, dy), dz)
  }

  let mut loader = SurroundingsLoader::new();
  let timers = TimerSet::new();
  let position = Pnt3::new(1, -4, 7);
  loader.update_queues(&timers, position);
  let mut loader = loader.load_queue.into_iter();

  // The load queue should contain cube shells in increasing order of radius, up to LOAD_DISTANCE radius.
  for radius in range_inclusive(0, LOAD_DISTANCE) {
    for _ in range(0, cube_shell_area(radius)) {
      let load_position = loader.next();
      // The next load position should be in the cube shell of the given radius, relative to the center position.
      assert_eq!(radius_between(&position, &load_position.unwrap()), radius);
    }
  }

  // The load queue should be exactly the shells specified above, and nothing else.
  assert!(loader.next().is_none());
}
<|endoftext|>


<commit_before>use bot::BotEnv;
use tasks::{RunsTask, Wait};
use chrono::prelude::*;
use chrono::Duration;
use database::models::Timeout;

use serenity::model::id::GuildId;

use diesel::prelude::*;

use std::sync::Arc;
use std::thread;

#[derive(Default)]
pub struct TimeoutCheckTask {
  next_sleep: i64
}

pub fn remove_timeout(timeout: &Timeout) {
  let mut member = match GuildId(*timeout.server_id).member(*timeout.user_id) {
    Ok(m) => m,
    Err(e) => {
      warn!("could not get member for timeout check: {}", e);
      return;
    }
  };
  if let Err(e) = member.remove_role(*timeout.role_id) {
    warn!("could not remove timeout role from {}: {}", *timeout.user_id, e);
  }
  if let Err(e) = ::bot::with_connection(|c| ::diesel::delete(timeout).execute(c)) {
    warn!("could not delete timeout {}: {}", timeout.id, e);
  }
}

impl RunsTask for TimeoutCheckTask {
  fn start(mut self, env: Arc<BotEnv>) {
    loop {
      thread::sleep(Duration::seconds(self.next_sleep).to_std().unwrap());
      if self.next_sleep == 0 {
        self.next_sleep = env.config.read().timeouts.role_check_interval.unwrap_or(300);
      }
      let now = Utc::now();
      let next_five_minutes = (now + Duration::minutes(5)).timestamp();
      let mut timeouts: Vec<Timeout> = match ::bot::with_connection(|c| ::database::schema::timeouts::dsl::timeouts.load(c)) {
        Ok(t) => t,
        Err(e) => {
          warn!("could not load timeouts: {}", e);
          continue;
        }
      };
      timeouts.retain(|t| t.ends() <= next_five_minutes);

      if timeouts.is_empty() {
        continue;
      }

      ::std::thread::spawn(move || {
        for (wait, timeout) in Wait::new(timeouts.into_iter().map(|t| (t.ends(), t))) {
          ::std::thread::sleep(Duration::seconds(wait).to_std().unwrap());
          remove_timeout(&timeout);
        }
      });
    }
  }
}
<commit_msg>fix: allow config changes to take place immediately<commit_after>use bot::BotEnv;
use tasks::{RunsTask, Wait};
use chrono::prelude::*;
use chrono::Duration;
use database::models::Timeout;

use serenity::model::id::GuildId;

use diesel::prelude::*;

use std::sync::Arc;
use std::thread;

#[derive(Default)]
pub struct TimeoutCheckTask {
  ran_once: bool
}

pub fn remove_timeout(timeout: &Timeout) {
  let mut member = match GuildId(*timeout.server_id).member(*timeout.user_id) {
    Ok(m) => m,
    Err(e) => {
      warn!("could not get member for timeout check: {}", e);
      return;
    }
  };
  if let Err(e) = member.remove_role(*timeout.role_id) {
    warn!("could not remove timeout role from {}: {}", *timeout.user_id, e);
  }
  if let Err(e) = ::bot::with_connection(|c| ::diesel::delete(timeout).execute(c)) {
    warn!("could not delete timeout {}: {}", timeout.id, e);
  }
}

impl RunsTask for TimeoutCheckTask {
  fn start(mut self, env: Arc<BotEnv>) {
    loop {
      let sleep = if self.ran_once {
        env.config.read().timeouts.role_check_interval.unwrap_or(300)
      } else {
        self.ran_once = true;
        0
      };
      thread::sleep(Duration::seconds(sleep).to_std().unwrap());
      let now = Utc::now();
      let next_five_minutes = (now + Duration::minutes(5)).timestamp();
      let mut timeouts: Vec<Timeout> = match ::bot::with_connection(|c| ::database::schema::timeouts::dsl::timeouts.load(c)) {
        Ok(t) => t,
        Err(e) => {
          warn!("could not load timeouts: {}", e);
          continue;
        }
      };
      timeouts.retain(|t| t.ends() <= next_five_minutes);

      if timeouts.is_empty() {
        continue;
      }

      ::std::thread::spawn(move || {
        for (wait, timeout) in Wait::new(timeouts.into_iter().map(|t| (t.ends(), t))) {
          ::std::thread::sleep(Duration::seconds(wait).to_std().unwrap());
          remove_timeout(&timeout);
        }
      });
    }
  }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern mod std;

use core::io;
use std::time;
use std::treemap::TreeMap;
use core::hashmap::linear::{LinearMap, LinearSet};
use core::trie::TrieMap;

fn timed(label: &str, f: &fn()) {
    let start = time::precise_time_s();
    f();
    let end = time::precise_time_s();
    io::println(fmt!("  %s: %f", label, end - start));
}

fn ascending<M: Map<uint, uint>>(map: &mut M, n_keys: uint) {
    io::println(" Ascending integers:");

    do timed("insert") {
        for uint::range(0, n_keys) |i| {
            map.insert(i, i + 1);
        }
    }

    do timed("search") {
        for uint::range(0, n_keys) |i| {
            fail_unless!(map.find(&i).unwrap() == &(i + 1));
        }
    }

    do timed("remove") {
        for uint::range(0, n_keys) |i| {
            fail_unless!(map.remove(&i));
        }
    }
}

fn descending<M: Map<uint, uint>>(map: &mut M, n_keys: uint) {
    io::println(" Descending integers:");

    do timed("insert") {
        for uint::range(0, n_keys) |i| {
            map.insert(i, i + 1);
        }
    }

    do timed("search") {
        for uint::range(0, n_keys) |i| {
            fail_unless!(map.find(&i).unwrap() == &(i + 1));
        }
    }

    do timed("remove") {
        for uint::range(0, n_keys) |i| {
            fail_unless!(map.remove(&i));
        }
    }
}

fn vector<M: Map<uint, uint>>(map: &mut M, n_keys: uint, dist: &[uint]) {

    do timed("insert") {
        for uint::range(0, n_keys) |i| {
            map.insert(dist[i], i + 1);
        }
    }

    do timed("search") {
        for uint::range(0, n_keys) |i| {
            fail_unless!(map.find(&dist[i]).unwrap() == &(i + 1));
        }
    }

    do timed("remove") {
        for uint::range(0, n_keys) |i| {
            fail_unless!(map.remove(&dist[i]));
        }
    }
}

fn main() {
    let args = os::args();
    let n_keys = {
        if args.len() == 2 {
            uint::from_str(args[1]).get()
        } else {
            1000000
        }
    };

    let mut rand = vec::with_capacity(n_keys);

    {
        let rng = core::rand::seeded_rng([1, 1, 1, 1, 1, 1, 1]);
        let mut set = LinearSet::new();
        while set.len() != n_keys {
            let next = rng.next() as uint;
            if set.insert(next) {
                rand.push(next);
            }
        }
    }

    io::println(fmt!("%? keys", n_keys));

    io::println("\nTreeMap:");

    {
        let mut map = TreeMap::new::<uint, uint>();
        ascending(&mut map, n_keys);
    }

    {
        let mut map = TreeMap::new::<uint, uint>();
        descending(&mut map, n_keys);
    }

    {
        io::println(" Random integers:");
        let mut map = TreeMap::new::<uint, uint>();
        vector(&mut map, n_keys, rand);
    }

    io::println("\nLinearMap:");

    {
        let mut map = LinearMap::new::<uint, uint>();
        ascending(&mut map, n_keys);
    }

    {
        let mut map = LinearMap::new::<uint, uint>();
        descending(&mut map, n_keys);
    }

    {
        io::println(" Random integers:");
        let mut map = LinearMap::new::<uint, uint>();
        vector(&mut map, n_keys, rand);
    }

    io::println("\nTrieMap:");

    {
        let mut map = TrieMap::new::<uint>();
        ascending(&mut map, n_keys);
    }

    {
        let mut map = TrieMap::new::<uint>();
        descending(&mut map, n_keys);
    }

    {
        io::println(" Random integers:");
        let mut map = TrieMap::new::<uint>();
        vector(&mut map, n_keys, rand);
    }
}
<commit_msg>auto merge of #5569 : thestinger/rust/map, r=catamorphism<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern mod std;

use core::io;
use std::time;
use std::treemap::TreeMap;
use core::hashmap::linear::{LinearMap, LinearSet};
use core::trie::TrieMap;

fn timed(label: &str, f: &fn()) {
    let start = time::precise_time_s();
    f();
    let end = time::precise_time_s();
    io::println(fmt!("  %s: %f", label, end - start));
}

fn ascending<M: Map<uint, uint>>(map: &mut M, n_keys: uint) {
    io::println(" Ascending integers:");

    do timed("insert") {
        for uint::range(0, n_keys) |i| {
            map.insert(i, i + 1);
        }
    }

    do timed("search") {
        for uint::range(0, n_keys) |i| {
            fail_unless!(map.find(&i).unwrap() == &(i + 1));
        }
    }

    do timed("remove") {
        for uint::range(0, n_keys) |i| {
            fail_unless!(map.remove(&i));
        }
    }
}

fn descending<M: Map<uint, uint>>(map: &mut M, n_keys: uint) {
    io::println(" Descending integers:");

    do timed("insert") {
        for uint::range_rev(n_keys, 0) |i| {
            map.insert(i, i + 1);
        }
    }

    do timed("search") {
        for uint::range_rev(n_keys, 0) |i| {
            fail_unless!(map.find(&i).unwrap() == &(i + 1));
        }
    }

    do timed("remove") {
        for uint::range_rev(n_keys, 0) |i| {
            fail_unless!(map.remove(&i));
        }
    }
}

fn vector<M: Map<uint, uint>>(map: &mut M, n_keys: uint, dist: &[uint]) {

    do timed("insert") {
        for uint::range(0, n_keys) |i| {
            map.insert(dist[i], i + 1);
        }
    }

    do timed("search") {
        for uint::range(0, n_keys) |i| {
            fail_unless!(map.find(&dist[i]).unwrap() == &(i + 1));
        }
    }

    do timed("remove") {
        for uint::range(0, n_keys) |i| {
            fail_unless!(map.remove(&dist[i]));
        }
    }
}

fn main() {
    let args = os::args();
    let n_keys = {
        if args.len() == 2 {
            uint::from_str(args[1]).get()
        } else {
            1000000
        }
    };

    let mut rand = vec::with_capacity(n_keys);

    {
        let rng = core::rand::seeded_rng([1, 1, 1, 1, 1, 1, 1]);
        let mut set = LinearSet::new();
        while set.len() != n_keys {
            let next = rng.next() as uint;
            if set.insert(next) {
                rand.push(next);
            }
        }
    }

    io::println(fmt!("%? keys", n_keys));

    io::println("\nTreeMap:");

    {
        let mut map = TreeMap::new::<uint, uint>();
        ascending(&mut map, n_keys);
    }

    {
        let mut map = TreeMap::new::<uint, uint>();
        descending(&mut map, n_keys);
    }

    {
        io::println(" Random integers:");
        let mut map = TreeMap::new::<uint, uint>();
        vector(&mut map, n_keys, rand);
    }

    io::println("\nLinearMap:");

    {
        let mut map = LinearMap::new::<uint, uint>();
        ascending(&mut map, n_keys);
    }

    {
        let mut map = LinearMap::new::<uint, uint>();
        descending(&mut map, n_keys);
    }

    {
        io::println(" Random integers:");
        let mut map = LinearMap::new::<uint, uint>();
        vector(&mut map, n_keys, rand);
    }

    io::println("\nTrieMap:");

    {
        let mut map = TrieMap::new::<uint>();
        ascending(&mut map, n_keys);
    }

    {
        let mut map = TrieMap::new::<uint>();
        descending(&mut map, n_keys);
    }

    {
        io::println(" Random integers:");
        let mut map = TrieMap::new::<uint>();
        vector(&mut map, n_keys, rand);
    }
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use common::{Span, Spanned};
use graphql_ir::{
    FragmentSpread, InlineFragment, LinkedField, Program, ScalarField, Selection, Transformed,
    Transformer,
};

use interner::{Intern, StringKey};
use schema::{FieldID, InterfaceID, ObjectID, Type};
use std::collections::HashMap;
use std::sync::Arc;
///
/// A transform that adds an `id` field on any type that has an id field but
/// where there is no unaliased `id` selection.
///
pub fn generate_id_field<'s>(program: &'s Program<'s>) -> Program<'s> {
    let mut transform = GenerateIDFieldTransform::new(program);
    transform
        .transform_program(program)
        .unwrap_or_else(|| program.clone())
}

struct GenerateIDFieldTransform<'s> {
    program: &'s Program<'s>,
    id_name: StringKey,
    node_interface_id: InterfaceID,
    cache: HashMap<Type, Option<FieldID>>,
    node_id_field_id: FieldID,
}

impl<'s> Transformer for GenerateIDFieldTransform<'s> {
    const NAME: &'static str = "GenerateIDFieldTransform";
    const VISIT_ARGUMENTS: bool = false;
    const VISIT_DIRECTIVES: bool = false;

    fn transform_linked_field(&mut self, field: &LinkedField) -> Transformed<Arc<LinkedField>> {
        let selections = self.transform_selections(&field.selections);

        let next_selections = if self.has_unaliased_id_field(&field.selections) {
            selections
        } else {
            let schema = self.program.schema();
            let type_ = self
                .program
                .schema()
                .field(field.definition.item)
                .type_
                .inner();
            match type_ {
                Type::Object(id) => {
                    let object = schema.object(id);
                    if let Some(id_field_id) = self.get_id_field_id(type_, &object.fields) {
                        let mut next_selections =
                            selections.unwrap_or_else(|| field.selections.clone());
                        next_selections.push(Selection::ScalarField(
                            self.create_id_field(field.definition.span, id_field_id),
                        ));
                        Some(next_selections)
                    } else {
                        selections
                    }
                }
                Type::Interface(id) => {
                    let interface = schema.interface(id);
                    if let Some(id_field_id) = self.get_id_field_id(type_, &interface.fields) {
                        let mut next_selections =
                            selections.unwrap_or_else(|| field.selections.clone());
                        next_selections.push(Selection::ScalarField(
                            self.create_id_field(field.definition.span, id_field_id),
                        ));
                        Some(next_selections)
                    } else {
                        let mut inline_fragments = self.create_id_inline_fragments(
                            field.definition.span,
                            &interface.implementors,
                        );
                        if inline_fragments.is_empty() {
                            selections
                        } else {
                            if let Some(selections) = selections {
                                inline_fragments.extend(selections.into_iter())
                            } else {
                                inline_fragments.extend(field.selections.iter().cloned());
                            }
                            Some(inline_fragments)
                        }
                    }
                }
                Type::Union(id) => {
                    let union = schema.union(id);
                    let mut inline_fragments =
                        self.create_id_inline_fragments(field.definition.span, &union.members);

                    if inline_fragments.is_empty() {
                        selections
                    } else {
                        if let Some(selections) = selections {
                            inline_fragments.extend(selections.into_iter())
                        } else {
                            inline_fragments.extend(field.selections.iter().cloned());
                        }
                        Some(inline_fragments)
                    }
                }
                _ => selections,
            }
        };
        match next_selections {
            None => Transformed::Keep,
            Some(selections) => Transformed::Replace(Arc::new(LinkedField {
                alias: field.alias,
                definition: field.definition,
                arguments: field.arguments.clone(),
                directives: field.directives.clone(),
                selections,
            })),
        }
    }

    fn transform_fragment_spread(
        &mut self,
        _spread: &FragmentSpread,
    ) -> Transformed<Arc<FragmentSpread>> {
        Transformed::Keep
    }

    fn transform_scalar_field(&mut self, _field: &ScalarField) -> Transformed<Arc<ScalarField>> {
        Transformed::Keep
    }
}

impl<'s> GenerateIDFieldTransform<'s> {
    fn new(program: &'s Program<'s>) -> Self {
        let schema = program.schema();
        let node_interface_id = match schema.get_type("Node".intern()) {
            Some(Type::Interface(id)) => id,
            _ => panic!("Expected the schema to contain an interface named `Node`."),
        };
        let node_interface = schema.interface(node_interface_id);
        let id_key = "id".intern();
        let node_id_field_id = *node_interface
            .fields
            .iter()
            .find(|&&id| schema.field(id).name == id_key)
            .expect("Expected `Node` to contain a field named `id`.");
        Self {
            program,
            id_name: schema.field(node_id_field_id).name,
            node_interface_id,
            cache: Default::default(),
            node_id_field_id,
        }
    }

    fn has_unaliased_id_field(&self, selections: &[Selection]) -> bool {
        selections.iter().any(|x| match x {
            Selection::ScalarField(child) => {
                child.alias.is_none()
                    && self.program.schema().field(child.definition.item).name == self.id_name
            }
            _ => false,
        })
    }

    fn get_id_field_id(&mut self, type_: Type, fields: &[FieldID]) -> Option<FieldID> {
        match self.cache.get(&type_) {
            Some(result) => *result,
            None => {
                for id in fields {
                    let field = self.program.schema().field(*id);
                    if field.name == self.id_name
                        && self.program.schema().is_id(field.type_.inner())
                    {
                        let result = Some(*id);
                        self.cache.insert(type_, result);
                        return result;
                    }
                }
                self.cache.insert(type_, None);
                None
            }
        }
    }

    /// For interfaces and unions: generate a `... on Node { id }`
    /// fragment if *any* concrete type implements Node. Then generate a
    /// `... on PossibleType { id }` for every concrete type that does *not*
    /// implement `Node`
    fn create_id_inline_fragments(
        &mut self,
        span: Span,
        concrete_ids: &[ObjectID],
    ) -> Vec<Selection> {
        let mut next_selections = vec![];
        let mut should_generate_node = false;

        for object_id in concrete_ids.clone() {
            let object = self.program.schema().object(*object_id);
            let implements_node = object
                .interfaces
                .iter()
                .any(|interface_id| interface_id == &self.node_interface_id);
            if implements_node {
                should_generate_node = true;
            } else if let Some(id_field_id) =
                self.get_id_field_id(Type::Object(*object_id), &object.fields)
            {
                next_selections.push(Selection::InlineFragment(self.create_inline_id_fragment(
                    span,
                    Type::Object(*object_id),
                    id_field_id,
                )));
            }
        }

        if should_generate_node {
            next_selections.push(Selection::InlineFragment(self.create_inline_id_fragment(
                span,
                Type::Interface(self.node_interface_id),
                self.node_id_field_id,
            )));
        }

        next_selections
    }

    fn create_id_field(&self, span: Span, id_field_id: FieldID) -> Arc<ScalarField> {
        Arc::new(ScalarField {
            alias: None,
            definition: Spanned::new(span, id_field_id),
            arguments: Default::default(),
            directives: Default::default(),
        })
    }

    fn create_inline_id_fragment(
        &self,
        span: Span,
        type_: Type,
        id_field_id: FieldID,
    ) -> Arc<InlineFragment> {
        Arc::new(InlineFragment {
            type_condition: Some(type_),
            directives: Default::default(),
            selections: vec![Selection::ScalarField(
                self.create_id_field(span, id_field_id),
            )],
        })
    }
}
<commit_msg>Address cargo clippy warning in generate_id_fields<commit_after>/*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use common::{Span, Spanned};
use graphql_ir::{
    FragmentSpread, InlineFragment, LinkedField, Program, ScalarField, Selection, Transformed,
    Transformer,
};

use interner::{Intern, StringKey};
use schema::{FieldID, InterfaceID, ObjectID, Type};
use std::collections::HashMap;
use std::sync::Arc;
///
/// A transform that adds an `id` field on any type that has an id field but
/// where there is no unaliased `id` selection.
///
pub fn generate_id_field<'s>(program: &'s Program<'s>) -> Program<'s> {
    let mut transform = GenerateIDFieldTransform::new(program);
    transform
        .transform_program(program)
        .unwrap_or_else(|| program.clone())
}

struct GenerateIDFieldTransform<'s> {
    program: &'s Program<'s>,
    id_name: StringKey,
    node_interface_id: InterfaceID,
    cache: HashMap<Type, Option<FieldID>>,
    node_id_field_id: FieldID,
}

impl<'s> Transformer for GenerateIDFieldTransform<'s> {
    const NAME: &'static str = "GenerateIDFieldTransform";
    const VISIT_ARGUMENTS: bool = false;
    const VISIT_DIRECTIVES: bool = false;

    fn transform_linked_field(&mut self, field: &LinkedField) -> Transformed<Arc<LinkedField>> {
        let selections = self.transform_selections(&field.selections);

        let next_selections = if self.has_unaliased_id_field(&field.selections) {
            selections
        } else {
            let schema = self.program.schema();
            let type_ = self
                .program
                .schema()
                .field(field.definition.item)
                .type_
                .inner();
            match type_ {
                Type::Object(id) => {
                    let object = schema.object(id);
                    if let Some(id_field_id) = self.get_id_field_id(type_, &object.fields) {
                        let mut next_selections =
                            selections.unwrap_or_else(|| field.selections.clone());
                        next_selections.push(Selection::ScalarField(
                            self.create_id_field(field.definition.span, id_field_id),
                        ));
                        Some(next_selections)
                    } else {
                        selections
                    }
                }
                Type::Interface(id) => {
                    let interface = schema.interface(id);
                    if let Some(id_field_id) = self.get_id_field_id(type_, &interface.fields) {
                        let mut next_selections =
                            selections.unwrap_or_else(|| field.selections.clone());
                        next_selections.push(Selection::ScalarField(
                            self.create_id_field(field.definition.span, id_field_id),
                        ));
                        Some(next_selections)
                    } else {
                        let mut inline_fragments = self.create_id_inline_fragments(
                            field.definition.span,
                            &interface.implementors,
                        );
                        if inline_fragments.is_empty() {
                            selections
                        } else {
                            if let Some(selections) = selections {
                                inline_fragments.extend(selections.into_iter())
                            } else {
                                inline_fragments.extend(field.selections.iter().cloned());
                            }
                            Some(inline_fragments)
                        }
                    }
                }
                Type::Union(id) => {
                    let union = schema.union(id);
                    let mut inline_fragments =
                        self.create_id_inline_fragments(field.definition.span, &union.members);

                    if inline_fragments.is_empty() {
                        selections
                    } else {
                        if let Some(selections) = selections {
                            inline_fragments.extend(selections.into_iter())
                        } else {
                            inline_fragments.extend(field.selections.iter().cloned());
                        }
                        Some(inline_fragments)
                    }
                }
                _ => selections,
            }
        };
        match next_selections {
            None => Transformed::Keep,
            Some(selections) => Transformed::Replace(Arc::new(LinkedField {
                alias: field.alias,
                definition: field.definition,
                arguments: field.arguments.clone(),
                directives: field.directives.clone(),
                selections,
            })),
        }
    }

    fn transform_fragment_spread(
        &mut self,
        _spread: &FragmentSpread,
    ) -> Transformed<Arc<FragmentSpread>> {
        Transformed::Keep
    }

    fn transform_scalar_field(&mut self, _field: &ScalarField) -> Transformed<Arc<ScalarField>> {
        Transformed::Keep
    }
}

impl<'s> GenerateIDFieldTransform<'s> {
    fn new(program: &'s Program<'s>) -> Self {
        let schema = program.schema();
        let node_interface_id = match schema.get_type("Node".intern()) {
            Some(Type::Interface(id)) => id,
            _ => panic!("Expected the schema to contain an interface named `Node`."),
        };
        let node_interface = schema.interface(node_interface_id);
        let id_key = "id".intern();
        let node_id_field_id = *node_interface
            .fields
            .iter()
            .find(|&&id| schema.field(id).name == id_key)
            .expect("Expected `Node` to contain a field named `id`.");
        Self {
            program,
            id_name: schema.field(node_id_field_id).name,
            node_interface_id,
            cache: Default::default(),
            node_id_field_id,
        }
    }

    fn has_unaliased_id_field(&self, selections: &[Selection]) -> bool {
        selections.iter().any(|x| match x {
            Selection::ScalarField(child) => {
                child.alias.is_none()
                    && self.program.schema().field(child.definition.item).name == self.id_name
            }
            _ => false,
        })
    }

    fn get_id_field_id(&mut self, type_: Type, fields: &[FieldID]) -> Option<FieldID> {
        match self.cache.get(&type_) {
            Some(result) => *result,
            None => {
                for id in fields {
                    let field = self.program.schema().field(*id);
                    if field.name == self.id_name
                        && self.program.schema().is_id(field.type_.inner())
                    {
                        let result = Some(*id);
                        self.cache.insert(type_, result);
                        return result;
                    }
                }
                self.cache.insert(type_, None);
                None
            }
        }
    }

    /// For interfaces and unions: generate a `... on Node { id }`
    /// fragment if *any* concrete type implements Node. Then generate a
    /// `... on PossibleType { id }` for every concrete type that does *not*
    /// implement `Node`
    fn create_id_inline_fragments(
        &mut self,
        span: Span,
        concrete_ids: &[ObjectID],
    ) -> Vec<Selection> {
        let mut next_selections = vec![];
        let mut should_generate_node = false;

        for object_id in concrete_ids {
            let object = self.program.schema().object(*object_id);
            let implements_node = object
                .interfaces
                .iter()
                .any(|interface_id| interface_id == &self.node_interface_id);
            if implements_node {
                should_generate_node = true;
            } else if let Some(id_field_id) =
                self.get_id_field_id(Type::Object(*object_id), &object.fields)
            {
                next_selections.push(Selection::InlineFragment(self.create_inline_id_fragment(
                    span,
                    Type::Object(*object_id),
                    id_field_id,
                )));
            }
        }

        if should_generate_node {
            next_selections.push(Selection::InlineFragment(self.create_inline_id_fragment(
                span,
                Type::Interface(self.node_interface_id),
                self.node_id_field_id,
            )));
        }

        next_selections
    }

    fn create_id_field(&self, span: Span, id_field_id: FieldID) -> Arc<ScalarField> {
        Arc::new(ScalarField {
            alias: None,
            definition: Spanned::new(span, id_field_id),
            arguments: Default::default(),
            directives: Default::default(),
        })
    }

    fn create_inline_id_fragment(
        &self,
        span: Span,
        type_: Type,
        id_field_id: FieldID,
    ) -> Arc<InlineFragment> {
        Arc::new(InlineFragment {
            type_condition: Some(type_),
            directives: Default::default(),
            selections: vec![Selection::ScalarField(
                self.create_id_field(span, id_field_id),
            )],
        })
    }
}
<|endoftext|>


<commit_before>use tools::{generate, gen_tests, run_rustfmt, Verify};

#[test]
fn generated_grammar_is_fresh() {
    if let Err(error) = generate(Verify) {
        panic!("{}. Please update it by running `cargo gen-syntax`", error);
    }
}

#[test]
fn generated_tests_are_fresh() {
    if let Err(error) = gen_tests(Verify) {
        panic!("{}. Please update tests by running `cargo gen-tests`", error);
    }
}

#[test]
fn check_code_formatting() {
    if let Err(error) = run_rustfmt(Verify) {
        panic!("{}. Please format the code by running `cargo format`", error);
    }
}
<commit_msg>forbid todo markers<commit_after>use walkdir::WalkDir;

use tools::{generate, gen_tests, run_rustfmt, Verify, project_root};

#[test]
fn generated_grammar_is_fresh() {
    if let Err(error) = generate(Verify) {
        panic!("{}. Please update it by running `cargo gen-syntax`", error);
    }
}

#[test]
fn generated_tests_are_fresh() {
    if let Err(error) = gen_tests(Verify) {
        panic!("{}. Please update tests by running `cargo gen-tests`", error);
    }
}

#[test]
fn check_code_formatting() {
    if let Err(error) = run_rustfmt(Verify) {
        panic!("{}. Please format the code by running `cargo format`", error);
    }
}

#[test]
fn no_todo() {
    WalkDir::new(project_root().join("crates")).into_iter().for_each(|e| {
        let e = e.unwrap();
        if e.path().extension().map(|it| it != "rs").unwrap_or(true) {
            return;
        }
        if e.path().ends_with("tests/cli.rs") {
            return;
        }
        let text = std::fs::read_to_string(e.path()).unwrap();
        if text.contains("TODO") {
            panic!(
                "\nTODO markers should not be commited to the master branch,\n\
                 use FIXME instead\n\
                 {}\n",
                e.path().display(),
            )
        }
    })
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

<%namespace name="helpers" file="/helpers.mako.rs" />

// SVG 1.1 (Second Edition)
// https://www.w3.org/TR/SVG/
<% data.new_style_struct("InheritedSVG", inherited=True, gecko_name="SVG") %>

// Section 10 - Text

${helpers.single_keyword(
    "dominant-baseline",
    """auto ideographic alphabetic hanging mathematical central middle
       text-after-edge text-before-edge""",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/css-inline-3/#propdef-dominant-baseline",
)}

${helpers.single_keyword(
    "text-anchor",
    "start middle end",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG/text.html#TextAnchorProperty",
    gecko_enum_prefix="StyleTextAnchor",
)}

// Section 11 - Painting: Filling, Stroking and Marker Symbols
${helpers.single_keyword(
    "color-interpolation",
    "srgb auto linearrgb",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#ColorInterpolationProperty",
)}

${helpers.single_keyword(
    "color-interpolation-filters",
    "linearrgb auto srgb",
    engines="gecko",
    gecko_constant_prefix="NS_STYLE_COLOR_INTERPOLATION",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#ColorInterpolationFiltersProperty",
)}

${helpers.predefined_type(
    "fill",
    "SVGPaint",
    "crate::values::computed::SVGPaint::black()",
    engines="gecko",
    animation_value_type="IntermediateSVGPaint",
    boxed=True,
    spec="https://www.w3.org/TR/SVG2/painting.html#SpecifyingFillPaint",
)}

${helpers.predefined_type(
    "fill-opacity",
    "SVGOpacity",
    "Default::default()",
    engines="gecko",
    animation_value_type="ComputedValue",
    spec="https://svgwg.org/svg2-draft/painting.html#FillOpacity",
)}

${helpers.predefined_type(
    "fill-rule",
    "FillRule",
    "Default::default()",
    engines="gecko",
    needs_context=False,
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#FillRuleProperty",
)}

${helpers.single_keyword(
    "shape-rendering",
    "auto optimizespeed crispedges geometricprecision",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#ShapeRenderingProperty",
    gecko_enum_prefix = "StyleShapeRendering",
)}

${helpers.predefined_type(
    "stroke",
    "SVGPaint",
    "Default::default()",
    engines="gecko",
    animation_value_type="IntermediateSVGPaint",
    boxed=True,
    spec="https://www.w3.org/TR/SVG2/painting.html#SpecifyingStrokePaint",
)}

${helpers.predefined_type(
    "stroke-width",
    "SVGWidth",
    "computed::SVGWidth::one()",
    engines="gecko",
    animation_value_type="crate::values::computed::SVGWidth",
    spec="https://www.w3.org/TR/SVG2/painting.html#StrokeWidth",
)}

${helpers.single_keyword(
    "stroke-linecap",
    "butt round square",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#StrokeLinecapProperty",
    gecko_enum_prefix = "StyleStrokeLinecap",
)}

${helpers.single_keyword(
    "stroke-linejoin",
    "miter round bevel",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#StrokeLinejoinProperty",
)}

${helpers.predefined_type(
    "stroke-miterlimit",
    "NonNegativeNumber",
    "From::from(4.0)",
    engines="gecko",
    animation_value_type="crate::values::computed::NonNegativeNumber",
    spec="https://www.w3.org/TR/SVG2/painting.html#StrokeMiterlimitProperty",
)}

${helpers.predefined_type(
    "stroke-opacity",
    "SVGOpacity",
    "Default::default()",
    engines="gecko",
    animation_value_type="ComputedValue",
    spec="https://svgwg.org/svg2-draft/painting.html#StrokeOpacity",
)}

${helpers.predefined_type(
    "stroke-dasharray",
    "SVGStrokeDashArray",
    "Default::default()",
    engines="gecko",
    animation_value_type="crate::values::computed::SVGStrokeDashArray",
    spec="https://www.w3.org/TR/SVG2/painting.html#StrokeDashing",
)}

${helpers.predefined_type(
    "stroke-dashoffset",
    "SVGLength",
    "computed::SVGLength::zero()",
    engines="gecko",
    animation_value_type="ComputedValue",
    spec="https://www.w3.org/TR/SVG2/painting.html#StrokeDashing",
)}

// Section 14 - Clipping, Masking and Compositing
${helpers.predefined_type(
    "clip-rule",
    "FillRule",
    "Default::default()",
    engines="gecko",
    needs_context=False,
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/masking.html#ClipRuleProperty",
)}

${helpers.predefined_type(
    "marker-start",
    "url::UrlOrNone",
    "computed::url::UrlOrNone::none()",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG2/painting.html#VertexMarkerProperties",
)}

${helpers.predefined_type(
    "marker-mid",
    "url::UrlOrNone",
    "computed::url::UrlOrNone::none()",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG2/painting.html#VertexMarkerProperties",
)}

${helpers.predefined_type(
    "marker-end",
    "url::UrlOrNone",
    "computed::url::UrlOrNone::none()",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG2/painting.html#VertexMarkerProperties",
)}

${helpers.predefined_type(
    "paint-order",
    "SVGPaintOrder",
    "computed::SVGPaintOrder::normal()",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG2/painting.html#PaintOrder",
)}

${helpers.predefined_type(
    "-moz-context-properties",
    "MozContextProperties",
    "computed::MozContextProperties::default()",
    engines="gecko",
    animation_value_type="none",
    spec="Nonstandard (https://developer.mozilla.org/en-US/docs/Web/CSS/-moz-context-properties)",
)}
<commit_msg>style: Convert stroke-linejoin #defines to an enum class.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

<%namespace name="helpers" file="/helpers.mako.rs" />

// SVG 1.1 (Second Edition)
// https://www.w3.org/TR/SVG/
<% data.new_style_struct("InheritedSVG", inherited=True, gecko_name="SVG") %>

// Section 10 - Text

${helpers.single_keyword(
    "dominant-baseline",
    """auto ideographic alphabetic hanging mathematical central middle
       text-after-edge text-before-edge""",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/css-inline-3/#propdef-dominant-baseline",
)}

${helpers.single_keyword(
    "text-anchor",
    "start middle end",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG/text.html#TextAnchorProperty",
    gecko_enum_prefix="StyleTextAnchor",
)}

// Section 11 - Painting: Filling, Stroking and Marker Symbols
${helpers.single_keyword(
    "color-interpolation",
    "srgb auto linearrgb",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#ColorInterpolationProperty",
)}

${helpers.single_keyword(
    "color-interpolation-filters",
    "linearrgb auto srgb",
    engines="gecko",
    gecko_constant_prefix="NS_STYLE_COLOR_INTERPOLATION",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#ColorInterpolationFiltersProperty",
)}

${helpers.predefined_type(
    "fill",
    "SVGPaint",
    "crate::values::computed::SVGPaint::black()",
    engines="gecko",
    animation_value_type="IntermediateSVGPaint",
    boxed=True,
    spec="https://www.w3.org/TR/SVG2/painting.html#SpecifyingFillPaint",
)}

${helpers.predefined_type(
    "fill-opacity",
    "SVGOpacity",
    "Default::default()",
    engines="gecko",
    animation_value_type="ComputedValue",
    spec="https://svgwg.org/svg2-draft/painting.html#FillOpacity",
)}

${helpers.predefined_type(
    "fill-rule",
    "FillRule",
    "Default::default()",
    engines="gecko",
    needs_context=False,
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#FillRuleProperty",
)}

${helpers.single_keyword(
    "shape-rendering",
    "auto optimizespeed crispedges geometricprecision",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#ShapeRenderingProperty",
    gecko_enum_prefix = "StyleShapeRendering",
)}

${helpers.predefined_type(
    "stroke",
    "SVGPaint",
    "Default::default()",
    engines="gecko",
    animation_value_type="IntermediateSVGPaint",
    boxed=True,
    spec="https://www.w3.org/TR/SVG2/painting.html#SpecifyingStrokePaint",
)}

${helpers.predefined_type(
    "stroke-width",
    "SVGWidth",
    "computed::SVGWidth::one()",
    engines="gecko",
    animation_value_type="crate::values::computed::SVGWidth",
    spec="https://www.w3.org/TR/SVG2/painting.html#StrokeWidth",
)}

${helpers.single_keyword(
    "stroke-linecap",
    "butt round square",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#StrokeLinecapProperty",
    gecko_enum_prefix = "StyleStrokeLinecap",
)}

${helpers.single_keyword(
    "stroke-linejoin",
    "miter round bevel",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/painting.html#StrokeLinejoinProperty",
    gecko_enum_prefix = "StyleStrokeLinejoin",
)}

${helpers.predefined_type(
    "stroke-miterlimit",
    "NonNegativeNumber",
    "From::from(4.0)",
    engines="gecko",
    animation_value_type="crate::values::computed::NonNegativeNumber",
    spec="https://www.w3.org/TR/SVG2/painting.html#StrokeMiterlimitProperty",
)}

${helpers.predefined_type(
    "stroke-opacity",
    "SVGOpacity",
    "Default::default()",
    engines="gecko",
    animation_value_type="ComputedValue",
    spec="https://svgwg.org/svg2-draft/painting.html#StrokeOpacity",
)}

${helpers.predefined_type(
    "stroke-dasharray",
    "SVGStrokeDashArray",
    "Default::default()",
    engines="gecko",
    animation_value_type="crate::values::computed::SVGStrokeDashArray",
    spec="https://www.w3.org/TR/SVG2/painting.html#StrokeDashing",
)}

${helpers.predefined_type(
    "stroke-dashoffset",
    "SVGLength",
    "computed::SVGLength::zero()",
    engines="gecko",
    animation_value_type="ComputedValue",
    spec="https://www.w3.org/TR/SVG2/painting.html#StrokeDashing",
)}

// Section 14 - Clipping, Masking and Compositing
${helpers.predefined_type(
    "clip-rule",
    "FillRule",
    "Default::default()",
    engines="gecko",
    needs_context=False,
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG11/masking.html#ClipRuleProperty",
)}

${helpers.predefined_type(
    "marker-start",
    "url::UrlOrNone",
    "computed::url::UrlOrNone::none()",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG2/painting.html#VertexMarkerProperties",
)}

${helpers.predefined_type(
    "marker-mid",
    "url::UrlOrNone",
    "computed::url::UrlOrNone::none()",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG2/painting.html#VertexMarkerProperties",
)}

${helpers.predefined_type(
    "marker-end",
    "url::UrlOrNone",
    "computed::url::UrlOrNone::none()",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG2/painting.html#VertexMarkerProperties",
)}

${helpers.predefined_type(
    "paint-order",
    "SVGPaintOrder",
    "computed::SVGPaintOrder::normal()",
    engines="gecko",
    animation_value_type="discrete",
    spec="https://www.w3.org/TR/SVG2/painting.html#PaintOrder",
)}

${helpers.predefined_type(
    "-moz-context-properties",
    "MozContextProperties",
    "computed::MozContextProperties::default()",
    engines="gecko",
    animation_value_type="none",
    spec="Nonstandard (https://developer.mozilla.org/en-US/docs/Web/CSS/-moz-context-properties)",
)}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    chained_bft::{
        chained_bft_smr::ChainedBftSMR, network::ConsensusNetworkImpl,
        persistent_storage::PersistentStorage,
    },
    consensus_provider::ConsensusProvider,
    counters,
    state_computer::ExecutionProxy,
    state_replication::StateMachineReplication,
    txn_manager::MempoolProxy,
};
use network::validator_network::{ConsensusNetworkEvents, ConsensusNetworkSender};
use nextgen_crypto::ed25519::*;

use crate::{
    chained_bft::{
        chained_bft_smr::ChainedBftSMRConfig, common::Author, persistent_storage::StorageWriteProxy,
    },
    state_synchronizer::{setup_state_synchronizer, StateSynchronizer},
};
use config::config::{ConsensusProposerType::FixedProposer, NodeConfig};
use execution_proto::proto::execution_grpc::ExecutionClient;
use failure::prelude::*;
use logger::prelude::*;
use mempool::proto::mempool_grpc::MempoolClient;
use std::{convert::TryFrom, sync::Arc};
use tokio::runtime;
use types::{
    account_address::AccountAddress, transaction::SignedTransaction,
    validator_signer::ValidatorSigner, validator_verifier::ValidatorVerifier,
};

struct InitialSetup {
    author: Author,
    signer: ValidatorSigner<Ed25519PrivateKey>,
    quorum_size: usize,
    peers: Arc<Vec<Author>>,
    validator: Arc<ValidatorVerifier<Ed25519PublicKey>>,
}

/// Supports the implementation of ConsensusProvider using LibraBFT.
pub struct ChainedBftProvider {
    smr: ChainedBftSMR<Vec<SignedTransaction>, Author>,
    mempool_client: Arc<MempoolClient>,
    execution_client: Arc<ExecutionClient>,
    synchronizer_client: Arc<StateSynchronizer>,
}

impl ChainedBftProvider {
    pub fn new(
        node_config: &mut NodeConfig,
        network_sender: ConsensusNetworkSender,
        network_events: ConsensusNetworkEvents,
        mempool_client: Arc<MempoolClient>,
        execution_client: Arc<ExecutionClient>,
    ) -> Self {
        let runtime = runtime::Builder::new()
            .name_prefix("consensus-")
            .build()
            .expect("Failed to create Tokio runtime!");

        let initial_setup = Self::initialize_setup(node_config);
        let network = ConsensusNetworkImpl::new(
            initial_setup.author,
            network_sender.clone(),
            network_events,
            Arc::clone(&initial_setup.peers),
            Arc::clone(&initial_setup.validator),
        );
        let synchronizer =
            setup_state_synchronizer(network_sender, runtime.executor(), node_config);
        let proposer = {
            if node_config.consensus.get_proposer_type() == FixedProposer {
                vec![Self::choose_leader(&initial_setup)]
            } else {
                initial_setup.validator.get_ordered_account_addresses()
            }
        };
        debug!("[Consensus] My peer: {:?}", initial_setup.author);
        debug!("[Consensus] Chosen proposer: {:?}", proposer);
        let config = ChainedBftSMRConfig::from_node_config(&node_config.consensus);
        let (storage, initial_data) = StorageWriteProxy::start(node_config);
        info!(
            "Starting up the consensus state machine with recovery data - {:?}, {:?}",
            initial_data.state(),
            initial_data.highest_timeout_certificates()
        );
        let smr = ChainedBftSMR::new(
            initial_setup.author,
            initial_setup.quorum_size,
            initial_setup.signer,
            proposer,
            network,
            runtime,
            config,
            storage,
            initial_data,
        );
        Self {
            smr,
            mempool_client,
            execution_client,
            synchronizer_client: Arc::new(synchronizer),
        }
    }

    /// Retrieve the initial "state" for consensus. This function is synchronous and returns after
    /// reading the local persistent store and retrieving the initial state from the executor.
    fn initialize_setup(node_config: &mut NodeConfig) -> InitialSetup {
        // Keeping the initial set of validators in a node config is embarrassing and we should
        // all feel bad about it.
        let peer_id_str = node_config.base.peer_id.clone();
        let author =
            AccountAddress::try_from(peer_id_str).expect("Failed to parse peer id of a validator");
        let private_key = node_config
            .base
            .peer_keypairs
            .take_consensus_private()
            .expect(
            "Failed to move a Consensus private key from a NodeConfig, key absent or already read",
        );

        let signer = ValidatorSigner::new(author, private_key);
        let peers_with_public_keys = node_config.base.trusted_peers.get_trusted_consensus_peers();
        let peers_with_nextgen_public_keys = peers_with_public_keys
            .clone()
            .into_iter()
            .map(|(k, v)| (AccountAddress::clone(&k), v))
            .collect();
        let peers = Arc::new(
            peers_with_public_keys
                .keys()
                .map(AccountAddress::clone)
                .collect(),
        );
        let validator = Arc::new(ValidatorVerifier::new(peers_with_nextgen_public_keys));
        counters::EPOCH_NUM.set(0); // No reconfiguration yet, so it is always zero
        counters::CURRENT_EPOCH_NUM_VALIDATORS.set(validator.len() as i64);
        counters::CURRENT_EPOCH_QUORUM_SIZE.set(validator.quorum_size() as i64);
        debug!("[Consensus]: quorum_size = {:?}", validator.quorum_size());
        InitialSetup {
            author,
            signer,
            quorum_size: validator.quorum_size(),
            peers,
            validator,
        }
    }

    /// Choose a proposer that is going to be the single leader (relevant for a mock fixed proposer
    /// election only).
    fn choose_leader(initial_setup: &InitialSetup) -> Author {
        // As it is just a tmp hack function, pick the smallest PeerId to be a proposer.
        *initial_setup
            .peers
            .iter()
            .max()
            .expect("No trusted peers found!")
    }
}

impl ConsensusProvider for ChainedBftProvider {
    fn start(&mut self) -> Result<()> {
        let txn_manager = Arc::new(MempoolProxy::new(self.mempool_client.clone()));
        let state_computer = Arc::new(ExecutionProxy::new(
            self.execution_client.clone(),
            self.synchronizer_client.clone(),
        ));
        debug!("Starting consensus provider.");
        self.smr.start(txn_manager, state_computer)
    }

    fn stop(&mut self) {
        self.smr.stop();
        debug!("Consensus provider stopped.");
    }
}
<commit_msg>[Consensus] Correct the comment<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    chained_bft::{
        chained_bft_smr::ChainedBftSMR, network::ConsensusNetworkImpl,
        persistent_storage::PersistentStorage,
    },
    consensus_provider::ConsensusProvider,
    counters,
    state_computer::ExecutionProxy,
    state_replication::StateMachineReplication,
    txn_manager::MempoolProxy,
};
use network::validator_network::{ConsensusNetworkEvents, ConsensusNetworkSender};
use nextgen_crypto::ed25519::*;

use crate::{
    chained_bft::{
        chained_bft_smr::ChainedBftSMRConfig, common::Author, persistent_storage::StorageWriteProxy,
    },
    state_synchronizer::{setup_state_synchronizer, StateSynchronizer},
};
use config::config::{ConsensusProposerType::FixedProposer, NodeConfig};
use execution_proto::proto::execution_grpc::ExecutionClient;
use failure::prelude::*;
use logger::prelude::*;
use mempool::proto::mempool_grpc::MempoolClient;
use std::{convert::TryFrom, sync::Arc};
use tokio::runtime;
use types::{
    account_address::AccountAddress, transaction::SignedTransaction,
    validator_signer::ValidatorSigner, validator_verifier::ValidatorVerifier,
};

struct InitialSetup {
    author: Author,
    signer: ValidatorSigner<Ed25519PrivateKey>,
    quorum_size: usize,
    peers: Arc<Vec<Author>>,
    validator: Arc<ValidatorVerifier<Ed25519PublicKey>>,
}

/// Supports the implementation of ConsensusProvider using LibraBFT.
pub struct ChainedBftProvider {
    smr: ChainedBftSMR<Vec<SignedTransaction>, Author>,
    mempool_client: Arc<MempoolClient>,
    execution_client: Arc<ExecutionClient>,
    synchronizer_client: Arc<StateSynchronizer>,
}

impl ChainedBftProvider {
    pub fn new(
        node_config: &mut NodeConfig,
        network_sender: ConsensusNetworkSender,
        network_events: ConsensusNetworkEvents,
        mempool_client: Arc<MempoolClient>,
        execution_client: Arc<ExecutionClient>,
    ) -> Self {
        let runtime = runtime::Builder::new()
            .name_prefix("consensus-")
            .build()
            .expect("Failed to create Tokio runtime!");

        let initial_setup = Self::initialize_setup(node_config);
        let network = ConsensusNetworkImpl::new(
            initial_setup.author,
            network_sender.clone(),
            network_events,
            Arc::clone(&initial_setup.peers),
            Arc::clone(&initial_setup.validator),
        );
        let synchronizer =
            setup_state_synchronizer(network_sender, runtime.executor(), node_config);
        let proposer = {
            if node_config.consensus.get_proposer_type() == FixedProposer {
                vec![Self::choose_leader(&initial_setup)]
            } else {
                initial_setup.validator.get_ordered_account_addresses()
            }
        };
        debug!("[Consensus] My peer: {:?}", initial_setup.author);
        debug!("[Consensus] Chosen proposer: {:?}", proposer);
        let config = ChainedBftSMRConfig::from_node_config(&node_config.consensus);
        let (storage, initial_data) = StorageWriteProxy::start(node_config);
        info!(
            "Starting up the consensus state machine with recovery data - {:?}, {:?}",
            initial_data.state(),
            initial_data.highest_timeout_certificates()
        );
        let smr = ChainedBftSMR::new(
            initial_setup.author,
            initial_setup.quorum_size,
            initial_setup.signer,
            proposer,
            network,
            runtime,
            config,
            storage,
            initial_data,
        );
        Self {
            smr,
            mempool_client,
            execution_client,
            synchronizer_client: Arc::new(synchronizer),
        }
    }

    /// Retrieve the initial "state" for consensus. This function is synchronous and returns after
    /// reading the local persistent store and retrieving the initial state from the executor.
    fn initialize_setup(node_config: &mut NodeConfig) -> InitialSetup {
        // Keeping the initial set of validators in a node config is embarrassing and we should
        // all feel bad about it.
        let peer_id_str = node_config.base.peer_id.clone();
        let author =
            AccountAddress::try_from(peer_id_str).expect("Failed to parse peer id of a validator");
        let private_key = node_config
            .base
            .peer_keypairs
            .take_consensus_private()
            .expect(
            "Failed to move a Consensus private key from a NodeConfig, key absent or already read",
        );

        let signer = ValidatorSigner::new(author, private_key);
        let peers_with_public_keys = node_config.base.trusted_peers.get_trusted_consensus_peers();
        let peers_with_nextgen_public_keys = peers_with_public_keys
            .clone()
            .into_iter()
            .map(|(k, v)| (AccountAddress::clone(&k), v))
            .collect();
        let peers = Arc::new(
            peers_with_public_keys
                .keys()
                .map(AccountAddress::clone)
                .collect(),
        );
        let validator = Arc::new(ValidatorVerifier::new(peers_with_nextgen_public_keys));
        counters::EPOCH_NUM.set(0); // No reconfiguration yet, so it is always zero
        counters::CURRENT_EPOCH_NUM_VALIDATORS.set(validator.len() as i64);
        counters::CURRENT_EPOCH_QUORUM_SIZE.set(validator.quorum_size() as i64);
        debug!("[Consensus]: quorum_size = {:?}", validator.quorum_size());
        InitialSetup {
            author,
            signer,
            quorum_size: validator.quorum_size(),
            peers,
            validator,
        }
    }

    /// Choose a proposer that is going to be the single leader (relevant for a mock fixed proposer
    /// election only).
    fn choose_leader(initial_setup: &InitialSetup) -> Author {
        // As it is just a tmp hack function, pick the max PeerId to be a proposer.
        // TODO: VRF will be integrated later.
        *initial_setup
            .peers
            .iter()
            .max()
            .expect("No trusted peers found!")
    }
}

impl ConsensusProvider for ChainedBftProvider {
    fn start(&mut self) -> Result<()> {
        let txn_manager = Arc::new(MempoolProxy::new(self.mempool_client.clone()));
        let state_computer = Arc::new(ExecutionProxy::new(
            self.execution_client.clone(),
            self.synchronizer_client.clone(),
        ));
        debug!("Starting consensus provider.");
        self.smr.start(txn_manager, state_computer)
    }

    fn stop(&mut self) {
        self.smr.stop();
        debug!("Consensus provider stopped.");
    }
}
<|endoftext|>