<commit_before>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

extern crate ctrlc;

use exec;
use fshelper;
use internal::{error, FdOptions, EXITCODE_SIGINT, MAX_BUFFER_LENGTH};
use output;

use std::process;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::channel;
use std::thread;
use std::time;

use ignore::{self, WalkBuilder};
use ignore::overrides::OverrideBuilder;
use regex::Regex;

/// The receiver thread can either be buffering results or directly streaming to the console.
enum ReceiverMode {
    /// Receiver is still buffering in order to sort the results, if the search finishes fast
    /// enough.
    Buffering,

    /// Receiver is directly printing results to the output.
    Streaming,
}

/// Recursively scan the given search path for files / pathnames matching the pattern.
///
/// If the `--exec` argument was supplied, this will create a thread pool for executing
/// jobs in parallel from a given command line and the discovered paths. Otherwise, each
/// path will simply be written to standard output.
pub fn scan(path_vec: &[PathBuf], pattern: Arc<Regex>, config: Arc<FdOptions>) {
    let mut path_iter = path_vec.iter();
    let first_path_buf = path_iter
        .next()
        .expect("Error: Path vector can not be empty");
    let (tx, rx) = channel();
    let threads = config.threads;

    let mut override_builder = OverrideBuilder::new(first_path_buf.as_path());

    for pattern in &config.exclude_patterns {
        let res = override_builder.add(pattern);
        if res.is_err() {
            error(&format!("Error: malformed exclude pattern '{}'", pattern));
        }
    }
    let overrides = override_builder.build().unwrap_or_else(|_| {
        error("Mismatch in exclude patterns");
    });

    let mut walker = WalkBuilder::new(first_path_buf.as_path());
    walker
        .hidden(config.ignore_hidden)
        .ignore(false)
        .parents(config.read_fdignore || config.read_vcsignore)
        .git_ignore(config.read_vcsignore)
        .git_global(config.read_vcsignore)
        .git_exclude(config.read_vcsignore)
        .overrides(overrides)
        .follow_links(config.follow_links)
        .max_depth(config.max_depth);

    if config.read_fdignore {
        walker.add_custom_ignore_filename(".fdignore");
    }

    for path_entry in path_iter {
        walker.add(path_entry.as_path());
    }

    let parallel_walker = walker.threads(threads).build_parallel();

    let wants_to_quit = Arc::new(AtomicBool::new(false));
    let receiver_wtq = Arc::clone(&wants_to_quit);
    let sender_wtq = Arc::clone(&wants_to_quit);
    if config.ls_colors.is_some() && config.command.is_none() {
        let wq = Arc::clone(&receiver_wtq);
        ctrlc::set_handler(move || {
            wq.store(true, Ordering::Relaxed);
        }).unwrap();
    }

    // Spawn the thread that receives all results through the channel.
    let rx_config = Arc::clone(&config);
    let receiver_thread = thread::spawn(move || {
        // This will be set to `Some` if the `--exec` argument was supplied.
        if let Some(ref cmd) = rx_config.command {
            let shared_rx = Arc::new(Mutex::new(rx));

            let out_perm = Arc::new(Mutex::new(()));

            // TODO: the following line is a workaround to replace the `unsafe` block that was
            // previously used here to avoid the (unnecessary?) cloning of the command. The
            // `unsafe` block caused problems on some platforms (SIGILL instructions on Linux) and
            // therefore had to be removed.
            let cmd = Arc::new(cmd.clone());

            // Each spawned job will store it's thread handle in here.
            let mut handles = Vec::with_capacity(threads);
            for _ in 0..threads {
                let rx = Arc::clone(&shared_rx);
                let cmd = Arc::clone(&cmd);
                let out_perm = Arc::clone(&out_perm);

                // Spawn a job thread that will listen for and execute inputs.
                let handle = thread::spawn(move || exec::job(rx, cmd, out_perm));

                // Push the handle of the spawned thread into the vector for later joining.
                handles.push(handle);
            }

            // Wait for all threads to exit before exiting the program.
            for h in handles {
                h.join().unwrap();
            }
        } else {
            let start = time::Instant::now();

            let mut buffer = vec![];

            // Start in buffering mode
            let mut mode = ReceiverMode::Buffering;

            // Maximum time to wait before we start streaming to the console.
            let max_buffer_time = rx_config
                .max_buffer_time
                .unwrap_or_else(|| time::Duration::from_millis(100));

            for value in rx {
                match mode {
                    ReceiverMode::Buffering => {
                        buffer.push(value);

                        // Have we reached the maximum buffer size or maximum buffering time?
                        if buffer.len() > MAX_BUFFER_LENGTH
                            || time::Instant::now() - start > max_buffer_time
                        {
                            // Flush the buffer
                            for v in &buffer {
                                output::print_entry(v, &rx_config, &receiver_wtq);
                            }
                            buffer.clear();

                            // Start streaming
                            mode = ReceiverMode::Streaming;
                        }
                    }
                    ReceiverMode::Streaming => {
                        output::print_entry(&value, &rx_config, &receiver_wtq);
                    }
                }
            }

            // If we have finished fast enough (faster than max_buffer_time), we haven't streamed
            // anything to the console, yet. In this case, sort the results and print them:
            if !buffer.is_empty() {
                buffer.sort();
                for value in buffer {
                    output::print_entry(&value, &rx_config, &receiver_wtq);
                }
            }
        }
    });

    // Spawn the sender threads.
    parallel_walker.run(|| {
        let config = Arc::clone(&config);
        let pattern = Arc::clone(&pattern);
        let tx_thread = tx.clone();
        let wants_to_quit = Arc::clone(&sender_wtq);

        Box::new(move |entry_o| {
            if wants_to_quit.load(Ordering::Relaxed) {
                return ignore::WalkState::Quit;
            }

            let entry = match entry_o {
                Ok(e) => e,
                Err(_) => return ignore::WalkState::Continue,
            };

            if entry.depth() == 0 {
                return ignore::WalkState::Continue;
            }

            // Filter out unwanted file types.
            if let Some(ref file_types) = config.file_types {
                if let Some(ref entry_type) = entry.file_type() {
                    if (entry_type.is_file() && !file_types.files)
                        || (entry_type.is_dir() && !file_types.directories)
                        || (entry_type.is_symlink() && !file_types.symlinks)
                    {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            let entry_path = entry.path();

            // Filter out unwanted extensions.
            if let Some(ref exts_regex) = config.extensions {
                if let Some(path_str) = entry_path.file_name().map_or(None, |s| s.to_str()) {
                    if !exts_regex.is_match(path_str) {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            let search_str_o = if config.search_full_path {
                match fshelper::path_absolute_form(entry_path) {
                    Ok(path_abs_buf) => Some(path_abs_buf.to_string_lossy().into_owned().into()),
                    Err(_) => error("Error: unable to get full path."),
                }
            } else {
                entry_path.file_name().map(|f| f.to_string_lossy())
            };

            if let Some(search_str) = search_str_o {
                if pattern.is_match(&*search_str) {
                    // TODO: take care of the unwrap call
                    tx_thread.send(entry_path.to_owned()).unwrap()
                }
            }

            ignore::WalkState::Continue
        })
    });

    // Drop the initial sender. If we don't do this, the receiver will block even
    // if all threads have finished, since there is still one sender around.
    drop(tx);

    // Wait for the receiver thread to print out all results.
    receiver_thread.join().unwrap();

    if wants_to_quit.load(Ordering::Relaxed) {
        process::exit(EXITCODE_SIGINT);
    }
}
<commit_msg>Skip fifos, sockets, .. when using --type f/d/l<commit_after>// Copyright (c) 2017 fd developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0>
// or the MIT license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

extern crate ctrlc;

use exec;
use fshelper;
use internal::{error, FdOptions, EXITCODE_SIGINT, MAX_BUFFER_LENGTH};
use output;

use std::process;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::channel;
use std::thread;
use std::time;

use ignore::{self, WalkBuilder};
use ignore::overrides::OverrideBuilder;
use regex::Regex;

/// The receiver thread can either be buffering results or directly streaming to the console.
enum ReceiverMode {
    /// Receiver is still buffering in order to sort the results, if the search finishes fast
    /// enough.
    Buffering,

    /// Receiver is directly printing results to the output.
    Streaming,
}

/// Recursively scan the given search path for files / pathnames matching the pattern.
///
/// If the `--exec` argument was supplied, this will create a thread pool for executing
/// jobs in parallel from a given command line and the discovered paths. Otherwise, each
/// path will simply be written to standard output.
pub fn scan(path_vec: &[PathBuf], pattern: Arc<Regex>, config: Arc<FdOptions>) {
    let mut path_iter = path_vec.iter();
    let first_path_buf = path_iter
        .next()
        .expect("Error: Path vector can not be empty");
    let (tx, rx) = channel();
    let threads = config.threads;

    let mut override_builder = OverrideBuilder::new(first_path_buf.as_path());

    for pattern in &config.exclude_patterns {
        let res = override_builder.add(pattern);
        if res.is_err() {
            error(&format!("Error: malformed exclude pattern '{}'", pattern));
        }
    }
    let overrides = override_builder.build().unwrap_or_else(|_| {
        error("Mismatch in exclude patterns");
    });

    let mut walker = WalkBuilder::new(first_path_buf.as_path());
    walker
        .hidden(config.ignore_hidden)
        .ignore(false)
        .parents(config.read_fdignore || config.read_vcsignore)
        .git_ignore(config.read_vcsignore)
        .git_global(config.read_vcsignore)
        .git_exclude(config.read_vcsignore)
        .overrides(overrides)
        .follow_links(config.follow_links)
        .max_depth(config.max_depth);

    if config.read_fdignore {
        walker.add_custom_ignore_filename(".fdignore");
    }

    for path_entry in path_iter {
        walker.add(path_entry.as_path());
    }

    let parallel_walker = walker.threads(threads).build_parallel();

    let wants_to_quit = Arc::new(AtomicBool::new(false));
    let receiver_wtq = Arc::clone(&wants_to_quit);
    let sender_wtq = Arc::clone(&wants_to_quit);
    if config.ls_colors.is_some() && config.command.is_none() {
        let wq = Arc::clone(&receiver_wtq);
        ctrlc::set_handler(move || {
            wq.store(true, Ordering::Relaxed);
        }).unwrap();
    }

    // Spawn the thread that receives all results through the channel.
    let rx_config = Arc::clone(&config);
    let receiver_thread = thread::spawn(move || {
        // This will be set to `Some` if the `--exec` argument was supplied.
        if let Some(ref cmd) = rx_config.command {
            let shared_rx = Arc::new(Mutex::new(rx));

            let out_perm = Arc::new(Mutex::new(()));

            // TODO: the following line is a workaround to replace the `unsafe` block that was
            // previously used here to avoid the (unnecessary?) cloning of the command. The
            // `unsafe` block caused problems on some platforms (SIGILL instructions on Linux) and
            // therefore had to be removed.
            let cmd = Arc::new(cmd.clone());

            // Each spawned job will store it's thread handle in here.
            let mut handles = Vec::with_capacity(threads);
            for _ in 0..threads {
                let rx = Arc::clone(&shared_rx);
                let cmd = Arc::clone(&cmd);
                let out_perm = Arc::clone(&out_perm);

                // Spawn a job thread that will listen for and execute inputs.
                let handle = thread::spawn(move || exec::job(rx, cmd, out_perm));

                // Push the handle of the spawned thread into the vector for later joining.
                handles.push(handle);
            }

            // Wait for all threads to exit before exiting the program.
            for h in handles {
                h.join().unwrap();
            }
        } else {
            let start = time::Instant::now();

            let mut buffer = vec![];

            // Start in buffering mode
            let mut mode = ReceiverMode::Buffering;

            // Maximum time to wait before we start streaming to the console.
            let max_buffer_time = rx_config
                .max_buffer_time
                .unwrap_or_else(|| time::Duration::from_millis(100));

            for value in rx {
                match mode {
                    ReceiverMode::Buffering => {
                        buffer.push(value);

                        // Have we reached the maximum buffer size or maximum buffering time?
                        if buffer.len() > MAX_BUFFER_LENGTH
                            || time::Instant::now() - start > max_buffer_time
                        {
                            // Flush the buffer
                            for v in &buffer {
                                output::print_entry(v, &rx_config, &receiver_wtq);
                            }
                            buffer.clear();

                            // Start streaming
                            mode = ReceiverMode::Streaming;
                        }
                    }
                    ReceiverMode::Streaming => {
                        output::print_entry(&value, &rx_config, &receiver_wtq);
                    }
                }
            }

            // If we have finished fast enough (faster than max_buffer_time), we haven't streamed
            // anything to the console, yet. In this case, sort the results and print them:
            if !buffer.is_empty() {
                buffer.sort();
                for value in buffer {
                    output::print_entry(&value, &rx_config, &receiver_wtq);
                }
            }
        }
    });

    // Spawn the sender threads.
    parallel_walker.run(|| {
        let config = Arc::clone(&config);
        let pattern = Arc::clone(&pattern);
        let tx_thread = tx.clone();
        let wants_to_quit = Arc::clone(&sender_wtq);

        Box::new(move |entry_o| {
            if wants_to_quit.load(Ordering::Relaxed) {
                return ignore::WalkState::Quit;
            }

            let entry = match entry_o {
                Ok(e) => e,
                Err(_) => return ignore::WalkState::Continue,
            };

            if entry.depth() == 0 {
                return ignore::WalkState::Continue;
            }

            // Filter out unwanted file types.
            if let Some(ref file_types) = config.file_types {
                if let Some(ref entry_type) = entry.file_type() {
                    if (entry_type.is_file() && !file_types.files)
                        || (entry_type.is_dir() && !file_types.directories)
                        || (entry_type.is_symlink() && !file_types.symlinks)
                    {
                        return ignore::WalkState::Continue;
                    } else if !(entry_type.is_file() || entry_type.is_dir()
                        || entry_type.is_symlink())
                    {
                        // This is probably a block device, char device, fifo or socket. Skip it.
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            let entry_path = entry.path();

            // Filter out unwanted extensions.
            if let Some(ref exts_regex) = config.extensions {
                if let Some(path_str) = entry_path.file_name().map_or(None, |s| s.to_str()) {
                    if !exts_regex.is_match(path_str) {
                        return ignore::WalkState::Continue;
                    }
                } else {
                    return ignore::WalkState::Continue;
                }
            }

            let search_str_o = if config.search_full_path {
                match fshelper::path_absolute_form(entry_path) {
                    Ok(path_abs_buf) => Some(path_abs_buf.to_string_lossy().into_owned().into()),
                    Err(_) => error("Error: unable to get full path."),
                }
            } else {
                entry_path.file_name().map(|f| f.to_string_lossy())
            };

            if let Some(search_str) = search_str_o {
                if pattern.is_match(&*search_str) {
                    // TODO: take care of the unwrap call
                    tx_thread.send(entry_path.to_owned()).unwrap()
                }
            }

            ignore::WalkState::Continue
        })
    });

    // Drop the initial sender. If we don't do this, the receiver will block even
    // if all threads have finished, since there is still one sender around.
    drop(tx);

    // Wait for the receiver thread to print out all results.
    receiver_thread.join().unwrap();

    if wants_to_quit.load(Ordering::Relaxed) {
        process::exit(EXITCODE_SIGINT);
    }
}
<|endoftext|>


<commit_before>// In EVM we implement the code execution model as far as the
// transaction level. Blockchain, blocks and accounts should be
// handled by the invoking Ethereum Classic client, as copying the
// world state every time the EVM gets invoked might be expensive.

pub trait Transaction {
    fn gas_price(&self) -> U256;
    fn gas_limit(&self) -> U256;
    fn sender(&self) -> Address;
    fn callee(&self) -> Address;
    fn originator(&self) -> Address;
    fn value(&self) -> U256;
    fn data(&self) -> Option<&[u8]>; // Only for message call transaction.
    fn init(&self) -> Option<&[u8]>; // Only for account initialization procedure.
}

pub struct FakeTransaction {
    gas_price: U256,
    gas_limit: U256,
    sender: Address,
    callee: Address,
    originator: Address,
    value: U256,
    data: Option<Vec<u8>>,
    init: Option<Vec<u8>>,
}

impl FakeTransaction {
    pub fn message_call(value: U256, data: &[u8]) {
        FakeTransaction {
            gas_price: U256::zero(),
            gas_limit: U256::zero(),
            sender: Address::default(),
            callee: Address::default(),
            originator: Address::default(),
            value: value,
            data: Some(data.into()),
            init: None
        }
    }
}

impl Transaction for FakeTransaction {
    type A = FakeAccount;

    fn gas_price(&self) -> U256 {
        self.gas_price
    }

    fn gas_limit(&self) -> U256 {
        self.gas_limit
    }

    fn sender(&self) -> &FakeAccount {
        &self.sender
    }

    fn callee(&self) -> &FakeAccount {
        &self.callee
    }

    fn callee_mut(&mut self) -> &mut FakeAccount {
        &mut self.callee
    }

    fn originator(&self) -> &FakeAccount {
        if self.originator.is_none() {
            self.sender()
        } else {
            self.originator.as_ref().unwrap()
        }
    }

    fn value(&self) -> U256 {
        self.value
    }

    fn data(&self) -> Option<&[u8]> {
        self.data.map(|s| s.into())
    }

    fn init(&self) -> Option<&[u8]> {
        self.init.map(|s| s.into())
    }
}
<commit_msg>Problem: Transaction uses U256 instead of Gas Solution: U256 -> Gas<commit_after>// In EVM we implement the code execution model as far as the
// transaction level. Blockchain, blocks and accounts should be
// handled by the invoking Ethereum Classic client, as copying the
// world state every time the EVM gets invoked might be expensive.

use utils::u256::U256;
use utils::address::Address;
use utils::gas::Gas;

pub trait Transaction {
    fn gas_price(&self) -> Gas;
    fn gas_limit(&self) -> Gas;
    fn sender(&self) -> Address;
    fn callee(&self) -> Address;
    fn originator(&self) -> Address;
    fn value(&self) -> U256;
    fn data(&self) -> Option<&[u8]>; // Only for message call transaction.
    fn init(&self) -> Option<&[u8]>; // Only for account initialization procedure.
}

pub struct VectorTransaction {
    gas_price: Gas,
    gas_limit: Gas,
    sender: Address,
    callee: Address,
    originator: Address,
    value: U256,
    data: Option<Vec<u8>>,
    init: Option<Vec<u8>>,
}

impl VectorTransaction {
    pub fn message_call(value: U256, data: &[u8], gas_limit: Gas) -> VectorTransaction {
        VectorTransaction {
            gas_price: Gas::zero(),
            gas_limit: gas_limit,
            sender: Address::default(),
            callee: Address::default(),
            originator: Address::default(),
            value: value,
            data: Some(data.into()),
            init: None
        }
    }
}

impl Transaction for VectorTransaction {
    fn gas_price(&self) -> Gas {
        self.gas_price
    }

    fn gas_limit(&self) -> Gas {
        self.gas_limit
    }

    fn sender(&self) -> Address {
        self.sender
    }

    fn callee(&self) -> Address {
        self.callee
    }

    fn originator(&self) -> Address {
        self.originator
    }

    fn value(&self) -> U256 {
        self.value
    }

    fn data(&self) -> Option<&[u8]> {
        if self.data.is_some() {
            Some(self.data.as_ref().unwrap().as_ref())
        } else {
            None
        }
    }

    fn init(&self) -> Option<&[u8]> {
        if self.init.is_some() {
            Some(self.init.as_ref().unwrap().as_ref())
        } else {
            None
        }
    }
}
<|endoftext|>


<commit_before>use crate::beacon_node_fallback::{BeaconNodeFallback, RequireSynced};
use crate::validator_store::{DoppelgangerStatus, ValidatorStore};
use bls::PublicKeyBytes;
use environment::RuntimeContext;
use parking_lot::RwLock;
use slog::{debug, error, info};
use slot_clock::SlotClock;
use std::collections::HashMap;
use std::hash::Hash;
use std::ops::Deref;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::time::{sleep, Duration};
use types::{
    Address, ChainSpec, EthSpec, ProposerPreparationData, SignedValidatorRegistrationData,
    ValidatorRegistrationData,
};

/// Number of epochs before the Bellatrix hard fork to begin posting proposer preparations.
const PROPOSER_PREPARATION_LOOKAHEAD_EPOCHS: u64 = 2;

/// Number of epochs to wait before re-submitting validator registration.
const EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION: u64 = 1;

/// The number of validator registrations to include per request to the beacon node.
const VALIDATOR_REGISTRATION_BATCH_SIZE: usize = 500;

/// Builds an `PreparationService`.
pub struct PreparationServiceBuilder<T: SlotClock + 'static, E: EthSpec> {
    validator_store: Option<Arc<ValidatorStore<T, E>>>,
    slot_clock: Option<T>,
    beacon_nodes: Option<Arc<BeaconNodeFallback<T, E>>>,
    context: Option<RuntimeContext<E>>,
    builder_registration_timestamp_override: Option<u64>,
}

impl<T: SlotClock + 'static, E: EthSpec> PreparationServiceBuilder<T, E> {
    pub fn new() -> Self {
        Self {
            validator_store: None,
            slot_clock: None,
            beacon_nodes: None,
            context: None,
            builder_registration_timestamp_override: None,
        }
    }

    pub fn validator_store(mut self, store: Arc<ValidatorStore<T, E>>) -> Self {
        self.validator_store = Some(store);
        self
    }

    pub fn slot_clock(mut self, slot_clock: T) -> Self {
        self.slot_clock = Some(slot_clock);
        self
    }

    pub fn beacon_nodes(mut self, beacon_nodes: Arc<BeaconNodeFallback<T, E>>) -> Self {
        self.beacon_nodes = Some(beacon_nodes);
        self
    }

    pub fn runtime_context(mut self, context: RuntimeContext<E>) -> Self {
        self.context = Some(context);
        self
    }

    pub fn builder_registration_timestamp_override(
        mut self,
        builder_registration_timestamp_override: Option<u64>,
    ) -> Self {
        self.builder_registration_timestamp_override = builder_registration_timestamp_override;
        self
    }

    pub fn build(self) -> Result<PreparationService<T, E>, String> {
        Ok(PreparationService {
            inner: Arc::new(Inner {
                validator_store: self
                    .validator_store
                    .ok_or("Cannot build PreparationService without validator_store")?,
                slot_clock: self
                    .slot_clock
                    .ok_or("Cannot build PreparationService without slot_clock")?,
                beacon_nodes: self
                    .beacon_nodes
                    .ok_or("Cannot build PreparationService without beacon_nodes")?,
                context: self
                    .context
                    .ok_or("Cannot build PreparationService without runtime_context")?,
                builder_registration_timestamp_override: self
                    .builder_registration_timestamp_override,
                validator_registration_cache: RwLock::new(HashMap::new()),
            }),
        })
    }
}

/// Helper to minimise `Arc` usage.
pub struct Inner<T, E: EthSpec> {
    validator_store: Arc<ValidatorStore<T, E>>,
    slot_clock: T,
    beacon_nodes: Arc<BeaconNodeFallback<T, E>>,
    context: RuntimeContext<E>,
    builder_registration_timestamp_override: Option<u64>,
    // Used to track unpublished validator registration changes.
    validator_registration_cache:
        RwLock<HashMap<ValidatorRegistrationKey, SignedValidatorRegistrationData>>,
}

#[derive(Hash, Eq, PartialEq, Debug, Clone)]
pub struct ValidatorRegistrationKey {
    pub fee_recipient: Address,
    pub gas_limit: u64,
    pub pubkey: PublicKeyBytes,
}

impl From<ValidatorRegistrationData> for ValidatorRegistrationKey {
    fn from(data: ValidatorRegistrationData) -> Self {
        let ValidatorRegistrationData {
            fee_recipient,
            gas_limit,
            timestamp: _,
            pubkey,
        } = data;
        Self {
            fee_recipient,
            gas_limit,
            pubkey,
        }
    }
}

/// Attempts to produce proposer preparations for all known validators at the beginning of each epoch.
pub struct PreparationService<T, E: EthSpec> {
    inner: Arc<Inner<T, E>>,
}

impl<T, E: EthSpec> Clone for PreparationService<T, E> {
    fn clone(&self) -> Self {
        Self {
            inner: self.inner.clone(),
        }
    }
}

impl<T, E: EthSpec> Deref for PreparationService<T, E> {
    type Target = Inner<T, E>;

    fn deref(&self) -> &Self::Target {
        self.inner.deref()
    }
}

impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
    pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> {
        self.clone().start_validator_registration_service(spec)?;
        self.start_proposer_prepare_service(spec)
    }

    /// Starts the service which periodically produces proposer preparations.
    pub fn start_proposer_prepare_service(self, spec: &ChainSpec) -> Result<(), String> {
        let log = self.context.log().clone();

        let slot_duration = Duration::from_secs(spec.seconds_per_slot);
        info!(
            log,
            "Proposer preparation service started";
        );

        let executor = self.context.executor.clone();
        let spec = spec.clone();

        let interval_fut = async move {
            loop {
                if self.should_publish_at_current_slot(&spec) {
                    // Poll the endpoint immediately to ensure fee recipients are received.
                    self.prepare_proposers_and_publish(&spec)
                        .await
                        .map_err(|e| {
                            error!(
                                log,
                                "Error during proposer preparation";
                                "error" => ?e,
                            )
                        })
                        .unwrap_or(());
                }

                if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
                    sleep(duration_to_next_slot).await;
                } else {
                    error!(log, "Failed to read slot clock");
                    // If we can't read the slot clock, just wait another slot.
                    sleep(slot_duration).await;
                }
            }
        };

        executor.spawn(interval_fut, "preparation_service");
        Ok(())
    }

    /// Starts the service which periodically sends connected beacon nodes validator registration information.
    pub fn start_validator_registration_service(self, spec: &ChainSpec) -> Result<(), String> {
        let log = self.context.log().clone();

        info!(
            log,
            "Validator registration service started";
        );

        let spec = spec.clone();
        let slot_duration = Duration::from_secs(spec.seconds_per_slot);

        let executor = self.context.executor.clone();

        let validator_registration_fut = async move {
            loop {
                // Poll the endpoint immediately to ensure fee recipients are received.
                if let Err(e) = self.register_validators().await {
                    error!(log,"Error during validator registration";"error" => ?e);
                }

                // Wait one slot if the register validator request fails or if we should not publish at the current slot.
                if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
                    sleep(duration_to_next_slot).await;
                } else {
                    error!(log, "Failed to read slot clock");
                    // If we can't read the slot clock, just wait another slot.
                    sleep(slot_duration).await;
                }
            }
        };
        executor.spawn(validator_registration_fut, "validator_registration_service");
        Ok(())
    }

    /// Return `true` if the current slot is close to or past the Bellatrix fork epoch.
    ///
    /// This avoids spamming the BN with preparations before the Bellatrix fork epoch, which may
    /// cause errors if it doesn't support the preparation API.
    fn should_publish_at_current_slot(&self, spec: &ChainSpec) -> bool {
        let current_epoch = self
            .slot_clock
            .now()
            .map_or(E::genesis_epoch(), |slot| slot.epoch(E::slots_per_epoch()));
        spec.bellatrix_fork_epoch.map_or(false, |fork_epoch| {
            current_epoch + PROPOSER_PREPARATION_LOOKAHEAD_EPOCHS >= fork_epoch
        })
    }

    /// Prepare proposer preparations and send to beacon node
    async fn prepare_proposers_and_publish(&self, spec: &ChainSpec) -> Result<(), String> {
        let preparation_data = self.collect_preparation_data(spec);
        if !preparation_data.is_empty() {
            self.publish_preparation_data(preparation_data).await?;
        }

        Ok(())
    }

    fn collect_preparation_data(&self, spec: &ChainSpec) -> Vec<ProposerPreparationData> {
        let log = self.context.log();
        self.collect_proposal_data(|pubkey, proposal_data| {
            if let Some(fee_recipient) = proposal_data.fee_recipient {
                Some(ProposerPreparationData {
                    // Ignore fee recipients for keys without indices, they are inactive.
                    validator_index: proposal_data.validator_index?,
                    fee_recipient,
                })
            } else {
                if spec.bellatrix_fork_epoch.is_some() {
                    error!(
                        log,
                        "Validator is missing fee recipient";
                        "msg" => "update validator_definitions.yml",
                        "pubkey" => ?pubkey
                    );
                }
                None
            }
        })
    }

    fn collect_validator_registration_keys(&self) -> Vec<ValidatorRegistrationKey> {
        self.collect_proposal_data(|pubkey, proposal_data| {
            // We don't log for missing fee recipients here because this will be logged more
            // frequently in `collect_preparation_data`.
            proposal_data.fee_recipient.and_then(|fee_recipient| {
                proposal_data
                    .builder_proposals
                    .then(|| ValidatorRegistrationKey {
                        fee_recipient,
                        gas_limit: proposal_data.gas_limit,
                        pubkey,
                    })
            })
        })
    }

    fn collect_proposal_data<G, U>(&self, map_fn: G) -> Vec<U>
    where
        G: Fn(PublicKeyBytes, ProposalData) -> Option<U>,
    {
        let all_pubkeys: Vec<_> = self
            .validator_store
            .voting_pubkeys(DoppelgangerStatus::ignored);

        all_pubkeys
            .into_iter()
            .filter_map(|pubkey| {
                let proposal_data = self.validator_store.proposal_data(&pubkey)?;
                map_fn(pubkey, proposal_data)
            })
            .collect()
    }

    async fn publish_preparation_data(
        &self,
        preparation_data: Vec<ProposerPreparationData>,
    ) -> Result<(), String> {
        let log = self.context.log();

        // Post the proposer preparations to the BN.
        let preparation_data_len = preparation_data.len();
        let preparation_entries = preparation_data.as_slice();
        match self
            .beacon_nodes
            .first_success(RequireSynced::Yes, |beacon_node| async move {
                beacon_node
                    .post_validator_prepare_beacon_proposer(preparation_entries)
                    .await
            })
            .await
        {
            Ok(()) => debug!(
                log,
                "Published proposer preparation";
                "count" => preparation_data_len,
            ),
            Err(e) => error!(
                log,
                "Unable to publish proposer preparation";
                "error" => %e,
            ),
        }
        Ok(())
    }

    /// Register validators with builders, used in the blinded block proposal flow.
    async fn register_validators(&self) -> Result<(), String> {
        let registration_keys = self.collect_validator_registration_keys();

        let mut changed_keys = vec![];

        // Need to scope this so the read lock is not held across an await point (I don't know why
        // but the explicit `drop` is not enough).
        {
            let guard = self.validator_registration_cache.read();
            for key in registration_keys.iter() {
                if !guard.contains_key(key) {
                    changed_keys.push(key.clone());
                }
            }
            drop(guard);
        }

        // Check if any have changed or it's been `EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION`.
        if let Some(slot) = self.slot_clock.now() {
            if slot % (E::slots_per_epoch() * EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION) == 0 {
                self.publish_validator_registration_data(registration_keys)
                    .await?;
            } else if !changed_keys.is_empty() {
                self.publish_validator_registration_data(changed_keys)
                    .await?;
            }
        }

        Ok(())
    }

    async fn publish_validator_registration_data(
        &self,
        registration_keys: Vec<ValidatorRegistrationKey>,
    ) -> Result<(), String> {
        let log = self.context.log();

        let registration_data_len = registration_keys.len();
        let mut signed = Vec::with_capacity(registration_data_len);

        for key in registration_keys {
            let cached_registration_opt =
                self.validator_registration_cache.read().get(&key).cloned();

            let signed_data = if let Some(signed_data) = cached_registration_opt {
                signed_data
            } else {
                let timestamp =
                    if let Some(timestamp) = self.builder_registration_timestamp_override {
                        timestamp
                    } else {
                        SystemTime::now()
                            .duration_since(UNIX_EPOCH)
                            .map_err(|e| format!("{e:?}"))?
                            .as_secs()
                    };

                let ValidatorRegistrationKey {
                    fee_recipient,
                    gas_limit,
                    pubkey,
                } = key.clone();

                let signed_data = match self
                    .validator_store
                    .sign_validator_registration_data(ValidatorRegistrationData {
                        fee_recipient,
                        gas_limit,
                        timestamp,
                        pubkey,
                    })
                    .await
                {
                    Ok(data) => data,
                    Err(e) => {
                        error!(log, "Unable to sign validator registration data"; "error" => ?e, "pubkey" => ?pubkey);
                        continue;
                    }
                };

                self.validator_registration_cache
                    .write()
                    .insert(key, signed_data.clone());

                signed_data
            };
            signed.push(signed_data);
        }

        if !signed.is_empty() {
            for batch in signed.chunks(VALIDATOR_REGISTRATION_BATCH_SIZE) {
                match self
                    .beacon_nodes
                    .first_success(RequireSynced::Yes, |beacon_node| async move {
                        beacon_node.post_validator_register_validator(batch).await
                    })
                    .await
                {
                    Ok(()) => info!(
                        log,
                        "Published validator registrations to the builder network";
                        "count" => registration_data_len,
                    ),
                    Err(e) => error!(
                        log,
                        "Unable to publish validator registrations to the builder network";
                        "error" => %e,
                    ),
                }
            }
        }
        Ok(())
    }
}

/// A helper struct, used for passing data from the validator store to services.
pub struct ProposalData {
    pub(crate) validator_index: Option<u64>,
    pub(crate) fee_recipient: Option<Address>,
    pub(crate) gas_limit: u64,
    pub(crate) builder_proposals: bool,
}
<commit_msg>Don't attempt to register validators that are pre-activation (#3441)<commit_after>use crate::beacon_node_fallback::{BeaconNodeFallback, RequireSynced};
use crate::validator_store::{DoppelgangerStatus, ValidatorStore};
use bls::PublicKeyBytes;
use environment::RuntimeContext;
use parking_lot::RwLock;
use slog::{debug, error, info};
use slot_clock::SlotClock;
use std::collections::HashMap;
use std::hash::Hash;
use std::ops::Deref;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::time::{sleep, Duration};
use types::{
    Address, ChainSpec, EthSpec, ProposerPreparationData, SignedValidatorRegistrationData,
    ValidatorRegistrationData,
};

/// Number of epochs before the Bellatrix hard fork to begin posting proposer preparations.
const PROPOSER_PREPARATION_LOOKAHEAD_EPOCHS: u64 = 2;

/// Number of epochs to wait before re-submitting validator registration.
const EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION: u64 = 1;

/// The number of validator registrations to include per request to the beacon node.
const VALIDATOR_REGISTRATION_BATCH_SIZE: usize = 500;

/// Builds an `PreparationService`.
pub struct PreparationServiceBuilder<T: SlotClock + 'static, E: EthSpec> {
    validator_store: Option<Arc<ValidatorStore<T, E>>>,
    slot_clock: Option<T>,
    beacon_nodes: Option<Arc<BeaconNodeFallback<T, E>>>,
    context: Option<RuntimeContext<E>>,
    builder_registration_timestamp_override: Option<u64>,
}

impl<T: SlotClock + 'static, E: EthSpec> PreparationServiceBuilder<T, E> {
    pub fn new() -> Self {
        Self {
            validator_store: None,
            slot_clock: None,
            beacon_nodes: None,
            context: None,
            builder_registration_timestamp_override: None,
        }
    }

    pub fn validator_store(mut self, store: Arc<ValidatorStore<T, E>>) -> Self {
        self.validator_store = Some(store);
        self
    }

    pub fn slot_clock(mut self, slot_clock: T) -> Self {
        self.slot_clock = Some(slot_clock);
        self
    }

    pub fn beacon_nodes(mut self, beacon_nodes: Arc<BeaconNodeFallback<T, E>>) -> Self {
        self.beacon_nodes = Some(beacon_nodes);
        self
    }

    pub fn runtime_context(mut self, context: RuntimeContext<E>) -> Self {
        self.context = Some(context);
        self
    }

    pub fn builder_registration_timestamp_override(
        mut self,
        builder_registration_timestamp_override: Option<u64>,
    ) -> Self {
        self.builder_registration_timestamp_override = builder_registration_timestamp_override;
        self
    }

    pub fn build(self) -> Result<PreparationService<T, E>, String> {
        Ok(PreparationService {
            inner: Arc::new(Inner {
                validator_store: self
                    .validator_store
                    .ok_or("Cannot build PreparationService without validator_store")?,
                slot_clock: self
                    .slot_clock
                    .ok_or("Cannot build PreparationService without slot_clock")?,
                beacon_nodes: self
                    .beacon_nodes
                    .ok_or("Cannot build PreparationService without beacon_nodes")?,
                context: self
                    .context
                    .ok_or("Cannot build PreparationService without runtime_context")?,
                builder_registration_timestamp_override: self
                    .builder_registration_timestamp_override,
                validator_registration_cache: RwLock::new(HashMap::new()),
            }),
        })
    }
}

/// Helper to minimise `Arc` usage.
pub struct Inner<T, E: EthSpec> {
    validator_store: Arc<ValidatorStore<T, E>>,
    slot_clock: T,
    beacon_nodes: Arc<BeaconNodeFallback<T, E>>,
    context: RuntimeContext<E>,
    builder_registration_timestamp_override: Option<u64>,
    // Used to track unpublished validator registration changes.
    validator_registration_cache:
        RwLock<HashMap<ValidatorRegistrationKey, SignedValidatorRegistrationData>>,
}

#[derive(Hash, Eq, PartialEq, Debug, Clone)]
pub struct ValidatorRegistrationKey {
    pub fee_recipient: Address,
    pub gas_limit: u64,
    pub pubkey: PublicKeyBytes,
}

impl From<ValidatorRegistrationData> for ValidatorRegistrationKey {
    fn from(data: ValidatorRegistrationData) -> Self {
        let ValidatorRegistrationData {
            fee_recipient,
            gas_limit,
            timestamp: _,
            pubkey,
        } = data;
        Self {
            fee_recipient,
            gas_limit,
            pubkey,
        }
    }
}

/// Attempts to produce proposer preparations for all known validators at the beginning of each epoch.
pub struct PreparationService<T, E: EthSpec> {
    inner: Arc<Inner<T, E>>,
}

impl<T, E: EthSpec> Clone for PreparationService<T, E> {
    fn clone(&self) -> Self {
        Self {
            inner: self.inner.clone(),
        }
    }
}

impl<T, E: EthSpec> Deref for PreparationService<T, E> {
    type Target = Inner<T, E>;

    fn deref(&self) -> &Self::Target {
        self.inner.deref()
    }
}

impl<T: SlotClock + 'static, E: EthSpec> PreparationService<T, E> {
    pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> {
        self.clone().start_validator_registration_service(spec)?;
        self.start_proposer_prepare_service(spec)
    }

    /// Starts the service which periodically produces proposer preparations.
    pub fn start_proposer_prepare_service(self, spec: &ChainSpec) -> Result<(), String> {
        let log = self.context.log().clone();

        let slot_duration = Duration::from_secs(spec.seconds_per_slot);
        info!(
            log,
            "Proposer preparation service started";
        );

        let executor = self.context.executor.clone();
        let spec = spec.clone();

        let interval_fut = async move {
            loop {
                if self.should_publish_at_current_slot(&spec) {
                    // Poll the endpoint immediately to ensure fee recipients are received.
                    self.prepare_proposers_and_publish(&spec)
                        .await
                        .map_err(|e| {
                            error!(
                                log,
                                "Error during proposer preparation";
                                "error" => ?e,
                            )
                        })
                        .unwrap_or(());
                }

                if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
                    sleep(duration_to_next_slot).await;
                } else {
                    error!(log, "Failed to read slot clock");
                    // If we can't read the slot clock, just wait another slot.
                    sleep(slot_duration).await;
                }
            }
        };

        executor.spawn(interval_fut, "preparation_service");
        Ok(())
    }

    /// Starts the service which periodically sends connected beacon nodes validator registration information.
    pub fn start_validator_registration_service(self, spec: &ChainSpec) -> Result<(), String> {
        let log = self.context.log().clone();

        info!(
            log,
            "Validator registration service started";
        );

        let spec = spec.clone();
        let slot_duration = Duration::from_secs(spec.seconds_per_slot);

        let executor = self.context.executor.clone();

        let validator_registration_fut = async move {
            loop {
                // Poll the endpoint immediately to ensure fee recipients are received.
                if let Err(e) = self.register_validators().await {
                    error!(log,"Error during validator registration";"error" => ?e);
                }

                // Wait one slot if the register validator request fails or if we should not publish at the current slot.
                if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() {
                    sleep(duration_to_next_slot).await;
                } else {
                    error!(log, "Failed to read slot clock");
                    // If we can't read the slot clock, just wait another slot.
                    sleep(slot_duration).await;
                }
            }
        };
        executor.spawn(validator_registration_fut, "validator_registration_service");
        Ok(())
    }

    /// Return `true` if the current slot is close to or past the Bellatrix fork epoch.
    ///
    /// This avoids spamming the BN with preparations before the Bellatrix fork epoch, which may
    /// cause errors if it doesn't support the preparation API.
    fn should_publish_at_current_slot(&self, spec: &ChainSpec) -> bool {
        let current_epoch = self
            .slot_clock
            .now()
            .map_or(E::genesis_epoch(), |slot| slot.epoch(E::slots_per_epoch()));
        spec.bellatrix_fork_epoch.map_or(false, |fork_epoch| {
            current_epoch + PROPOSER_PREPARATION_LOOKAHEAD_EPOCHS >= fork_epoch
        })
    }

    /// Prepare proposer preparations and send to beacon node
    async fn prepare_proposers_and_publish(&self, spec: &ChainSpec) -> Result<(), String> {
        let preparation_data = self.collect_preparation_data(spec);
        if !preparation_data.is_empty() {
            self.publish_preparation_data(preparation_data).await?;
        }

        Ok(())
    }

    fn collect_preparation_data(&self, spec: &ChainSpec) -> Vec<ProposerPreparationData> {
        let log = self.context.log();
        self.collect_proposal_data(|pubkey, proposal_data| {
            if let Some(fee_recipient) = proposal_data.fee_recipient {
                Some(ProposerPreparationData {
                    // Ignore fee recipients for keys without indices, they are inactive.
                    validator_index: proposal_data.validator_index?,
                    fee_recipient,
                })
            } else {
                if spec.bellatrix_fork_epoch.is_some() {
                    error!(
                        log,
                        "Validator is missing fee recipient";
                        "msg" => "update validator_definitions.yml",
                        "pubkey" => ?pubkey
                    );
                }
                None
            }
        })
    }

    fn collect_validator_registration_keys(&self) -> Vec<ValidatorRegistrationKey> {
        self.collect_proposal_data(|pubkey, proposal_data| {
            // Ignore fee recipients for keys without indices, they are inactive.
            proposal_data.validator_index?;

            // We don't log for missing fee recipients here because this will be logged more
            // frequently in `collect_preparation_data`.
            proposal_data.fee_recipient.and_then(|fee_recipient| {
                proposal_data
                    .builder_proposals
                    .then(|| ValidatorRegistrationKey {
                        fee_recipient,
                        gas_limit: proposal_data.gas_limit,
                        pubkey,
                    })
            })
        })
    }

    fn collect_proposal_data<G, U>(&self, map_fn: G) -> Vec<U>
    where
        G: Fn(PublicKeyBytes, ProposalData) -> Option<U>,
    {
        let all_pubkeys: Vec<_> = self
            .validator_store
            .voting_pubkeys(DoppelgangerStatus::ignored);

        all_pubkeys
            .into_iter()
            .filter_map(|pubkey| {
                let proposal_data = self.validator_store.proposal_data(&pubkey)?;
                map_fn(pubkey, proposal_data)
            })
            .collect()
    }

    async fn publish_preparation_data(
        &self,
        preparation_data: Vec<ProposerPreparationData>,
    ) -> Result<(), String> {
        let log = self.context.log();

        // Post the proposer preparations to the BN.
        let preparation_data_len = preparation_data.len();
        let preparation_entries = preparation_data.as_slice();
        match self
            .beacon_nodes
            .first_success(RequireSynced::Yes, |beacon_node| async move {
                beacon_node
                    .post_validator_prepare_beacon_proposer(preparation_entries)
                    .await
            })
            .await
        {
            Ok(()) => debug!(
                log,
                "Published proposer preparation";
                "count" => preparation_data_len,
            ),
            Err(e) => error!(
                log,
                "Unable to publish proposer preparation";
                "error" => %e,
            ),
        }
        Ok(())
    }

    /// Register validators with builders, used in the blinded block proposal flow.
    async fn register_validators(&self) -> Result<(), String> {
        let registration_keys = self.collect_validator_registration_keys();

        let mut changed_keys = vec![];

        // Need to scope this so the read lock is not held across an await point (I don't know why
        // but the explicit `drop` is not enough).
        {
            let guard = self.validator_registration_cache.read();
            for key in registration_keys.iter() {
                if !guard.contains_key(key) {
                    changed_keys.push(key.clone());
                }
            }
            drop(guard);
        }

        // Check if any have changed or it's been `EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION`.
        if let Some(slot) = self.slot_clock.now() {
            if slot % (E::slots_per_epoch() * EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION) == 0 {
                self.publish_validator_registration_data(registration_keys)
                    .await?;
            } else if !changed_keys.is_empty() {
                self.publish_validator_registration_data(changed_keys)
                    .await?;
            }
        }

        Ok(())
    }

    async fn publish_validator_registration_data(
        &self,
        registration_keys: Vec<ValidatorRegistrationKey>,
    ) -> Result<(), String> {
        let log = self.context.log();

        let registration_data_len = registration_keys.len();
        let mut signed = Vec::with_capacity(registration_data_len);

        for key in registration_keys {
            let cached_registration_opt =
                self.validator_registration_cache.read().get(&key).cloned();

            let signed_data = if let Some(signed_data) = cached_registration_opt {
                signed_data
            } else {
                let timestamp =
                    if let Some(timestamp) = self.builder_registration_timestamp_override {
                        timestamp
                    } else {
                        SystemTime::now()
                            .duration_since(UNIX_EPOCH)
                            .map_err(|e| format!("{e:?}"))?
                            .as_secs()
                    };

                let ValidatorRegistrationKey {
                    fee_recipient,
                    gas_limit,
                    pubkey,
                } = key.clone();

                let signed_data = match self
                    .validator_store
                    .sign_validator_registration_data(ValidatorRegistrationData {
                        fee_recipient,
                        gas_limit,
                        timestamp,
                        pubkey,
                    })
                    .await
                {
                    Ok(data) => data,
                    Err(e) => {
                        error!(log, "Unable to sign validator registration data"; "error" => ?e, "pubkey" => ?pubkey);
                        continue;
                    }
                };

                self.validator_registration_cache
                    .write()
                    .insert(key, signed_data.clone());

                signed_data
            };
            signed.push(signed_data);
        }

        if !signed.is_empty() {
            for batch in signed.chunks(VALIDATOR_REGISTRATION_BATCH_SIZE) {
                match self
                    .beacon_nodes
                    .first_success(RequireSynced::Yes, |beacon_node| async move {
                        beacon_node.post_validator_register_validator(batch).await
                    })
                    .await
                {
                    Ok(()) => info!(
                        log,
                        "Published validator registrations to the builder network";
                        "count" => registration_data_len,
                    ),
                    Err(e) => error!(
                        log,
                        "Unable to publish validator registrations to the builder network";
                        "error" => %e,
                    ),
                }
            }
        }
        Ok(())
    }
}

/// A helper struct, used for passing data from the validator store to services.
pub struct ProposalData {
    pub(crate) validator_index: Option<u64>,
    pub(crate) fee_recipient: Option<Address>,
    pub(crate) gas_limit: u64,
    pub(crate) builder_proposals: bool,
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate clap;
extern crate sputnikvm;
extern crate bigint;
extern crate serde_json;
extern crate gethrpc;

use serde_json::{Value};
use std::process;
use std::fs::File;
use std::path::Path;
use std::io::{BufReader};
use std::str::FromStr;
use std::collections::{HashMap, HashSet};

use sputnikvm::{Gas, Address};
use bigint::{U256, M256, read_hex};
use sputnikvm::vm::{BlockHeader, Context, SeqTransactionVM, Transaction, VM, Log, Patch, AccountCommitment, Account, FRONTIER_PATCH};
use sputnikvm::vm::errors::RequireError;
use gethrpc::{regression, GethRPCClient, RPCCall, RPCBlock, RPCTransaction, RPCLog};

fn from_rpc_block(block: &RPCBlock) -> BlockHeader {
    BlockHeader {
        coinbase: Address::from_str(&block.miner).unwrap(),
        timestamp: M256::from_str(&block.timestamp).unwrap(),
        number: M256::from_str(&block.number).unwrap(),
        difficulty: M256::from_str(&block.difficulty).unwrap(),
        gas_limit: Gas::from_str(&block.gasLimit).unwrap(),
    }
}

fn from_rpc_transaction(transaction: &RPCTransaction) -> Transaction {
    if transaction.to.is_empty() {
        Transaction::ContractCreation {
            caller: Address::from_str(&transaction.from).unwrap(),
            value: U256::from_str(&transaction.value).unwrap(),
            gas_limit: Gas::from_str(&transaction.gas).unwrap(),
            gas_price: Gas::from_str(&transaction.gasPrice).unwrap(),
            init: read_hex(&transaction.input).unwrap(),
        }
    } else {
        Transaction::MessageCall {
            caller: Address::from_str(&transaction.from).unwrap(),
            address: Address::from_str(&transaction.to).unwrap(),
            value: U256::from_str(&transaction.value).unwrap(),
            gas_limit: Gas::from_str(&transaction.gas).unwrap(),
            gas_price: Gas::from_str(&transaction.gasPrice).unwrap(),
            data: read_hex(&transaction.input).unwrap(),
        }
    }
}

fn from_rpc_log(log: &RPCLog) -> Log {
    let mut topics: Vec<M256> = Vec::new();
    for topic in &log.topics {
        topics.push(M256::from_str(&topic).unwrap());
    }
    Log {
        address: Address::from_str(&log.address).unwrap(),
        data: read_hex(&log.data).unwrap(),
        topics: topics,
    }
}

/// Accounts in this list should always be considered exists.
fn irregular_addresses(number: usize) -> HashSet<Address> {
    let config: HashMap<Address, usize> = {
        let mut config = HashMap::new();
        config.insert(Address::from_str("0x0000000000000000000000000000000000000000").unwrap(), 0);
        config.insert(Address::from_str("0x0000000000000000000000000000000000000001").unwrap(), 0);
        config.insert(Address::from_str("0x0000000000000000000000000000000000000002").unwrap(), 505137);
        config.insert(Address::from_str("0x0000000000000000000000000000000000000003").unwrap(), 0);
        config.insert(Address::from_str("0x0000000000000000000000000000000000000004").unwrap(), 0);

        config
    };

    let mut result: HashSet<Address> = HashSet::new();
    for (key, value) in config {
        if number > value {
            result.insert(key);
        }
    }
    result
}

fn handle_fire(client: &mut GethRPCClient, vm: &mut SeqTransactionVM, last_block_number: &str,
               irregular_addresses: &HashSet<Address>) {
    loop {
        match vm.fire() {
            Ok(()) => {
                println!("VM exited with {:?}.", vm.status());
                break;
            },
            Err(RequireError::Account(address)) => {
                println!("Feeding VM account at 0x{:x} ...", address);
                let nonce = M256::from_str(&client.get_transaction_count(&format!("0x{:x}", address),
                                                                         &last_block_number)).unwrap();
                let balance = U256::from_str(&client.get_balance(&format!("0x{:x}", address),
                                                                 &last_block_number)).unwrap();
                let code = read_hex(&client.get_code(&format!("0x{:x}", address),
                                                     &last_block_number)).unwrap();
                if nonce == M256::zero() && balance == U256::zero() && code.len() == 0 &&
                    !irregular_addresses.contains(&address)
                {
                    vm.commit_account(AccountCommitment::Nonexist(address)).unwrap();
                } else {
                    vm.commit_account(AccountCommitment::Full {
                        nonce: nonce,
                        address: address,
                        balance: balance,
                        code: code,
                    }).unwrap();
                }
            },
            Err(RequireError::AccountStorage(address, index)) => {
                println!("Feeding VM account storage at 0x{:x} with index 0x{:x} ...", address, index);
                let value = M256::from_str(&client.get_storage_at(&format!("0x{:x}", address),
                                                                  &format!("0x{:x}", index),
                                                                  &last_block_number)).unwrap();
                vm.commit_account(AccountCommitment::Storage {
                    address: address,
                    index: index,
                    value: value,
                }).unwrap();
            },
            Err(RequireError::AccountCode(address)) => {
                println!("Feeding VM account code at 0x{:x} ...", address);
                let code = read_hex(&client.get_code(&format!("0x{:x}", address),
                                                     &last_block_number)).unwrap();
                vm.commit_account(AccountCommitment::Code {
                    address: address,
                    code: code,
                }).unwrap();
            },
            Err(RequireError::Blockhash(number)) => {
                println!("Feeding blockhash with number 0x{:x} ...", number);
                let hash = M256::from_str(&client.get_block_by_number(&format!("0x{:x}", number))
                                          .hash).unwrap();
                vm.commit_blockhash(number, hash);
            },
        }
    }
}

fn is_miner_or_uncle(address: Address, block: &RPCBlock, client: &mut GethRPCClient) -> bool {
    // Give up balance testing if the address is a miner or an uncle.

    let miner = Address::from_str(&block.miner).unwrap();
    if miner == address {
        return true;
    }
    if block.uncles.len() > 0 {
        for i in 0..block.uncles.len() {
            let uncle = client.get_uncle_by_block_number_and_index(&block.number,
                                                                   &format!("0x{:x}", i));
            let uncle_miner = Address::from_str(&uncle.miner).unwrap();
            if uncle_miner == address {
                return true;
            }
        }
    }

    return false;
}

fn test_block(client: &mut GethRPCClient, number: usize) {
    let block = client.get_block_by_number(format!("0x{:x}", number).as_str());
    println!("block {}, transaction count: {}", block.number, block.transactions.len());
    let last_number = format!("0x{:x}", number - 1);
    let cur_number = block.number.to_string();
    let block_header = from_rpc_block(&block);
    let irregular_addresses = irregular_addresses(number);

    let mut last_vm: Option<SeqTransactionVM> = None;
    for transaction_hash in &block.transactions {
        println!("\nworking on transaction {}", transaction_hash);
        let transaction = from_rpc_transaction(&client.get_transaction_by_hash(&transaction_hash));
        let receipt = client.get_transaction_receipt(&transaction_hash);

        let mut vm = if last_vm.is_none() {
            SeqTransactionVM::new(transaction, block_header.clone(), &FRONTIER_PATCH)
        } else {
            SeqTransactionVM::with_previous(transaction, block_header.clone(), &FRONTIER_PATCH, last_vm.as_ref().unwrap())
        };

        handle_fire(client, &mut vm, &last_number, &irregular_addresses);

        assert!(Gas::from_str(&receipt.gasUsed).unwrap() == vm.real_used_gas());
        assert!(receipt.logs.len() == vm.logs().len());
        for i in 0..receipt.logs.len() {
            assert!(from_rpc_log(&receipt.logs[i]) == vm.logs()[i]);
        }

        last_vm = Some(vm);
    }

    if last_vm.is_some() {
        for account in last_vm.as_ref().unwrap().accounts() {
            match account {
                &Account::Full {
                    address,
                    balance,
                    ref changing_storage,
                    ..
                } => {
                    if !is_miner_or_uncle(address, &block, client) {
                        let expected_balance = client.get_balance(&format!("0x{:x}", address),
                                                                  &cur_number);
                        assert!(U256::from_str(&expected_balance).unwrap() == balance);
                    }
                    let changing_storage: HashMap<M256, M256> = changing_storage.clone().into();
                    for (key, value) in changing_storage {
                        let expected_value = client.get_storage_at(&format!("0x{:x}", address),
                                                                   &format!("0x{:x}", key),
                                                                   &cur_number);
                        assert!(M256::from_str(&expected_value).unwrap() == value);
                    }
                },
                &Account::Create {
                    address,
                    balance,
                    ref storage,
                    ..
                } => {
                    if !is_miner_or_uncle(address, &block, client) {
                        let expected_balance = client.get_balance(&format!("0x{:x}", address),
                                                                  &cur_number);
                        assert!(U256::from_str(&expected_balance).unwrap() == balance);
                    }
                    let storage: HashMap<M256, M256> = storage.clone().into();
                    for (key, value) in storage {
                        let expected_value = client.get_storage_at(&format!("0x{:x}", address),
                                                                   &format!("0x{:x}", key),
                                                                   &cur_number);
                        assert!(M256::from_str(&expected_value).unwrap() == value);
                    }
                },
                &Account::IncreaseBalance(address, balance) => {
                    if !is_miner_or_uncle(address, &block, client) {
                        let last_balance = client.get_balance(&format!("0x{:x}", address),
                                                              &last_number);
                        let cur_balance = client.get_balance(&format!("0x{:x}", address),
                                                             &cur_number);

                        assert!(U256::from_str(&last_balance).unwrap() + balance ==
                                U256::from_str(&cur_balance).unwrap());
                    }
                },
                &Account::DecreaseBalance(address, balance) => {
                    if !is_miner_or_uncle(address, &block, client) {
                        let last_balance = client.get_balance(&format!("0x{:x}", address),
                                                              &last_number);
                        let cur_balance = client.get_balance(&format!("0x{:x}", address),
                                                             &cur_number);

                        assert!(U256::from_str(&last_balance).unwrap() - balance ==
                                U256::from_str(&cur_balance).unwrap());
                    }
                },
            }
        }
    }
}

fn main() {
    let matches = clap_app!(regtests =>
        (version: "0.1")
        (author: "Ethereum Classic Contributors")
        (about: "Performs an regression test on the entire Ethereum Classic blockchain.\n\nSteps to reproduce:\n* Install Ethereum Classic Geth: `$ go install github.com/ethereumproject/go-ethereum/cmd/geth`.\n* Run Geth with this command: `$ ~/go/bin/geth --rpc --rpcaddr 127.0.0.1 --rpcport 8545`.\n* Wait for the chain to sync.\n* Change directory into the `regtests` directory `$ cd regtests`\n* Run this command: `$ RUST_BACKTRACE=1 cargo run --bin regtests -- -r http://127.0.0.1:8545")
        (@arg RPC: -r --rpc +takes_value +required "Domain of Ethereum Classic Geth's RPC endpoint. e.g. `-r http://127.0.0.1:8545`.")
        (@arg NUMBER: -n --number +takes_value + required "Block number to run this test. Radix is 10. e.g. `-n 49439`.")
    ).get_matches();

    let address = matches.value_of("RPC").unwrap();
    let number = matches.value_of("NUMBER").unwrap();
    let mut client = GethRPCClient::new(address);

    if number.contains("..") {
        let number: Vec<&str> = number.split("..").collect();
        let from = usize::from_str_radix(&number[0], 10).unwrap();
        let to = usize::from_str_radix(&number[1], 10).unwrap();
        for n in from..to {
            test_block(&mut client, n);
        }
    } else {
        let number = usize::from_str_radix(&number, 10).unwrap();
        test_block(&mut client, number);
    }
}
<commit_msg>Irregular accounts: add<commit_after>#[macro_use]
extern crate clap;
extern crate sputnikvm;
extern crate bigint;
extern crate serde_json;
extern crate gethrpc;

use serde_json::{Value};
use std::process;
use std::fs::File;
use std::path::Path;
use std::io::{BufReader};
use std::str::FromStr;
use std::collections::{HashMap, HashSet};

use sputnikvm::{Gas, Address};
use bigint::{U256, M256, read_hex};
use sputnikvm::vm::{BlockHeader, Context, SeqTransactionVM, Transaction, VM, Log, Patch, AccountCommitment, Account, FRONTIER_PATCH};
use sputnikvm::vm::errors::RequireError;
use gethrpc::{regression, GethRPCClient, RPCCall, RPCBlock, RPCTransaction, RPCLog};

fn from_rpc_block(block: &RPCBlock) -> BlockHeader {
    BlockHeader {
        coinbase: Address::from_str(&block.miner).unwrap(),
        timestamp: M256::from_str(&block.timestamp).unwrap(),
        number: M256::from_str(&block.number).unwrap(),
        difficulty: M256::from_str(&block.difficulty).unwrap(),
        gas_limit: Gas::from_str(&block.gasLimit).unwrap(),
    }
}

fn from_rpc_transaction(transaction: &RPCTransaction) -> Transaction {
    if transaction.to.is_empty() {
        Transaction::ContractCreation {
            caller: Address::from_str(&transaction.from).unwrap(),
            value: U256::from_str(&transaction.value).unwrap(),
            gas_limit: Gas::from_str(&transaction.gas).unwrap(),
            gas_price: Gas::from_str(&transaction.gasPrice).unwrap(),
            init: read_hex(&transaction.input).unwrap(),
        }
    } else {
        Transaction::MessageCall {
            caller: Address::from_str(&transaction.from).unwrap(),
            address: Address::from_str(&transaction.to).unwrap(),
            value: U256::from_str(&transaction.value).unwrap(),
            gas_limit: Gas::from_str(&transaction.gas).unwrap(),
            gas_price: Gas::from_str(&transaction.gasPrice).unwrap(),
            data: read_hex(&transaction.input).unwrap(),
        }
    }
}

fn from_rpc_log(log: &RPCLog) -> Log {
    let mut topics: Vec<M256> = Vec::new();
    for topic in &log.topics {
        topics.push(M256::from_str(&topic).unwrap());
    }
    Log {
        address: Address::from_str(&log.address).unwrap(),
        data: read_hex(&log.data).unwrap(),
        topics: topics,
    }
}

/// Accounts in this list should always be considered exists.
fn irregular_addresses(number: usize) -> HashSet<Address> {
    let config: HashMap<Address, usize> = {
        let mut config = HashMap::new();
        config.insert(Address::from_str("0x0000000000000000000000000000000000000000").unwrap(), 0);
        config.insert(Address::from_str("0x0000000000000000000000000000000000000001").unwrap(), 0);
        config.insert(Address::from_str("0x0000000000000000000000000000000000000002").unwrap(), 505137);
        config.insert(Address::from_str("0x0000000000000000000000000000000000000003").unwrap(), 0);
        config.insert(Address::from_str("0x0000000000000000000000000000000000000004").unwrap(), 0);

        config.insert(Address::from_str("0xbfe465e7eb5a2928b5bf22bef93ad06089dc6179").unwrap(), 229894);

        config
    };

    let mut result: HashSet<Address> = HashSet::new();
    for (key, value) in config {
        if number > value {
            result.insert(key);
        }
    }
    result
}

fn handle_fire(client: &mut GethRPCClient, vm: &mut SeqTransactionVM, last_block_number: &str,
               irregular_addresses: &HashSet<Address>) {
    loop {
        match vm.fire() {
            Ok(()) => {
                println!("VM exited with {:?}.", vm.status());
                break;
            },
            Err(RequireError::Account(address)) => {
                println!("Feeding VM account at 0x{:x} ...", address);
                let nonce = M256::from_str(&client.get_transaction_count(&format!("0x{:x}", address),
                                                                         &last_block_number)).unwrap();
                let balance = U256::from_str(&client.get_balance(&format!("0x{:x}", address),
                                                                 &last_block_number)).unwrap();
                let code = read_hex(&client.get_code(&format!("0x{:x}", address),
                                                     &last_block_number)).unwrap();
                if nonce == M256::zero() && balance == U256::zero() && code.len() == 0 &&
                    !irregular_addresses.contains(&address)
                {
                    vm.commit_account(AccountCommitment::Nonexist(address)).unwrap();
                } else {
                    vm.commit_account(AccountCommitment::Full {
                        nonce: nonce,
                        address: address,
                        balance: balance,
                        code: code,
                    }).unwrap();
                }
            },
            Err(RequireError::AccountStorage(address, index)) => {
                println!("Feeding VM account storage at 0x{:x} with index 0x{:x} ...", address, index);
                let value = M256::from_str(&client.get_storage_at(&format!("0x{:x}", address),
                                                                  &format!("0x{:x}", index),
                                                                  &last_block_number)).unwrap();
                vm.commit_account(AccountCommitment::Storage {
                    address: address,
                    index: index,
                    value: value,
                }).unwrap();
            },
            Err(RequireError::AccountCode(address)) => {
                println!("Feeding VM account code at 0x{:x} ...", address);
                let code = read_hex(&client.get_code(&format!("0x{:x}", address),
                                                     &last_block_number)).unwrap();
                vm.commit_account(AccountCommitment::Code {
                    address: address,
                    code: code,
                }).unwrap();
            },
            Err(RequireError::Blockhash(number)) => {
                println!("Feeding blockhash with number 0x{:x} ...", number);
                let hash = M256::from_str(&client.get_block_by_number(&format!("0x{:x}", number))
                                          .hash).unwrap();
                vm.commit_blockhash(number, hash);
            },
        }
    }
}

fn is_miner_or_uncle(address: Address, block: &RPCBlock, client: &mut GethRPCClient) -> bool {
    // Give up balance testing if the address is a miner or an uncle.

    let miner = Address::from_str(&block.miner).unwrap();
    if miner == address {
        return true;
    }
    if block.uncles.len() > 0 {
        for i in 0..block.uncles.len() {
            let uncle = client.get_uncle_by_block_number_and_index(&block.number,
                                                                   &format!("0x{:x}", i));
            let uncle_miner = Address::from_str(&uncle.miner).unwrap();
            if uncle_miner == address {
                return true;
            }
        }
    }

    return false;
}

fn test_block(client: &mut GethRPCClient, number: usize) {
    let block = client.get_block_by_number(format!("0x{:x}", number).as_str());
    println!("block {}, transaction count: {}", block.number, block.transactions.len());
    let last_number = format!("0x{:x}", number - 1);
    let cur_number = block.number.to_string();
    let block_header = from_rpc_block(&block);
    let irregular_addresses = irregular_addresses(number);

    let mut last_vm: Option<SeqTransactionVM> = None;
    for transaction_hash in &block.transactions {
        println!("\nworking on transaction {}", transaction_hash);
        let transaction = from_rpc_transaction(&client.get_transaction_by_hash(&transaction_hash));
        let receipt = client.get_transaction_receipt(&transaction_hash);

        let mut vm = if last_vm.is_none() {
            SeqTransactionVM::new(transaction, block_header.clone(), &FRONTIER_PATCH)
        } else {
            SeqTransactionVM::with_previous(transaction, block_header.clone(), &FRONTIER_PATCH, last_vm.as_ref().unwrap())
        };

        handle_fire(client, &mut vm, &last_number, &irregular_addresses);

        assert!(Gas::from_str(&receipt.gasUsed).unwrap() == vm.real_used_gas());
        assert!(receipt.logs.len() == vm.logs().len());
        for i in 0..receipt.logs.len() {
            assert!(from_rpc_log(&receipt.logs[i]) == vm.logs()[i]);
        }

        last_vm = Some(vm);
    }

    if last_vm.is_some() {
        for account in last_vm.as_ref().unwrap().accounts() {
            match account {
                &Account::Full {
                    address,
                    balance,
                    ref changing_storage,
                    ..
                } => {
                    if !is_miner_or_uncle(address, &block, client) {
                        let expected_balance = client.get_balance(&format!("0x{:x}", address),
                                                                  &cur_number);
                        assert!(U256::from_str(&expected_balance).unwrap() == balance);
                    }
                    let changing_storage: HashMap<M256, M256> = changing_storage.clone().into();
                    for (key, value) in changing_storage {
                        let expected_value = client.get_storage_at(&format!("0x{:x}", address),
                                                                   &format!("0x{:x}", key),
                                                                   &cur_number);
                        assert!(M256::from_str(&expected_value).unwrap() == value);
                    }
                },
                &Account::Create {
                    address,
                    balance,
                    ref storage,
                    ..
                } => {
                    if !is_miner_or_uncle(address, &block, client) {
                        let expected_balance = client.get_balance(&format!("0x{:x}", address),
                                                                  &cur_number);
                        assert!(U256::from_str(&expected_balance).unwrap() == balance);
                    }
                    let storage: HashMap<M256, M256> = storage.clone().into();
                    for (key, value) in storage {
                        let expected_value = client.get_storage_at(&format!("0x{:x}", address),
                                                                   &format!("0x{:x}", key),
                                                                   &cur_number);
                        assert!(M256::from_str(&expected_value).unwrap() == value);
                    }
                },
                &Account::IncreaseBalance(address, balance) => {
                    if !is_miner_or_uncle(address, &block, client) {
                        let last_balance = client.get_balance(&format!("0x{:x}", address),
                                                              &last_number);
                        let cur_balance = client.get_balance(&format!("0x{:x}", address),
                                                             &cur_number);

                        assert!(U256::from_str(&last_balance).unwrap() + balance ==
                                U256::from_str(&cur_balance).unwrap());
                    }
                },
                &Account::DecreaseBalance(address, balance) => {
                    if !is_miner_or_uncle(address, &block, client) {
                        let last_balance = client.get_balance(&format!("0x{:x}", address),
                                                              &last_number);
                        let cur_balance = client.get_balance(&format!("0x{:x}", address),
                                                             &cur_number);

                        assert!(U256::from_str(&last_balance).unwrap() - balance ==
                                U256::from_str(&cur_balance).unwrap());
                    }
                },
            }
        }
    }
}

fn main() {
    let matches = clap_app!(regtests =>
        (version: "0.1")
        (author: "Ethereum Classic Contributors")
        (about: "Performs an regression test on the entire Ethereum Classic blockchain.\n\nSteps to reproduce:\n* Install Ethereum Classic Geth: `$ go install github.com/ethereumproject/go-ethereum/cmd/geth`.\n* Run Geth with this command: `$ ~/go/bin/geth --rpc --rpcaddr 127.0.0.1 --rpcport 8545`.\n* Wait for the chain to sync.\n* Change directory into the `regtests` directory `$ cd regtests`\n* Run this command: `$ RUST_BACKTRACE=1 cargo run --bin regtests -- -r http://127.0.0.1:8545")
        (@arg RPC: -r --rpc +takes_value +required "Domain of Ethereum Classic Geth's RPC endpoint. e.g. `-r http://127.0.0.1:8545`.")
        (@arg NUMBER: -n --number +takes_value + required "Block number to run this test. Radix is 10. e.g. `-n 49439`.")
    ).get_matches();

    let address = matches.value_of("RPC").unwrap();
    let number = matches.value_of("NUMBER").unwrap();
    let mut client = GethRPCClient::new(address);

    if number.contains("..") {
        let number: Vec<&str> = number.split("..").collect();
        let from = usize::from_str_radix(&number[0], 10).unwrap();
        let to = usize::from_str_radix(&number[1], 10).unwrap();
        for n in from..to {
            test_block(&mut client, n);
        }
    } else {
        let number = usize::from_str_radix(&number, 10).unwrap();
        test_block(&mut client, number);
    }
}
<|endoftext|>