<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{Error, KeyManager, LibraInterface};
use executor::Executor;
use libra_config::config::NodeConfig;
use libra_crypto::{ed25519::Ed25519PrivateKey, HashValue, PrivateKey, Uniform};
use libra_secure_storage::{InMemoryStorage, KVStorage, Policy, Value};
use libra_types::{
    account_address::AccountAddress,
    account_config,
    account_state::AccountState,
    block_info::BlockInfo,
    block_metadata::{BlockMetadata, LibraBlockResource},
    discovery_set::DiscoverySet,
    ledger_info::{LedgerInfo, LedgerInfoWithSignatures},
    transaction::Transaction,
    validator_config::ValidatorConfig,
    validator_info::ValidatorInfo,
    validator_set::ValidatorSetResource,
};
use libra_vm::LibraVM;
use libradb::{LibraDB, LibraDBTrait};
use rand::{rngs::StdRng, SeedableRng};
use std::{cell::RefCell, collections::BTreeMap, convert::TryFrom, sync::Arc, time::SystemTime};
use storage_client::{StorageReadServiceClient, StorageWriteServiceClient};
use tokio::runtime::Runtime;

struct Node {
    account: AccountAddress,
    executor: Executor<LibraVM>,
    libra: TestLibraInterface,
    key_manager: KeyManager<TestLibraInterface, InMemoryStorage>,
    storage: Arc<LibraDB>,
    _storage_service: Runtime,
}

fn setup_secure_storage(config: &NodeConfig) -> InMemoryStorage {
    let mut sec_storage = InMemoryStorage::new();
    let test_config = config.clone().test.unwrap();

    let mut a_keypair = test_config.account_keypair.unwrap();
    let a_prikey = Value::Ed25519PrivateKey(a_keypair.take_private().unwrap());

    sec_storage
        .create(crate::ACCOUNT_KEY, a_prikey, &Policy::public())
        .unwrap();

    let mut c_keypair = test_config.consensus_keypair.unwrap();
    let c_prikey = c_keypair.take_private().unwrap();
    let c_prikey0 = Value::Ed25519PrivateKey(c_prikey.clone());
    let c_prikey1 = Value::Ed25519PrivateKey(c_prikey);

    sec_storage
        .create(crate::CONSENSUS_KEY, c_prikey0, &Policy::public())
        .unwrap();
    // Ugly hack but we need this until we support retrieving a policy from within storage and that
    // currently is not easy, since we would need to convert from Vault -> Libra policy.
    sec_storage
        .create(
            &format!("{}_previous", crate::CONSENSUS_KEY),
            c_prikey1,
            &Policy::public(),
        )
        .unwrap();
    sec_storage
}

fn setup(config: &NodeConfig) -> Node {
    let storage = storage_service::init_libra_db(config);
    let storage_service = storage_service::start_storage_service_with_db(&config, storage.clone());
    let executor = Executor::new(
        Arc::new(StorageReadServiceClient::new(&config.storage.address)),
        Arc::new(StorageWriteServiceClient::new(&config.storage.address)),
        config,
    );
    let libra = TestLibraInterface {
        queued_transactions: Arc::new(RefCell::new(Vec::new())),
        storage: storage.clone(),
    };
    let account = config.validator_network.as_ref().unwrap().peer_id;
    let key_manager = KeyManager::new(
        account,
        "consensus_key".to_owned(),
        libra.clone(),
        setup_secure_storage(&config),
    );

    Node {
        account,
        executor,
        key_manager,
        libra,
        storage,
        _storage_service: storage_service,
    }
}

#[derive(Clone)]
struct TestLibraInterface {
    queued_transactions: Arc<RefCell<Vec<Transaction>>>,
    storage: Arc<LibraDB>,
}

impl TestLibraInterface {
    fn retrieve_account_state(&self, account: AccountAddress) -> Result<AccountState, Error> {
        let blob = self
            .storage
            .get_latest_account_state(account)?
            .ok_or(Error::DataDoesNotExist("AccountState"))?;
        Ok(AccountState::try_from(&blob)?)
    }

    fn retrieve_discovery_set(&self) -> Result<DiscoverySet, Error> {
        let account = account_config::discovery_set_address();
        let account_state = self.retrieve_account_state(account)?;
        Ok(account_state
            .get_discovery_set_resource()?
            .ok_or(Error::DataDoesNotExist("DiscoverySetResource"))?
            .discovery_set()
            .clone())
    }

    fn retrieve_validator_set_resource(&self) -> Result<ValidatorSetResource, Error> {
        let account = account_config::validator_set_address();
        let account_state = self.retrieve_account_state(account)?;
        account_state
            .get_validator_set_resource()?
            .ok_or(Error::DataDoesNotExist("ValidatorSetResource"))
    }

    fn retrieve_libra_block_resource(&self) -> Result<LibraBlockResource, Error> {
        let account = account_config::association_address();
        let account_state = self.retrieve_account_state(account)?;
        account_state
            .get_libra_block_resource()?
            .ok_or(Error::DataDoesNotExist("BlockMetadata"))
    }

    fn take_all_transactions(&self) -> Vec<Transaction> {
        self.queued_transactions.borrow_mut().drain(0..).collect()
    }
}

impl LibraInterface for TestLibraInterface {
    fn last_reconfiguration(&self) -> Result<u64, Error> {
        self.retrieve_validator_set_resource()
            .map(|v| v.last_reconfiguration_time())
    }

    fn libra_timestamp(&self) -> Result<u64, Error> {
        let account = account_config::association_address();
        let blob = self
            .storage
            .get_latest_account_state(account)?
            .ok_or(Error::DataDoesNotExist("AccountState"))?;
        let account_state = AccountState::try_from(&blob).unwrap();
        Ok(account_state
            .get_libra_timestamp_resource()?
            .ok_or(Error::DataDoesNotExist("LibraTimestampResource"))?
            .libra_timestamp
            .microseconds)
    }

    fn retrieve_sequence_number(&self, account: AccountAddress) -> Result<u64, Error> {
        let blob = self
            .storage
            .get_latest_account_state(account)?
            .ok_or(Error::DataDoesNotExist("AccountState"))?;
        let account_state = AccountState::try_from(&blob).unwrap();
        Ok(account_state
            .get_account_resource()?
            .ok_or(Error::DataDoesNotExist("AccountResource"))?
            .sequence_number())
    }

    fn submit_transaction(&self, transaction: Transaction) -> Result<(), Error> {
        self.queued_transactions.borrow_mut().push(transaction);
        Ok(())
    }

    fn retrieve_validator_config(&self, account: AccountAddress) -> Result<ValidatorConfig, Error> {
        let blob = self
            .storage
            .get_latest_account_state(account)?
            .ok_or(Error::DataDoesNotExist("AccountState"))?;
        let account_state = AccountState::try_from(&blob).unwrap();
        Ok(account_state
            .get_validator_config_resource()?
            .ok_or(Error::DataDoesNotExist("ValidatorConfigResource"))?
            .validator_config)
    }

    fn retrieve_validator_info(
        &self,
        validator_account: AccountAddress,
    ) -> Result<ValidatorInfo, Error> {
        self.retrieve_validator_set_resource()?
            .validator_set()
            .iter()
            .find(|vi| vi.account_address() == &validator_account)
            .cloned()
            .ok_or(Error::ValidatorInfoNotFound(validator_account))
    }
}

// TODO(davidiw): Notify external upon reconfigure and update epoch via ledger_info and also add
// next validator set if appropriate...
fn execute_and_commit(node: &mut Node, mut block: Vec<Transaction>) {
    let block_id = HashValue::zero();
    let startup_info = node.storage.get_startup_info().unwrap().unwrap();
    // Clock is supposed to be in microseconds
    let clock = (startup_info.committed_tree_state.version + 1) * 1_000_000;

    // This will set the current clock to be equal to the current version + 1, this guarantees that
    // the clock is always refreshed for this set of transactions.
    let block_metadata = BlockMetadata::new(block_id, 0, clock, vec![], node.account);
    let prologue = Transaction::BlockMetadata(block_metadata);
    block.insert(0, prologue);

    let result = node
        .executor
        .execute_block((block_id, block), node.executor.committed_block_id())
        .unwrap();

    let ledger_info = LedgerInfo::new(
        BlockInfo::new(
            0,
            0,
            block_id,
            result.root_hash(),
            result.version(),
            0,
            None,
        ),
        HashValue::zero(),
    );
    let ledger_info_with_sigs = LedgerInfoWithSignatures::new(ledger_info, BTreeMap::new());

    node.executor
        .commit_blocks(vec![block_id], ledger_info_with_sigs)
        .unwrap();
}

#[test]
// This simple test just proves that genesis took effect and the values are in storage
fn test_ability_to_read_move_data() {
    let (config, _genesis_key) = config_builder::test_config();
    let node = setup(&config);

    node.libra.last_reconfiguration().unwrap();
    node.libra.retrieve_discovery_set().unwrap();
    node.libra.retrieve_validator_config(node.account).unwrap();
    node.libra.retrieve_discovery_set().unwrap();
    node.libra.retrieve_validator_info(node.account).unwrap();
    assert!(node.libra.retrieve_libra_block_resource().is_ok());
}

#[test]
// This test verifies that a node can rotate its key and it results in a new validator set
fn test_consensus_rotation() {
    let (config, _genesis_key) = config_builder::test_config();
    let mut node = setup(&config);

    let test_config = config.test.unwrap();
    let mut keypair = test_config.consensus_keypair.unwrap();

    let genesis_prikey = keypair.take_private().unwrap();
    let genesis_pubkey = genesis_prikey.public_key();
    let account_prikey = test_config.account_keypair.unwrap().take_private().unwrap();

    let genesis_config = node.libra.retrieve_validator_config(node.account).unwrap();
    let genesis_info = node.libra.retrieve_validator_info(node.account).unwrap();

    assert_eq!(genesis_pubkey, genesis_config.consensus_pubkey);
    assert_eq!(&genesis_pubkey, genesis_info.consensus_public_key());
    assert_eq!(&node.account, genesis_info.account_address());

    let mut rng = StdRng::from_seed([44u8; 32]);
    let new_prikey = Ed25519PrivateKey::generate_for_testing(&mut rng);
    let new_pubkey = new_prikey.public_key();

    let txn = crate::build_transaction(node.account, 0, &account_prikey, &new_pubkey);

    execute_and_commit(&mut node, vec![txn]);

    let new_config = node.libra.retrieve_validator_config(node.account).unwrap();
    let new_info = node.libra.retrieve_validator_info(node.account).unwrap();

    assert!(new_pubkey != genesis_pubkey);
    assert_eq!(new_pubkey, new_config.consensus_pubkey);
    assert_eq!(&new_pubkey, new_info.consensus_public_key());
}

#[test]
// This verifies the key manager is properly setup and that a
fn test_key_manager_init_and_basic_rotation() {
    let (config, _genesis_key) = config_builder::test_config();
    let mut node = setup(&config);
    node.key_manager.compare_storage_to_config().unwrap();
    node.key_manager.compare_info_to_config().unwrap();
    let now = SystemTime::now()
        .duration_since(SystemTime::UNIX_EPOCH)
        .unwrap()
        .as_secs();
    assert!(now - 600 <= node.key_manager.last_rotation().unwrap());
    // No reconfiguration yet
    assert_eq!(0, node.key_manager.last_reconfiguration().unwrap());
    // No executions yet
    assert_eq!(0, node.key_manager.libra_timestamp().unwrap());

    let genesis_info = node.libra.retrieve_validator_info(node.account).unwrap();
    let new_key = node.key_manager.rotate_consensus_key().unwrap();
    let pre_exe_rotated_info = node.libra.retrieve_validator_info(node.account).unwrap();
    assert!(genesis_info.consensus_public_key() == pre_exe_rotated_info.consensus_public_key());
    assert!(pre_exe_rotated_info.consensus_public_key() != &new_key);

    let txns = node.libra.take_all_transactions();
    execute_and_commit(&mut node, txns);
    let rotated_info = node.libra.retrieve_validator_info(node.account).unwrap();
    assert!(genesis_info.consensus_public_key() != rotated_info.consensus_public_key());
    assert!(rotated_info.consensus_public_key() == &new_key);

    // Executions have occurred but nothing after our rotation
    assert!(0 != node.key_manager.libra_timestamp().unwrap());
    assert_eq!(
        node.key_manager.last_reconfiguration(),
        node.key_manager.libra_timestamp()
    );

    // TODO(davidiw): Enable once reconfiguration is correclty implemented above...
    // Executions have occurred after our rotation
    // execute_and_commit(&node, node.libra.take_all_transactions());
    // assert!(node.key_manager.last_reconfiguration() != node.key_manager.libra_timestamp());
}
<commit_msg>[key-manager] improve test code<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{Error, KeyManager, LibraInterface};
use executor::Executor;
use libra_config::config::NodeConfig;
use libra_crypto::{ed25519::Ed25519PrivateKey, HashValue, PrivateKey, Uniform};
use libra_secure_storage::{InMemoryStorage, KVStorage, Policy, Value};
use libra_types::{
    account_address::AccountAddress,
    account_config,
    account_state::AccountState,
    block_info::BlockInfo,
    block_metadata::{BlockMetadata, LibraBlockResource},
    discovery_set::DiscoverySet,
    ledger_info::{LedgerInfo, LedgerInfoWithSignatures},
    transaction::Transaction,
    validator_config::ValidatorConfig,
    validator_info::ValidatorInfo,
    validator_set::ValidatorSetResource,
};
use libra_vm::LibraVM;
use libradb::{LibraDB, LibraDBTrait};
use rand::{rngs::StdRng, SeedableRng};
use std::{cell::RefCell, collections::BTreeMap, convert::TryFrom, sync::Arc, time::SystemTime};
use storage_client::{StorageReadServiceClient, StorageWriteServiceClient};
use tokio::runtime::Runtime;

struct Node {
    account: AccountAddress,
    executor: Executor<LibraVM>,
    libra: TestLibraInterface,
    key_manager: KeyManager<TestLibraInterface, InMemoryStorage>,
    storage: Arc<LibraDB>,
    _storage_service: Runtime,
}

fn setup_secure_storage(config: &NodeConfig) -> InMemoryStorage {
    let mut sec_storage = InMemoryStorage::new();
    let test_config = config.clone().test.unwrap();

    let mut a_keypair = test_config.account_keypair.unwrap();
    let a_prikey = Value::Ed25519PrivateKey(a_keypair.take_private().unwrap());

    sec_storage
        .create(crate::ACCOUNT_KEY, a_prikey, &Policy::public())
        .unwrap();

    let mut c_keypair = test_config.consensus_keypair.unwrap();
    let c_prikey = c_keypair.take_private().unwrap();
    let c_prikey0 = Value::Ed25519PrivateKey(c_prikey.clone());
    let c_prikey1 = Value::Ed25519PrivateKey(c_prikey);

    sec_storage
        .create(crate::CONSENSUS_KEY, c_prikey0, &Policy::public())
        .unwrap();
    // Ugly hack but we need this until we support retrieving a policy from within storage and that
    // currently is not easy, since we would need to convert from Vault -> Libra policy.
    sec_storage
        .create(
            &format!("{}_previous", crate::CONSENSUS_KEY),
            c_prikey1,
            &Policy::public(),
        )
        .unwrap();
    sec_storage
}

impl Node {
    fn setup(config: &NodeConfig) -> Self {
        let storage = storage_service::init_libra_db(config);
        let storage_service =
            storage_service::start_storage_service_with_db(&config, storage.clone());
        let executor = Executor::new(
            Arc::new(StorageReadServiceClient::new(&config.storage.address)),
            Arc::new(StorageWriteServiceClient::new(&config.storage.address)),
            config,
        );
        let libra = TestLibraInterface {
            queued_transactions: Arc::new(RefCell::new(Vec::new())),
            storage: storage.clone(),
        };
        let account = config.validator_network.as_ref().unwrap().peer_id;
        let key_manager = KeyManager::new(
            account,
            "consensus_key".to_owned(),
            libra.clone(),
            setup_secure_storage(&config),
        );

        Self {
            account,
            executor,
            key_manager,
            libra,
            storage,
            _storage_service: storage_service,
        }
    }

    // TODO(davidiw): Notify external upon reconfigure and update epoch via ledger_info and also add
    // next validator set if appropriate...
    fn execute_and_commit(&mut self, mut block: Vec<Transaction>) {
        let block_id = HashValue::zero();
        let startup_info = self.storage.get_startup_info().unwrap().unwrap();
        // Clock is supposed to be in microseconds
        let clock = (startup_info.committed_tree_state.version + 1) * 1_000_000;

        // the clock is always refreshed for this set of transactions.
        let block_metadata = BlockMetadata::new(block_id, 0, clock, vec![], self.account);
        let prologue = Transaction::BlockMetadata(block_metadata);
        block.insert(0, prologue);

        let output = self
            .executor
            .execute_block((block_id, block), self.executor.committed_block_id())
            .unwrap();

        let ledger_info = LedgerInfo::new(
            BlockInfo::new(
                0,
                0,
                block_id,
                output.root_hash(),
                output.version(),
                0,
                output.validators().clone(),
            ),
            HashValue::zero(),
        );
        let ledger_info_with_sigs = LedgerInfoWithSignatures::new(ledger_info, BTreeMap::new());
        self.executor.commit_blocks(vec![block_id], ledger_info_with_sigs).unwrap();
    }
}

#[derive(Clone)]
struct TestLibraInterface {
    queued_transactions: Arc<RefCell<Vec<Transaction>>>,
    storage: Arc<LibraDB>,
}

impl TestLibraInterface {
    fn retrieve_account_state(&self, account: AccountAddress) -> Result<AccountState, Error> {
        let blob = self
            .storage
            .get_latest_account_state(account)?
            .ok_or(Error::DataDoesNotExist("AccountState"))?;
        Ok(AccountState::try_from(&blob)?)
    }

    fn retrieve_discovery_set(&self) -> Result<DiscoverySet, Error> {
        let account = account_config::discovery_set_address();
        let account_state = self.retrieve_account_state(account)?;
        Ok(account_state
            .get_discovery_set_resource()?
            .ok_or(Error::DataDoesNotExist("DiscoverySetResource"))?
            .discovery_set()
            .clone())
    }

    fn retrieve_validator_set_resource(&self) -> Result<ValidatorSetResource, Error> {
        let account = account_config::validator_set_address();
        let account_state = self.retrieve_account_state(account)?;
        account_state
            .get_validator_set_resource()?
            .ok_or(Error::DataDoesNotExist("ValidatorSetResource"))
    }

    fn retrieve_libra_block_resource(&self) -> Result<LibraBlockResource, Error> {
        let account = account_config::association_address();
        let account_state = self.retrieve_account_state(account)?;
        account_state
            .get_libra_block_resource()?
            .ok_or(Error::DataDoesNotExist("BlockMetadata"))
    }

    fn take_all_transactions(&self) -> Vec<Transaction> {
        self.queued_transactions.borrow_mut().drain(0..).collect()
    }
}

impl LibraInterface for TestLibraInterface {
    fn last_reconfiguration(&self) -> Result<u64, Error> {
        self.retrieve_validator_set_resource()
            .map(|v| v.last_reconfiguration_time())
    }

    fn libra_timestamp(&self) -> Result<u64, Error> {
        let account = account_config::association_address();
        let blob = self
            .storage
            .get_latest_account_state(account)?
            .ok_or(Error::DataDoesNotExist("AccountState"))?;
        let account_state = AccountState::try_from(&blob).unwrap();
        Ok(account_state
            .get_libra_timestamp_resource()?
            .ok_or(Error::DataDoesNotExist("LibraTimestampResource"))?
            .libra_timestamp
            .microseconds)
    }

    fn retrieve_sequence_number(&self, account: AccountAddress) -> Result<u64, Error> {
        let blob = self
            .storage
            .get_latest_account_state(account)?
            .ok_or(Error::DataDoesNotExist("AccountState"))?;
        let account_state = AccountState::try_from(&blob).unwrap();
        Ok(account_state
            .get_account_resource()?
            .ok_or(Error::DataDoesNotExist("AccountResource"))?
            .sequence_number())
    }

    fn submit_transaction(&self, transaction: Transaction) -> Result<(), Error> {
        self.queued_transactions.borrow_mut().push(transaction);
        Ok(())
    }

    fn retrieve_validator_config(&self, account: AccountAddress) -> Result<ValidatorConfig, Error> {
        let blob = self
            .storage
            .get_latest_account_state(account)?
            .ok_or(Error::DataDoesNotExist("AccountState"))?;
        let account_state = AccountState::try_from(&blob).unwrap();
        Ok(account_state
            .get_validator_config_resource()?
            .ok_or(Error::DataDoesNotExist("ValidatorConfigResource"))?
            .validator_config)
    }

    fn retrieve_validator_info(
        &self,
        validator_account: AccountAddress,
    ) -> Result<ValidatorInfo, Error> {
        self.retrieve_validator_set_resource()?
            .validator_set()
            .iter()
            .find(|vi| vi.account_address() == &validator_account)
            .cloned()
            .ok_or(Error::ValidatorInfoNotFound(validator_account))
    }
}

#[test]
// This simple test just proves that genesis took effect and the values are in storage
fn test_ability_to_read_move_data() {
    let (config, _genesis_key) = config_builder::test_config();
    let node = Node::setup(&config);

    node.libra.last_reconfiguration().unwrap();
    node.libra.retrieve_discovery_set().unwrap();
    node.libra.retrieve_validator_config(node.account).unwrap();
    node.libra.retrieve_discovery_set().unwrap();
    node.libra.retrieve_validator_info(node.account).unwrap();
    assert!(node.libra.retrieve_libra_block_resource().is_ok());
}

#[test]
// This test verifies that a node can rotate its key and it results in a new validator set
fn test_consensus_rotation() {
    let (config, _genesis_key) = config_builder::test_config();
    let mut node = Node::setup(&config);

    let test_config = config.test.unwrap();
    let mut keypair = test_config.consensus_keypair.unwrap();

    let genesis_prikey = keypair.take_private().unwrap();
    let genesis_pubkey = genesis_prikey.public_key();
    let account_prikey = test_config.account_keypair.unwrap().take_private().unwrap();

    let genesis_config = node.libra.retrieve_validator_config(node.account).unwrap();
    let genesis_info = node.libra.retrieve_validator_info(node.account).unwrap();

    assert_eq!(genesis_pubkey, genesis_config.consensus_pubkey);
    assert_eq!(&genesis_pubkey, genesis_info.consensus_public_key());
    assert_eq!(&node.account, genesis_info.account_address());

    let mut rng = StdRng::from_seed([44u8; 32]);
    let new_prikey = Ed25519PrivateKey::generate_for_testing(&mut rng);
    let new_pubkey = new_prikey.public_key();

    let txn = crate::build_transaction(node.account, 0, &account_prikey, &new_pubkey);

    node.execute_and_commit(vec![txn]);

    let new_config = node.libra.retrieve_validator_config(node.account).unwrap();
    let new_info = node.libra.retrieve_validator_info(node.account).unwrap();

    assert!(new_pubkey != genesis_pubkey);
    assert_eq!(new_pubkey, new_config.consensus_pubkey);
    assert_eq!(&new_pubkey, new_info.consensus_public_key());
}

#[test]
// This verifies the key manager is properly setup and that a
fn test_key_manager_init_and_basic_rotation() {
    let (config, _genesis_key) = config_builder::test_config();
    let mut node = Node::setup(&config);
    node.key_manager.compare_storage_to_config().unwrap();
    node.key_manager.compare_info_to_config().unwrap();
    let now = SystemTime::now()
        .duration_since(SystemTime::UNIX_EPOCH)
        .unwrap()
        .as_secs();
    assert!(now - 600 <= node.key_manager.last_rotation().unwrap());
    // No reconfiguration yet
    assert_eq!(0, node.key_manager.last_reconfiguration().unwrap());
    // No executions yet
    assert_eq!(0, node.key_manager.libra_timestamp().unwrap());

    let genesis_info = node.libra.retrieve_validator_info(node.account).unwrap();
    let new_key = node.key_manager.rotate_consensus_key().unwrap();
    let pre_exe_rotated_info = node.libra.retrieve_validator_info(node.account).unwrap();
    assert!(genesis_info.consensus_public_key() == pre_exe_rotated_info.consensus_public_key());
    assert!(pre_exe_rotated_info.consensus_public_key() != &new_key);

    node.execute_and_commit(node.libra.take_all_transactions());
    let rotated_info = node.libra.retrieve_validator_info(node.account).unwrap();
    assert!(genesis_info.consensus_public_key() != rotated_info.consensus_public_key());
    assert!(rotated_info.consensus_public_key() == &new_key);

    // Executions have occurred but nothing after our rotation
    assert!(0 != node.key_manager.libra_timestamp().unwrap());
    assert_eq!(
        node.key_manager.last_reconfiguration(),
        node.key_manager.libra_timestamp()
    );

    // TODO(davidiw): Enable once reconfiguration is correclty implemented above...
    // Executions have occurred after our rotation
    node.execute_and_commit(node.libra.take_all_transactions());
    assert!(node.key_manager.last_reconfiguration() != node.key_manager.libra_timestamp());
}
<|endoftext|>


<commit_before>//!
//! https://tools.ietf.org/html/rfc5161
//!
//! The IMAP ENABLE Extension
//!

// rustfmt doesn't do a very good job on nom parser invocations.
#![cfg_attr(rustfmt, rustfmt_skip)]
#![cfg_attr(feature = "cargo-clippy", allow(clippy::redundant_closure))]

use crate::types::*;
use crate::parser::core::atom;

// The ENABLED response lists capabilities that were enabled in response
// to a ENABLE command.
// [RFC5161 - 3.2 The ENABLED Response](https://tools.ietf.org/html/rfc5161#section-3.2)
named!(pub (crate) resp_enabled<Response>, map!(
    enabled_data,
    |c| Response::Capabilities(c)
));

named!(enabled_data<Vec<Capability>>, do_parse!(
        tag_no_case!("ENABLED") >>
        capabilities: many0!(preceded!(char!(' '), capability)) >>
        (capabilities)
));

named!(capability<Capability>,
       map!(atom, |a| Capability::Atom(a))
);
<commit_msg>Remove usage of redundant closures<commit_after>//!
//! https://tools.ietf.org/html/rfc5161
//!
//! The IMAP ENABLE Extension
//!

// rustfmt doesn't do a very good job on nom parser invocations.
#![cfg_attr(rustfmt, rustfmt_skip)]

use crate::types::*;
use crate::parser::core::atom;

// The ENABLED response lists capabilities that were enabled in response
// to a ENABLE command.
// [RFC5161 - 3.2 The ENABLED Response](https://tools.ietf.org/html/rfc5161#section-3.2)
named!(pub (crate) resp_enabled<Response>, map!(
    enabled_data,
    Response::Capabilities
));

named!(enabled_data<Vec<Capability>>, do_parse!(
        tag_no_case!("ENABLED") >>
        capabilities: many0!(preceded!(char!(' '), capability)) >>
        (capabilities)
));

named!(capability<Capability>,
       map!(atom, Capability::Atom)
);
<|endoftext|>


<commit_before>//  * This file is part of the uutils coreutils package.
//  *
//  * (c) Alex Lyon <arcterus@mail.com>
//  *
//  * For the full copyright and license information, please view the LICENSE
//  * file that was distributed with this source code.

// spell-checker:ignore (ToDO) bitor ulong

#[macro_use]
extern crate uucore;

use clap::{crate_version, App, Arg};
use remove_dir_all::remove_dir_all;
use std::collections::VecDeque;
use std::fs;
use std::io::{stderr, stdin, BufRead, Write};
use std::ops::BitOr;
use std::path::{Path, PathBuf};
use walkdir::{DirEntry, WalkDir};

#[derive(Eq, PartialEq, Clone, Copy)]
enum InteractiveMode {
    None,
    Once,
    Always,
}

struct Options {
    force: bool,
    interactive: InteractiveMode,
    #[allow(dead_code)]
    one_fs: bool,
    preserve_root: bool,
    recursive: bool,
    dir: bool,
    verbose: bool,
}

static ABOUT: &str = "Remove (unlink) the FILE(s)";
static OPT_DIR: &str = "dir";
static OPT_INTERACTIVE: &str = "interactive";
static OPT_FORCE: &str = "force";
static OPT_NO_PRESERVE_ROOT: &str = "no-preserve-root";
static OPT_ONE_FILE_SYSTEM: &str = "one-file-system";
static OPT_PRESERVE_ROOT: &str = "preserve-root";
static OPT_PROMPT: &str = "prompt";
static OPT_PROMPT_MORE: &str = "prompt-more";
static OPT_RECURSIVE: &str = "recursive";
static OPT_RECURSIVE_R: &str = "recursive_R";
static OPT_VERBOSE: &str = "verbose";

static ARG_FILES: &str = "files";

fn get_usage() -> String {
    format!("{0} [OPTION]... FILE...", executable!())
}

fn get_long_usage() -> String {
    String::from(
        "By default, rm does not remove directories.  Use the --recursive (-r or -R)
        option to remove each listed directory, too, along with all of its contents

        To remove a file whose name starts with a '-', for example '-foo',
        use one of these commands:
        rm -- -foo

        rm ./-foo

        Note that if you use rm to remove a file, it might be possible to recover
        some of its contents, given sufficient expertise and/or time.  For greater
        assurance that the contents are truly unrecoverable, consider using shred.",
    )
}

pub fn uumain(args: impl uucore::Args) -> i32 {
    let usage = get_usage();
    let long_usage = get_long_usage();

    let matches = App::new(executable!())
        .version(crate_version!())
        .about(ABOUT)
        .usage(&usage[..])
        .after_help(&long_usage[..])

        .arg(
            Arg::with_name(OPT_FORCE)
            .short("f")
            .long(OPT_FORCE)
            .multiple(true)
            .help("ignore nonexistent files and arguments, never prompt")
        )
        .arg(
            Arg::with_name(OPT_PROMPT)
            .short("i")
            .long("prompt before every removal")
        )
        .arg(
            Arg::with_name(OPT_PROMPT_MORE)
            .short("I")
            .help("prompt once before removing more than three files, or when removing recursively. Less intrusive than -i, while still giving some protection against most mistakes")
        )
        .arg(
            Arg::with_name(OPT_INTERACTIVE)
            .long(OPT_INTERACTIVE)
            .help("prompt according to WHEN: never, once (-I), or always (-i). Without WHEN, prompts always")
            .value_name("WHEN")
            .takes_value(true)
        )
        .arg(
            Arg::with_name(OPT_ONE_FILE_SYSTEM)
            .long(OPT_ONE_FILE_SYSTEM)
            .help("when removing a hierarchy recursively, skip any directory that is on a file system different from that of the corresponding command line argument (NOT IMPLEMENTED)")
        )
        .arg(
            Arg::with_name(OPT_NO_PRESERVE_ROOT)
            .long(OPT_NO_PRESERVE_ROOT)
            .help("do not treat '/' specially")
        )
        .arg(
            Arg::with_name(OPT_PRESERVE_ROOT)
            .long(OPT_PRESERVE_ROOT)
            .help("do not remove '/' (default)")
        )
        .arg(
            Arg::with_name(OPT_RECURSIVE).short("r")
            .long(OPT_RECURSIVE)
            .help("remove directories and their contents recursively")
        )
        .arg(
            // To mimic GNU's behavior we also want the '-R' flag. However, using clap's
            // alias method 'visible_alias("R")' would result in a long '--R' flag.
            Arg::with_name(OPT_RECURSIVE_R).short("R")
            .help("Equivalent to -r")
        )
        .arg(
            Arg::with_name(OPT_DIR)
            .short("d")
            .long(OPT_DIR)
            .help("remove empty directories")
        )
        .arg(
            Arg::with_name(OPT_VERBOSE)
            .short("v")
            .long(OPT_VERBOSE)
            .help("explain what is being done")
        )
        .arg(
            Arg::with_name(ARG_FILES)
            .multiple(true)
            .takes_value(true)
            .min_values(1)
        )
        .get_matches_from(args);

    let files: Vec<String> = matches
        .values_of(ARG_FILES)
        .map(|v| v.map(ToString::to_string).collect())
        .unwrap_or_default();

    let force = matches.is_present(OPT_FORCE);

    if files.is_empty() && !force {
        // Still check by hand and not use clap
        // Because "rm -f" is a thing
        show_error!("missing an argument");
        show_error!("for help, try '{0} --help'", executable!());
        return 1;
    } else {
        let options = Options {
            force,
            interactive: {
                if matches.is_present(OPT_PROMPT) {
                    InteractiveMode::Always
                } else if matches.is_present(OPT_PROMPT_MORE) {
                    InteractiveMode::Once
                } else if matches.is_present(OPT_INTERACTIVE) {
                    match matches.value_of(OPT_INTERACTIVE).unwrap() {
                        "none" => InteractiveMode::None,
                        "once" => InteractiveMode::Once,
                        "always" => InteractiveMode::Always,
                        val => crash!(1, "Invalid argument to interactive ({})", val),
                    }
                } else {
                    InteractiveMode::None
                }
            },
            one_fs: matches.is_present(OPT_ONE_FILE_SYSTEM),
            preserve_root: !matches.is_present(OPT_NO_PRESERVE_ROOT),
            recursive: matches.is_present(OPT_RECURSIVE) || matches.is_present(OPT_RECURSIVE_R),
            dir: matches.is_present(OPT_DIR),
            verbose: matches.is_present(OPT_VERBOSE),
        };
        if options.interactive == InteractiveMode::Once && (options.recursive || files.len() > 3) {
            let msg = if options.recursive {
                "Remove all arguments recursively? "
            } else {
                "Remove all arguments? "
            };
            if !prompt(msg) {
                return 0;
            }
        }

        if remove(files, options) {
            return 1;
        }
    }

    0
}

// TODO: implement one-file-system (this may get partially implemented in walkdir)
fn remove(files: Vec<String>, options: Options) -> bool {
    let mut had_err = false;

    for filename in &files {
        let file = Path::new(filename);
        had_err = match file.symlink_metadata() {
            Ok(metadata) => {
                if metadata.is_dir() {
                    handle_dir(file, &options)
                } else if is_symlink_dir(&metadata) {
                    remove_dir(file, &options)
                } else {
                    remove_file(file, &options)
                }
            }
            Err(_e) => {
                // TODO: actually print out the specific error
                // TODO: When the error is not about missing files
                // (e.g., permission), even rm -f should fail with
                // outputting the error, but there's no easy eay.
                if !options.force {
                    show_error!("cannot remove '{}': No such file or directory", filename);
                    true
                } else {
                    false
                }
            }
        }
        .bitor(had_err);
    }

    had_err
}

fn handle_dir(path: &Path, options: &Options) -> bool {
    let mut had_err = false;

    let is_root = path.has_root() && path.parent().is_none();
    if options.recursive && (!is_root || !options.preserve_root) {
        if options.interactive != InteractiveMode::Always && !options.verbose {
            // we need the extra crate because apparently fs::remove_dir_all() does not function
            // correctly on Windows
            if let Err(e) = remove_dir_all(path) {
                had_err = true;
                show_error!("could not remove '{}': {}", path.display(), e);
            }
        } else {
            let mut dirs: VecDeque<DirEntry> = VecDeque::new();

            for entry in WalkDir::new(path) {
                match entry {
                    Ok(entry) => {
                        let file_type = entry.file_type();
                        if file_type.is_dir() {
                            dirs.push_back(entry);
                        } else {
                            had_err = remove_file(entry.path(), options).bitor(had_err);
                        }
                    }
                    Err(e) => {
                        had_err = true;
                        show_error!("recursing in '{}': {}", path.display(), e);
                    }
                }
            }

            for dir in dirs.iter().rev() {
                had_err = remove_dir(dir.path(), options).bitor(had_err);
            }
        }
    } else if options.dir && (!is_root || !options.preserve_root) {
        had_err = remove_dir(path, options).bitor(had_err);
    } else if options.recursive {
        show_error!("could not remove directory '{}'", path.display());
        had_err = true;
    } else {
        show_error!(
            "cannot remove '{}': Is a directory", // GNU's rm error message does not include help
            path.display()
        );
        had_err = true;
    }

    had_err
}

fn remove_dir(path: &Path, options: &Options) -> bool {
    let response = if options.interactive == InteractiveMode::Always {
        prompt_file(path, true)
    } else {
        true
    };
    if response {
        if let Ok(mut read_dir) = fs::read_dir(path) {
            if options.dir || options.recursive {
                if read_dir.next().is_none() {
                    match fs::remove_dir(path) {
                        Ok(_) => {
                            if options.verbose {
                                println!("removed directory '{}'", normalize(path).display());
                            }
                        }
                        Err(e) => {
                            show_error!("cannot remove '{}': {}", path.display(), e);
                            return true;
                        }
                    }
                } else {
                    // directory can be read but is not empty
                    show_error!("cannot remove '{}': Directory not empty", path.display());
                    return true;
                }
            } else {
                // called to remove a symlink_dir (windows) without "-r"/"-R" or "-d"
                show_error!("cannot remove '{}': Is a directory", path.display());
                return true;
            }
        } else {
            // GNU's rm shows this message if directory is empty but not readable
            show_error!("cannot remove '{}': Directory not empty", path.display());
            return true;
        }
    }

    false
}

fn remove_file(path: &Path, options: &Options) -> bool {
    let response = if options.interactive == InteractiveMode::Always {
        prompt_file(path, false)
    } else {
        true
    };
    if response {
        match fs::remove_file(path) {
            Ok(_) => {
                if options.verbose {
                    println!("removed '{}'", normalize(path).display());
                }
            }
            Err(e) => {
                show_error!("removing '{}': {}", path.display(), e);
                return true;
            }
        }
    }

    false
}

fn prompt_file(path: &Path, is_dir: bool) -> bool {
    if is_dir {
        prompt(&(format!("rm: remove directory '{}'? ", path.display())))
    } else {
        prompt(&(format!("rm: remove file '{}'? ", path.display())))
    }
}

fn normalize(path: &Path) -> PathBuf {
    // copied from https://github.com/rust-lang/cargo/blob/2e4cfc2b7d43328b207879228a2ca7d427d188bb/src/cargo/util/paths.rs#L65-L90
    // both projects are MIT https://github.com/rust-lang/cargo/blob/master/LICENSE-MIT
    // for std impl progress see rfc https://github.com/rust-lang/rfcs/issues/2208
    // TODO: replace this once that lands
    uucore::fs::normalize_path(path)
}

fn prompt(msg: &str) -> bool {
    let _ = stderr().write_all(msg.as_bytes());
    let _ = stderr().flush();

    let mut buf = Vec::new();
    let stdin = stdin();
    let mut stdin = stdin.lock();

    match stdin.read_until(b'\n', &mut buf) {
        Ok(x) if x > 0 => matches!(buf[0], b'y' | b'Y'),
        _ => false,
    }
}

#[cfg(not(windows))]
fn is_symlink_dir(_metadata: &fs::Metadata) -> bool {
    false
}

#[cfg(windows)]
use std::os::windows::prelude::MetadataExt;

#[cfg(windows)]
fn is_symlink_dir(metadata: &fs::Metadata) -> bool {
    use std::os::raw::c_ulong;
    pub type DWORD = c_ulong;
    pub const FILE_ATTRIBUTE_DIRECTORY: DWORD = 0x10;

    metadata.file_type().is_symlink()
        && ((metadata.file_attributes() & FILE_ATTRIBUTE_DIRECTORY) != 0)
}
<commit_msg>gnu/rm: match gnu's output<commit_after>//  * This file is part of the uutils coreutils package.
//  *
//  * (c) Alex Lyon <arcterus@mail.com>
//  *
//  * For the full copyright and license information, please view the LICENSE
//  * file that was distributed with this source code.

// spell-checker:ignore (ToDO) bitor ulong

#[macro_use]
extern crate uucore;

use clap::{crate_version, App, Arg};
use remove_dir_all::remove_dir_all;
use std::collections::VecDeque;
use std::fs;
use std::io::{stderr, stdin, BufRead, Write};
use std::ops::BitOr;
use std::path::{Path, PathBuf};
use walkdir::{DirEntry, WalkDir};

#[derive(Eq, PartialEq, Clone, Copy)]
enum InteractiveMode {
    None,
    Once,
    Always,
}

struct Options {
    force: bool,
    interactive: InteractiveMode,
    #[allow(dead_code)]
    one_fs: bool,
    preserve_root: bool,
    recursive: bool,
    dir: bool,
    verbose: bool,
}

static ABOUT: &str = "Remove (unlink) the FILE(s)";
static OPT_DIR: &str = "dir";
static OPT_INTERACTIVE: &str = "interactive";
static OPT_FORCE: &str = "force";
static OPT_NO_PRESERVE_ROOT: &str = "no-preserve-root";
static OPT_ONE_FILE_SYSTEM: &str = "one-file-system";
static OPT_PRESERVE_ROOT: &str = "preserve-root";
static OPT_PROMPT: &str = "prompt";
static OPT_PROMPT_MORE: &str = "prompt-more";
static OPT_RECURSIVE: &str = "recursive";
static OPT_RECURSIVE_R: &str = "recursive_R";
static OPT_VERBOSE: &str = "verbose";

static ARG_FILES: &str = "files";

fn get_usage() -> String {
    format!("{0} [OPTION]... FILE...", executable!())
}

fn get_long_usage() -> String {
    String::from(
        "By default, rm does not remove directories.  Use the --recursive (-r or -R)
        option to remove each listed directory, too, along with all of its contents

        To remove a file whose name starts with a '-', for example '-foo',
        use one of these commands:
        rm -- -foo

        rm ./-foo

        Note that if you use rm to remove a file, it might be possible to recover
        some of its contents, given sufficient expertise and/or time.  For greater
        assurance that the contents are truly unrecoverable, consider using shred.",
    )
}

pub fn uumain(args: impl uucore::Args) -> i32 {
    let usage = get_usage();
    let long_usage = get_long_usage();

    let matches = App::new(executable!())
        .version(crate_version!())
        .about(ABOUT)
        .usage(&usage[..])
        .after_help(&long_usage[..])

        .arg(
            Arg::with_name(OPT_FORCE)
            .short("f")
            .long(OPT_FORCE)
            .multiple(true)
            .help("ignore nonexistent files and arguments, never prompt")
        )
        .arg(
            Arg::with_name(OPT_PROMPT)
            .short("i")
            .long("prompt before every removal")
        )
        .arg(
            Arg::with_name(OPT_PROMPT_MORE)
            .short("I")
            .help("prompt once before removing more than three files, or when removing recursively. Less intrusive than -i, while still giving some protection against most mistakes")
        )
        .arg(
            Arg::with_name(OPT_INTERACTIVE)
            .long(OPT_INTERACTIVE)
            .help("prompt according to WHEN: never, once (-I), or always (-i). Without WHEN, prompts always")
            .value_name("WHEN")
            .takes_value(true)
        )
        .arg(
            Arg::with_name(OPT_ONE_FILE_SYSTEM)
            .long(OPT_ONE_FILE_SYSTEM)
            .help("when removing a hierarchy recursively, skip any directory that is on a file system different from that of the corresponding command line argument (NOT IMPLEMENTED)")
        )
        .arg(
            Arg::with_name(OPT_NO_PRESERVE_ROOT)
            .long(OPT_NO_PRESERVE_ROOT)
            .help("do not treat '/' specially")
        )
        .arg(
            Arg::with_name(OPT_PRESERVE_ROOT)
            .long(OPT_PRESERVE_ROOT)
            .help("do not remove '/' (default)")
        )
        .arg(
            Arg::with_name(OPT_RECURSIVE).short("r")
            .long(OPT_RECURSIVE)
            .help("remove directories and their contents recursively")
        )
        .arg(
            // To mimic GNU's behavior we also want the '-R' flag. However, using clap's
            // alias method 'visible_alias("R")' would result in a long '--R' flag.
            Arg::with_name(OPT_RECURSIVE_R).short("R")
            .help("Equivalent to -r")
        )
        .arg(
            Arg::with_name(OPT_DIR)
            .short("d")
            .long(OPT_DIR)
            .help("remove empty directories")
        )
        .arg(
            Arg::with_name(OPT_VERBOSE)
            .short("v")
            .long(OPT_VERBOSE)
            .help("explain what is being done")
        )
        .arg(
            Arg::with_name(ARG_FILES)
            .multiple(true)
            .takes_value(true)
            .min_values(1)
        )
        .get_matches_from(args);

    let files: Vec<String> = matches
        .values_of(ARG_FILES)
        .map(|v| v.map(ToString::to_string).collect())
        .unwrap_or_default();

    let force = matches.is_present(OPT_FORCE);

    if files.is_empty() && !force {
        // Still check by hand and not use clap
        // Because "rm -f" is a thing
        show_error!("missing an argument");
        show_error!("for help, try '{0} --help'", executable!());
        return 1;
    } else {
        let options = Options {
            force,
            interactive: {
                if matches.is_present(OPT_PROMPT) {
                    InteractiveMode::Always
                } else if matches.is_present(OPT_PROMPT_MORE) {
                    InteractiveMode::Once
                } else if matches.is_present(OPT_INTERACTIVE) {
                    match matches.value_of(OPT_INTERACTIVE).unwrap() {
                        "none" => InteractiveMode::None,
                        "once" => InteractiveMode::Once,
                        "always" => InteractiveMode::Always,
                        val => crash!(1, "Invalid argument to interactive ({})", val),
                    }
                } else {
                    InteractiveMode::None
                }
            },
            one_fs: matches.is_present(OPT_ONE_FILE_SYSTEM),
            preserve_root: !matches.is_present(OPT_NO_PRESERVE_ROOT),
            recursive: matches.is_present(OPT_RECURSIVE) || matches.is_present(OPT_RECURSIVE_R),
            dir: matches.is_present(OPT_DIR),
            verbose: matches.is_present(OPT_VERBOSE),
        };
        if options.interactive == InteractiveMode::Once && (options.recursive || files.len() > 3) {
            let msg = if options.recursive {
                "Remove all arguments recursively? "
            } else {
                "Remove all arguments? "
            };
            if !prompt(msg) {
                return 0;
            }
        }

        if remove(files, options) {
            return 1;
        }
    }

    0
}

// TODO: implement one-file-system (this may get partially implemented in walkdir)
fn remove(files: Vec<String>, options: Options) -> bool {
    let mut had_err = false;

    for filename in &files {
        let file = Path::new(filename);
        had_err = match file.symlink_metadata() {
            Ok(metadata) => {
                if metadata.is_dir() {
                    handle_dir(file, &options)
                } else if is_symlink_dir(&metadata) {
                    remove_dir(file, &options)
                } else {
                    remove_file(file, &options)
                }
            }
            Err(_e) => {
                // TODO: actually print out the specific error
                // TODO: When the error is not about missing files
                // (e.g., permission), even rm -f should fail with
                // outputting the error, but there's no easy eay.
                if !options.force {
                    show_error!("cannot remove '{}': No such file or directory", filename);
                    true
                } else {
                    false
                }
            }
        }
        .bitor(had_err);
    }

    had_err
}

fn handle_dir(path: &Path, options: &Options) -> bool {
    let mut had_err = false;

    let is_root = path.has_root() && path.parent().is_none();
    if options.recursive && (!is_root || !options.preserve_root) {
        if options.interactive != InteractiveMode::Always && !options.verbose {
            // we need the extra crate because apparently fs::remove_dir_all() does not function
            // correctly on Windows
            if let Err(e) = remove_dir_all(path) {
                had_err = true;
                if e.kind() == std::io::ErrorKind::PermissionDenied {
                    // GNU compatibility (rm/fail-eacces.sh)
                    // here, GNU doesn't use some kind of remove_dir_all
                    // It will show directory+file
                    show_error!(
                        "cannot remove '{}': {}",
                        path.display(),
                        "Permission denied"
                    );
                } else {
                    show_error!("cannot remove '{}': {}", path.display(), e);
                }
            }
        } else {
            let mut dirs: VecDeque<DirEntry> = VecDeque::new();

            for entry in WalkDir::new(path) {
                match entry {
                    Ok(entry) => {
                        let file_type = entry.file_type();
                        if file_type.is_dir() {
                            dirs.push_back(entry);
                        } else {
                            had_err = remove_file(entry.path(), options).bitor(had_err);
                        }
                    }
                    Err(e) => {
                        had_err = true;
                        show_error!("recursing in '{}': {}", path.display(), e);
                    }
                }
            }

            for dir in dirs.iter().rev() {
                had_err = remove_dir(dir.path(), options).bitor(had_err);
            }
        }
    } else if options.dir && (!is_root || !options.preserve_root) {
        had_err = remove_dir(path, options).bitor(had_err);
    } else if options.recursive {
        show_error!("could not remove directory '{}'", path.display());
        had_err = true;
    } else {
        show_error!(
            "cannot remove '{}': Is a directory", // GNU's rm error message does not include help
            path.display()
        );
        had_err = true;
    }

    had_err
}

fn remove_dir(path: &Path, options: &Options) -> bool {
    let response = if options.interactive == InteractiveMode::Always {
        prompt_file(path, true)
    } else {
        true
    };
    if response {
        if let Ok(mut read_dir) = fs::read_dir(path) {
            if options.dir || options.recursive {
                if read_dir.next().is_none() {
                    match fs::remove_dir(path) {
                        Ok(_) => {
                            if options.verbose {
                                println!("removed directory '{}'", normalize(path).display());
                            }
                        }
                        Err(e) => {
                            if e.kind() == std::io::ErrorKind::PermissionDenied {
                                // GNU compatibility (rm/fail-eacces.sh)
                                show_error!(
                                    "cannot remove '{}': {}",
                                    path.display(),
                                    "Permission denied"
                                );
                            } else {
                                show_error!("cannot remove '{}': {}", path.display(), e);
                            }
                            return true;
                        }
                    }
                } else {
                    // directory can be read but is not empty
                    show_error!("cannot remove '{}': Directory not empty", path.display());
                    return true;
                }
            } else {
                // called to remove a symlink_dir (windows) without "-r"/"-R" or "-d"
                show_error!("cannot remove '{}': Is a directory", path.display());
                return true;
            }
        } else {
            // GNU's rm shows this message if directory is empty but not readable
            show_error!("cannot remove '{}': Directory not empty", path.display());
            return true;
        }
    }

    false
}

fn remove_file(path: &Path, options: &Options) -> bool {
    let response = if options.interactive == InteractiveMode::Always {
        prompt_file(path, false)
    } else {
        true
    };
    if response {
        match fs::remove_file(path) {
            Ok(_) => {
                if options.verbose {
                    println!("removed '{}'", normalize(path).display());
                }
            }
            Err(e) => {
                if e.kind() == std::io::ErrorKind::PermissionDenied {
                    // GNU compatibility (rm/fail-eacces.sh)
                    show_error!(
                        "cannot remove '{}': {}",
                        path.display(),
                        "Permission denied"
                    );
                } else {
                    show_error!("cannot remove '{}': {}", path.display(), e);
                }
                return true;
            }
        }
    }

    false
}

fn prompt_file(path: &Path, is_dir: bool) -> bool {
    if is_dir {
        prompt(&(format!("rm: remove directory '{}'? ", path.display())))
    } else {
        prompt(&(format!("rm: remove file '{}'? ", path.display())))
    }
}

fn normalize(path: &Path) -> PathBuf {
    // copied from https://github.com/rust-lang/cargo/blob/2e4cfc2b7d43328b207879228a2ca7d427d188bb/src/cargo/util/paths.rs#L65-L90
    // both projects are MIT https://github.com/rust-lang/cargo/blob/master/LICENSE-MIT
    // for std impl progress see rfc https://github.com/rust-lang/rfcs/issues/2208
    // TODO: replace this once that lands
    uucore::fs::normalize_path(path)
}

fn prompt(msg: &str) -> bool {
    let _ = stderr().write_all(msg.as_bytes());
    let _ = stderr().flush();

    let mut buf = Vec::new();
    let stdin = stdin();
    let mut stdin = stdin.lock();

    match stdin.read_until(b'\n', &mut buf) {
        Ok(x) if x > 0 => matches!(buf[0], b'y' | b'Y'),
        _ => false,
    }
}

#[cfg(not(windows))]
fn is_symlink_dir(_metadata: &fs::Metadata) -> bool {
    false
}

#[cfg(windows)]
use std::os::windows::prelude::MetadataExt;

#[cfg(windows)]
fn is_symlink_dir(metadata: &fs::Metadata) -> bool {
    use std::os::raw::c_ulong;
    pub type DWORD = c_ulong;
    pub const FILE_ATTRIBUTE_DIRECTORY: DWORD = 0x10;

    metadata.file_type().is_symlink()
        && ((metadata.file_attributes() & FILE_ATTRIBUTE_DIRECTORY) != 0)
}
<|endoftext|>


<commit_before>//  * This file is part of the uutils coreutils package.
//  *
//  * (c) Alex Lyon <arcterus@mail.com>
//  *
//  * For the full copyright and license information, please view the LICENSE
//  * file that was distributed with this source code.

// spell-checker:ignore (ToDO) bitor ulong

#[macro_use]
extern crate uucore;

use clap::{App, Arg};
use remove_dir_all::remove_dir_all;
use std::collections::VecDeque;
use std::fs;
use std::io::{stderr, stdin, BufRead, Write};
use std::ops::BitOr;
use std::path::Path;
use walkdir::{DirEntry, WalkDir};

#[derive(Eq, PartialEq, Clone, Copy)]
enum InteractiveMode {
    None,
    Once,
    Always,
}

struct Options {
    force: bool,
    interactive: InteractiveMode,
    #[allow(dead_code)]
    one_fs: bool,
    preserve_root: bool,
    recursive: bool,
    dir: bool,
    verbose: bool,
}

static ABOUT: &str = "Remove (unlink) the FILE(s)";
static VERSION: &str = env!("CARGO_PKG_VERSION");
static OPT_DIR: &str = "dir";
static OPT_INTERACTIVE: &str = "interactive";
static OPT_FORCE: &str = "force";
static OPT_NO_PRESERVE_ROOT: &str = "no-preserve-root";
static OPT_ONE_FILE_SYSTEM: &str = "one-file-system";
static OPT_PRESERVE_ROOT: &str = "preserve-root";
static OPT_PROMPT: &str = "prompt";
static OPT_PROMPT_MORE: &str = "prompt-more";
static OPT_RECURSIVE: &str = "recursive";
static OPT_VERBOSE: &str = "verbose";

static ARG_FILES: &str = "files";

fn get_usage() -> String {
    format!("{0} [OPTION]... FILE...", executable!())
}

fn get_long_usage() -> String {
    String::from(
        "By default, rm does not remove directories.  Use the --recursive (-r)
        option to remove each listed directory, too, along with all of its contents

        To remove a file whose name starts with a '-', for example '-foo',
        use one of these commands:
        rm -- -foo

        rm ./-foo

        Note that if you use rm to remove a file, it might be possible to recover
        some of its contents, given sufficient expertise and/or time.  For greater
        assurance that the contents are truly unrecoverable, consider using shred.",
    )
}

pub fn uumain(args: impl uucore::Args) -> i32 {
    let usage = get_usage();
    let long_usage = get_long_usage();

    let matches = App::new(executable!())
        .version(VERSION)
        .about(ABOUT)
        .usage(&usage[..])
        .after_help(&long_usage[..])
    // TODO: make getopts support -R in addition to -r

        .arg(
            Arg::with_name(OPT_FORCE)
            .short("f")
            .long(OPT_FORCE)
            .multiple(true)
            .help("ignore nonexistent files and arguments, never prompt")
        )
        .arg(
            Arg::with_name(OPT_PROMPT)
            .short("i")
            .long("prompt before every removal")
        )
        .arg(
            Arg::with_name(OPT_PROMPT_MORE)
            .short("I")
            .help("prompt once before removing more than three files, or when removing recursively. Less intrusive than -i, while still giving some protection against most mistakes")
        )
        .arg(
            Arg::with_name(OPT_INTERACTIVE)
            .long(OPT_INTERACTIVE)
            .help("prompt according to WHEN: never, once (-I), or always (-i). Without WHEN, prompts always")
            .value_name("WHEN")
            .takes_value(true)
        )
        .arg(
            Arg::with_name(OPT_ONE_FILE_SYSTEM)
            .long(OPT_ONE_FILE_SYSTEM)
            .help("when removing a hierarchy recursively, skip any directory that is on a file system different from that of the corresponding command line argument (NOT IMPLEMENTED)")
        )
        .arg(
            Arg::with_name(OPT_NO_PRESERVE_ROOT)
            .long(OPT_NO_PRESERVE_ROOT)
            .help("do not treat '/' specially")
        )
        .arg(
            Arg::with_name(OPT_PRESERVE_ROOT)
            .long(OPT_PRESERVE_ROOT)
            .help("do not remove '/' (default)")
        )
        .arg(
            Arg::with_name(OPT_RECURSIVE).short("r")
            .long(OPT_RECURSIVE)
            .help("remove directories and their contents recursively")
        )
        .arg(
            Arg::with_name(OPT_DIR)
            .short("d")
            .long(OPT_DIR)
            .help("remove empty directories")
        )
        .arg(
            Arg::with_name(OPT_VERBOSE)
            .short("v")
            .long(OPT_VERBOSE)
            .help("explain what is being done")
        )
        .arg(
            Arg::with_name(ARG_FILES)
            .multiple(true)
            .takes_value(true)
            .min_values(1)
        )
        .get_matches_from(args);

    let files: Vec<String> = matches
        .values_of(ARG_FILES)
        .map(|v| v.map(ToString::to_string).collect())
        .unwrap_or_default();

    let force = matches.is_present(OPT_FORCE);

    if files.is_empty() && !force {
        // Still check by hand and not use clap
        // Because "rm -f" is a thing
        show_error!("missing an argument");
        show_error!("for help, try '{0} --help'", executable!());
        return 1;
    } else {
        let options = Options {
            force,
            interactive: {
                if matches.is_present(OPT_PROMPT) {
                    InteractiveMode::Always
                } else if matches.is_present(OPT_PROMPT_MORE) {
                    InteractiveMode::Once
                } else if matches.is_present(OPT_INTERACTIVE) {
                    match &matches.value_of(OPT_INTERACTIVE).unwrap()[..] {
                        "none" => InteractiveMode::None,
                        "once" => InteractiveMode::Once,
                        "always" => InteractiveMode::Always,
                        val => crash!(1, "Invalid argument to interactive ({})", val),
                    }
                } else {
                    InteractiveMode::None
                }
            },
            one_fs: matches.is_present(OPT_ONE_FILE_SYSTEM),
            preserve_root: !matches.is_present(OPT_NO_PRESERVE_ROOT),
            recursive: matches.is_present(OPT_RECURSIVE),
            dir: matches.is_present(OPT_DIR),
            verbose: matches.is_present(OPT_VERBOSE),
        };
        if options.interactive == InteractiveMode::Once && (options.recursive || files.len() > 3) {
            let msg = if options.recursive {
                "Remove all arguments recursively? "
            } else {
                "Remove all arguments? "
            };
            if !prompt(msg) {
                return 0;
            }
        }

        if remove(files, options) {
            return 1;
        }
    }

    0
}

// TODO: implement one-file-system (this may get partially implemented in walkdir)
fn remove(files: Vec<String>, options: Options) -> bool {
    let mut had_err = false;

    for filename in &files {
        let file = Path::new(filename);
        had_err = match file.symlink_metadata() {
            Ok(metadata) => {
                if metadata.is_dir() {
                    handle_dir(file, &options)
                } else if is_symlink_dir(&metadata) {
                    remove_dir(file, &options)
                } else {
                    remove_file(file, &options)
                }
            }
            Err(_e) => {
                // TODO: actually print out the specific error
                // TODO: When the error is not about missing files
                // (e.g., permission), even rm -f should fail with
                // outputting the error, but there's no easy eay.
                if !options.force {
                    show_error!("no such file or directory '{}'", filename);
                    true
                } else {
                    false
                }
            }
        }
        .bitor(had_err);
    }

    had_err
}

fn handle_dir(path: &Path, options: &Options) -> bool {
    let mut had_err = false;

    let is_root = path.has_root() && path.parent().is_none();
    if options.recursive && (!is_root || !options.preserve_root) {
        if options.interactive != InteractiveMode::Always {
            // we need the extra crate because apparently fs::remove_dir_all() does not function
            // correctly on Windows
            if let Err(e) = remove_dir_all(path) {
                had_err = true;
                show_error!("could not remove '{}': {}", path.display(), e);
            }
        } else {
            let mut dirs: VecDeque<DirEntry> = VecDeque::new();

            for entry in WalkDir::new(path) {
                match entry {
                    Ok(entry) => {
                        let file_type = entry.file_type();
                        if file_type.is_dir() {
                            dirs.push_back(entry);
                        } else {
                            had_err = remove_file(entry.path(), options).bitor(had_err);
                        }
                    }
                    Err(e) => {
                        had_err = true;
                        show_error!("recursing in '{}': {}", path.display(), e);
                    }
                }
            }

            for dir in dirs.iter().rev() {
                had_err = remove_dir(dir.path(), options).bitor(had_err);
            }
        }
    } else if options.dir && (!is_root || !options.preserve_root) {
        had_err = remove_dir(path, options).bitor(had_err);
    } else if options.recursive {
        show_error!("could not remove directory '{}'", path.display());
        had_err = true;
    } else {
        show_error!(
            "could not remove directory '{}' (did you mean to pass '-r'?)",
            path.display()
        );
        had_err = true;
    }

    had_err
}

fn remove_dir(path: &Path, options: &Options) -> bool {
    let response = if options.interactive == InteractiveMode::Always {
        prompt_file(path, true)
    } else {
        true
    };
    if response {
        match fs::remove_dir(path) {
            Ok(_) => {
                if options.verbose {
                    println!("removed '{}'", path.display());
                }
            }
            Err(e) => {
                show_error!("removing '{}': {}", path.display(), e);
                return true;
            }
        }
    }

    false
}

fn remove_file(path: &Path, options: &Options) -> bool {
    let response = if options.interactive == InteractiveMode::Always {
        prompt_file(path, false)
    } else {
        true
    };
    if response {
        match fs::remove_file(path) {
            Ok(_) => {
                if options.verbose {
                    println!("removed '{}'", path.display());
                }
            }
            Err(e) => {
                show_error!("removing '{}': {}", path.display(), e);
                return true;
            }
        }
    }

    false
}

fn prompt_file(path: &Path, is_dir: bool) -> bool {
    if is_dir {
        prompt(&(format!("rm: remove directory '{}'? ", path.display())))
    } else {
        prompt(&(format!("rm: remove file '{}'? ", path.display())))
    }
}

fn prompt(msg: &str) -> bool {
    let _ = stderr().write_all(msg.as_bytes());
    let _ = stderr().flush();

    let mut buf = Vec::new();
    let stdin = stdin();
    let mut stdin = stdin.lock();

    #[allow(clippy::match_like_matches_macro)]
    // `matches!(...)` macro not stabilized until rust v1.42
    match stdin.read_until(b'\n', &mut buf) {
        Ok(x) if x > 0 => match buf[0] {
            b'y' | b'Y' => true,
            _ => false,
        },
        _ => false,
    }
}

#[cfg(not(windows))]
fn is_symlink_dir(_metadata: &fs::Metadata) -> bool {
    false
}

#[cfg(windows)]
use std::os::windows::prelude::MetadataExt;

#[cfg(windows)]
fn is_symlink_dir(metadata: &fs::Metadata) -> bool {
    use std::os::raw::c_ulong;
    pub type DWORD = c_ulong;
    pub const FILE_ATTRIBUTE_DIRECTORY: DWORD = 0x10;

    metadata.file_type().is_symlink()
        && ((metadata.file_attributes() & FILE_ATTRIBUTE_DIRECTORY) != 0)
}
<commit_msg>rm: add an additional flag -R for --recursive<commit_after>//  * This file is part of the uutils coreutils package.
//  *
//  * (c) Alex Lyon <arcterus@mail.com>
//  *
//  * For the full copyright and license information, please view the LICENSE
//  * file that was distributed with this source code.

// spell-checker:ignore (ToDO) bitor ulong

#[macro_use]
extern crate uucore;

use clap::{App, Arg};
use remove_dir_all::remove_dir_all;
use std::collections::VecDeque;
use std::fs;
use std::io::{stderr, stdin, BufRead, Write};
use std::ops::BitOr;
use std::path::Path;
use walkdir::{DirEntry, WalkDir};

#[derive(Eq, PartialEq, Clone, Copy)]
enum InteractiveMode {
    None,
    Once,
    Always,
}

struct Options {
    force: bool,
    interactive: InteractiveMode,
    #[allow(dead_code)]
    one_fs: bool,
    preserve_root: bool,
    recursive: bool,
    dir: bool,
    verbose: bool,
}

static ABOUT: &str = "Remove (unlink) the FILE(s)";
static VERSION: &str = env!("CARGO_PKG_VERSION");
static OPT_DIR: &str = "dir";
static OPT_INTERACTIVE: &str = "interactive";
static OPT_FORCE: &str = "force";
static OPT_NO_PRESERVE_ROOT: &str = "no-preserve-root";
static OPT_ONE_FILE_SYSTEM: &str = "one-file-system";
static OPT_PRESERVE_ROOT: &str = "preserve-root";
static OPT_PROMPT: &str = "prompt";
static OPT_PROMPT_MORE: &str = "prompt-more";
static OPT_RECURSIVE: &str = "recursive";
static OPT_RECURSIVE_R: &str = "recursive_R";
static OPT_VERBOSE: &str = "verbose";

static ARG_FILES: &str = "files";

fn get_usage() -> String {
    format!("{0} [OPTION]... FILE...", executable!())
}

fn get_long_usage() -> String {
    String::from(
        "By default, rm does not remove directories.  Use the --recursive (-r or -R)
        option to remove each listed directory, too, along with all of its contents

        To remove a file whose name starts with a '-', for example '-foo',
        use one of these commands:
        rm -- -foo

        rm ./-foo

        Note that if you use rm to remove a file, it might be possible to recover
        some of its contents, given sufficient expertise and/or time.  For greater
        assurance that the contents are truly unrecoverable, consider using shred.",
    )
}

pub fn uumain(args: impl uucore::Args) -> i32 {
    let usage = get_usage();
    let long_usage = get_long_usage();

    let matches = App::new(executable!())
        .version(VERSION)
        .about(ABOUT)
        .usage(&usage[..])
        .after_help(&long_usage[..])

        .arg(
            Arg::with_name(OPT_FORCE)
            .short("f")
            .long(OPT_FORCE)
            .multiple(true)
            .help("ignore nonexistent files and arguments, never prompt")
        )
        .arg(
            Arg::with_name(OPT_PROMPT)
            .short("i")
            .long("prompt before every removal")
        )
        .arg(
            Arg::with_name(OPT_PROMPT_MORE)
            .short("I")
            .help("prompt once before removing more than three files, or when removing recursively. Less intrusive than -i, while still giving some protection against most mistakes")
        )
        .arg(
            Arg::with_name(OPT_INTERACTIVE)
            .long(OPT_INTERACTIVE)
            .help("prompt according to WHEN: never, once (-I), or always (-i). Without WHEN, prompts always")
            .value_name("WHEN")
            .takes_value(true)
        )
        .arg(
            Arg::with_name(OPT_ONE_FILE_SYSTEM)
            .long(OPT_ONE_FILE_SYSTEM)
            .help("when removing a hierarchy recursively, skip any directory that is on a file system different from that of the corresponding command line argument (NOT IMPLEMENTED)")
        )
        .arg(
            Arg::with_name(OPT_NO_PRESERVE_ROOT)
            .long(OPT_NO_PRESERVE_ROOT)
            .help("do not treat '/' specially")
        )
        .arg(
            Arg::with_name(OPT_PRESERVE_ROOT)
            .long(OPT_PRESERVE_ROOT)
            .help("do not remove '/' (default)")
        )
        .arg(
            Arg::with_name(OPT_RECURSIVE).short("r")
            .long(OPT_RECURSIVE)
            .help("remove directories and their contents recursively")
        )
        .arg(
            Arg::with_name(OPT_RECURSIVE_R).short("R")
            .help("Equivalent to -r")
        )
        .arg(
            Arg::with_name(OPT_DIR)
            .short("d")
            .long(OPT_DIR)
            .help("remove empty directories")
        )
        .arg(
            Arg::with_name(OPT_VERBOSE)
            .short("v")
            .long(OPT_VERBOSE)
            .help("explain what is being done")
        )
        .arg(
            Arg::with_name(ARG_FILES)
            .multiple(true)
            .takes_value(true)
            .min_values(1)
        )
        .get_matches_from(args);

    let files: Vec<String> = matches
        .values_of(ARG_FILES)
        .map(|v| v.map(ToString::to_string).collect())
        .unwrap_or_default();

    let force = matches.is_present(OPT_FORCE);

    if files.is_empty() && !force {
        // Still check by hand and not use clap
        // Because "rm -f" is a thing
        show_error!("missing an argument");
        show_error!("for help, try '{0} --help'", executable!());
        return 1;
    } else {
        let options = Options {
            force,
            interactive: {
                if matches.is_present(OPT_PROMPT) {
                    InteractiveMode::Always
                } else if matches.is_present(OPT_PROMPT_MORE) {
                    InteractiveMode::Once
                } else if matches.is_present(OPT_INTERACTIVE) {
                    match &matches.value_of(OPT_INTERACTIVE).unwrap()[..] {
                        "none" => InteractiveMode::None,
                        "once" => InteractiveMode::Once,
                        "always" => InteractiveMode::Always,
                        val => crash!(1, "Invalid argument to interactive ({})", val),
                    }
                } else {
                    InteractiveMode::None
                }
            },
            one_fs: matches.is_present(OPT_ONE_FILE_SYSTEM),
            preserve_root: !matches.is_present(OPT_NO_PRESERVE_ROOT),
            recursive: matches.is_present(OPT_RECURSIVE) || matches.is_present(OPT_RECURSIVE_R),
            dir: matches.is_present(OPT_DIR),
            verbose: matches.is_present(OPT_VERBOSE),
        };
        if options.interactive == InteractiveMode::Once && (options.recursive || files.len() > 3) {
            let msg = if options.recursive {
                "Remove all arguments recursively? "
            } else {
                "Remove all arguments? "
            };
            if !prompt(msg) {
                return 0;
            }
        }

        if remove(files, options) {
            return 1;
        }
    }

    0
}

// TODO: implement one-file-system (this may get partially implemented in walkdir)
fn remove(files: Vec<String>, options: Options) -> bool {
    let mut had_err = false;

    for filename in &files {
        let file = Path::new(filename);
        had_err = match file.symlink_metadata() {
            Ok(metadata) => {
                if metadata.is_dir() {
                    handle_dir(file, &options)
                } else if is_symlink_dir(&metadata) {
                    remove_dir(file, &options)
                } else {
                    remove_file(file, &options)
                }
            }
            Err(_e) => {
                // TODO: actually print out the specific error
                // TODO: When the error is not about missing files
                // (e.g., permission), even rm -f should fail with
                // outputting the error, but there's no easy eay.
                if !options.force {
                    show_error!("no such file or directory '{}'", filename);
                    true
                } else {
                    false
                }
            }
        }
        .bitor(had_err);
    }

    had_err
}

fn handle_dir(path: &Path, options: &Options) -> bool {
    let mut had_err = false;

    let is_root = path.has_root() && path.parent().is_none();
    if options.recursive && (!is_root || !options.preserve_root) {
        if options.interactive != InteractiveMode::Always {
            // we need the extra crate because apparently fs::remove_dir_all() does not function
            // correctly on Windows
            if let Err(e) = remove_dir_all(path) {
                had_err = true;
                show_error!("could not remove '{}': {}", path.display(), e);
            }
        } else {
            let mut dirs: VecDeque<DirEntry> = VecDeque::new();

            for entry in WalkDir::new(path) {
                match entry {
                    Ok(entry) => {
                        let file_type = entry.file_type();
                        if file_type.is_dir() {
                            dirs.push_back(entry);
                        } else {
                            had_err = remove_file(entry.path(), options).bitor(had_err);
                        }
                    }
                    Err(e) => {
                        had_err = true;
                        show_error!("recursing in '{}': {}", path.display(), e);
                    }
                }
            }

            for dir in dirs.iter().rev() {
                had_err = remove_dir(dir.path(), options).bitor(had_err);
            }
        }
    } else if options.dir && (!is_root || !options.preserve_root) {
        had_err = remove_dir(path, options).bitor(had_err);
    } else if options.recursive {
        show_error!("could not remove directory '{}'", path.display());
        had_err = true;
    } else {
        show_error!(
            "could not remove directory '{}' (did you mean to pass '-r' or '-R'?)",
            path.display()
        );
        had_err = true;
    }

    had_err
}

fn remove_dir(path: &Path, options: &Options) -> bool {
    let response = if options.interactive == InteractiveMode::Always {
        prompt_file(path, true)
    } else {
        true
    };
    if response {
        match fs::remove_dir(path) {
            Ok(_) => {
                if options.verbose {
                    println!("removed '{}'", path.display());
                }
            }
            Err(e) => {
                show_error!("removing '{}': {}", path.display(), e);
                return true;
            }
        }
    }

    false
}

fn remove_file(path: &Path, options: &Options) -> bool {
    let response = if options.interactive == InteractiveMode::Always {
        prompt_file(path, false)
    } else {
        true
    };
    if response {
        match fs::remove_file(path) {
            Ok(_) => {
                if options.verbose {
                    println!("removed '{}'", path.display());
                }
            }
            Err(e) => {
                show_error!("removing '{}': {}", path.display(), e);
                return true;
            }
        }
    }

    false
}

fn prompt_file(path: &Path, is_dir: bool) -> bool {
    if is_dir {
        prompt(&(format!("rm: remove directory '{}'? ", path.display())))
    } else {
        prompt(&(format!("rm: remove file '{}'? ", path.display())))
    }
}

fn prompt(msg: &str) -> bool {
    let _ = stderr().write_all(msg.as_bytes());
    let _ = stderr().flush();

    let mut buf = Vec::new();
    let stdin = stdin();
    let mut stdin = stdin.lock();

    #[allow(clippy::match_like_matches_macro)]
    // `matches!(...)` macro not stabilized until rust v1.42
    match stdin.read_until(b'\n', &mut buf) {
        Ok(x) if x > 0 => match buf[0] {
            b'y' | b'Y' => true,
            _ => false,
        },
        _ => false,
    }
}

#[cfg(not(windows))]
fn is_symlink_dir(_metadata: &fs::Metadata) -> bool {
    false
}

#[cfg(windows)]
use std::os::windows::prelude::MetadataExt;

#[cfg(windows)]
fn is_symlink_dir(metadata: &fs::Metadata) -> bool {
    use std::os::raw::c_ulong;
    pub type DWORD = c_ulong;
    pub const FILE_ATTRIBUTE_DIRECTORY: DWORD = 0x10;

    metadata.file_type().is_symlink()
        && ((metadata.file_attributes() & FILE_ATTRIBUTE_DIRECTORY) != 0)
}
<|endoftext|>


<commit_before>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn transpose(filename: String, index: usize, peers: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let graph = GraphMMap::<u32>::new(&filename);

    let mut edges = Vec::new();
    let mut deg = vec![];

    for node in 0..graph.nodes() {
        if node % peers == index {
            deg.push(graph.edges(index + peers * node).len() as u32);
            for &b in graph.edges(index + peers * node) {
                edges.push((b as u32, node as u32));
            }
        }
    }

    edges.sort();

    let mut rev = vec![(0,0);0];
    let mut reversed = vec![];
    for (d, s) in edges.drain_temp() {
        let len = rev.len();
        if len == 0 || rev[len-1].1 < d {
            rev.push((d, 0));
        }

        let len = rev.len();
        rev[len-1].1 += 1;
        reversed.push(s);
    }

    return (deg, rev, reversed);
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let mut start = time::precise_time_s();

        let (deg, rev, edges) = transpose(filename, index, peers);

        println!("sorted {} edges; {}s", edges.len(), time::precise_time_s() - start);
        start = time::precise_time_s();

        let mut src = vec![0.0; deg.len()];

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node] / deg[node] as f32;
                    }

                    let mut session = output.session(&iter);

                    let mut counter = 0;
                    let mut slice = &edges[..];
                    for &(dst, deg) in &rev {
                        let mut accum = 0.0;
                        for &s in &slice[..deg as usize] {
                            accum += src[s as usize];
                        }
                        slice = &slice[deg as usize..];
                        session.give((dst, accum));
                        counter += 1;
                    }

                    for s in &mut src { *s = 0.0; }

                    println!("iteration {:?}: {}s; sent {} updates", iter, time::precise_time_s() - start, counter);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        // let local_index = index as usize % workers;
        // let mut acc = vec![0.0; src.len()];

        ranks
        // .unary_notify(
        //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for (node, rank) in data.drain_temp() {
        //                 acc[node as usize / workers] += rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
        //
        //             for _ in 0..(1 + (nodes/workers)) { acc.push(0.0); }
        //             assert!(acc.len() == (1 + (nodes/workers)));
        //         }
        //     }
        // )
        .connect_loop(helper);
    }

    while root.step() { }
}
<commit_msg>updated transpose implementation<commit_after>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn transpose(filename: String, index: usize, peers: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let graph = GraphMMap::<u32>::new(&filename);

    let mut edges = Vec::new();
    let mut deg = vec![];

    for node in 0..graph.nodes() {
        if node % peers == index {
            deg.push(graph.edges(node).len() as u32);
            for &b in graph.edges(node) {
                edges.push((b as u32, node as u32));
            }
        }
    }

    println!("slice {} of {} extracted {} edges", index, peers, edges.len());

    edges.sort();

    let mut rev = vec![(0,0);0];
    let mut reversed = vec![];
    for (d, s) in edges.drain_temp() {
        let len = rev.len();
        if len == 0 || rev[len-1].1 < d {
            rev.push((d, 0));
        }

        let len = rev.len();
        rev[len-1].1 += 1;
        reversed.push(s);
    }

    return (deg, rev, reversed);
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let mut start = time::precise_time_s();

        let (deg, rev, edges) = transpose(filename, index, peers);

        println!("sorted {} edges; {}s", edges.len(), time::precise_time_s() - start);
        start = time::precise_time_s();

        let mut src = vec![0.0; deg.len()];

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node] / deg[node] as f32;
                    }

                    let mut session = output.session(&iter);

                    let mut counter = 0;
                    let mut slice = &edges[..];
                    for &(dst, deg) in &rev {
                        let mut accum = 0.0;
                        for &s in &slice[..deg as usize] {
                            accum += src[s as usize];
                        }
                        slice = &slice[deg as usize..];
                        session.give((dst, accum));
                        counter += 1;
                    }

                    for s in &mut src { *s = 0.0; }

                    println!("iteration {:?}: {}s; sent {} updates", iter, time::precise_time_s() - start, counter);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        // let local_index = index as usize % workers;
        // let mut acc = vec![0.0; src.len()];

        ranks
        // .unary_notify(
        //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for (node, rank) in data.drain_temp() {
        //                 acc[node as usize / workers] += rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
        //
        //             for _ in 0..(1 + (nodes/workers)) { acc.push(0.0); }
        //             assert!(acc.len() == (1 + (nodes/workers)));
        //         }
        //     }
        // )
        .connect_loop(helper);
    }

    while root.step() { }
}
<|endoftext|>