<commit_before>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

//! Tasks that are executed by coordinators (short-lived compared to coordinators)
use crate::{
    core_mempool::{CoreMempool, TimelineState, TxnPointer},
    counters,
    logging::{LogEntry, LogEvent, LogSchema},
    network::MempoolSyncMsg,
    shared_mempool::types::{
        notify_subscribers, ScheduledBroadcast, SharedMempool, SharedMempoolNotification,
        SubmissionStatusBundle, TransactionSummary,
    },
    ConsensusRequest, ConsensusResponse, SubmissionStatus,
};
use anyhow::Result;
use diem_config::network_id::PeerNetworkId;
use diem_infallible::{Mutex, RwLock};
use diem_logger::prelude::*;
use diem_metrics::HistogramTimer;
use diem_types::{
    mempool_status::{MempoolStatus, MempoolStatusCode},
    on_chain_config::OnChainConfigPayload,
    transaction::SignedTransaction,
    vm_status::DiscardedVMStatus,
};
use futures::{channel::oneshot, stream::FuturesUnordered};
use network::application::interface::NetworkInterface;
use rayon::prelude::*;
use short_hex_str::AsShortHexStr;
use std::{
    cmp,
    collections::HashSet,
    sync::Arc,
    time::{Duration, Instant},
};
use tokio::runtime::Handle;
use vm_validator::vm_validator::{get_account_sequence_number, TransactionValidation};

// ============================== //
//  broadcast_coordinator tasks  //
// ============================== //

/// Attempts broadcast to `peer` and schedules the next broadcast.
pub(crate) fn execute_broadcast<V>(
    peer: PeerNetworkId,
    backoff: bool,
    smp: &mut SharedMempool<V>,
    scheduled_broadcasts: &mut FuturesUnordered<ScheduledBroadcast>,
    executor: Handle,
) where
    V: TransactionValidation,
{
    let network_interface = &smp.network_interface.clone();
    network_interface.execute_broadcast(peer, backoff, smp);
    let schedule_backoff = network_interface.is_backoff_mode(&peer);

    let interval_ms = if schedule_backoff {
        smp.config.shared_mempool_backoff_interval_ms
    } else {
        smp.config.shared_mempool_tick_interval_ms
    };

    scheduled_broadcasts.push(ScheduledBroadcast::new(
        Instant::now() + Duration::from_millis(interval_ms),
        peer,
        schedule_backoff,
        executor,
    ))
}

// =============================== //
// Tasks processing txn submission //
// =============================== //

/// Processes transactions directly submitted by client.
pub(crate) async fn process_client_transaction_submission<V>(
    smp: SharedMempool<V>,
    transaction: SignedTransaction,
    callback: oneshot::Sender<Result<SubmissionStatus>>,
    timer: HistogramTimer,
) where
    V: TransactionValidation,
{
    timer.stop_and_record();
    let _timer =
        counters::process_txn_submit_latency_timer(counters::CLIENT_LABEL, counters::CLIENT_LABEL);
    let statuses =
        process_incoming_transactions(&smp, vec![transaction], TimelineState::NotReady).await;
    log_txn_process_results(&statuses, None);

    if let Some(status) = statuses.get(0) {
        if callback.send(Ok(status.1.clone())).is_err() {
            error!(LogSchema::event_log(
                LogEntry::JsonRpc,
                LogEvent::CallbackFail
            ));
            counters::CLIENT_CALLBACK_FAIL.inc();
        }
    }
}

/// Processes transactions from other nodes.
pub(crate) async fn process_transaction_broadcast<V>(
    smp: SharedMempool<V>,
    transactions: Vec<SignedTransaction>,
    request_id: Vec<u8>,
    timeline_state: TimelineState,
    peer: PeerNetworkId,
    timer: HistogramTimer,
) where
    V: TransactionValidation,
{
    timer.stop_and_record();
    let _timer = counters::process_txn_submit_latency_timer(
        peer.network_id().as_str(),
        peer.peer_id().short_str().as_str(),
    );
    let results = process_incoming_transactions(&smp, transactions.clone(), timeline_state).await;
    log_txn_process_results(&results, Some(peer));

    let ack_response = gen_ack_response(request_id, results, &peer);
    let mut network_sender = smp.network_interface.sender();
    if let Err(e) = network_sender.send_to(peer, ack_response) {
        counters::network_send_fail_inc(counters::ACK_TXNS);
        error!(
            LogSchema::event_log(LogEntry::BroadcastACK, LogEvent::NetworkSendFail)
                .peer(&peer)
                .error(&e.into())
        );
        return;
    }
    notify_subscribers(SharedMempoolNotification::ACK, &smp.subscribers);
}

fn gen_ack_response(
    request_id: Vec<u8>,
    results: Vec<SubmissionStatusBundle>,
    peer: &PeerNetworkId,
) -> MempoolSyncMsg {
    let mut backoff = false;
    let mut retry = false;
    for r in results.into_iter() {
        let submission_status = r.1;
        if submission_status.0.code == MempoolStatusCode::MempoolIsFull {
            backoff = true;
        }
        if is_txn_retryable(submission_status) {
            retry = true;
        }

        if backoff && retry {
            break;
        }
    }

    update_ack_counter(peer, counters::SENT_LABEL, retry, backoff);
    MempoolSyncMsg::BroadcastTransactionsResponse {
        request_id,
        retry,
        backoff,
    }
}

pub(crate) fn update_ack_counter(
    peer: &PeerNetworkId,
    direction_label: &str,
    retry: bool,
    backoff: bool,
) {
    if retry {
        counters::shared_mempool_ack_inc(peer, direction_label, counters::RETRY_BROADCAST_LABEL);
    }
    if backoff {
        counters::shared_mempool_ack_inc(
            peer,
            direction_label,
            counters::BACKPRESSURE_BROADCAST_LABEL,
        );
    }
}

fn is_txn_retryable(result: SubmissionStatus) -> bool {
    result.0.code == MempoolStatusCode::MempoolIsFull
}

/// Submits a list of SignedTransaction to the local mempool
/// and returns a vector containing AdmissionControlStatus.
pub(crate) async fn process_incoming_transactions<V>(
    smp: &SharedMempool<V>,
    transactions: Vec<SignedTransaction>,
    timeline_state: TimelineState,
) -> Vec<SubmissionStatusBundle>
where
    V: TransactionValidation,
{
    let mut statuses = vec![];

    let start_storage_read = Instant::now();
    // Track latency: fetching seq number
    let seq_numbers = transactions
        .par_iter()
        .map(|t| {
            get_account_sequence_number(smp.db.as_ref(), t.sender()).map_err(|e| {
                error!(LogSchema::new(LogEntry::DBError).error(&e));
                counters::DB_ERROR.inc();
                e
            })
        })
        .collect::<Vec<_>>();
    // Track latency for storage read fetching sequence number
    let storage_read_latency = start_storage_read.elapsed();
    counters::PROCESS_TXN_BREAKDOWN_LATENCY
        .with_label_values(&[counters::FETCH_SEQ_NUM_LABEL])
        .observe(storage_read_latency.as_secs_f64() / transactions.len() as f64);

    let transactions: Vec<_> = transactions
        .into_iter()
        .enumerate()
        .filter_map(|(idx, t)| {
            if let Ok(crsn_or_seqno) = seq_numbers[idx] {
                if t.sequence_number() >= crsn_or_seqno.min_seq() {
                    return Some((t, crsn_or_seqno));
                } else {
                    statuses.push((
                        t,
                        (
                            MempoolStatus::new(MempoolStatusCode::VmError),
                            Some(DiscardedVMStatus::SEQUENCE_NUMBER_TOO_OLD),
                        ),
                    ));
                }
            } else {
                // Failed to get transaction
                statuses.push((
                    t,
                    (
                        MempoolStatus::new(MempoolStatusCode::VmError),
                        Some(DiscardedVMStatus::RESOURCE_DOES_NOT_EXIST),
                    ),
                ));
            }
            None
        })
        .collect();

    // Track latency: VM validation
    let vm_validation_timer = counters::PROCESS_TXN_BREAKDOWN_LATENCY
        .with_label_values(&[counters::VM_VALIDATION_LABEL])
        .start_timer();
    let validation_results = transactions
        .par_iter()
        .map(|t| smp.validator.read().validate_transaction(t.0.clone()))
        .collect::<Vec<_>>();
    vm_validation_timer.stop_and_record();

    {
        let mut mempool = smp.mempool.lock();
        for (idx, (transaction, crsn_or_seqno)) in transactions.into_iter().enumerate() {
            if let Ok(validation_result) = &validation_results[idx] {
                match validation_result.status() {
                    None => {
                        let gas_amount = transaction.max_gas_amount();
                        let ranking_score = validation_result.score();
                        let governance_role = validation_result.governance_role();
                        let mempool_status = mempool.add_txn(
                            transaction.clone(),
                            gas_amount,
                            ranking_score,
                            crsn_or_seqno,
                            timeline_state,
                            governance_role,
                        );
                        statuses.push((transaction, (mempool_status, None)));
                    }
                    Some(validation_status) => {
                        statuses.push((
                            transaction.clone(),
                            (
                                MempoolStatus::new(MempoolStatusCode::VmError),
                                Some(validation_status),
                            ),
                        ));
                    }
                }
            }
        }
    }
    notify_subscribers(SharedMempoolNotification::NewTransactions, &smp.subscribers);
    statuses
}

fn log_txn_process_results(results: &[SubmissionStatusBundle], sender: Option<PeerNetworkId>) {
    let (network, sender) = match sender {
        Some(peer) => (
            peer.network_id().to_string(),
            peer.peer_id().short_str().to_string(),
        ),
        None => (
            counters::CLIENT_LABEL.to_string(),
            counters::CLIENT_LABEL.to_string(),
        ),
    };
    for (txn, (mempool_status, maybe_vm_status)) in results.iter() {
        if let Some(vm_status) = maybe_vm_status {
            trace!(
                SecurityEvent::InvalidTransactionMempool,
                failed_transaction = txn,
                vm_status = vm_status,
                sender = sender,
            );
            counters::shared_mempool_transactions_processed_inc(
                counters::VM_VALIDATION_LABEL,
                &network,
                &sender,
            );
            continue;
        }
        match mempool_status.code {
            MempoolStatusCode::Accepted => counters::shared_mempool_transactions_processed_inc(
                counters::SUCCESS_LABEL,
                &network,
                &sender,
            ),
            _ => counters::shared_mempool_transactions_processed_inc(
                &mempool_status.code.to_string(),
                &network,
                &sender,
            ),
        }
    }
}

// ================================= //
// intra-node communication handlers //
// ================================= //

pub(crate) fn process_consensus_request(mempool: &Mutex<CoreMempool>, req: ConsensusRequest) {
    // Start latency timer
    let start_time = Instant::now();
    debug!(LogSchema::event_log(LogEntry::Consensus, LogEvent::Received).consensus_msg(&req));

    let (resp, callback, counter_label) = match req {
        ConsensusRequest::GetBlockRequest(max_block_size, transactions, callback) => {
            let exclude_transactions: HashSet<TxnPointer> = transactions
                .iter()
                .map(|txn| (txn.sender, txn.sequence_number))
                .collect();
            let mut txns;
            {
                let mut mempool = mempool.lock();
                // gc before pulling block as extra protection against txns that may expire in consensus
                // Note: this gc operation relies on the fact that consensus uses the system time to determine block timestamp
                let curr_time = diem_infallible::duration_since_epoch();
                mempool.gc_by_expiration_time(curr_time);
                let block_size = cmp::max(max_block_size, 1);
                txns = mempool.get_block(block_size, exclude_transactions);
            }
            counters::mempool_service_transactions(counters::GET_BLOCK_LABEL, txns.len());
            txns.len();
            let pulled_block = txns.drain(..).map(SignedTransaction::into).collect();

            (
                ConsensusResponse::GetBlockResponse(pulled_block),
                callback,
                counters::GET_BLOCK_LABEL,
            )
        }
        ConsensusRequest::RejectNotification(transactions, callback) => {
            counters::mempool_service_transactions(
                counters::COMMIT_CONSENSUS_LABEL,
                transactions.len(),
            );
            commit_txns(mempool, transactions, 0, true);
            (
                ConsensusResponse::CommitResponse(),
                callback,
                counters::COMMIT_CONSENSUS_LABEL,
            )
        }
    };
    // Send back to callback
    let result = if callback.send(Ok(resp)).is_err() {
        error!(LogSchema::event_log(
            LogEntry::Consensus,
            LogEvent::CallbackFail
        ));
        counters::REQUEST_FAIL_LABEL
    } else {
        counters::REQUEST_SUCCESS_LABEL
    };
    let latency = start_time.elapsed();
    counters::mempool_service_latency(counter_label, result, latency);
}

pub fn commit_txns(
    mempool: &Mutex<CoreMempool>,
    transactions: Vec<TransactionSummary>,
    block_timestamp_usecs: u64,
    is_rejected: bool,
) {
    let mut pool = mempool.lock();

    for transaction in transactions {
        pool.remove_transaction(
            &transaction.sender,
            transaction.sequence_number,
            is_rejected,
        );
    }

    if block_timestamp_usecs > 0 {
        pool.gc_by_expiration_time(Duration::from_micros(block_timestamp_usecs));
    }
}

/// Processes on-chain reconfiguration notification.
pub(crate) async fn process_config_update<V>(
    config_update: OnChainConfigPayload,
    validator: Arc<RwLock<V>>,
) where
    V: TransactionValidation,
{
    info!(
        LogSchema::event_log(LogEntry::ReconfigUpdate, LogEvent::Process)
            .reconfig_update(config_update.clone())
    );

    if let Err(e) = validator.write().restart(config_update) {
        counters::VM_RECONFIG_UPDATE_FAIL_COUNT.inc();
        error!(LogSchema::event_log(LogEntry::ReconfigUpdate, LogEvent::VMUpdateFail).error(&e));
    }
}
<commit_msg>[mempool] Make process_incoming_transactions not async<commit_after>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

//! Tasks that are executed by coordinators (short-lived compared to coordinators)
use crate::{
    core_mempool::{CoreMempool, TimelineState, TxnPointer},
    counters,
    logging::{LogEntry, LogEvent, LogSchema},
    network::MempoolSyncMsg,
    shared_mempool::types::{
        notify_subscribers, ScheduledBroadcast, SharedMempool, SharedMempoolNotification,
        SubmissionStatusBundle, TransactionSummary,
    },
    ConsensusRequest, ConsensusResponse, SubmissionStatus,
};
use anyhow::Result;
use diem_config::network_id::PeerNetworkId;
use diem_infallible::{Mutex, RwLock};
use diem_logger::prelude::*;
use diem_metrics::HistogramTimer;
use diem_types::{
    mempool_status::{MempoolStatus, MempoolStatusCode},
    on_chain_config::OnChainConfigPayload,
    transaction::SignedTransaction,
    vm_status::DiscardedVMStatus,
};
use futures::{channel::oneshot, stream::FuturesUnordered};
use network::application::interface::NetworkInterface;
use rayon::prelude::*;
use short_hex_str::AsShortHexStr;
use std::{
    cmp,
    collections::HashSet,
    sync::Arc,
    time::{Duration, Instant},
};
use tokio::runtime::Handle;
use vm_validator::vm_validator::{get_account_sequence_number, TransactionValidation};

// ============================== //
//  broadcast_coordinator tasks  //
// ============================== //

/// Attempts broadcast to `peer` and schedules the next broadcast.
pub(crate) fn execute_broadcast<V>(
    peer: PeerNetworkId,
    backoff: bool,
    smp: &mut SharedMempool<V>,
    scheduled_broadcasts: &mut FuturesUnordered<ScheduledBroadcast>,
    executor: Handle,
) where
    V: TransactionValidation,
{
    let network_interface = &smp.network_interface.clone();
    network_interface.execute_broadcast(peer, backoff, smp);
    let schedule_backoff = network_interface.is_backoff_mode(&peer);

    let interval_ms = if schedule_backoff {
        smp.config.shared_mempool_backoff_interval_ms
    } else {
        smp.config.shared_mempool_tick_interval_ms
    };

    scheduled_broadcasts.push(ScheduledBroadcast::new(
        Instant::now() + Duration::from_millis(interval_ms),
        peer,
        schedule_backoff,
        executor,
    ))
}

// =============================== //
// Tasks processing txn submission //
// =============================== //

/// Processes transactions directly submitted by client.
pub(crate) async fn process_client_transaction_submission<V>(
    smp: SharedMempool<V>,
    transaction: SignedTransaction,
    callback: oneshot::Sender<Result<SubmissionStatus>>,
    timer: HistogramTimer,
) where
    V: TransactionValidation,
{
    timer.stop_and_record();
    let _timer =
        counters::process_txn_submit_latency_timer(counters::CLIENT_LABEL, counters::CLIENT_LABEL);
    let statuses = process_incoming_transactions(&smp, vec![transaction], TimelineState::NotReady);
    log_txn_process_results(&statuses, None);

    if let Some(status) = statuses.get(0) {
        if callback.send(Ok(status.1.clone())).is_err() {
            error!(LogSchema::event_log(
                LogEntry::JsonRpc,
                LogEvent::CallbackFail
            ));
            counters::CLIENT_CALLBACK_FAIL.inc();
        }
    }
}

/// Processes transactions from other nodes.
pub(crate) async fn process_transaction_broadcast<V>(
    smp: SharedMempool<V>,
    transactions: Vec<SignedTransaction>,
    request_id: Vec<u8>,
    timeline_state: TimelineState,
    peer: PeerNetworkId,
    timer: HistogramTimer,
) where
    V: TransactionValidation,
{
    timer.stop_and_record();
    let _timer = counters::process_txn_submit_latency_timer(
        peer.network_id().as_str(),
        peer.peer_id().short_str().as_str(),
    );
    let results = process_incoming_transactions(&smp, transactions, timeline_state);
    log_txn_process_results(&results, Some(peer));

    let ack_response = gen_ack_response(request_id, results, &peer);
    let mut network_sender = smp.network_interface.sender();
    if let Err(e) = network_sender.send_to(peer, ack_response) {
        counters::network_send_fail_inc(counters::ACK_TXNS);
        error!(
            LogSchema::event_log(LogEntry::BroadcastACK, LogEvent::NetworkSendFail)
                .peer(&peer)
                .error(&e.into())
        );
        return;
    }
    notify_subscribers(SharedMempoolNotification::ACK, &smp.subscribers);
}

fn gen_ack_response(
    request_id: Vec<u8>,
    results: Vec<SubmissionStatusBundle>,
    peer: &PeerNetworkId,
) -> MempoolSyncMsg {
    let mut backoff = false;
    let mut retry = false;
    for r in results.into_iter() {
        let submission_status = r.1;
        if submission_status.0.code == MempoolStatusCode::MempoolIsFull {
            backoff = true;
        }
        if is_txn_retryable(submission_status) {
            retry = true;
        }

        if backoff && retry {
            break;
        }
    }

    update_ack_counter(peer, counters::SENT_LABEL, retry, backoff);
    MempoolSyncMsg::BroadcastTransactionsResponse {
        request_id,
        retry,
        backoff,
    }
}

pub(crate) fn update_ack_counter(
    peer: &PeerNetworkId,
    direction_label: &str,
    retry: bool,
    backoff: bool,
) {
    if retry {
        counters::shared_mempool_ack_inc(peer, direction_label, counters::RETRY_BROADCAST_LABEL);
    }
    if backoff {
        counters::shared_mempool_ack_inc(
            peer,
            direction_label,
            counters::BACKPRESSURE_BROADCAST_LABEL,
        );
    }
}

fn is_txn_retryable(result: SubmissionStatus) -> bool {
    result.0.code == MempoolStatusCode::MempoolIsFull
}

/// Submits a list of SignedTransaction to the local mempool
/// and returns a vector containing AdmissionControlStatus.
pub(crate) fn process_incoming_transactions<V>(
    smp: &SharedMempool<V>,
    transactions: Vec<SignedTransaction>,
    timeline_state: TimelineState,
) -> Vec<SubmissionStatusBundle>
where
    V: TransactionValidation,
{
    let mut statuses = vec![];

    let start_storage_read = Instant::now();
    // Track latency: fetching seq number
    let seq_numbers = transactions
        .par_iter()
        .map(|t| {
            get_account_sequence_number(smp.db.as_ref(), t.sender()).map_err(|e| {
                error!(LogSchema::new(LogEntry::DBError).error(&e));
                counters::DB_ERROR.inc();
                e
            })
        })
        .collect::<Vec<_>>();
    // Track latency for storage read fetching sequence number
    let storage_read_latency = start_storage_read.elapsed();
    counters::PROCESS_TXN_BREAKDOWN_LATENCY
        .with_label_values(&[counters::FETCH_SEQ_NUM_LABEL])
        .observe(storage_read_latency.as_secs_f64() / transactions.len() as f64);

    let transactions: Vec<_> = transactions
        .into_iter()
        .enumerate()
        .filter_map(|(idx, t)| {
            if let Ok(crsn_or_seqno) = seq_numbers[idx] {
                if t.sequence_number() >= crsn_or_seqno.min_seq() {
                    return Some((t, crsn_or_seqno));
                } else {
                    statuses.push((
                        t,
                        (
                            MempoolStatus::new(MempoolStatusCode::VmError),
                            Some(DiscardedVMStatus::SEQUENCE_NUMBER_TOO_OLD),
                        ),
                    ));
                }
            } else {
                // Failed to get transaction
                statuses.push((
                    t,
                    (
                        MempoolStatus::new(MempoolStatusCode::VmError),
                        Some(DiscardedVMStatus::RESOURCE_DOES_NOT_EXIST),
                    ),
                ));
            }
            None
        })
        .collect();

    // Track latency: VM validation
    let vm_validation_timer = counters::PROCESS_TXN_BREAKDOWN_LATENCY
        .with_label_values(&[counters::VM_VALIDATION_LABEL])
        .start_timer();
    let validation_results = transactions
        .par_iter()
        .map(|t| smp.validator.read().validate_transaction(t.0.clone()))
        .collect::<Vec<_>>();
    vm_validation_timer.stop_and_record();

    {
        let mut mempool = smp.mempool.lock();
        for (idx, (transaction, crsn_or_seqno)) in transactions.into_iter().enumerate() {
            if let Ok(validation_result) = &validation_results[idx] {
                match validation_result.status() {
                    None => {
                        let gas_amount = transaction.max_gas_amount();
                        let ranking_score = validation_result.score();
                        let governance_role = validation_result.governance_role();
                        let mempool_status = mempool.add_txn(
                            transaction.clone(),
                            gas_amount,
                            ranking_score,
                            crsn_or_seqno,
                            timeline_state,
                            governance_role,
                        );
                        statuses.push((transaction, (mempool_status, None)));
                    }
                    Some(validation_status) => {
                        statuses.push((
                            transaction.clone(),
                            (
                                MempoolStatus::new(MempoolStatusCode::VmError),
                                Some(validation_status),
                            ),
                        ));
                    }
                }
            }
        }
    }
    notify_subscribers(SharedMempoolNotification::NewTransactions, &smp.subscribers);
    statuses
}

fn log_txn_process_results(results: &[SubmissionStatusBundle], sender: Option<PeerNetworkId>) {
    let (network, sender) = match sender {
        Some(peer) => (
            peer.network_id().to_string(),
            peer.peer_id().short_str().to_string(),
        ),
        None => (
            counters::CLIENT_LABEL.to_string(),
            counters::CLIENT_LABEL.to_string(),
        ),
    };
    for (txn, (mempool_status, maybe_vm_status)) in results.iter() {
        if let Some(vm_status) = maybe_vm_status {
            trace!(
                SecurityEvent::InvalidTransactionMempool,
                failed_transaction = txn,
                vm_status = vm_status,
                sender = sender,
            );
            counters::shared_mempool_transactions_processed_inc(
                counters::VM_VALIDATION_LABEL,
                &network,
                &sender,
            );
            continue;
        }
        match mempool_status.code {
            MempoolStatusCode::Accepted => counters::shared_mempool_transactions_processed_inc(
                counters::SUCCESS_LABEL,
                &network,
                &sender,
            ),
            _ => counters::shared_mempool_transactions_processed_inc(
                &mempool_status.code.to_string(),
                &network,
                &sender,
            ),
        }
    }
}

// ================================= //
// intra-node communication handlers //
// ================================= //

pub(crate) fn process_consensus_request(mempool: &Mutex<CoreMempool>, req: ConsensusRequest) {
    // Start latency timer
    let start_time = Instant::now();
    debug!(LogSchema::event_log(LogEntry::Consensus, LogEvent::Received).consensus_msg(&req));

    let (resp, callback, counter_label) = match req {
        ConsensusRequest::GetBlockRequest(max_block_size, transactions, callback) => {
            let exclude_transactions: HashSet<TxnPointer> = transactions
                .iter()
                .map(|txn| (txn.sender, txn.sequence_number))
                .collect();
            let mut txns;
            {
                let mut mempool = mempool.lock();
                // gc before pulling block as extra protection against txns that may expire in consensus
                // Note: this gc operation relies on the fact that consensus uses the system time to determine block timestamp
                let curr_time = diem_infallible::duration_since_epoch();
                mempool.gc_by_expiration_time(curr_time);
                let block_size = cmp::max(max_block_size, 1);
                txns = mempool.get_block(block_size, exclude_transactions);
            }
            counters::mempool_service_transactions(counters::GET_BLOCK_LABEL, txns.len());
            txns.len();
            let pulled_block = txns.drain(..).map(SignedTransaction::into).collect();

            (
                ConsensusResponse::GetBlockResponse(pulled_block),
                callback,
                counters::GET_BLOCK_LABEL,
            )
        }
        ConsensusRequest::RejectNotification(transactions, callback) => {
            counters::mempool_service_transactions(
                counters::COMMIT_CONSENSUS_LABEL,
                transactions.len(),
            );
            commit_txns(mempool, transactions, 0, true);
            (
                ConsensusResponse::CommitResponse(),
                callback,
                counters::COMMIT_CONSENSUS_LABEL,
            )
        }
    };
    // Send back to callback
    let result = if callback.send(Ok(resp)).is_err() {
        error!(LogSchema::event_log(
            LogEntry::Consensus,
            LogEvent::CallbackFail
        ));
        counters::REQUEST_FAIL_LABEL
    } else {
        counters::REQUEST_SUCCESS_LABEL
    };
    let latency = start_time.elapsed();
    counters::mempool_service_latency(counter_label, result, latency);
}

pub fn commit_txns(
    mempool: &Mutex<CoreMempool>,
    transactions: Vec<TransactionSummary>,
    block_timestamp_usecs: u64,
    is_rejected: bool,
) {
    let mut pool = mempool.lock();

    for transaction in transactions {
        pool.remove_transaction(
            &transaction.sender,
            transaction.sequence_number,
            is_rejected,
        );
    }

    if block_timestamp_usecs > 0 {
        pool.gc_by_expiration_time(Duration::from_micros(block_timestamp_usecs));
    }
}

/// Processes on-chain reconfiguration notification.
pub(crate) async fn process_config_update<V>(
    config_update: OnChainConfigPayload,
    validator: Arc<RwLock<V>>,
) where
    V: TransactionValidation,
{
    info!(
        LogSchema::event_log(LogEntry::ReconfigUpdate, LogEvent::Process)
            .reconfig_update(config_update.clone())
    );

    if let Err(e) = validator.write().restart(config_update) {
        counters::VM_RECONFIG_UPDATE_FAIL_COUNT.inc();
        error!(LogSchema::event_log(LogEntry::ReconfigUpdate, LogEvent::VMUpdateFail).error(&e));
    }
}
<|endoftext|>


<commit_before>
use std::collections::HashMap;
use std::collections::hash_map::{Entry};
use std::hash::Hash;
use std::fmt;

/// An iterator adapter to filter out duplicate elements.
///
/// See [`.unique_by()`](../trait.Itertools.html#method.unique) for more information.
#[derive(Clone)]
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
pub struct UniqueBy<I: Iterator, V, F> {
    iter: I,
    // Use a hashmap for the entry API
    used: HashMap<V, ()>,
    f: F,
}

impl<I, V, F> fmt::Debug for UniqueBy<I, V, F>
    where I: Iterator + fmt::Debug,
          V: fmt::Debug + Hash + Eq,
{
    debug_fmt_fields!(UniqueBy, iter, used);
}

/// Create a new `UniqueBy` iterator.
pub fn unique_by<I, V, F>(iter: I, f: F) -> UniqueBy<I, V, F>
    where V: Eq + Hash,
          F: FnMut(&I::Item) -> V,
          I: Iterator,
{
    UniqueBy {
        iter: iter,
        used: HashMap::new(),
        f: f,
    }
}

// count the number of new unique keys in iterable (`used` is the set already seen)
fn count_new_keys<I, K>(mut used: HashMap<K, ()>, iterable: I) -> usize
    where I: IntoIterator<Item=K>,
          K: Hash + Eq,
{
    let iter = iterable.into_iter();
    let current_used = used.len();
    used.extend(iter.map(|key| (key, ())));
    used.len() - current_used
}

impl<I, V, F> Iterator for UniqueBy<I, V, F>
    where I: Iterator,
          V: Eq + Hash,
          F: FnMut(&I::Item) -> V
{
    type Item = I::Item;

    fn next(&mut self) -> Option<I::Item> {
        loop {
            match self.iter.next() {
                None => return None,
                Some(v) => {
                    let key = (self.f)(&v);
                    if self.used.insert(key, ()).is_none() {
                        return Some(v);
                    }
                }
            }
        }
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        let (low, hi) = self.iter.size_hint();
        ((low > 0 && self.used.is_empty()) as usize, hi)
    }

    fn count(self) -> usize {
        let mut key_f = self.f;
        count_new_keys(self.used, self.iter.map(move |elt| key_f(&elt)))
    }
}

impl<I> Iterator for Unique<I>
    where I: Iterator,
          I::Item: Eq + Hash + Clone
{
    type Item = I::Item;

    fn next(&mut self) -> Option<I::Item> {
        loop {
            match self.iter.iter.next() {
                None => return None,
                Some(v) => {
                    match self.iter.used.entry(v) {
                        Entry::Occupied(_) => { }
                        Entry::Vacant(entry) => {
                            let elt = entry.key().clone();
                            entry.insert(());
                            return Some(elt);
                        }
                    }
                }
            }
        }
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        let (low, hi) = self.iter.iter.size_hint();
        ((low > 0 && self.iter.used.is_empty()) as usize, hi)
    }

    fn count(self) -> usize {
        count_new_keys(self.iter.used, self.iter.iter)
    }
}

/// An iterator adapter to filter out duplicate elements.
///
/// See [`.unique()`](../trait.Itertools.html#method.unique) for more information.
#[derive(Clone)]
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
pub struct Unique<I: Iterator> {
    iter: UniqueBy<I, I::Item, ()>,
}

impl<I> fmt::Debug for Unique<I>
    where I: Iterator + fmt::Debug,
          I::Item: Hash + Eq + fmt::Debug,
{
    debug_fmt_fields!(Unique, iter);
}

pub fn unique<I>(iter: I) -> Unique<I>
    where I: Iterator,
          I::Item: Eq + Hash,
{
    Unique {
        iter: UniqueBy {
            iter: iter,
            used: HashMap::new(),
            f: (),
        }
    }
}
<commit_msg>REFAC: Use while let/if let in unique/_by<commit_after>
use std::collections::HashMap;
use std::collections::hash_map::{Entry};
use std::hash::Hash;
use std::fmt;

/// An iterator adapter to filter out duplicate elements.
///
/// See [`.unique_by()`](../trait.Itertools.html#method.unique) for more information.
#[derive(Clone)]
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
pub struct UniqueBy<I: Iterator, V, F> {
    iter: I,
    // Use a hashmap for the entry API
    used: HashMap<V, ()>,
    f: F,
}

impl<I, V, F> fmt::Debug for UniqueBy<I, V, F>
    where I: Iterator + fmt::Debug,
          V: fmt::Debug + Hash + Eq,
{
    debug_fmt_fields!(UniqueBy, iter, used);
}

/// Create a new `UniqueBy` iterator.
pub fn unique_by<I, V, F>(iter: I, f: F) -> UniqueBy<I, V, F>
    where V: Eq + Hash,
          F: FnMut(&I::Item) -> V,
          I: Iterator,
{
    UniqueBy {
        iter: iter,
        used: HashMap::new(),
        f: f,
    }
}

// count the number of new unique keys in iterable (`used` is the set already seen)
fn count_new_keys<I, K>(mut used: HashMap<K, ()>, iterable: I) -> usize
    where I: IntoIterator<Item=K>,
          K: Hash + Eq,
{
    let iter = iterable.into_iter();
    let current_used = used.len();
    used.extend(iter.map(|key| (key, ())));
    used.len() - current_used
}

impl<I, V, F> Iterator for UniqueBy<I, V, F>
    where I: Iterator,
          V: Eq + Hash,
          F: FnMut(&I::Item) -> V
{
    type Item = I::Item;

    fn next(&mut self) -> Option<I::Item> {
        while let Some(v) = self.iter.next() {
            let key = (self.f)(&v);
            if self.used.insert(key, ()).is_none() {
                return Some(v);
            }
        }
        None
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        let (low, hi) = self.iter.size_hint();
        ((low > 0 && self.used.is_empty()) as usize, hi)
    }

    fn count(self) -> usize {
        let mut key_f = self.f;
        count_new_keys(self.used, self.iter.map(move |elt| key_f(&elt)))
    }
}

impl<I> Iterator for Unique<I>
    where I: Iterator,
          I::Item: Eq + Hash + Clone
{
    type Item = I::Item;

    fn next(&mut self) -> Option<I::Item> {
        while let Some(v) = self.iter.iter.next() {
            if let Entry::Vacant(entry) = self.iter.used.entry(v) {
                let elt = entry.key().clone();
                entry.insert(());
                return Some(elt);
            }
        }
        None
    }

    #[inline]
    fn size_hint(&self) -> (usize, Option<usize>) {
        let (low, hi) = self.iter.iter.size_hint();
        ((low > 0 && self.iter.used.is_empty()) as usize, hi)
    }

    fn count(self) -> usize {
        count_new_keys(self.iter.used, self.iter.iter)
    }
}

/// An iterator adapter to filter out duplicate elements.
///
/// See [`.unique()`](../trait.Itertools.html#method.unique) for more information.
#[derive(Clone)]
#[must_use = "iterator adaptors are lazy and do nothing unless consumed"]
pub struct Unique<I: Iterator> {
    iter: UniqueBy<I, I::Item, ()>,
}

impl<I> fmt::Debug for Unique<I>
    where I: Iterator + fmt::Debug,
          I::Item: Hash + Eq + fmt::Debug,
{
    debug_fmt_fields!(Unique, iter);
}

pub fn unique<I>(iter: I) -> Unique<I>
    where I: Iterator,
          I::Item: Eq + Hash,
{
    Unique {
        iter: UniqueBy {
            iter: iter,
            used: HashMap::new(),
            f: (),
        }
    }
}
<|endoftext|>


<commit_before>use self::configs::{Configures, RaftMember, CONFIG_SM_ID};
use super::super::*;
use super::*;
use std::collections::HashMap;
use utils::bincode;

#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum ExecError {
    SmNotFound,
    FnNotFound,
    ServersUnreachable,
    CannotConstructClient,
    NotCommitted,
    Unknown,
    TooManyRetry,
}

pub enum RegisterResult {
    OK,
    EXISTED,
    RESERVED,
}

pub type ExecOk = Vec<u8>;
pub type ExecResult = Result<ExecOk, ExecError>;
pub type SubStateMachine = Box<StateMachineCtl>;
pub type SnapshotDataItem = (u64, Vec<u8>);
pub type SnapshotDataItems = Vec<SnapshotDataItem>;

raft_state_machine!{}

pub struct MasterStateMachine {
    subs: HashMap<u64, SubStateMachine>,
    pub configs: Configures,
}

impl StateMachineCmds for MasterStateMachine {}

impl StateMachineCtl for MasterStateMachine {
    raft_sm_complete!();
    fn id(&self) -> u64 {
        0
    }
    fn snapshot(&self) -> Option<Vec<u8>> {
        let mut sms: SnapshotDataItems = Vec::with_capacity(self.subs.len());
        for (sm_id, smc) in self.subs.iter() {
            let sub_snapshot = smc.snapshot();
            if let Some(snapshot) = sub_snapshot {
                sms.push((*sm_id, snapshot));
            }
        }
        sms.push((self.configs.id(), self.configs.snapshot().unwrap()));
        let data = bincode::serialize(&sms);
        Some(data)
    }
    fn recover(&mut self, data: Vec<u8>) {
        let mut sms: SnapshotDataItems = bincode::deserialize(&data);
        for (sm_id, snapshot) in sms {
            if let Some(sm) = self.subs.get_mut(&sm_id) {
                sm.recover(snapshot);
            } else if sm_id == self.configs.id() {
                self.configs.recover(snapshot);
            }
        }
    }
}

fn parse_output(r: Option<Vec<u8>>) -> ExecResult {
    if let Some(d) = r {
        Ok(d)
    } else {
        Err(ExecError::FnNotFound)
    }
}

impl MasterStateMachine {
    pub fn new(service_id: u64) -> MasterStateMachine {
        let mut msm = MasterStateMachine {
            subs: HashMap::new(),
            configs: Configures::new(service_id),
        };
        msm
    }

    pub fn register(&mut self, smc: SubStateMachine) -> RegisterResult {
        let id = smc.id();
        if id < 2 {
            return RegisterResult::RESERVED;
        }
        if self.subs.contains_key(&id) {
            return RegisterResult::EXISTED;
        };
        self.subs.insert(id, smc);
        RegisterResult::OK
    }

    pub fn members(&self) -> &HashMap<u64, RaftMember> {
        &self.configs.members
    }

    pub fn commit_cmd(&mut self, entry: &LogEntry) -> ExecResult {
        match entry.sm_id {
            CONFIG_SM_ID => parse_output(self.configs.fn_dispatch_cmd(entry.fn_id, &entry.data)),
            _ => {
                if let Some(sm) = self.subs.get_mut(&entry.sm_id) {
                    parse_output(sm.as_mut().fn_dispatch_cmd(entry.fn_id, &entry.data))
                } else {
                    Err(ExecError::SmNotFound)
                }
            }
        }
    }
    pub fn exec_qry(&self, entry: &LogEntry) -> ExecResult {
        match entry.sm_id {
            CONFIG_SM_ID => parse_output(self.configs.fn_dispatch_qry(entry.fn_id, &entry.data)),
            _ => {
                if let Some(sm) = self.subs.get(&entry.sm_id) {
                    parse_output(sm.fn_dispatch_qry(entry.fn_id, &entry.data))
                } else {
                    Err(ExecError::SmNotFound)
                }
            }
        }
    }
    pub fn clear_subs(&mut self) {
        self.subs.clear()
    }
}
<commit_msg>impl error for SM ExecError<commit_after>use self::configs::{Configures, RaftMember, CONFIG_SM_ID};
use super::super::*;
use super::*;
use std::collections::HashMap;
use utils::bincode;
use std::error::Error;

#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum ExecError {
    SmNotFound,
    FnNotFound,
    ServersUnreachable,
    CannotConstructClient,
    NotCommitted,
    Unknown,
    TooManyRetry,
}

pub enum RegisterResult {
    OK,
    EXISTED,
    RESERVED,
}

pub type ExecOk = Vec<u8>;
pub type ExecResult = Result<ExecOk, ExecError>;
pub type SubStateMachine = Box<StateMachineCtl>;
pub type SnapshotDataItem = (u64, Vec<u8>);
pub type SnapshotDataItems = Vec<SnapshotDataItem>;

raft_state_machine!{}

pub struct MasterStateMachine {
    subs: HashMap<u64, SubStateMachine>,
    pub configs: Configures,
}

impl StateMachineCmds for MasterStateMachine {}

impl StateMachineCtl for MasterStateMachine {
    raft_sm_complete!();
    fn id(&self) -> u64 {
        0
    }
    fn snapshot(&self) -> Option<Vec<u8>> {
        let mut sms: SnapshotDataItems = Vec::with_capacity(self.subs.len());
        for (sm_id, smc) in self.subs.iter() {
            let sub_snapshot = smc.snapshot();
            if let Some(snapshot) = sub_snapshot {
                sms.push((*sm_id, snapshot));
            }
        }
        sms.push((self.configs.id(), self.configs.snapshot().unwrap()));
        let data = bincode::serialize(&sms);
        Some(data)
    }
    fn recover(&mut self, data: Vec<u8>) {
        let mut sms: SnapshotDataItems = bincode::deserialize(&data);
        for (sm_id, snapshot) in sms {
            if let Some(sm) = self.subs.get_mut(&sm_id) {
                sm.recover(snapshot);
            } else if sm_id == self.configs.id() {
                self.configs.recover(snapshot);
            }
        }
    }
}

fn parse_output(r: Option<Vec<u8>>) -> ExecResult {
    if let Some(d) = r {
        Ok(d)
    } else {
        Err(ExecError::FnNotFound)
    }
}

impl MasterStateMachine {
    pub fn new(service_id: u64) -> MasterStateMachine {
        let mut msm = MasterStateMachine {
            subs: HashMap::new(),
            configs: Configures::new(service_id),
        };
        msm
    }

    pub fn register(&mut self, smc: SubStateMachine) -> RegisterResult {
        let id = smc.id();
        if id < 2 {
            return RegisterResult::RESERVED;
        }
        if self.subs.contains_key(&id) {
            return RegisterResult::EXISTED;
        };
        self.subs.insert(id, smc);
        RegisterResult::OK
    }

    pub fn members(&self) -> &HashMap<u64, RaftMember> {
        &self.configs.members
    }

    pub fn commit_cmd(&mut self, entry: &LogEntry) -> ExecResult {
        match entry.sm_id {
            CONFIG_SM_ID => parse_output(self.configs.fn_dispatch_cmd(entry.fn_id, &entry.data)),
            _ => {
                if let Some(sm) = self.subs.get_mut(&entry.sm_id) {
                    parse_output(sm.as_mut().fn_dispatch_cmd(entry.fn_id, &entry.data))
                } else {
                    Err(ExecError::SmNotFound)
                }
            }
        }
    }
    pub fn exec_qry(&self, entry: &LogEntry) -> ExecResult {
        match entry.sm_id {
            CONFIG_SM_ID => parse_output(self.configs.fn_dispatch_qry(entry.fn_id, &entry.data)),
            _ => {
                if let Some(sm) = self.subs.get(&entry.sm_id) {
                    parse_output(sm.fn_dispatch_qry(entry.fn_id, &entry.data))
                } else {
                    Err(ExecError::SmNotFound)
                }
            }
        }
    }
    pub fn clear_subs(&mut self) {
        self.subs.clear()
    }
}

impl Error for ExecError {}<|endoftext|>


<commit_before>use crate::context::Context;
use crate::tera::Tera;

#[test]
fn can_remove_whitespace_basic() {
    let mut context = Context::new();
    context.insert("numbers", &vec![1, 2, 3]);

    let inputs = vec![
        ("  {%- for n in numbers %}{{n}}{% endfor -%} ", "123"),
        ("{%- for n in numbers %} {{n}}{%- endfor -%} ", " 1 2 3"),
        ("{%- for n in numbers -%}\n {{n}}\n {%- endfor -%} ", "123"),
        ("{%- if true -%}\n {{numbers}}\n {%- endif -%} ", "[1, 2, 3]"),
        ("{%- if false -%}\n {{numbers}}\n {% else %} Nope{%- endif -%} ", " Nope"),
        ("  {%- if false -%}\n {{numbers}}\n {% else -%} Nope {%- endif -%} ", "Nope"),
        ("  {%- if false -%}\n {{numbers}}\n {% elif true -%} Nope {%- endif -%} ", "Nope"),
        ("  {%- if false -%}\n {{numbers}}\n {% elif false -%} Nope {% else %} else {%- endif -%} ", " else"),
        ("  {%- set var = 2 -%} {{var}}", "2"),
        ("  {% set var = 2 -%} {{var}}", "  2"),
        ("  {% raw -%}{{2}} {% endraw -%} ", "  {{2}}"),
        ("  {% filter upper -%} hey {%- endfilter -%} ", "  HEY"),
        ("  {{ \"hello\" -}} ", "  hello"),
        ("  {{- \"hello\" }} ", "hello "),
        ("  {{- \"hello\" -}} ", "hello"),
    ];

    for (input, expected) in inputs {
        let mut tera = Tera::default();
        tera.add_raw_template("tpl", input).unwrap();
        assert_eq!(tera.render("tpl", &context).unwrap(), expected);
    }
}

#[test]
fn can_remove_whitespace_include() {
    let mut context = Context::new();
    context.insert("numbers", &vec![1, 2, 3]);

    let inputs = vec![
        (r#"Hi {%- include "include" -%} "#, "HiIncluded"),
        (r#"Hi {% include "include" -%} "#, "Hi Included"),
        (r#"Hi {% include "include" %} "#, "Hi Included "),
    ];

    for (input, expected) in inputs {
        let mut tera = Tera::default();
        tera.add_raw_templates(vec![("include", "Included"), ("tpl", input)]).unwrap();
        assert_eq!(tera.render("tpl", &context).unwrap(), expected);
    }
}

#[test]
fn can_remove_whitespace_macros() {
    let mut context = Context::new();
    context.insert("numbers", &vec![1, 2, 3]);

    let inputs = vec![
        (r#" {%- import "macros" as macros -%} {{macros::hey()}}"#, "Hey!"),
        (r#" {% import "macros" as macros %} {{macros::hey()}}"#, "Hey!"),
        (r#" {%- import "macros" as macros %} {%- set hey = macros::hey() -%} {{hey}}"#, "Hey!"),
    ];

    for (input, expected) in inputs {
        let mut tera = Tera::default();
        tera.add_raw_templates(vec![
            ("macros", "{% macro hey() -%} Hey! {%- endmacro %}"),
            ("tpl", input),
        ])
        .unwrap();
        assert_eq!(tera.render("tpl", &context).unwrap(), expected);
    }
}

#[test]
fn can_remove_whitespace_inheritance() {
    let mut context = Context::new();
    context.insert("numbers", &vec![1, 2, 3]);

    let inputs = vec![
        (r#"{%- extends "base" -%} {% block content %}{{super()}}{% endblock %}"#, " Hey! "),
        (r#"{%- extends "base" -%} {% block content -%}{{super()}}{%- endblock %}"#, " Hey! "),
        (r#"{%- extends "base" %} {%- block content -%}{{super()}}{%- endblock -%} "#, " Hey! "),
    ];

    for (input, expected) in inputs {
        let mut tera = Tera::default();
        tera.add_raw_templates(vec![
            ("base", "{% block content %} Hey! {% endblock %}"),
            ("tpl", input),
        ])
        .unwrap();
        assert_eq!(tera.render("tpl", &context).unwrap(), expected);
    }
}

// https://github.com/Keats/tera/issues/475
#[test]
fn works_with_filter_section() {
    let mut context = Context::new();
    context.insert("d", "d");
    let input = r#"{% filter upper %}  {{ "c" }}   d{% endfilter %}"#;
    let res = Tera::one_off(input, &context, true).unwrap();
    assert_eq!(res, "  C   D");
}
<commit_msg>add a tests to check inappropriate whitespace deletion<commit_after>use crate::context::Context;
use crate::tera::Tera;

#[test]
fn can_remove_whitespace_basic() {
    let mut context = Context::new();
    context.insert("numbers", &vec![1, 2, 3]);

    let inputs = vec![
        ("  {%- for n in numbers %}{{n}}{% endfor -%} ", "123"),
        ("{%- for n in numbers %} {{n}}{%- endfor -%} ", " 1 2 3"),
        ("{%- for n in numbers -%}\n {{n}}\n {%- endfor -%} ", "123"),
        ("{%- if true -%}\n {{numbers}}\n {%- endif -%} ", "[1, 2, 3]"),
        ("{%- if false -%}\n {{numbers}}\n {% else %} Nope{%- endif -%} ", " Nope"),
        ("  {%- if false -%}\n {{numbers}}\n {% else -%} Nope {%- endif -%} ", "Nope"),
        ("  {%- if false -%}\n {{numbers}}\n {% elif true -%} Nope {%- endif -%} ", "Nope"),
        ("  {%- if false -%}\n {{numbers}}\n {% elif false -%} Nope {% else %} else {%- endif -%} ", " else"),
        ("  {%- set var = 2 -%} {{var}}", "2"),
        ("  {% set var = 2 -%} {{var}}", "  2"),
        ("  {% raw -%}{{2}} {% endraw -%} ", "  {{2}}"),
        ("  {% filter upper -%} hey {%- endfilter -%} ", "  HEY"),
        ("  {{ \"hello\" -}} ", "  hello"),
        ("  {{- \"hello\" }} ", "hello "),
        ("  {{- \"hello\" -}} ", "hello"),
    ];

    for (input, expected) in inputs {
        let mut tera = Tera::default();
        tera.add_raw_template("tpl", input).unwrap();
        assert_eq!(tera.render("tpl", &context).unwrap(), expected);
    }
}

#[test]
fn can_remove_whitespace_include() {
    let mut context = Context::new();
    context.insert("numbers", &vec![1, 2, 3]);

    let inputs = vec![
        (r#"Hi {%- include "include" -%} "#, "HiIncluded"),
        (r#"Hi {% include "include" -%} "#, "Hi Included"),
        (r#"Hi {% include "include" %} "#, "Hi Included "),
    ];

    for (input, expected) in inputs {
        let mut tera = Tera::default();
        tera.add_raw_templates(vec![("include", "Included"), ("tpl", input)]).unwrap();
        assert_eq!(tera.render("tpl", &context).unwrap(), expected);
    }
}

#[test]
fn can_remove_whitespace_macros() {
    let mut context = Context::new();
    context.insert("numbers", &vec![1, 2, 3]);

    let inputs = vec![
        (r#" {%- import "macros" as macros -%} {{macros::hey()}}"#, "Hey!"),
        (r#" {% import "macros" as macros %} {{macros::hey()}}"#, "Hey!"),
        (r#" {%- import "macros" as macros %} {%- set hey = macros::hey() -%} {{hey}}"#, "Hey!"),
    ];

    for (input, expected) in inputs {
        let mut tera = Tera::default();
        tera.add_raw_templates(vec![
            ("macros", "{% macro hey() -%} Hey! {%- endmacro %}"),
            ("tpl", input),
        ])
        .unwrap();
        assert_eq!(tera.render("tpl", &context).unwrap(), expected);
    }
}

#[test]
fn can_remove_whitespace_inheritance() {
    let mut context = Context::new();
    context.insert("numbers", &vec![1, 2, 3]);

    let inputs = vec![
        (r#"{%- extends "base" -%} {% block content %}{{super()}}{% endblock %}"#, " Hey! "),
        (r#"{%- extends "base" -%} {% block content -%}{{super()}}{%- endblock %}"#, " Hey! "),
        (r#"{%- extends "base" %} {%- block content -%}{{super()}}{%- endblock -%} "#, " Hey! "),
    ];

    for (input, expected) in inputs {
        let mut tera = Tera::default();
        tera.add_raw_templates(vec![
            ("base", "{% block content %} Hey! {% endblock %}"),
            ("tpl", input),
        ])
        .unwrap();
        assert_eq!(tera.render("tpl", &context).unwrap(), expected);
    }
}

// https://github.com/Keats/tera/issues/475
#[test]
fn works_with_filter_section() {
    let mut context = Context::new();
    context.insert("d", "d");
    let input = r#"{% filter upper %}  {{ "c" }}   d{% endfilter %}"#;
    let res = Tera::one_off(input, &context, true).unwrap();
    assert_eq!(res, "  C   D");
}

#[test]
fn make_sure_not_to_deleting_whitespaces() {
    let mut context = Context::new();
    context.insert("d", "d");
    let input = r#"{% raw %}    encoder:     {% endraw %}"#;
    let res = Tera::one_off(input, &context, true).unwrap();
    assert_eq!(res, "    \nc    \n");
}
<|endoftext|>


<commit_before>mod crc;
#[macro_use]
mod macros;
mod types;

pub use self::crc::{crc8, crc16};
pub use self::types::{ErrorKind, ByteStream, ReadStream};

use nom::{self, IResult};
use metadata::{Metadata, metadata_parser};

use std::ops::{Add, AddAssign, BitAnd, BitOr, Mul, Sub, Shl, ShlAssign, Shr};

/// An interface for parsing through some type of producer to a byte stream.
///
/// External parsers get passed in and consumes the bytes held internally
/// and outputs the `Result` of that parser.
pub trait StreamProducer {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T, ErrorKind>;
}

/// An abstraction trait for keeping different sized integers.
pub trait Sample: PartialEq + Eq + Sized + Clone + Copy +
                  Add<Output = Self> + AddAssign +
                  BitAnd<Self, Output = Self> + BitOr<Self, Output = Self> +
                  Mul<Output = Self> + Shl<u32, Output = Self>  +
                  ShlAssign<u32> + Shr<u32, Output = Self> +
                  Shr<i8, Output = Self> + Shr<i32, Output = Self> +
                  Sub<Output = Self> {
  /// The normal size for the current a `Sample`.
  type Normal;

  /// The size, in bits, for the `Sample::Normal`.
  fn size() -> usize;

  /// The size, in bits, for the `Sample`.
  fn size_extended() -> usize;

  /// Convert the extended `Sample` to the normal.
  fn to_normal(sample: Self) -> Option<Self::Normal>;

  /// Convert an i8 into a `Sample`.
  fn from_i8(sample: i8) -> Self;

  /// Convert an i16 into a `Sample`.
  fn from_i16(sample: i16) -> Self;

  /// Convert an i32 into a `Sample`.
  ///
  /// With `Sample` sometimes being smaller than a i32, there is a chance
  /// for this function to return an incorrect number. So when the number is
  /// larger of smaller than the current `Sample`, it returns `None`
  /// otherwise `Some(sample)`.
  fn from_i32(sample: i32) -> Option<Self>;

  /// Convert an i32 into a `Sample`.
  fn from_i32_lossy(sample: i32) -> Self;
}

pub trait SampleSize {
  type Extended: Sample;
}

impl SampleSize for i8 {
  type Extended = i16;
}

impl SampleSize for i16 {
  type Extended = i32;
}

impl SampleSize for i32 {
  type Extended = i64;
}

// Convert one to four byte slices into an unsigned 32-bit number.
//
// NOTE: This assumes big-endian since most numbers in the FLAC binary are
// that endianness.
#[inline]
pub fn to_u32(bytes: &[u8]) -> u32 {
  let length = bytes.len();

  debug_assert!(length <= 4);

  (0..length).fold(0, |result, i|
    result + ((bytes[i] as u32) << ((length - 1 - i) * 8))
  )
}

// Extends a signed value of a specific bit size to a larger bit size.
//
// NOTE: This assumes that the larger bit size will be 32 bit since that is
// the largest sample size supported in FLAC.
pub fn extend_sign(value: u32, bit_count: usize) -> i32 {
  if bit_count >= 32 || value < (1 << (bit_count - 1)) {
    value as i32
  } else {
    (value as i32).wrapping_sub(1 << bit_count)
  }
}

// Bit shifted version for two to the power of a given exponent.
#[inline]
pub fn power_of_two(exponent: u32) -> u32 {
  debug_assert!(exponent <= 31);

  1 << exponent
}

#[derive(PartialEq, Eq)]
enum ParserState {
  Header,
  StreamInfo,
  Metadata
}

fn parser<'a>(input: &'a [u8], state: &mut ParserState)
              -> IResult<&'a [u8], Metadata, ErrorKind> {
  let mut slice = input;
  let error     = nom::Err::Code(nom::ErrorKind::Custom(ErrorKind::Unknown));

  if *state == ParserState::Header {
    let (i, _) = try_parser! {
      to_custom_error!(slice, tag!("fLaC"), HeaderParser)
    };

    slice  = i;
    *state = ParserState::StreamInfo;
  }

  match *state {
    ParserState::StreamInfo => {
      let (i, block) = try_parse!(slice, metadata_parser);

      if block.is_stream_info() {
        *state = ParserState::Metadata;

        IResult::Done(i, block)
      } else {
        IResult::Error(error)
      }
    }
    ParserState::Metadata   => metadata_parser(slice),
    _                       => IResult::Error(error),
  }
}

pub fn many_metadata<S, F>(stream: &mut S, mut f: F) -> Result<(), ErrorKind>
 where S: StreamProducer,
       F: FnMut(Metadata) {
  let mut state  = ParserState::Header;
  let mut result = Ok(());

  loop {
    match stream.parse(|i| parser(i, &mut state)) {
      Ok(block)                => {
        let is_last = block.is_last();

        f(block);

        if is_last {
          break;
        }
      }
      Err(ErrorKind::Continue) => continue,
      Err(e)                   => {
        result = Err(e);

        break;
      }
    }
  }

  result
}

#[cfg(test)]
mod tests {
  use super::*;

  #[test]
  #[should_panic]
  fn test_panic_to_u32() {
    to_u32(&[0x00, 0x00, 0x00, 0x00, 0x00]);
  }

  #[test]
  fn test_to_u32() {
    let bytes = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef];

    assert_eq!(to_u32(&bytes[0..1]), 0x00000001);
    assert_eq!(to_u32(&bytes[3..5]), 0x00006789);
    assert_eq!(to_u32(&bytes[1..4]), 0x00234567);
    assert_eq!(to_u32(&bytes[4..]), 0x89abcdef);
  }

  #[test]
  fn test_extend_sign() {
    assert_eq!(extend_sign(32, 6), -32);
    assert_eq!(extend_sign(31, 6), 31);
    assert_eq!(extend_sign(128, 8), -128);
    assert_eq!(extend_sign(127, 8), 127);

    assert_eq!(extend_sign(2048, 12), -2048);
    assert_eq!(extend_sign(2047, 12), 2047);
    assert_eq!(extend_sign(32768, 16), -32768);
    assert_eq!(extend_sign(32767, 16), 32767);

    assert_eq!(extend_sign(8388608, 24), -8388608);
    assert_eq!(extend_sign(8388607, 24), 8388607);
    assert_eq!(extend_sign(2147483648, 32), -2147483648);
    assert_eq!(extend_sign(2147483647, 32), 2147483647);
  }

  #[test]
  #[should_panic]
  fn test_panic_power_of_two() {
    power_of_two(32);
  }

  #[test]
  fn test_power_of_two() {
    assert_eq!(power_of_two(0), 1);
    assert_eq!(power_of_two(1), 2);
    assert_eq!(power_of_two(2), 4);
    assert_eq!(power_of_two(10), 1024);
    assert_eq!(power_of_two(31), 2147483648);
  }
}
<commit_msg>Add documentation for `SampleSize` trait<commit_after>mod crc;
#[macro_use]
mod macros;
mod types;

pub use self::crc::{crc8, crc16};
pub use self::types::{ErrorKind, ByteStream, ReadStream};

use nom::{self, IResult};
use metadata::{Metadata, metadata_parser};

use std::ops::{Add, AddAssign, BitAnd, BitOr, Mul, Sub, Shl, ShlAssign, Shr};

/// An interface for parsing through some type of producer to a byte stream.
///
/// External parsers get passed in and consumes the bytes held internally
/// and outputs the `Result` of that parser.
pub trait StreamProducer {
  fn parse<F, T>(&mut self, f: F) -> Result<T, ErrorKind>
   where F: FnOnce(&[u8]) -> IResult<&[u8], T, ErrorKind>;
}

/// An abstraction trait for keeping different sized integers.
pub trait Sample: PartialEq + Eq + Sized + Clone + Copy +
                  Add<Output = Self> + AddAssign +
                  BitAnd<Self, Output = Self> + BitOr<Self, Output = Self> +
                  Mul<Output = Self> + Shl<u32, Output = Self>  +
                  ShlAssign<u32> + Shr<u32, Output = Self> +
                  Shr<i8, Output = Self> + Shr<i32, Output = Self> +
                  Sub<Output = Self> {
  /// The normal size for the current a `Sample`.
  type Normal;

  /// The size, in bits, for the `Sample::Normal`.
  fn size() -> usize;

  /// The size, in bits, for the `Sample`.
  fn size_extended() -> usize;

  /// Convert the extended `Sample` to the normal.
  fn to_normal(sample: Self) -> Option<Self::Normal>;

  /// Convert an i8 into a `Sample`.
  fn from_i8(sample: i8) -> Self;

  /// Convert an i16 into a `Sample`.
  fn from_i16(sample: i16) -> Self;

  /// Convert an i32 into a `Sample`.
  ///
  /// With `Sample` sometimes being smaller than a i32, there is a chance
  /// for this function to return an incorrect number. So when the number is
  /// larger of smaller than the current `Sample`, it returns `None`
  /// otherwise `Some(sample)`.
  fn from_i32(sample: i32) -> Option<Self>;

  /// Convert an i32 into a `Sample`.
  fn from_i32_lossy(sample: i32) -> Self;
}

/// A trait for defining the size of a sample.
pub trait SampleSize {
  type Extended: Sample;
}

impl SampleSize for i8 {
  type Extended = i16;
}

impl SampleSize for i16 {
  type Extended = i32;
}

impl SampleSize for i32 {
  type Extended = i64;
}

// Convert one to four byte slices into an unsigned 32-bit number.
//
// NOTE: This assumes big-endian since most numbers in the FLAC binary are
// that endianness.
#[inline]
pub fn to_u32(bytes: &[u8]) -> u32 {
  let length = bytes.len();

  debug_assert!(length <= 4);

  (0..length).fold(0, |result, i|
    result + ((bytes[i] as u32) << ((length - 1 - i) * 8))
  )
}

// Extends a signed value of a specific bit size to a larger bit size.
//
// NOTE: This assumes that the larger bit size will be 32 bit since that is
// the largest sample size supported in FLAC.
pub fn extend_sign(value: u32, bit_count: usize) -> i32 {
  if bit_count >= 32 || value < (1 << (bit_count - 1)) {
    value as i32
  } else {
    (value as i32).wrapping_sub(1 << bit_count)
  }
}

// Bit shifted version for two to the power of a given exponent.
#[inline]
pub fn power_of_two(exponent: u32) -> u32 {
  debug_assert!(exponent <= 31);

  1 << exponent
}

#[derive(PartialEq, Eq)]
enum ParserState {
  Header,
  StreamInfo,
  Metadata
}

fn parser<'a>(input: &'a [u8], state: &mut ParserState)
              -> IResult<&'a [u8], Metadata, ErrorKind> {
  let mut slice = input;
  let error     = nom::Err::Code(nom::ErrorKind::Custom(ErrorKind::Unknown));

  if *state == ParserState::Header {
    let (i, _) = try_parser! {
      to_custom_error!(slice, tag!("fLaC"), HeaderParser)
    };

    slice  = i;
    *state = ParserState::StreamInfo;
  }

  match *state {
    ParserState::StreamInfo => {
      let (i, block) = try_parse!(slice, metadata_parser);

      if block.is_stream_info() {
        *state = ParserState::Metadata;

        IResult::Done(i, block)
      } else {
        IResult::Error(error)
      }
    }
    ParserState::Metadata   => metadata_parser(slice),
    _                       => IResult::Error(error),
  }
}

pub fn many_metadata<S, F>(stream: &mut S, mut f: F) -> Result<(), ErrorKind>
 where S: StreamProducer,
       F: FnMut(Metadata) {
  let mut state  = ParserState::Header;
  let mut result = Ok(());

  loop {
    match stream.parse(|i| parser(i, &mut state)) {
      Ok(block)                => {
        let is_last = block.is_last();

        f(block);

        if is_last {
          break;
        }
      }
      Err(ErrorKind::Continue) => continue,
      Err(e)                   => {
        result = Err(e);

        break;
      }
    }
  }

  result
}

#[cfg(test)]
mod tests {
  use super::*;

  #[test]
  #[should_panic]
  fn test_panic_to_u32() {
    to_u32(&[0x00, 0x00, 0x00, 0x00, 0x00]);
  }

  #[test]
  fn test_to_u32() {
    let bytes = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef];

    assert_eq!(to_u32(&bytes[0..1]), 0x00000001);
    assert_eq!(to_u32(&bytes[3..5]), 0x00006789);
    assert_eq!(to_u32(&bytes[1..4]), 0x00234567);
    assert_eq!(to_u32(&bytes[4..]), 0x89abcdef);
  }

  #[test]
  fn test_extend_sign() {
    assert_eq!(extend_sign(32, 6), -32);
    assert_eq!(extend_sign(31, 6), 31);
    assert_eq!(extend_sign(128, 8), -128);
    assert_eq!(extend_sign(127, 8), 127);

    assert_eq!(extend_sign(2048, 12), -2048);
    assert_eq!(extend_sign(2047, 12), 2047);
    assert_eq!(extend_sign(32768, 16), -32768);
    assert_eq!(extend_sign(32767, 16), 32767);

    assert_eq!(extend_sign(8388608, 24), -8388608);
    assert_eq!(extend_sign(8388607, 24), 8388607);
    assert_eq!(extend_sign(2147483648, 32), -2147483648);
    assert_eq!(extend_sign(2147483647, 32), 2147483647);
  }

  #[test]
  #[should_panic]
  fn test_panic_power_of_two() {
    power_of_two(32);
  }

  #[test]
  fn test_power_of_two() {
    assert_eq!(power_of_two(0), 1);
    assert_eq!(power_of_two(1), 2);
    assert_eq!(power_of_two(2), 4);
    assert_eq!(power_of_two(10), 1024);
    assert_eq!(power_of_two(31), 2147483648);
  }
}
<|endoftext|>


<commit_before>//! Utilities for interacting with the build queue

use crate::error::Result;
use postgres::Connection;

const DEFAULT_PRIORITY: i32 = 0;

/// Get the build queue priority for a crate
pub fn get_crate_priority(conn: &Connection, name: &str) -> Result<i32> {
    // Search the `priority` table for a priority where the crate name matches the stored pattern
    let query = conn.query(
        "SELECT priority FROM crate_priorities WHERE $1 LIKE pattern LIMIT 1",
        &[&name],
    )?;

    // If no match is found, return the default priority
    if let Some(row) = query.iter().next() {
        Ok(row.get(0))
    } else {
        Ok(DEFAULT_PRIORITY)
    }
}

/// Set all crates that match [`pattern`] to have a certain priority
///
/// Note: `pattern` is used in a `LIKE` statement, so it must follow the postgres like syntax
///
/// [`pattern`]: https://www.postgresql.org/docs/8.3/functions-matching.html
pub fn set_crate_priority(conn: &Connection, pattern: &str, priority: i32) -> Result<()> {
    conn.query(
        "INSERT INTO crate_priorities (pattern, priority) VALUES ($1, $2)",
        &[&pattern, &priority],
    )?;

    Ok(())
}

/// Remove a pattern from the priority table, returning the priority that it was associated with or `None`
/// if nothing was removed
pub fn remove_crate_priority(conn: &Connection, pattern: &str) -> Result<Option<i32>> {
    let query = conn.query(
        "DELETE FROM crate_priorities WHERE pattern = $1 RETURNING priority",
        &[&pattern],
    )?;

    Ok(query.iter().next().map(|row| row.get(0)))
}
/// Adds a crate to the build queue to be built by rustdoc. `priority` should be gotten from `get_crate_priority`
pub fn add_crate_to_queue(
    conn: &Connection,
    name: &str,
    version: &str,
    priority: i32,
) -> Result<()> {
    conn.execute(
        "INSERT INTO queue (name, version, priority) VALUES ($1, $2, $3)",
        &[&name, &version, &priority],
    )?;

    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::test::wrapper;

    #[test]
    fn set_priority() {
        wrapper(|env| {
            let db = env.db();

            set_crate_priority(&db.conn(), "cratesfyi-%", -100)?;
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-database")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-s3")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-webserver")?, -100);
            assert_eq!(
                get_crate_priority(&db.conn(), "cratesfyi")?,
                DEFAULT_PRIORITY
            );

            set_crate_priority(&db.conn(), "_c_", 100)?;
            assert_eq!(get_crate_priority(&db.conn(), "rcc")?, 100);
            assert_eq!(get_crate_priority(&db.conn(), "rc")?, DEFAULT_PRIORITY);

            set_crate_priority(&db.conn(), "hexponent", 10)?;
            assert_eq!(get_crate_priority(&db.conn(), "hexponent")?, 10);
            assert_eq!(
                get_crate_priority(&db.conn(), "hexponents")?,
                DEFAULT_PRIORITY
            );
            assert_eq!(
                get_crate_priority(&db.conn(), "floathexponent")?,
                DEFAULT_PRIORITY
            );

            Ok(())
        })
    }

    #[test]
    fn remove_priority() {
        wrapper(|env| {
            let db = env.db();

            set_crate_priority(&db.conn(), "cratesfyi-%", -100)?;
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-")?, -100);

            assert_eq!(
                remove_crate_priority(&db.conn(), "cratesfyi-%")?,
                Some(-100)
            );
            assert_eq!(
                get_crate_priority(&db.conn(), "cratesfyi-")?,
                DEFAULT_PRIORITY
            );

            Ok(())
        })
    }

    #[test]
    fn get_priority() {
        wrapper(|env| {
            let db = env.db();

            set_crate_priority(&db.conn(), "cratesfyi-%", -100)?;

            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-database")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-s3")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-webserver")?, -100);
            assert_eq!(
                get_crate_priority(&db.conn(), "unrelated")?,
                DEFAULT_PRIORITY
            );

            Ok(())
        })
    }

    #[test]
    fn get_default_priority() {
        wrapper(|env| {
            let db = env.db();

            assert_eq!(
                get_crate_priority(&db.conn(), "cratesfyi")?,
                DEFAULT_PRIORITY
            );
            assert_eq!(get_crate_priority(&db.conn(), "rcc")?, DEFAULT_PRIORITY);
            assert_eq!(get_crate_priority(&db.conn(), "lasso")?, DEFAULT_PRIORITY);
            assert_eq!(
                get_crate_priority(&db.conn(), "hexponent")?,
                DEFAULT_PRIORITY
            );
            assert_eq!(
                get_crate_priority(&db.conn(), "rust4lyfe")?,
                DEFAULT_PRIORITY
            );

            Ok(())
        })
    }

    #[test]
    fn add_to_queue() {
        wrapper(|env| {
            let db = env.db();

            let test_crates = [
                ("rcc", "0.1.0", 2),
                ("lasso", "0.1.0", -1),
                ("hexponent", "0.1.0", 0),
                ("destroy-all-humans", "0.0.0-alpha", -100000),
                ("totally-not-destroying-humans", "0.0.1", 0),
            ];

            for (name, version, priority) in test_crates.iter() {
                add_crate_to_queue(&db.conn(), name, version, *priority)?;

                let query = db.conn().query(
                    "SELECT name, version, priority FROM queue WHERE name = $1",
                    &[&name],
                )?;

                assert!(query.len() == 1);
                let row = query.iter().next().unwrap();

                assert_eq!(&row.get::<_, String>(0), name);
                assert_eq!(&row.get::<_, String>(1), version);
                assert_eq!(row.get::<_, i32>(2), *priority);
            }

            Ok(())
        })
    }
}
<commit_msg>Exposed priority commands to the binary<commit_after>//! Utilities for interacting with the build queue

use crate::error::Result;
use postgres::Connection;

const DEFAULT_PRIORITY: i32 = 0;

/// Get the build queue priority for a crate
pub fn get_crate_priority(conn: &Connection, name: &str) -> Result<i32> {
    // Search the `priority` table for a priority where the crate name matches the stored pattern
    let query = conn.query(
        "SELECT priority FROM crate_priorities WHERE $1 LIKE pattern LIMIT 1",
        &[&name],
    )?;

    // If no match is found, return the default priority
    if let Some(row) = query.iter().next() {
        Ok(row.get(0))
    } else {
        Ok(DEFAULT_PRIORITY)
    }
}

/// Set all crates that match [`pattern`] to have a certain priority
///
/// Note: `pattern` is used in a `LIKE` statement, so it must follow the postgres like syntax
///
/// [`pattern`]: https://www.postgresql.org/docs/8.3/functions-matching.html
pub fn set_crate_priority(conn: &Connection, pattern: &str, priority: i32) -> Result<()> {
    conn.query(
        "INSERT INTO crate_priorities (pattern, priority) VALUES ($1, $2)",
        &[&pattern, &priority],
    )?;

    Ok(())
}

/// Remove a pattern from the priority table, returning the priority that it was associated with or `None`
/// if nothing was removed
pub fn remove_crate_priority(conn: &Connection, pattern: &str) -> Result<Option<i32>> {
    let query = conn.query(
        "DELETE FROM crate_priorities WHERE pattern = $1 RETURNING priority",
        &[&pattern],
    )?;

    Ok(query.iter().next().map(|row| row.get(0)))
}

/// Adds a crate to the build queue to be built by rustdoc. `priority` should be gotten from `get_crate_priority`
pub fn add_crate_to_queue(
    conn: &Connection,
    name: &str,
    version: &str,
    priority: i32,
) -> Result<()> {
    conn.execute(
        "INSERT INTO queue (name, version, priority) VALUES ($1, $2, $3)",
        &[&name, &version, &priority],
    )?;

    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::test::wrapper;

    #[test]
    fn set_priority() {
        wrapper(|env| {
            let db = env.db();

            set_crate_priority(&db.conn(), "cratesfyi-%", -100)?;
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-database")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-s3")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-webserver")?, -100);
            assert_eq!(
                get_crate_priority(&db.conn(), "cratesfyi")?,
                DEFAULT_PRIORITY
            );

            set_crate_priority(&db.conn(), "_c_", 100)?;
            assert_eq!(get_crate_priority(&db.conn(), "rcc")?, 100);
            assert_eq!(get_crate_priority(&db.conn(), "rc")?, DEFAULT_PRIORITY);

            set_crate_priority(&db.conn(), "hexponent", 10)?;
            assert_eq!(get_crate_priority(&db.conn(), "hexponent")?, 10);
            assert_eq!(
                get_crate_priority(&db.conn(), "hexponents")?,
                DEFAULT_PRIORITY
            );
            assert_eq!(
                get_crate_priority(&db.conn(), "floathexponent")?,
                DEFAULT_PRIORITY
            );

            Ok(())
        })
    }

    #[test]
    fn remove_priority() {
        wrapper(|env| {
            let db = env.db();

            set_crate_priority(&db.conn(), "cratesfyi-%", -100)?;
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-")?, -100);

            assert_eq!(
                remove_crate_priority(&db.conn(), "cratesfyi-%")?,
                Some(-100)
            );
            assert_eq!(
                get_crate_priority(&db.conn(), "cratesfyi-")?,
                DEFAULT_PRIORITY
            );

            Ok(())
        })
    }

    #[test]
    fn get_priority() {
        wrapper(|env| {
            let db = env.db();

            set_crate_priority(&db.conn(), "cratesfyi-%", -100)?;

            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-database")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-s3")?, -100);
            assert_eq!(get_crate_priority(&db.conn(), "cratesfyi-webserver")?, -100);
            assert_eq!(
                get_crate_priority(&db.conn(), "unrelated")?,
                DEFAULT_PRIORITY
            );

            Ok(())
        })
    }

    #[test]
    fn get_default_priority() {
        wrapper(|env| {
            let db = env.db();

            assert_eq!(
                get_crate_priority(&db.conn(), "cratesfyi")?,
                DEFAULT_PRIORITY
            );
            assert_eq!(get_crate_priority(&db.conn(), "rcc")?, DEFAULT_PRIORITY);
            assert_eq!(get_crate_priority(&db.conn(), "lasso")?, DEFAULT_PRIORITY);
            assert_eq!(
                get_crate_priority(&db.conn(), "hexponent")?,
                DEFAULT_PRIORITY
            );
            assert_eq!(
                get_crate_priority(&db.conn(), "rust4lyfe")?,
                DEFAULT_PRIORITY
            );

            Ok(())
        })
    }

    #[test]
    fn add_to_queue() {
        wrapper(|env| {
            let db = env.db();

            let test_crates = [
                ("rcc", "0.1.0", 2),
                ("lasso", "0.1.0", -1),
                ("hexponent", "0.1.0", 0),
                ("destroy-all-humans", "0.0.0-alpha", -100000),
                ("totally-not-destroying-humans", "0.0.1", 0),
            ];

            for (name, version, priority) in test_crates.iter() {
                add_crate_to_queue(&db.conn(), name, version, *priority)?;

                let query = db.conn().query(
                    "SELECT name, version, priority FROM queue WHERE name = $1",
                    &[&name],
                )?;

                assert!(query.len() == 1);
                let row = query.iter().next().unwrap();

                assert_eq!(&row.get::<_, String>(0), name);
                assert_eq!(&row.get::<_, String>(1), version);
                assert_eq!(row.get::<_, i32>(2), *priority);
            }

            Ok(())
        })
    }
}
<|endoftext|>


<commit_before>use std::env;
use std::error::Error;
use std::fs::File;
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};

fn main() -> Result<(), Box<dyn Error>> {
    println!("cargo:rerun-if-changed=colors.txt");
    let path = Path::new(&env::var("OUT_DIR").unwrap()).join("colors.rs");
    let mut file = BufWriter::new(File::create(&path).unwrap());

    let rdr = BufReader::new(File::open(
        PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("colors.txt"),
    )?);

    let mut map = phf_codegen::Map::new();
    let mut recs: Vec<(String, String)> = Vec::new();

    for result in rdr.lines() {
        let record = result?;
        let record = record.split("//").next().unwrap().trim();
        if record.is_empty() {
            continue;
        }
        let mut record = record.split_whitespace();
        let id = record.next().unwrap().to_owned();
        let hex = record.next().unwrap().to_owned();
        let clean_hex = hex.trim_start_matches('#');
        let color = match clean_hex.len() {
            3 => {
                let d = match u32::from_str_radix(&clean_hex, 16) {
                    Ok(x) => x,
                    Err(_) => 0,
                };

                let mut r = (d & 0xF) << 4;
                let mut g = ((d >> 4) & 0xF) << 4;
                let mut b = ((d >> 8) & 0xF) << 4;
                r |= r >> 4;
                g |= g >> 4;
                b |= b >> 4;

                format!("Color::rgb({}, {}, {})", r, g, b)
            }
            6 => {
                let x = match u32::from_str_radix(&clean_hex, 16) {
                    Ok(x) => x,
                    Err(_) => 0,
                };

                format!(
                    "Color::rgb({}, {}, {})",
                    ((x >> 16) & 0xFF) as u8,
                    ((x >> 8) & 0xFF) as u8,
                    (x & 0xFF) as u8
                )
            }
            _ => panic!(""),
        };
        recs.push((id, color));
    }

    for rec in recs.iter() {
        map.entry(&*rec.0, &rec.1);
    }

    map.entry("transparent", "Color::rgba(0, 0, 0, 0)");

    writeln!(
        &mut file,
        "static COLORS: phf::Map<&'static str, Color> = \n{};\n",
        map.build()
    )
    .unwrap();

    Ok(())
}
<commit_msg>lint fix<commit_after>use std::env;
use std::error::Error;
use std::fs::File;
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};

fn main() -> Result<(), Box<dyn Error>> {
    println!("cargo:rerun-if-changed=colors.txt");
    let path = Path::new(&env::var("OUT_DIR").unwrap()).join("colors.rs");
    let mut file = BufWriter::new(File::create(&path).unwrap());

    let rdr = BufReader::new(File::open(
        PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()).join("colors.txt"),
    )?);

    let mut map = phf_codegen::Map::new();
    let mut recs: Vec<(String, String)> = Vec::new();

    for result in rdr.lines() {
        let record = result?;
        let record = record.split("//").next().unwrap().trim();
        if record.is_empty() {
            continue;
        }
        let mut record = record.split_whitespace();
        let id = record.next().unwrap().to_owned();
        let hex = record.next().unwrap().to_owned();
        let clean_hex = hex.trim_start_matches('#');
        let color = match clean_hex.len() {
            3 => {
                let d = u32::from_str_radix(&clean_hex, 16).unwrap_or(0);

                let mut r = (d & 0xF) << 4;
                let mut g = ((d >> 4) & 0xF) << 4;
                let mut b = ((d >> 8) & 0xF) << 4;
                r |= r >> 4;
                g |= g >> 4;
                b |= b >> 4;

                format!("Color::rgb({}, {}, {})", r, g, b)
            }
            6 => {
                let x = u32::from_str_radix(&clean_hex, 16).unwrap_or(0);

                format!(
                    "Color::rgb({}, {}, {})",
                    ((x >> 16) & 0xFF) as u8,
                    ((x >> 8) & 0xFF) as u8,
                    (x & 0xFF) as u8
                )
            }
            _ => panic!(""),
        };
        recs.push((id, color));
    }

    for rec in recs.iter() {
        map.entry(&*rec.0, &rec.1);
    }

    map.entry("transparent", "Color::rgba(0, 0, 0, 0)");

    writeln!(
        &mut file,
        "static COLORS: phf::Map<&'static str, Color> = \n{};\n",
        map.build()
    )
    .unwrap();

    Ok(())
}
<|endoftext|>


<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::io::{Command, fs, USER_RWX};
use std::os;

fn main() {
    // If we're the child, make sure we were invoked correctly
    let args = os::args();
    if args.len() > 1 && args[1].as_slice() == "child" {
        return assert_eq!(args[0].as_slice(), "mytest");
    }

    test();
}

fn test() {
    // If we're the parent, copy our own binary to a new directory.
    let my_path = os::self_exe_name().unwrap();
    let my_dir  = my_path.dir_path();

    let child_dir = Path::new(my_dir.join("issue-15149-child"));
    drop(fs::mkdir(&child_dir, USER_RWX));

    let child_path = child_dir.join(format!("mytest{}",
                                            os::consts::EXE_SUFFIX));
    fs::copy(&my_path, &child_path).unwrap();

    // Append the new directory to our own PATH.
    let mut path = os::split_paths(os::getenv("PATH").unwrap_or(String::new()));
    path.push(child_dir.clone());
    let path = os::join_paths(path.as_slice()).unwrap();

    assert!(Command::new("mytest").env("PATH", path.as_slice())
                                  .arg("child")
                                  .status().unwrap().success());
}
<commit_msg>Attempt fix for assertion on Windows, and add extra output for debugging.<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::path::BytesContainer;
use std::io::{Command, fs, USER_RWX};
use std::os;

fn main() {
    // If we're the child, make sure we were invoked correctly
    let args = os::args();
    if args.len() > 1 && args[1].as_slice() == "child" {
        return assert_eq!(args[0],
                          format!("mytest{}", os::consts::EXE_SUFFIX));
    }

    test();
}

fn test() {
    // If we're the parent, copy our own binary to a new directory.
    let my_path = os::self_exe_name().unwrap();
    let my_dir  = my_path.dir_path();

    let child_dir = Path::new(my_dir.join("issue-15149-child"));
    drop(fs::mkdir(&child_dir, USER_RWX));

    let child_path = child_dir.join(format!("mytest{}",
                                            os::consts::EXE_SUFFIX));
    fs::copy(&my_path, &child_path).unwrap();

    // Append the new directory to our own PATH.
    let mut path = os::split_paths(os::getenv("PATH").unwrap_or(String::new()));
    path.push(child_dir.clone());
    let path = os::join_paths(path.as_slice()).unwrap();

    let child_output = Command::new("mytest").env("PATH", path.as_slice())
                                             .arg("child")
                                             .output().unwrap();

    assert!(child_output.status.success(),
            format!("child assertion failed\n child stdout:\n {}\n child stderr:\n {}",
                    child_output.output.container_as_str().unwrap(),
                    child_output.error.container_as_str().unwrap()));
}
<|endoftext|>


<commit_before>use cairo;
use cairo::{PatternTrait, Rectangle};
use std::cell::{Cell, RefCell};

use crate::allowed_url::Href;
use crate::aspect_ratio::AspectRatio;
use crate::attributes::Attribute;
use crate::bbox::BoundingBox;
use crate::drawing_ctx::{ClipMode, DrawingCtx};
use crate::error::{NodeError, RenderingError};
use crate::float_eq_cairo::ApproxEqCairo;
use crate::length::{LengthHorizontal, LengthVertical};
use crate::node::*;
use crate::parsers::{ParseError, ParseValue};
use crate::property_bag::PropertyBag;
use crate::rect::RectangleExt;
use crate::viewbox::ViewBox;

pub struct NodeImage {
    aspect: Cell<AspectRatio>,
    x: Cell<LengthHorizontal>,
    y: Cell<LengthVertical>,
    w: Cell<LengthHorizontal>,
    h: Cell<LengthVertical>,
    href: RefCell<Option<Href>>,
}

impl NodeImage {
    pub fn new() -> NodeImage {
        NodeImage {
            aspect: Cell::new(AspectRatio::default()),
            x: Cell::new(Default::default()),
            y: Cell::new(Default::default()),
            w: Cell::new(Default::default()),
            h: Cell::new(Default::default()),
            href: RefCell::new(None),
        }
    }
}

impl NodeTrait for NodeImage {
    fn set_atts(&self, node: &RsvgNode, pbag: &PropertyBag<'_>) -> NodeResult {
        // SVG element has overflow:hidden
        // https://www.w3.org/TR/SVG/styling.html#UAStyleSheet
        node.set_overflow_hidden();

        for (attr, value) in pbag.iter() {
            match attr {
                Attribute::X => self.x.set(attr.parse(value)?),
                Attribute::Y => self.y.set(attr.parse(value)?),
                Attribute::Width => self
                    .w
                    .set(attr.parse_and_validate(value, LengthHorizontal::check_nonnegative)?),
                Attribute::Height => self
                    .h
                    .set(attr.parse_and_validate(value, LengthVertical::check_nonnegative)?),

                Attribute::PreserveAspectRatio => self.aspect.set(attr.parse(value)?),

                // "path" is used by some older Adobe Illustrator versions
                Attribute::XlinkHref | Attribute::Path => {
                    let href = Href::parse(value).map_err(|_| {
                        NodeError::parse_error(attr, ParseError::new("could not parse href"))
                    })?;

                    *self.href.borrow_mut() = Some(href);
                }

                _ => (),
            }
        }

        Ok(())
    }

    fn draw(
        &self,
        node: &RsvgNode,
        cascaded: &CascadedValues<'_>,
        draw_ctx: &mut DrawingCtx,
        clipping: bool,
    ) -> Result<(), RenderingError> {
        let values = cascaded.get();
        let params = draw_ctx.get_view_params();

        let x = self.x.get().normalize(values, &params);
        let y = self.y.get().normalize(values, &params);
        let w = self.w.get().normalize(values, &params);
        let h = self.h.get().normalize(values, &params);

        if w.approx_eq_cairo(&0.0) || h.approx_eq_cairo(&0.0) {
            return Ok(());
        }

        draw_ctx.with_discrete_layer(node, values, clipping, &mut |dc| {
            let surface = if let Some(Href::PlainUrl(ref url)) = *self.href.borrow() {
                dc.lookup_image(&url)?
            } else {
                return Ok(());
            };

            let aspect = self.aspect.get();

            let clip_mode = if !values.is_overflow() && aspect.is_slice() {
                Some(ClipMode::ClipToViewport)
            } else {
                None
            };

            let image_width = surface.width();
            let image_height = surface.height();
            if clipping || image_width == 0 || image_height == 0 {
                return Ok(());
            }

            // The bounding box for <image> is decided by the values of x, y, w, h and not by
            // the final computed image bounds.
            let bbox = BoundingBox::new(&dc.get_cairo_context().get_matrix()).with_rect(Some(
                cairo::Rectangle {
                    x,
                    y,
                    width: w,
                    height: h,
                },
            ));

            let cr = dc.get_cairo_context();
            cr.save();

            let image_width = f64::from(image_width);
            let image_height = f64::from(image_height);

            if let Some(_params) = dc.push_new_viewport(
                Some(ViewBox::new(0.0, 0.0, image_width, image_height)),
                &Rectangle::new(x, y, w, h),
                aspect,
                clip_mode,
            ) {
                // We need to set extend appropriately, so can't use cr.set_source_surface().
                //
                // If extend is left at its default value (None), then bilinear scaling uses
                // transparency outside of the image producing incorrect results.
                // For example, in svg1.1/filters-blend-01-b.svgthere's a completely
                // opaque 100×1 image of a gradient scaled to 100×98 which ends up
                // transparent almost everywhere without this fix (which it shouldn't).
                let ptn = surface.to_cairo_pattern();
                ptn.set_extend(cairo::Extend::Pad);
                cr.set_source(&ptn);

                // Clip is needed due to extend being set to pad.
                cr.rectangle(0.0, 0.0, image_width, image_height);
                cr.clip();

                cr.paint();
            }

            cr.restore();
            dc.insert_bbox(&bbox);
            Ok(())
        })
    }
}
<commit_msg>image.rs: use DrawingCtx.with_saved_cr()<commit_after>use cairo;
use cairo::{PatternTrait, Rectangle};
use std::cell::{Cell, RefCell};

use crate::allowed_url::Href;
use crate::aspect_ratio::AspectRatio;
use crate::attributes::Attribute;
use crate::bbox::BoundingBox;
use crate::drawing_ctx::{ClipMode, DrawingCtx};
use crate::error::{NodeError, RenderingError};
use crate::float_eq_cairo::ApproxEqCairo;
use crate::length::{LengthHorizontal, LengthVertical};
use crate::node::*;
use crate::parsers::{ParseError, ParseValue};
use crate::property_bag::PropertyBag;
use crate::rect::RectangleExt;
use crate::viewbox::ViewBox;

pub struct NodeImage {
    aspect: Cell<AspectRatio>,
    x: Cell<LengthHorizontal>,
    y: Cell<LengthVertical>,
    w: Cell<LengthHorizontal>,
    h: Cell<LengthVertical>,
    href: RefCell<Option<Href>>,
}

impl NodeImage {
    pub fn new() -> NodeImage {
        NodeImage {
            aspect: Cell::new(AspectRatio::default()),
            x: Cell::new(Default::default()),
            y: Cell::new(Default::default()),
            w: Cell::new(Default::default()),
            h: Cell::new(Default::default()),
            href: RefCell::new(None),
        }
    }
}

impl NodeTrait for NodeImage {
    fn set_atts(&self, node: &RsvgNode, pbag: &PropertyBag<'_>) -> NodeResult {
        // SVG element has overflow:hidden
        // https://www.w3.org/TR/SVG/styling.html#UAStyleSheet
        node.set_overflow_hidden();

        for (attr, value) in pbag.iter() {
            match attr {
                Attribute::X => self.x.set(attr.parse(value)?),
                Attribute::Y => self.y.set(attr.parse(value)?),
                Attribute::Width => self
                    .w
                    .set(attr.parse_and_validate(value, LengthHorizontal::check_nonnegative)?),
                Attribute::Height => self
                    .h
                    .set(attr.parse_and_validate(value, LengthVertical::check_nonnegative)?),

                Attribute::PreserveAspectRatio => self.aspect.set(attr.parse(value)?),

                // "path" is used by some older Adobe Illustrator versions
                Attribute::XlinkHref | Attribute::Path => {
                    let href = Href::parse(value).map_err(|_| {
                        NodeError::parse_error(attr, ParseError::new("could not parse href"))
                    })?;

                    *self.href.borrow_mut() = Some(href);
                }

                _ => (),
            }
        }

        Ok(())
    }

    fn draw(
        &self,
        node: &RsvgNode,
        cascaded: &CascadedValues<'_>,
        draw_ctx: &mut DrawingCtx,
        clipping: bool,
    ) -> Result<(), RenderingError> {
        let values = cascaded.get();
        let params = draw_ctx.get_view_params();

        let x = self.x.get().normalize(values, &params);
        let y = self.y.get().normalize(values, &params);
        let w = self.w.get().normalize(values, &params);
        let h = self.h.get().normalize(values, &params);

        if w.approx_eq_cairo(&0.0) || h.approx_eq_cairo(&0.0) {
            return Ok(());
        }

        draw_ctx.with_discrete_layer(node, values, clipping, &mut |dc| {
            let surface = if let Some(Href::PlainUrl(ref url)) = *self.href.borrow() {
                dc.lookup_image(&url)?
            } else {
                return Ok(());
            };

            let aspect = self.aspect.get();

            let clip_mode = if !values.is_overflow() && aspect.is_slice() {
                Some(ClipMode::ClipToViewport)
            } else {
                None
            };

            let image_width = surface.width();
            let image_height = surface.height();
            if clipping || image_width == 0 || image_height == 0 {
                return Ok(());
            }

            // The bounding box for <image> is decided by the values of x, y, w, h and not by
            // the final computed image bounds.
            let bbox = BoundingBox::new(&dc.get_cairo_context().get_matrix()).with_rect(Some(
                cairo::Rectangle {
                    x,
                    y,
                    width: w,
                    height: h,
                },
            ));

            dc.with_saved_cr(&mut |dc| {
                let cr = dc.get_cairo_context();

                let image_width = f64::from(image_width);
                let image_height = f64::from(image_height);

                if let Some(_params) = dc.push_new_viewport(
                    Some(ViewBox::new(0.0, 0.0, image_width, image_height)),
                    &Rectangle::new(x, y, w, h),
                    aspect,
                    clip_mode,
                ) {
                    // We need to set extend appropriately, so can't use cr.set_source_surface().
                    //
                    // If extend is left at its default value (None), then bilinear scaling uses
                    // transparency outside of the image producing incorrect results.
                    // For example, in svg1.1/filters-blend-01-b.svgthere's a completely
                    // opaque 100×1 image of a gradient scaled to 100×98 which ends up
                    // transparent almost everywhere without this fix (which it shouldn't).
                    let ptn = surface.to_cairo_pattern();
                    ptn.set_extend(cairo::Extend::Pad);
                    cr.set_source(&ptn);

                    // Clip is needed due to extend being set to pad.
                    cr.rectangle(0.0, 0.0, image_width, image_height);
                    cr.clip();

                    cr.paint();
                }

                Ok(())
            }).and_then(|()| {
                dc.insert_bbox(&bbox);
                Ok(())
            })
        })
    }
}
<|endoftext|>