<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::counters::{FETCHED_TRANSACTION, UNABLE_TO_FETCH_TRANSACTION};
use aptos_rest_client::{Client as RestClient, State, Transaction};
use futures::channel::mpsc;
use futures::{SinkExt, StreamExt};
use serde_json::Value;
use std::time::Duration;
use tokio::task::JoinHandle;
use url::Url;

// TODO: make this configurable
const RETRY_TIME_MILLIS: u64 = 1000;
const TRANSACTION_FETCH_BATCH_SIZE: u16 = 500;
const TRANSACTION_CHANNEL_SIZE: usize = 35;
const MAX_THREADS: usize = 10;
const MAX_RETRIES: usize = 5;

#[derive(Debug)]
pub struct Fetcher {
    client: RestClient,
    chain_id: u8,
    current_version: u64,
    highest_known_version: u64,
    transactions_sender: mpsc::Sender<Vec<Transaction>>,
}

impl Fetcher {
    pub fn new(
        client: RestClient,
        current_version: u64,
        transactions_sender: mpsc::Sender<Vec<Transaction>>,
    ) -> Self {
        Self {
            client,
            chain_id: 0,
            current_version,
            highest_known_version: current_version,
            transactions_sender,
        }
    }

    pub async fn set_highest_known_version(&mut self) {
        let info = self.client.get_ledger_information().await;
        let res = info.unwrap();
        let state = res.state();
        self.highest_known_version = state.version;
        self.chain_id = state.chain_id;
    }

    pub async fn run(&mut self) {
        loop {
            if self.current_version >= self.highest_known_version {
                tokio::time::sleep(Duration::from_millis(200)).await;
                self.set_highest_known_version().await;
            }

            let num_missing = self.highest_known_version - self.current_version;
            let num_batches = std::cmp::min(
                (num_missing as f64 / TRANSACTION_FETCH_BATCH_SIZE as f64).ceil() as u64,
                MAX_THREADS as u64,
            ) as usize;
            let mut futures = vec![];
            for i in 0..num_batches {
                futures.push(fetch_nexts(
                    self.client.clone(),
                    self.current_version + (i as u64 * TRANSACTION_FETCH_BATCH_SIZE as u64),
                ));
            }
            let mut res: Vec<Vec<Transaction>> = futures::future::join_all(futures).await;
            res.sort_by(|a, b| {
                a.first()
                    .unwrap()
                    .version()
                    .unwrap()
                    .cmp(&b.first().unwrap().version().unwrap())
            });

            for batch in res {
                self.current_version = batch.last().unwrap().version().unwrap();
                self.transactions_sender.send(batch).await.unwrap();
            }
        }
    }
}

/// Fetches the next version based on its internal version counter
/// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
/// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
async fn fetch_nexts(client: RestClient, starting_version: u64) -> Vec<Transaction> {
    let mut retries = 0;
    while retries < MAX_RETRIES {
        retries += 1;

        let res = client
            .get_transactions(Some(starting_version), Some(TRANSACTION_FETCH_BATCH_SIZE))
            .await;

        match res {
            Ok(response) => {
                FETCHED_TRANSACTION.inc();
                return remove_null_bytes_from_txns(response.into_inner());
            }
            Err(err) => {
                let err_str = err.to_string();
                // If it's a 404, then we're all caught up; no need to increment the `UNABLE_TO_FETCH_TRANSACTION` counter
                if err_str.contains("404") {
                    aptos_logger::debug!(
                            "Could not fetch {} transactions starting at {}: all caught up. Will check again in {}ms.",
                            TRANSACTION_FETCH_BATCH_SIZE,
                            starting_version,
                            RETRY_TIME_MILLIS,
                        );
                }
                UNABLE_TO_FETCH_TRANSACTION.inc();
                aptos_logger::error!(
                    "Could not fetch {} transactions starting at {}, will retry in {}ms. Err: {:?}",
                    TRANSACTION_FETCH_BATCH_SIZE,
                    starting_version,
                    RETRY_TIME_MILLIS,
                    err
                );
            }
        }
        tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
    }

    panic!(
        "Could not fetch {} transactions starting at {}!",
        TRANSACTION_FETCH_BATCH_SIZE, starting_version
    );
}

#[derive(Debug)]
pub struct TransactionFetcher {
    starting_version: u64,
    client: RestClient,
    fetcher_handle: Option<JoinHandle<()>>,
    transactions_sender: Option<mpsc::Sender<Vec<Transaction>>>,
    transaction_receiver: mpsc::Receiver<Vec<Transaction>>,
}

impl TransactionFetcher {
    pub fn new(node_url: Url, starting_version: Option<u64>) -> Self {
        let (transactions_sender, transaction_receiver) =
            mpsc::channel::<Vec<Transaction>>(TRANSACTION_CHANNEL_SIZE);

        let client = RestClient::new(node_url);

        Self {
            starting_version: starting_version.unwrap_or(0),
            client,
            fetcher_handle: None,
            transactions_sender: Some(transactions_sender),
            transaction_receiver,
        }
    }
}

#[async_trait::async_trait]
impl TransactionFetcherTrait for TransactionFetcher {
    /// Fetches the next batch based on its internal version counter
    /// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
    /// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_next_batch(&mut self) -> Vec<Transaction> {
        self.transaction_receiver.next().await.unwrap()
    }

    /// fetches one version; this used for error checking/repair/etc
    /// In the event it can't, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_version(&self, version: u64) -> Transaction {
        loop {
            let res = self.client.get_transaction_by_version(version).await;
            match res {
                Ok(response) => {
                    FETCHED_TRANSACTION.inc();
                    return response.into_inner();
                }
                Err(err) => {
                    UNABLE_TO_FETCH_TRANSACTION.inc();
                    aptos_logger::error!(
                        "Could not fetch version {}, will retry in {}ms. Err: {:?}",
                        version,
                        RETRY_TIME_MILLIS,
                        err
                    );
                    tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
                }
            };
        }
    }

    async fn fetch_ledger_info(&mut self) -> State {
        let mut retries = 0;
        while retries < MAX_RETRIES {
            retries += 1;
            match self.client.get_ledger_information().await {
                Ok(inner) => return inner.into_inner(),
                Err(err) => {
                    aptos_logger::error!(
                        "Failed to get ledger info, will retry in {}ms. Err: {:?}",
                        RETRY_TIME_MILLIS,
                        err
                    );
                    tokio::time::sleep(Duration::from_secs(RETRY_TIME_MILLIS)).await
                }
            }
        }
        panic!("Failed to get ledger info.",);
    }

    async fn set_version(&mut self, version: u64) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        self.starting_version = version;
    }

    async fn start(&mut self) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        let client = self.client.clone();
        let transactions_sender = self.transactions_sender.take().unwrap();
        let starting_version = self.starting_version;
        let fetcher_handle = tokio::spawn(async move {
            let mut fetcher = Fetcher::new(client, starting_version, transactions_sender);
            fetcher.run().await;
        });
        self.fetcher_handle = Some(fetcher_handle);
    }
}

pub fn string_null_byte_replacement(value: &mut str) -> String {
    value.replace('\u{0000}', "").replace("\\u0000", "")
}

pub fn recurse_remove_null_bytes_from_json(sub_json: &mut Value) {
    match sub_json {
        Value::Array(array) => {
            for item in array {
                recurse_remove_null_bytes_from_json(item);
            }
        }
        Value::Object(object) => {
            for (_key, value) in object {
                recurse_remove_null_bytes_from_json(value);
            }
        }
        Value::String(str) => {
            if !str.is_empty() {
                let replacement = string_null_byte_replacement(str);
                *str = replacement;
            }
        }
        _ => {}
    }
}

pub fn remove_null_bytes_from_txns(txns: Vec<Transaction>) -> Vec<Transaction> {
    txns.iter()
        .map(|txn| {
            let mut txn_json = serde_json::to_value(txn).unwrap();
            recurse_remove_null_bytes_from_json(&mut txn_json);
            serde_json::from_value::<Transaction>(txn_json).unwrap()
        })
        .collect::<Vec<Transaction>>()
}

/// For mocking TransactionFetcher in tests
#[async_trait::async_trait]
pub trait TransactionFetcherTrait: Send + Sync {
    async fn fetch_next_batch(&mut self) -> Vec<Transaction>;

    async fn fetch_version(&self, version: u64) -> Transaction;

    async fn fetch_ledger_info(&mut self) -> State;

    async fn set_version(&mut self, version: u64);

    async fn start(&mut self);
}
<commit_msg>format<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::counters::{FETCHED_TRANSACTION, UNABLE_TO_FETCH_TRANSACTION};
use aptos_rest_client::{Client as RestClient, State, Transaction};
use futures::channel::mpsc;
use futures::{SinkExt, StreamExt};
use serde_json::Value;
use std::time::Duration;
use tokio::task::JoinHandle;
use url::Url;

// TODO: make this configurable
const RETRY_TIME_MILLIS: u64 = 1000;
const TRANSACTION_FETCH_BATCH_SIZE: u16 = 500;
const TRANSACTION_CHANNEL_SIZE: usize = 35;
const MAX_THREADS: usize = 10;
const MAX_RETRIES: usize = 5;

#[derive(Debug)]
pub struct Fetcher {
    client: RestClient,
    chain_id: u8,
    current_version: u64,
    highest_known_version: u64,
    transactions_sender: mpsc::Sender<Vec<Transaction>>,
}

impl Fetcher {
    pub fn new(
        client: RestClient,
        current_version: u64,
        transactions_sender: mpsc::Sender<Vec<Transaction>>,
    ) -> Self {
        Self {
            client,
            chain_id: 0,
            current_version,
            highest_known_version: current_version,
            transactions_sender,
        }
    }

    pub async fn set_highest_known_version(&mut self) {
        let info = self.client.get_ledger_information().await;
        let res = info.unwrap();
        let state = res.state();
        self.highest_known_version = state.version;
        self.chain_id = state.chain_id;
    }

    pub async fn run(&mut self) {
        loop {
            if self.current_version >= self.highest_known_version {
                tokio::time::sleep(Duration::from_millis(200)).await;
                self.set_highest_known_version().await;
            }

            let num_missing = self.highest_known_version - self.current_version;
            let num_batches = std::cmp::min(
                (num_missing as f64 / TRANSACTION_FETCH_BATCH_SIZE as f64).ceil() as u64,
                MAX_THREADS as u64,
            ) as usize;
            let mut futures = vec![];
            for i in 0..num_batches {
                futures.push(fetch_nexts(
                    self.client.clone(),
                    self.current_version + (i as u64 * TRANSACTION_FETCH_BATCH_SIZE as u64),
                ));
            }
            let mut res: Vec<Vec<Transaction>> = futures::future::join_all(futures).await;
            res.sort_by(|a, b| {
                a.first()
                    .unwrap()
                    .version()
                    .unwrap()
                    .cmp(&b.first().unwrap().version().unwrap())
            });

            for batch in res {
                self.current_version = batch.last().unwrap().version().unwrap();
                self.transactions_sender.send(batch).await.unwrap();
            }
        }
    }
}

/// Fetches the next version based on its internal version counter
/// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
/// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
async fn fetch_nexts(client: RestClient, starting_version: u64) -> Vec<Transaction> {
    let mut retries = 0;
    while retries < MAX_RETRIES {
        retries += 1;

        let res = client
            .get_transactions(Some(starting_version), Some(TRANSACTION_FETCH_BATCH_SIZE))
            .await;

        match res {
            Ok(response) => {
                FETCHED_TRANSACTION.inc();
                return remove_null_bytes_from_txns(response.into_inner());
            }
            Err(err) => {
                let err_str = err.to_string();
                // If it's a 404, then we're all caught up; no need to increment the `UNABLE_TO_FETCH_TRANSACTION` counter
                if err_str.contains("404") {
                    aptos_logger::debug!(
                            "Could not fetch {} transactions starting at {}: all caught up. Will check again in {}ms.",
                            TRANSACTION_FETCH_BATCH_SIZE,
                            starting_version,
                            RETRY_TIME_MILLIS,
                        );
                }
                UNABLE_TO_FETCH_TRANSACTION.inc();
                aptos_logger::error!(
                    "Could not fetch {} transactions starting at {}, will retry in {}ms. Err: {:?}",
                    TRANSACTION_FETCH_BATCH_SIZE,
                    starting_version,
                    RETRY_TIME_MILLIS,
                    err
                );
            }
        }
        tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
    }

    panic!(
        "Could not fetch {} transactions starting at {}!",
        TRANSACTION_FETCH_BATCH_SIZE, starting_version
    );
}

#[derive(Debug)]
pub struct TransactionFetcher {
    starting_version: u64,
    client: RestClient,
    fetcher_handle: Option<JoinHandle<()>>,
    transactions_sender: Option<mpsc::Sender<Vec<Transaction>>>,
    transaction_receiver: mpsc::Receiver<Vec<Transaction>>,
}

impl TransactionFetcher {
    pub fn new(node_url: Url, starting_version: Option<u64>) -> Self {
        let (transactions_sender, transaction_receiver) =
            mpsc::channel::<Vec<Transaction>>(TRANSACTION_CHANNEL_SIZE);

        let client = RestClient::new(node_url);

        Self {
            starting_version: starting_version.unwrap_or(0),
            client,
            fetcher_handle: None,
            transactions_sender: Some(transactions_sender),
            transaction_receiver,
        }
    }
}

#[async_trait::async_trait]
impl TransactionFetcherTrait for TransactionFetcher {
    /// Fetches the next batch based on its internal version counter
    /// Under the hood, it fetches TRANSACTION_FETCH_BATCH_SIZE versions in bulk (when needed), and uses that buffer to feed out
    /// In the event it can't fetch, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_next_batch(&mut self) -> Vec<Transaction> {
        self.transaction_receiver.next().await.unwrap()
    }

    /// fetches one version; this used for error checking/repair/etc
    /// In the event it can't, it will keep retrying every RETRY_TIME_MILLIS ms
    async fn fetch_version(&self, version: u64) -> Transaction {
        loop {
            let res = self.client.get_transaction_by_version(version).await;
            match res {
                Ok(response) => {
                    FETCHED_TRANSACTION.inc();
                    return response.into_inner();
                }
                Err(err) => {
                    UNABLE_TO_FETCH_TRANSACTION.inc();
                    aptos_logger::error!(
                        "Could not fetch version {}, will retry in {}ms. Err: {:?}",
                        version,
                        RETRY_TIME_MILLIS,
                        err
                    );
                    tokio::time::sleep(Duration::from_millis(RETRY_TIME_MILLIS)).await;
                }
            };
        }
    }

    async fn fetch_ledger_info(&mut self) -> State {
        let mut retries = 0;
        while retries < MAX_RETRIES {
            retries += 1;
            match self.client.get_ledger_information().await {
                Ok(inner) => return inner.into_inner(),
                Err(err) => {
                    aptos_logger::error!(
                        "Failed to get ledger info, will retry in {}ms. Err: {:?}",
                        RETRY_TIME_MILLIS,
                        err
                    );
                    tokio::time::sleep(Duration::from_secs(RETRY_TIME_MILLIS)).await
                }
            }
        }
        panic!("Failed to get ledger info.");
    }

    async fn set_version(&mut self, version: u64) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        self.starting_version = version;
    }

    async fn start(&mut self) {
        if self.fetcher_handle.is_some() {
            panic!("TransactionFetcher already started!");
        }
        let client = self.client.clone();
        let transactions_sender = self.transactions_sender.take().unwrap();
        let starting_version = self.starting_version;
        let fetcher_handle = tokio::spawn(async move {
            let mut fetcher = Fetcher::new(client, starting_version, transactions_sender);
            fetcher.run().await;
        });
        self.fetcher_handle = Some(fetcher_handle);
    }
}

pub fn string_null_byte_replacement(value: &mut str) -> String {
    value.replace('\u{0000}', "").replace("\\u0000", "")
}

pub fn recurse_remove_null_bytes_from_json(sub_json: &mut Value) {
    match sub_json {
        Value::Array(array) => {
            for item in array {
                recurse_remove_null_bytes_from_json(item);
            }
        }
        Value::Object(object) => {
            for (_key, value) in object {
                recurse_remove_null_bytes_from_json(value);
            }
        }
        Value::String(str) => {
            if !str.is_empty() {
                let replacement = string_null_byte_replacement(str);
                *str = replacement;
            }
        }
        _ => {}
    }
}

pub fn remove_null_bytes_from_txns(txns: Vec<Transaction>) -> Vec<Transaction> {
    txns.iter()
        .map(|txn| {
            let mut txn_json = serde_json::to_value(txn).unwrap();
            recurse_remove_null_bytes_from_json(&mut txn_json);
            serde_json::from_value::<Transaction>(txn_json).unwrap()
        })
        .collect::<Vec<Transaction>>()
}

/// For mocking TransactionFetcher in tests
#[async_trait::async_trait]
pub trait TransactionFetcherTrait: Send + Sync {
    async fn fetch_next_batch(&mut self) -> Vec<Transaction>;

    async fn fetch_version(&self, version: u64) -> Transaction;

    async fn fetch_ledger_info(&mut self) -> State;

    async fn set_version(&mut self, version: u64);

    async fn start(&mut self);
}
<|endoftext|>


<commit_before>// no-system-llvm
// assembly-output: emit-asm
// compile-flags: --target mips-unknown-linux-gnu
// needs-llvm-components: mips

#![feature(no_core, lang_items, rustc_attrs, repr_simd)]
#![crate_type = "rlib"]
#![no_core]
#![allow(asm_sub_register, non_camel_case_types)]

#[rustc_builtin_macro]
macro_rules! asm {
    () => {};
}
#[rustc_builtin_macro]
macro_rules! concat {
    () => {};
}
#[rustc_builtin_macro]
macro_rules! stringify {
    () => {};
}

#[lang = "sized"]
trait Sized {}
#[lang = "copy"]
trait Copy {}

type ptr = *const i32;

impl Copy for i8 {}
impl Copy for u8 {}
impl Copy for i16 {}
impl Copy for i32 {}
impl Copy for f32 {}
impl Copy for ptr {}
extern "C" {
    fn extern_func();
    static extern_static: u8;
}

// Hack to avoid function merging
extern "Rust" {
    fn dont_merge(s: &str);
}

macro_rules! check { ($func:ident, $ty:ty, $class:ident) => {
    #[no_mangle]
    pub unsafe fn $func(x: $ty) -> $ty {
        dont_merge(stringify!($func));

        let y;
        asm!("move {}, {}", out($class) y, in($class) x);
        y
    }
};}

macro_rules! check_reg { ($func:ident, $ty:ty, $reg:tt) => {
    #[no_mangle]
    pub unsafe fn $func(x: $ty) -> $ty {
        dont_merge(stringify!($func));

        let y;
        asm!(concat!("move ", $reg, ", ", $reg), lateout($reg) y, in($reg) x);
        y
    }
};}

// CHECK-LABEL: sym_static:
// CHECK: #APP
// CHECK: lw $3, %got(extern_static)
// CHECK: #NO_APP
#[no_mangle]
pub unsafe fn sym_static() {
    dont_merge(stringify!($func));

    asm!("la $v1, {}", sym extern_static);
}

// CHECK-LABEL: sym_fn:
// CHECK: #APP
// CHECK: lw $3, %got(extern_func)
// CHECK: #NO_APP
#[no_mangle]
pub unsafe fn sym_fn() {
    dont_merge(stringify!($func));

    asm!("la $v1, {}", sym extern_func);
}

// CHECK-LABEL: reg_f32:
// CHECK: #APP
// CHECK: mov.s $f{{[0-9]+}}, $f{{[0-9]+}}
// CHECK: #NO_APP
#[no_mangle]
pub unsafe fn reg_f32(x: f32) -> f32 {
    dont_merge("reg_f32");
    let y;
    asm!("mov.s {}, {}", out(freg) y, in(freg) x);
    y
}

// CHECK-LABEL: f0_f32:
// CHECK: #APP
// CHECK: mov.s $f0, $f0
// CHECK: #NO_APP
#[no_mangle]
pub unsafe fn f0_f32(x: f32) -> f32 {
    dont_merge("f0_f32");
    let y;
    asm!("mov.s $f0, $f0", lateout("$f0") y, in("$f0") x);
    y
}

// CHECK-LABEL: reg_ptr:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_ptr, ptr, reg);

// CHECK-LABEL: reg_i32:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_i32, i32, reg);

// CHECK-LABEL: reg_f32_soft:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_f32_soft, f32, reg);

// CHECK-LABEL: reg_i8:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_i8, i8, reg);

// CHECK-LABEL: reg_u8:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_u8, u8, reg);

// CHECK-LABEL: reg_i16:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_i16, i16, reg);

// CHECK-LABEL: t0_ptr:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_ptr, ptr, "$t0");

// CHECK-LABEL: t0_i32:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_i32, i32, "$t0");

// CHECK-LABEL: t0_f32:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_f32, f32, "$t0");

// CHECK-LABEL: t0_i8:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_i8, i8, "$t0");

// CHECK-LABEL: t0_u8:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_u8, u8, "$t0");

// CHECK-LABEL: t0_i16:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_i16, i16, "$t0");

// CHECK-LABEL: r8_i16:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(r8_i16, i16, "$8");
<commit_msg>Remove useless stringify<commit_after>// no-system-llvm
// assembly-output: emit-asm
// compile-flags: --target mips-unknown-linux-gnu
// needs-llvm-components: mips

#![feature(no_core, lang_items, rustc_attrs, repr_simd)]
#![crate_type = "rlib"]
#![no_core]
#![allow(asm_sub_register, non_camel_case_types)]

#[rustc_builtin_macro]
macro_rules! asm {
    () => {};
}
#[rustc_builtin_macro]
macro_rules! concat {
    () => {};
}
#[rustc_builtin_macro]
macro_rules! stringify {
    () => {};
}

#[lang = "sized"]
trait Sized {}
#[lang = "copy"]
trait Copy {}

type ptr = *const i32;

impl Copy for i8 {}
impl Copy for u8 {}
impl Copy for i16 {}
impl Copy for i32 {}
impl Copy for f32 {}
impl Copy for ptr {}
extern "C" {
    fn extern_func();
    static extern_static: u8;
}

// Hack to avoid function merging
extern "Rust" {
    fn dont_merge(s: &str);
}

macro_rules! check { ($func:ident, $ty:ty, $class:ident) => {
    #[no_mangle]
    pub unsafe fn $func(x: $ty) -> $ty {
        dont_merge(stringify!($func));

        let y;
        asm!("move {}, {}", out($class) y, in($class) x);
        y
    }
};}

macro_rules! check_reg { ($func:ident, $ty:ty, $reg:tt) => {
    #[no_mangle]
    pub unsafe fn $func(x: $ty) -> $ty {
        dont_merge(stringify!($func));

        let y;
        asm!(concat!("move ", $reg, ", ", $reg), lateout($reg) y, in($reg) x);
        y
    }
};}

// CHECK-LABEL: sym_static:
// CHECK: #APP
// CHECK: lw $3, %got(extern_static)
// CHECK: #NO_APP
#[no_mangle]
pub unsafe fn sym_static() {
    asm!("la $v1, {}", sym extern_static);
}

// CHECK-LABEL: sym_fn:
// CHECK: #APP
// CHECK: lw $3, %got(extern_func)
// CHECK: #NO_APP
#[no_mangle]
pub unsafe fn sym_fn() {
    asm!("la $v1, {}", sym extern_func);
}

// CHECK-LABEL: reg_f32:
// CHECK: #APP
// CHECK: mov.s $f{{[0-9]+}}, $f{{[0-9]+}}
// CHECK: #NO_APP
#[no_mangle]
pub unsafe fn reg_f32(x: f32) -> f32 {
    dont_merge("reg_f32");
    let y;
    asm!("mov.s {}, {}", out(freg) y, in(freg) x);
    y
}

// CHECK-LABEL: f0_f32:
// CHECK: #APP
// CHECK: mov.s $f0, $f0
// CHECK: #NO_APP
#[no_mangle]
pub unsafe fn f0_f32(x: f32) -> f32 {
    dont_merge("f0_f32");
    let y;
    asm!("mov.s $f0, $f0", lateout("$f0") y, in("$f0") x);
    y
}

// CHECK-LABEL: reg_ptr:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_ptr, ptr, reg);

// CHECK-LABEL: reg_i32:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_i32, i32, reg);

// CHECK-LABEL: reg_f32_soft:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_f32_soft, f32, reg);

// CHECK-LABEL: reg_i8:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_i8, i8, reg);

// CHECK-LABEL: reg_u8:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_u8, u8, reg);

// CHECK-LABEL: reg_i16:
// CHECK: #APP
// CHECK: move ${{[0-9]+}}, ${{[0-9]+}}
// CHECK: #NO_APP
check!(reg_i16, i16, reg);

// CHECK-LABEL: t0_ptr:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_ptr, ptr, "$t0");

// CHECK-LABEL: t0_i32:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_i32, i32, "$t0");

// CHECK-LABEL: t0_f32:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_f32, f32, "$t0");

// CHECK-LABEL: t0_i8:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_i8, i8, "$t0");

// CHECK-LABEL: t0_u8:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_u8, u8, "$t0");

// CHECK-LABEL: t0_i16:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(t0_i16, i16, "$t0");

// CHECK-LABEL: r8_i16:
// CHECK: #APP
// CHECK: move $8, $8
// CHECK: #NO_APP
check_reg!(r8_i16, i16, "$8");
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

struct Monster {
    damage: isize
}

fn main() {
    let _m = Monster(); //~ ERROR `Monster` is a structure name, but
    //~^ HELP did you mean to write: `Monster { /* fields */ }`?
}
<commit_msg>Fixup test<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

struct Monster {
    damage: isize
}

fn main() {
    let _m = Monster(); //~ ERROR `Monster` is a struct variant name, but
    //~^ HELP did you mean to write: `Monster { /* fields */ }`?
}
<|endoftext|>


<commit_before>//! Functions to aid the `Api::to_tokens` method.

use proc_macro2::{Span, TokenStream};
use quote::quote;
use std::collections::BTreeSet;
use syn::{
    AngleBracketedGenericArguments, GenericArgument, Ident, Lifetime,
    ParenthesizedGenericArguments, PathArguments, Type, TypePath, TypeReference,
};

use crate::api::{metadata::Metadata, request::Request};

pub fn collect_lifetime_ident(lifetimes: &mut BTreeSet<Lifetime>, ty: &Type) {
    match ty {
        Type::Path(TypePath { path, .. }) => {
            for seg in &path.segments {
                match &seg.arguments {
                    PathArguments::AngleBracketed(AngleBracketedGenericArguments {
                        args, ..
                    }) => {
                        for gen in args {
                            if let GenericArgument::Type(ty) = gen {
                                collect_lifetime_ident(lifetimes, &ty)
                            } else if let GenericArgument::Lifetime(lt) = gen {
                                lifetimes.insert(lt.clone());
                            }
                        }
                    }
                    PathArguments::Parenthesized(ParenthesizedGenericArguments {
                        inputs, ..
                    }) => {
                        for ty in inputs {
                            collect_lifetime_ident(lifetimes, ty)
                        }
                    }
                    _ => {}
                }
            }
        }
        Type::Reference(TypeReference { elem, lifetime, .. }) => {
            collect_lifetime_ident(lifetimes, &*elem);
            if let Some(lt) = lifetime {
                lifetimes.insert(lt.clone());
            }
        }
        _ => {}
    }
}

/// Generates a `TokenStream` of lifetime identifiers `<'lifetime>`.
pub fn generics_to_tokens<'a, I: Iterator<Item = &'a Lifetime>>(lifetimes: I) -> TokenStream {
    let lifetimes = lifetimes.collect::<Vec<_>>();
    if lifetimes.is_empty() {
        TokenStream::new()
    } else {
        let lifetimes = quote! { #( #lifetimes ),* };
        quote! { < #lifetimes > }
    }
}

/// The first item in the tuple generates code for the request path from
/// the `Metadata` and `Request` structs. The second item in the returned tuple
/// is the code to generate a Request struct field created from any segments
/// of the path that start with ":".
///
/// The first `TokenStream` returned is the constructed url path. The second `TokenStream` is
/// used for implementing `TryFrom<http::Request<Vec<u8>>>`, from path strings deserialized to Ruma
/// types.
pub(crate) fn request_path_string_and_parse(
    request: &Request,
    metadata: &Metadata,
) -> (TokenStream, TokenStream) {
    if request.has_path_fields() {
        let path_string = metadata.path.value();

        assert!(path_string.starts_with('/'), "path needs to start with '/'");
        assert!(
            path_string.chars().filter(|c| *c == ':').count() == request.path_field_count(),
            "number of declared path parameters needs to match amount of placeholders in path"
        );

        let format_call = {
            let mut format_string = path_string.clone();
            let mut format_args = Vec::new();

            while let Some(start_of_segment) = format_string.find(':') {
                // ':' should only ever appear at the start of a segment
                assert_eq!(&format_string[start_of_segment - 1..start_of_segment], "/");

                let end_of_segment = match format_string[start_of_segment..].find('/') {
                    Some(rel_pos) => start_of_segment + rel_pos,
                    None => format_string.len(),
                };

                let path_var = Ident::new(
                    &format_string[start_of_segment + 1..end_of_segment],
                    Span::call_site(),
                );
                format_args.push(quote! {
                    ::ruma_api::exports::percent_encoding::utf8_percent_encode(
                        &self.#path_var.to_string(),
                        ::ruma_api::exports::percent_encoding::NON_ALPHANUMERIC,
                    )
                });
                format_string.replace_range(start_of_segment..end_of_segment, "{}");
            }

            quote! {
                format_args!(#format_string, #(#format_args),*)
            }
        };

        let path_fields =
            path_string[1..].split('/').enumerate().filter(|(_, s)| s.starts_with(':')).map(
                |(i, segment)| {
                    let path_var = &segment[1..];
                    let path_var_ident = Ident::new(path_var, Span::call_site());
                    quote! {
                        #path_var_ident: {
                            use ::ruma_api::error::RequestDeserializationError;

                            let segment = path_segments.get(#i).unwrap().as_bytes();
                            let decoded = ::ruma_api::try_deserialize!(
                                request,
                                ::ruma_api::exports::percent_encoding::percent_decode(segment)
                                    .decode_utf8(),
                            );

                            ::ruma_api::try_deserialize!(request, ::std::convert::TryFrom::try_from(&*decoded))
                        }
                    }
                },
            );

        (format_call, quote! { #(#path_fields,)* })
    } else {
        (quote! { metadata.path.to_owned() }, TokenStream::new())
    }
}

/// The function determines the type of query string that needs to be built
/// and then builds it using `ruma_serde::urlencoded::to_string`.
pub(crate) fn build_query_string(request: &Request) -> TokenStream {
    if let Some(field) = request.query_map_field() {
        let field_name = field.ident.as_ref().expect("expected field to have identifier");

        quote!({
            // This function exists so that the compiler will throw an
            // error when the type of the field with the query_map
            // attribute doesn't implement IntoIterator<Item = (String, String)>
            //
            // This is necessary because the ruma_serde::urlencoded::to_string
            // call will result in a runtime error when the type cannot be
            // encoded as a list key-value pairs (?key1=value1&key2=value2)
            //
            // By asserting that it implements the iterator trait, we can
            // ensure that it won't fail.
            fn assert_trait_impl<T>(_: &T)
            where
                T: ::std::iter::IntoIterator<Item = (::std::string::String, ::std::string::String)>,
            {}

            let request_query = RequestQuery(self.#field_name);
            assert_trait_impl(&request_query.0);

            format_args!(
                "?{}",
                ::ruma_api::exports::ruma_serde::urlencoded::to_string(request_query)?
            )
        })
    } else if request.has_query_fields() {
        let request_query_init_fields = request.request_query_init_fields();

        quote!({
            let request_query = RequestQuery {
                #request_query_init_fields
            };

            format_args!(
                "?{}",
                ::ruma_api::exports::ruma_serde::urlencoded::to_string(request_query)?
            )
        })
    } else {
        quote! { "" }
    }
}

/// Deserialize the query string.
pub(crate) fn extract_request_query(request: &Request) -> TokenStream {
    if request.query_map_field().is_some() {
        quote! {
            let request_query = ::ruma_api::try_deserialize!(
                request,
                ::ruma_api::exports::ruma_serde::urlencoded::from_str(
                    &request.uri().query().unwrap_or("")
                ),
            );
        }
    } else if request.has_query_fields() {
        let request_query_type = if request.has_query_lifetimes() {
            quote! { IncomingRequestQuery }
        } else {
            quote! { RequestQuery }
        };
        quote! {
            let request_query: #request_query_type = ::ruma_api::try_deserialize!(
                request,
                ::ruma_api::exports::ruma_serde::urlencoded::from_str(
                    &request.uri().query().unwrap_or("")
                ),
            );
        }
    } else {
        TokenStream::new()
    }
}

/// Generates the code to initialize a `Request`.
///
/// Used to construct an `http::Request`s body.
pub(crate) fn build_request_body(request: &Request) -> TokenStream {
    if let Some(field) = request.newtype_raw_body_field() {
        let field_name = field.ident.as_ref().expect("expected field to have an identifier");
        quote!(self.#field_name)
    } else if request.has_body_fields() || request.newtype_body_field().is_some() {
        let request_body_initializers = if let Some(field) = request.newtype_body_field() {
            let field_name = field.ident.as_ref().expect("expected field to have an identifier");
            quote! { (self.#field_name) }
        } else {
            let initializers = request.request_body_init_fields();
            quote! { { #initializers } }
        };

        quote! {
            {
                let request_body = RequestBody #request_body_initializers;
                ::ruma_api::exports::serde_json::to_vec(&request_body)?
            }
        }
    } else {
        quote!(Vec::new())
    }
}

pub(crate) fn parse_request_body(request: &Request) -> TokenStream {
    if let Some(field) = request.newtype_body_field() {
        let field_name = field.ident.as_ref().expect("expected field to have an identifier");
        quote! {
            #field_name: request_body.0,
        }
    } else if let Some(field) = request.newtype_raw_body_field() {
        let field_name = field.ident.as_ref().expect("expected field to have an identifier");
        quote! {
            #field_name: request.into_body(),
        }
    } else {
        request.request_init_body_fields()
    }
}

pub(crate) fn req_res_meta_word<T>(
    attr_kind: &str,
    field: &syn::Field,
    newtype_body_field: &mut Option<syn::Field>,
    body_field_kind: T,
    raw_field_kind: T,
) -> syn::Result<T> {
    if let Some(f) = &newtype_body_field {
        let mut error = syn::Error::new_spanned(field, "There can only be one newtype body field");
        error.combine(syn::Error::new_spanned(f, "Previous newtype body field"));
        return Err(error);
    }

    *newtype_body_field = Some(field.clone());
    Ok(match attr_kind {
        "body" => body_field_kind,
        "raw_body" => raw_field_kind,
        _ => unreachable!(),
    })
}

pub(crate) fn req_res_name_value<T>(
    name: Ident,
    value: Ident,
    header: &mut Option<Ident>,
    field_kind: T,
) -> syn::Result<T> {
    if name != "header" {
        return Err(syn::Error::new_spanned(
            name,
            "Invalid #[ruma_api] argument with value, expected `header`",
        ));
    }

    *header = Some(value);
    Ok(field_kind)
}

pub(crate) fn is_valid_endpoint_path(string: &str) -> bool {
    string.as_bytes().iter().all(|b| (0x21..=0x7E).contains(b))
}
<commit_msg>Handle tuple types in collect_lifetime_idents<commit_after>//! Functions to aid the `Api::to_tokens` method.

use proc_macro2::{Span, TokenStream};
use quote::quote;
use std::collections::BTreeSet;
use syn::{
    AngleBracketedGenericArguments, GenericArgument, Ident, Lifetime,
    ParenthesizedGenericArguments, PathArguments, Type, TypePath, TypeReference,
};

use crate::api::{metadata::Metadata, request::Request};

pub fn collect_lifetime_ident(lifetimes: &mut BTreeSet<Lifetime>, ty: &Type) {
    match ty {
        Type::Path(TypePath { path, .. }) => {
            for seg in &path.segments {
                match &seg.arguments {
                    PathArguments::AngleBracketed(AngleBracketedGenericArguments {
                        args, ..
                    }) => {
                        for gen in args {
                            if let GenericArgument::Type(ty) = gen {
                                collect_lifetime_ident(lifetimes, &ty)
                            } else if let GenericArgument::Lifetime(lt) = gen {
                                lifetimes.insert(lt.clone());
                            }
                        }
                    }
                    PathArguments::Parenthesized(ParenthesizedGenericArguments {
                        inputs, ..
                    }) => {
                        for ty in inputs {
                            collect_lifetime_ident(lifetimes, ty)
                        }
                    }
                    _ => {}
                }
            }
        }
        Type::Reference(TypeReference { elem, lifetime, .. }) => {
            collect_lifetime_ident(lifetimes, &*elem);
            if let Some(lt) = lifetime {
                lifetimes.insert(lt.clone());
            }
        }
        Type::Tuple(syn::TypeTuple { elems, .. }) => {
            for ty in elems {
                collect_lifetime_ident(lifetimes, ty)
            }
        }
        _ => {}
    }
}

/// Generates a `TokenStream` of lifetime identifiers `<'lifetime>`.
pub fn generics_to_tokens<'a, I: Iterator<Item = &'a Lifetime>>(lifetimes: I) -> TokenStream {
    let lifetimes = lifetimes.collect::<Vec<_>>();
    if lifetimes.is_empty() {
        TokenStream::new()
    } else {
        let lifetimes = quote! { #( #lifetimes ),* };
        quote! { < #lifetimes > }
    }
}

/// The first item in the tuple generates code for the request path from
/// the `Metadata` and `Request` structs. The second item in the returned tuple
/// is the code to generate a Request struct field created from any segments
/// of the path that start with ":".
///
/// The first `TokenStream` returned is the constructed url path. The second `TokenStream` is
/// used for implementing `TryFrom<http::Request<Vec<u8>>>`, from path strings deserialized to Ruma
/// types.
pub(crate) fn request_path_string_and_parse(
    request: &Request,
    metadata: &Metadata,
) -> (TokenStream, TokenStream) {
    if request.has_path_fields() {
        let path_string = metadata.path.value();

        assert!(path_string.starts_with('/'), "path needs to start with '/'");
        assert!(
            path_string.chars().filter(|c| *c == ':').count() == request.path_field_count(),
            "number of declared path parameters needs to match amount of placeholders in path"
        );

        let format_call = {
            let mut format_string = path_string.clone();
            let mut format_args = Vec::new();

            while let Some(start_of_segment) = format_string.find(':') {
                // ':' should only ever appear at the start of a segment
                assert_eq!(&format_string[start_of_segment - 1..start_of_segment], "/");

                let end_of_segment = match format_string[start_of_segment..].find('/') {
                    Some(rel_pos) => start_of_segment + rel_pos,
                    None => format_string.len(),
                };

                let path_var = Ident::new(
                    &format_string[start_of_segment + 1..end_of_segment],
                    Span::call_site(),
                );
                format_args.push(quote! {
                    ::ruma_api::exports::percent_encoding::utf8_percent_encode(
                        &self.#path_var.to_string(),
                        ::ruma_api::exports::percent_encoding::NON_ALPHANUMERIC,
                    )
                });
                format_string.replace_range(start_of_segment..end_of_segment, "{}");
            }

            quote! {
                format_args!(#format_string, #(#format_args),*)
            }
        };

        let path_fields =
            path_string[1..].split('/').enumerate().filter(|(_, s)| s.starts_with(':')).map(
                |(i, segment)| {
                    let path_var = &segment[1..];
                    let path_var_ident = Ident::new(path_var, Span::call_site());
                    quote! {
                        #path_var_ident: {
                            use ::ruma_api::error::RequestDeserializationError;

                            let segment = path_segments.get(#i).unwrap().as_bytes();
                            let decoded = ::ruma_api::try_deserialize!(
                                request,
                                ::ruma_api::exports::percent_encoding::percent_decode(segment)
                                    .decode_utf8(),
                            );

                            ::ruma_api::try_deserialize!(request, ::std::convert::TryFrom::try_from(&*decoded))
                        }
                    }
                },
            );

        (format_call, quote! { #(#path_fields,)* })
    } else {
        (quote! { metadata.path.to_owned() }, TokenStream::new())
    }
}

/// The function determines the type of query string that needs to be built
/// and then builds it using `ruma_serde::urlencoded::to_string`.
pub(crate) fn build_query_string(request: &Request) -> TokenStream {
    if let Some(field) = request.query_map_field() {
        let field_name = field.ident.as_ref().expect("expected field to have identifier");

        quote!({
            // This function exists so that the compiler will throw an
            // error when the type of the field with the query_map
            // attribute doesn't implement IntoIterator<Item = (String, String)>
            //
            // This is necessary because the ruma_serde::urlencoded::to_string
            // call will result in a runtime error when the type cannot be
            // encoded as a list key-value pairs (?key1=value1&key2=value2)
            //
            // By asserting that it implements the iterator trait, we can
            // ensure that it won't fail.
            fn assert_trait_impl<T>(_: &T)
            where
                T: ::std::iter::IntoIterator<Item = (::std::string::String, ::std::string::String)>,
            {}

            let request_query = RequestQuery(self.#field_name);
            assert_trait_impl(&request_query.0);

            format_args!(
                "?{}",
                ::ruma_api::exports::ruma_serde::urlencoded::to_string(request_query)?
            )
        })
    } else if request.has_query_fields() {
        let request_query_init_fields = request.request_query_init_fields();

        quote!({
            let request_query = RequestQuery {
                #request_query_init_fields
            };

            format_args!(
                "?{}",
                ::ruma_api::exports::ruma_serde::urlencoded::to_string(request_query)?
            )
        })
    } else {
        quote! { "" }
    }
}

/// Deserialize the query string.
pub(crate) fn extract_request_query(request: &Request) -> TokenStream {
    if request.query_map_field().is_some() {
        quote! {
            let request_query = ::ruma_api::try_deserialize!(
                request,
                ::ruma_api::exports::ruma_serde::urlencoded::from_str(
                    &request.uri().query().unwrap_or("")
                ),
            );
        }
    } else if request.has_query_fields() {
        let request_query_type = if request.has_query_lifetimes() {
            quote! { IncomingRequestQuery }
        } else {
            quote! { RequestQuery }
        };
        quote! {
            let request_query: #request_query_type = ::ruma_api::try_deserialize!(
                request,
                ::ruma_api::exports::ruma_serde::urlencoded::from_str(
                    &request.uri().query().unwrap_or("")
                ),
            );
        }
    } else {
        TokenStream::new()
    }
}

/// Generates the code to initialize a `Request`.
///
/// Used to construct an `http::Request`s body.
pub(crate) fn build_request_body(request: &Request) -> TokenStream {
    if let Some(field) = request.newtype_raw_body_field() {
        let field_name = field.ident.as_ref().expect("expected field to have an identifier");
        quote!(self.#field_name)
    } else if request.has_body_fields() || request.newtype_body_field().is_some() {
        let request_body_initializers = if let Some(field) = request.newtype_body_field() {
            let field_name = field.ident.as_ref().expect("expected field to have an identifier");
            quote! { (self.#field_name) }
        } else {
            let initializers = request.request_body_init_fields();
            quote! { { #initializers } }
        };

        quote! {
            {
                let request_body = RequestBody #request_body_initializers;
                ::ruma_api::exports::serde_json::to_vec(&request_body)?
            }
        }
    } else {
        quote!(Vec::new())
    }
}

pub(crate) fn parse_request_body(request: &Request) -> TokenStream {
    if let Some(field) = request.newtype_body_field() {
        let field_name = field.ident.as_ref().expect("expected field to have an identifier");
        quote! {
            #field_name: request_body.0,
        }
    } else if let Some(field) = request.newtype_raw_body_field() {
        let field_name = field.ident.as_ref().expect("expected field to have an identifier");
        quote! {
            #field_name: request.into_body(),
        }
    } else {
        request.request_init_body_fields()
    }
}

pub(crate) fn req_res_meta_word<T>(
    attr_kind: &str,
    field: &syn::Field,
    newtype_body_field: &mut Option<syn::Field>,
    body_field_kind: T,
    raw_field_kind: T,
) -> syn::Result<T> {
    if let Some(f) = &newtype_body_field {
        let mut error = syn::Error::new_spanned(field, "There can only be one newtype body field");
        error.combine(syn::Error::new_spanned(f, "Previous newtype body field"));
        return Err(error);
    }

    *newtype_body_field = Some(field.clone());
    Ok(match attr_kind {
        "body" => body_field_kind,
        "raw_body" => raw_field_kind,
        _ => unreachable!(),
    })
}

pub(crate) fn req_res_name_value<T>(
    name: Ident,
    value: Ident,
    header: &mut Option<Ident>,
    field_kind: T,
) -> syn::Result<T> {
    if name != "header" {
        return Err(syn::Error::new_spanned(
            name,
            "Invalid #[ruma_api] argument with value, expected `header`",
        ));
    }

    *header = Some(value);
    Ok(field_kind)
}

pub(crate) fn is_valid_endpoint_path(string: &str) -> bool {
    string.as_bytes().iter().all(|b| (0x21..=0x7E).contains(b))
}
<|endoftext|>


<commit_before>use crate::AHasher;
use core::hash::BuildHasher;
use core::sync::atomic::AtomicUsize;
use core::sync::atomic::Ordering;

#[cfg(feature = "compile-time-rng")]
use const_random::const_random;

///This constant come from Kunth's prng
pub(crate) const MULTIPLE: u64 = 6364136223846793005;

// Const random provides randomized starting key with no runtime cost.
#[cfg(feature = "compile-time-rng")]
static SEED: AtomicUsize = AtomicUsize::new(const_random!(u64));

#[cfg(not(feature = "compile-time-rng"))]
static SEED: AtomicUsize = AtomicUsize::new(MULTIPLE as usize);

/// Provides a [Hasher] factory. This is typically used (e.g. by [`HashMap`]) to create
/// [AHasher]s in order to hash the keys of the map. See `build_hasher` below.
///
/// [build_hasher]: ahash::
/// [Hasher]: std::hash::Hasher
/// [BuildHasher]: std::hash::BuildHasher
/// [HashMap]: std::collections::HashMap
#[derive(Clone)]
pub struct RandomState {
    pub(crate) k0: u64,
    pub(crate) k1: u64,
}

impl RandomState {
    #[inline]
    pub fn new() -> RandomState {
        //Using a self pointer. When running with ASLR this is a random value.
        let previous = SEED.load(Ordering::Relaxed) as u64;
        let stack_mem_loc = &previous as *const _ as u64;
        //This is similar to the update function in the fallback.
        //only one multiply is needed because memory locations are not under an attackers control.
        let current_seed = previous
            .wrapping_mul(MULTIPLE)
            .wrapping_add(stack_mem_loc)
            .rotate_left(31);
        SEED.store(current_seed as usize, Ordering::Relaxed);
        let (k0, k1) = scramble_keys(&SEED as *const _ as u64, current_seed);
        RandomState { k0, k1 }
    }
}

pub(crate) fn scramble_keys(k0: u64, k1: u64) -> (u64, u64) {
    //Scramble seeds (based on xoroshiro128+)
    //This is intentionally not similar the hash algorithm
    let result1 = k0.wrapping_add(k1);
    let k1 = k1 ^ k0;
    let k0 = k0.rotate_left(24) ^ k1 ^ (k1.wrapping_shl(16));
    let result2 = k0.wrapping_add(k1.rotate_left(37));
    (result2, result1)
}

impl Default for RandomState {
    #[inline]
    fn default() -> Self {
        Self::new()
    }
}

impl BuildHasher for RandomState {
    type Hasher = AHasher;

    /// Constructs a new [AHasher] with keys based on compile time generated constants** and the location
    /// of the this object in memory. This means that two different [BuildHasher]s will will generate
    /// [AHasher]s that will return different hashcodes, but [Hasher]s created from the same [BuildHasher]
    /// will generate the same hashes for the same input data.
    ///
    /// ** - only if the `compile-time-rng` feature is enabled.
    ///
    /// # Examples
    ///
    /// ```
    /// use ahash::{AHasher, RandomState};
    /// use std::hash::{Hasher, BuildHasher};
    ///
    /// let build_hasher = RandomState::new();
    /// let mut hasher_1 = build_hasher.build_hasher();
    /// let mut hasher_2 = build_hasher.build_hasher();
    ///
    /// hasher_1.write_u32(1234);
    /// hasher_2.write_u32(1234);
    ///
    /// assert_eq!(hasher_1.finish(), hasher_2.finish());
    ///
    /// let other_build_hasher = RandomState::new();
    /// let mut different_hasher = other_build_hasher.build_hasher();
    /// different_hasher.write_u32(1234);
    /// assert_ne!(different_hasher.finish(), hasher_1.finish());
    /// ```
    /// [Hasher]: std::hash::Hasher
    /// [BuildHasher]: std::hash::BuildHasher
    /// [HashMap]: std::collections::HashMap
    #[inline]
    fn build_hasher(&self) -> AHasher {
        AHasher::new_with_keys(self.k0, self.k1)
    }
}
<commit_msg>Fix RandomState::build_hasher documentation<commit_after>use crate::AHasher;
use core::hash::BuildHasher;
use core::sync::atomic::AtomicUsize;
use core::sync::atomic::Ordering;

#[cfg(feature = "compile-time-rng")]
use const_random::const_random;

///This constant come from Kunth's prng
pub(crate) const MULTIPLE: u64 = 6364136223846793005;

// Const random provides randomized starting key with no runtime cost.
#[cfg(feature = "compile-time-rng")]
static SEED: AtomicUsize = AtomicUsize::new(const_random!(u64));

#[cfg(not(feature = "compile-time-rng"))]
static SEED: AtomicUsize = AtomicUsize::new(MULTIPLE as usize);

/// Provides a [Hasher] factory. This is typically used (e.g. by [`HashMap`]) to create
/// [AHasher]s in order to hash the keys of the map. See `build_hasher` below.
///
/// [build_hasher]: ahash::
/// [Hasher]: std::hash::Hasher
/// [BuildHasher]: std::hash::BuildHasher
/// [HashMap]: std::collections::HashMap
#[derive(Clone)]
pub struct RandomState {
    pub(crate) k0: u64,
    pub(crate) k1: u64,
}

impl RandomState {
    #[inline]
    pub fn new() -> RandomState {
        //Using a self pointer. When running with ASLR this is a random value.
        let previous = SEED.load(Ordering::Relaxed) as u64;
        let stack_mem_loc = &previous as *const _ as u64;
        //This is similar to the update function in the fallback.
        //only one multiply is needed because memory locations are not under an attackers control.
        let current_seed = previous
            .wrapping_mul(MULTIPLE)
            .wrapping_add(stack_mem_loc)
            .rotate_left(31);
        SEED.store(current_seed as usize, Ordering::Relaxed);
        let (k0, k1) = scramble_keys(&SEED as *const _ as u64, current_seed);
        RandomState { k0, k1 }
    }
}

pub(crate) fn scramble_keys(k0: u64, k1: u64) -> (u64, u64) {
    //Scramble seeds (based on xoroshiro128+)
    //This is intentionally not similar the hash algorithm
    let result1 = k0.wrapping_add(k1);
    let k1 = k1 ^ k0;
    let k0 = k0.rotate_left(24) ^ k1 ^ (k1.wrapping_shl(16));
    let result2 = k0.wrapping_add(k1.rotate_left(37));
    (result2, result1)
}

impl Default for RandomState {
    #[inline]
    fn default() -> Self {
        Self::new()
    }
}

impl BuildHasher for RandomState {
    type Hasher = AHasher;

    /// Constructs a new [AHasher] with keys based on compile time generated constants** and the location
    /// this object was constructed at in memory. This means that two different [BuildHasher]s will will generate
    /// [AHasher]s that will return different hashcodes, but [Hasher]s created from the same [BuildHasher]
    /// will generate the same hashes for the same input data.
    ///
    /// ** - only if the `compile-time-rng` feature is enabled.
    ///
    /// # Examples
    ///
    /// ```
    /// use ahash::{AHasher, RandomState};
    /// use std::hash::{Hasher, BuildHasher};
    ///
    /// let build_hasher = RandomState::new();
    /// let mut hasher_1 = build_hasher.build_hasher();
    /// let mut hasher_2 = build_hasher.build_hasher();
    ///
    /// hasher_1.write_u32(1234);
    /// hasher_2.write_u32(1234);
    ///
    /// assert_eq!(hasher_1.finish(), hasher_2.finish());
    ///
    /// let other_build_hasher = RandomState::new();
    /// let mut different_hasher = other_build_hasher.build_hasher();
    /// different_hasher.write_u32(1234);
    /// assert_ne!(different_hasher.finish(), hasher_1.finish());
    /// ```
    /// [Hasher]: std::hash::Hasher
    /// [BuildHasher]: std::hash::BuildHasher
    /// [HashMap]: std::collections::HashMap
    #[inline]
    fn build_hasher(&self) -> AHasher {
        AHasher::new_with_keys(self.k0, self.k1)
    }
}
<|endoftext|>


<commit_before>#[cfg(doctest)]
use num_cpus as _;

#[cfg(test)]
compile_error!("Miri should not touch me");
<commit_msg>Use `compile_error!` instead of `use num_cpus`<commit_after>#[cfg(doctest)]
compile_error!("rustdoc should not touch me");

#[cfg(test)]
compile_error!("Miri should not touch me");
<|endoftext|>


<commit_before>use std::process::Command;
use std::env;

fn main() {
    let mut path = match env::var("WELD_HOME") {
        Ok(val) => val,
        Err(_) => "..".to_string(),
    };

    if path.chars().last().unwrap() != '/' {
        path = path + &"/";
    }

    let cpp_path = path + &"weld_rt/cpp";

    Command::new("make").arg("clean").arg("-C").arg(cpp_path.clone()).status().unwrap();
    Command::new("make").arg("-C").arg(cpp_path.clone()).status().unwrap();
    println!("{}", format!("cargo:rustc-link-search=native={}", cpp_path));

    let target = env::var("TARGET").unwrap();
    if target == "x86_64-apple-darwin" {
        let libs = vec!["z", "c++"];
        for lib in libs {
            println!("cargo:rustc-link-lib={}", lib);
        }
    }
}
<commit_msg>Add stdc++ for building weld_rt on linux (#91)<commit_after>use std::process::Command;
use std::env;

fn main() {
    let mut path = match env::var("WELD_HOME") {
        Ok(val) => val,
        Err(_) => "..".to_string(),
    };

    if path.chars().last().unwrap() != '/' {
        path = path + &"/";
    }

    let cpp_path = path + &"weld_rt/cpp";

    Command::new("make").arg("clean").arg("-C").arg(cpp_path.clone()).status().unwrap();
    Command::new("make").arg("-C").arg(cpp_path.clone()).status().unwrap();
    println!("{}", format!("cargo:rustc-link-search=native={}", cpp_path));

    let target = env::var("TARGET").unwrap();
    if target == "x86_64-apple-darwin" {
        let libs = vec!["z", "c++"];
        for lib in libs {
            println!("cargo:rustc-link-lib={}", lib);
        }
    }
    else if target == "x86_64-unknown-linux-gnu" {
        let libs = vec!["z", "stdc++"];
        for lib in libs {
            println!("cargo:rustc-link-lib={}", lib);
        }
    }
}
<|endoftext|>


<commit_before>///! Utilities for universally unique identifiers.
///!
///! The identifiers generated by this module:
///!
///! - are URL safe
///! - contain information on the type of object that they
///!   are identifying
///! - have an extremely low probability of collision
///!
///! Generated identifiers have a fixed length of 32 characters made up
///! of two parts separated by a hyphen:
///!
///! - 2 characters in the range `[a-z]` that identifying the "family" of
///!   identifiers, usually the type of object the identifier is for
///!   e.g. `fi` = file, `re` = request
///!
///! - 20 characters in the range `[0-9A-Za-z]` that are randomly generated
///!
///! For project identifiers (those starting with 'pr') only lowercase
///! letters are used for compatibility with Docker image naming rules.
///!
///! The total size of the generated ids is 23 bytes which allows it to fit
///! inside a [`SmartString`](https://lib.rs/crates/smartstring) for better
///! performance that a plain old `String`.
///!
///! See
///!  - https://segment.com/blog/a-brief-history-of-the-uuid/
///!  - https://zelark.github.io/nano-id-cc/
///!  - https://gist.github.com/fnky/76f533366f75cf75802c8052b577e2a5
use eyre::{bail, Result};
use nanoid::nanoid;
use regex::Regex;
use smartstring::{Compact, SmartString};

pub type Uuid = SmartString<Compact>;

/// The separator between the family and random parts of the identifier
///
/// A hyphen provides for better readability than a dot or colon when used
/// in pubsub topic strings and elsewhere.
const SEPARATOR: &str = "-";

/// The characters used in the random part of the identifier
const CHARACTERS: [char; 62] = [
    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',
    'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B',
    'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U',
    'V', 'W', 'X', 'Y', 'Z',
];

/// Create a family of UUIDs
///
/// ```
/// use uuids::uuid_family;
///
/// uuid_family!(MyId, "my");
/// let id = MyId::new();
/// ```
#[macro_export]
macro_rules! uuid_family {
    ($name:ident, $prefix:literal) => {
        #[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
        pub struct $name(uuids::Uuid);

        impl $name {
            pub fn new() -> Self {
                Self(uuids::generate($prefix))
            }
        }

        impl Default for $name {
            fn default() -> Self {
                Self::new()
            }
        }

        impl std::fmt::Display for $name {
            fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                write!(formatter, "{}", self.0.to_string())
            }
        }

        impl std::cmp::PartialEq for $name {
            fn eq(&self, other: &Self) -> bool {
                self.0 == other.0
            }
        }

        impl std::cmp::Eq for $name {}

        impl std::cmp::PartialOrd for $name {
            fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
                self.0.partial_cmp(&other.0)
            }
        }

        impl std::cmp::Ord for $name {
            fn cmp(&self, other: &Self) -> std::cmp::Ordering {
                self.0.cmp(&other.0)
            }
        }

        impl std::hash::Hash for $name {
            fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
                self.0.hash(state);
            }
        }

        impl std::ops::Deref for $name {
            type Target = uuids::Uuid;

            fn deref(&self) -> &Self::Target {
                &self.0
            }
        }

        impl std::convert::TryFrom<&str> for $name {
            type Error = uuids::InvalidUuid;

            fn try_from(id: &str) -> std::result::Result<Self, Self::Error> {
                match uuids::matches($prefix, id) {
                    true => Ok($name(id.into())),
                    false => Err(uuids::InvalidUuid {}),
                }
            }
        }
    };
}

// Generate a universally unique identifier
pub fn generate(prefix: &str) -> Uuid {
    let chars = nanoid!(20, &CHARACTERS);
    [prefix, SEPARATOR, &chars].concat().into()
}

// Generate a universally unique identifier with only lowercase letters and digits
pub fn generate_lower(prefix: &str) -> Uuid {
    let chars = nanoid!(20, &CHARACTERS[..36]);
    [prefix, SEPARATOR, &chars].concat().into()
}

// Test whether a string is an identifier for a particular prefix
pub fn matches(prefix: &str, id: &str) -> bool {
    let re = ["^", prefix, SEPARATOR, "[0-9a-zA-Z]{20}$"].concat();
    let re = Regex::new(&re).expect("Should be a valid regex");
    re.is_match(id)
}

// Assert that a `Uuid` is an identifer for a particular family
pub fn assert(family: &str, id: Uuid) -> Result<Uuid> {
    match matches(family, &id) {
        true => Ok(id),
        false => bail!(
            "Invalid UUID `{}`, family does not match `{}`",
            family.to_string(),
            id
        ),
    }
}

pub struct InvalidUuid {}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn normal() {
        let id = generate("no");
        assert_eq!(id.len(), 23);
        assert!(matches("no", &id));
        assert("no", id).unwrap();
    }

    #[test]
    fn lower() {
        let id = generate_lower("pr");
        assert_eq!(id.len(), 23);
        assert!(matches("pr", &id));
        assert("pr", id).unwrap();
    }
}
<commit_msg>chore(UUIDs): Fix up doc comments<commit_after>///! Utilities for universally unique identifiers.
///!
///! The identifiers generated by this module:
///!
///! - are URL safe
///! - contain information on the type of object that they
///!   are identifying
///! - have an extremely low probability of collision
///!
///! By default, generated identifiers have a fixed length of 23 characters made up
///! of two parts separated by a hyphen:
///!
///! - 2 characters in the range `[a-z]` that identifying the "family" of
///!   identifiers, usually the type of object the identifier is for
///!   e.g. `fi` = file, `re` = request
///!
///! - 20 characters in the range `[0-9A-Za-z]` that are randomly generated
///!
///! For project identifiers (those starting with 'pr') only lowercase
///! letters are used for compatibility with Docker image naming rules.
///!
///! The default size of 23 bytes allows ids to fit inside a [`SmartString`](https://lib.rs/crates/smartstring)
///! for better performance (particularly for `HashMap` lookups) than a plain old `String`.
///!
///! See
///!  - https://segment.com/blog/a-brief-history-of-the-uuid/
///!  - https://zelark.github.io/nano-id-cc/
///!  - https://gist.github.com/fnky/76f533366f75cf75802c8052b577e2a5
use eyre::{bail, Result};
use nanoid::nanoid;
use regex::Regex;
use smartstring::{Compact, SmartString};

pub type Uuid = SmartString<Compact>;

/// The separator between the family and random parts of the identifier
///
/// A hyphen provides for better readability than a dot or colon when used
/// in pubsub topic strings and elsewhere.
const SEPARATOR: &str = "-";

/// The characters used in the random part of the identifier
const CHARACTERS: [char; 62] = [
    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',
    'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B',
    'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U',
    'V', 'W', 'X', 'Y', 'Z',
];

/// Create a family of UUIDs
///
/// ```
/// use uuids::uuid_family;
///
/// uuid_family!(MyId, "my");
/// let id = MyId::new();
/// ```
#[macro_export]
macro_rules! uuid_family {
    ($name:ident, $prefix:literal) => {
        #[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
        pub struct $name(uuids::Uuid);

        impl $name {
            pub fn new() -> Self {
                Self(uuids::generate($prefix))
            }
        }

        impl Default for $name {
            fn default() -> Self {
                Self::new()
            }
        }

        impl std::fmt::Display for $name {
            fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                write!(formatter, "{}", self.0.to_string())
            }
        }

        impl std::cmp::PartialEq for $name {
            fn eq(&self, other: &Self) -> bool {
                self.0 == other.0
            }
        }

        impl std::cmp::Eq for $name {}

        impl std::cmp::PartialOrd for $name {
            fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
                self.0.partial_cmp(&other.0)
            }
        }

        impl std::cmp::Ord for $name {
            fn cmp(&self, other: &Self) -> std::cmp::Ordering {
                self.0.cmp(&other.0)
            }
        }

        impl std::hash::Hash for $name {
            fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
                self.0.hash(state);
            }
        }

        impl std::ops::Deref for $name {
            type Target = uuids::Uuid;

            fn deref(&self) -> &Self::Target {
                &self.0
            }
        }

        impl std::convert::TryFrom<&str> for $name {
            type Error = uuids::InvalidUuid;

            fn try_from(id: &str) -> std::result::Result<Self, Self::Error> {
                match uuids::matches($prefix, id) {
                    true => Ok($name(id.into())),
                    false => Err(uuids::InvalidUuid {}),
                }
            }
        }
    };
}

// Generate a universally unique identifier
pub fn generate(prefix: &str) -> Uuid {
    let chars = nanoid!(20, &CHARACTERS);
    [prefix, SEPARATOR, &chars].concat().into()
}

// Generate a universally unique identifier with only lowercase letters and digits
pub fn generate_lower(prefix: &str) -> Uuid {
    let chars = nanoid!(20, &CHARACTERS[..36]);
    [prefix, SEPARATOR, &chars].concat().into()
}

// Test whether a string is an identifier for a particular prefix
pub fn matches(prefix: &str, id: &str) -> bool {
    let re = ["^", prefix, SEPARATOR, "[0-9a-zA-Z]{20}$"].concat();
    let re = Regex::new(&re).expect("Should be a valid regex");
    re.is_match(id)
}

// Assert that a `Uuid` is an identifer for a particular family
pub fn assert(family: &str, id: Uuid) -> Result<Uuid> {
    match matches(family, &id) {
        true => Ok(id),
        false => bail!(
            "Invalid UUID `{}`, family does not match `{}`",
            family.to_string(),
            id
        ),
    }
}

pub struct InvalidUuid {}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn normal() {
        let id = generate("no");
        assert_eq!(id.len(), 23);
        assert!(matches("no", &id));
        assert("no", id).unwrap();
    }

    #[test]
    fn lower() {
        let id = generate_lower("pr");
        assert_eq!(id.len(), 23);
        assert!(matches("pr", &id));
        assert("pr", id).unwrap();
    }
}
<|endoftext|>


<commit_before>// Copyright (C) 2020 O.S. Systems Sofware LTDA
//
// SPDX-License-Identifier: Apache-2.0

use mockito::{mock, Mock};
use regex::Regex;
use serde_json::json;
use std::{env, net::TcpListener, path::PathBuf};

pub enum FakeServer {
    NoUpdate,
    HasUpdate(String),
    CheckRequirementsTest(String),
}

pub enum StopMessage {
    Custom(String),
    Polling(Polling),
}

pub enum Polling {
    Enable,
    Disable,
}

pub enum Server {
    Custom(String),
    Standard,
}

pub struct Settings {
    polling: bool,
    listen_socket: String,
    server_address: String,
    download_dir: Option<PathBuf>,
    config_file: Option<PathBuf>,
    timeout: Option<u64>,
    install_modes: Option<Vec<&'static str>>,
    state_change_callback: Option<&'static str>,
}

impl Default for Settings {
    fn default() -> Self {
        Settings {
            polling: false,
            listen_socket: format!(
                "localhost:{}",
                listen_available_port().expect("failed to bind a socket.")
            ),
            server_address: mockito::server_url(),
            download_dir: None,
            config_file: None,
            timeout: None,
            install_modes: None,
            state_change_callback: None,
        }
    }
}

impl Settings {
    pub fn init_server(self) -> (rexpect::session::PtySession, updatehub::tests::TestEnvironment) {
        let mut setup = updatehub::tests::TestEnvironment::build()
            .listen_socket(self.listen_socket)
            .server_address(self.server_address)
            .add_echo_binary("reboot");
        if let Some(s) = self.state_change_callback {
            setup = setup.state_change_callback(s.to_owned());
        }
        if let Some(l) = self.install_modes {
            setup = setup.supported_install_modes(l)
        }
        if !self.polling {
            setup = setup.disable_polling()
        }
        let mut setup = setup.finish();

        if let Some(download_dir) = self.download_dir {
            setup.settings.data.update.download_dir = download_dir;

            let content = toml::ser::to_string_pretty(&setup.settings.data.0)
                .expect("fail to convert the data to toml");
            std::fs::write(&setup.settings.stored_path, content)
                .expect("fail to write the content on settings file");
        }

        if let Some(config_file) = self.config_file {
            setup.settings.stored_path = config_file;
        }

        let cmd = format!(
            "{} server -v trace -c {}",
            cargo_bin("updatehub").to_string_lossy(),
            setup.settings.stored_path.to_string_lossy()
        );

        let handle = rexpect::spawn(&cmd, self.timeout).expect("fail to spawn server command");

        (handle, setup)
    }

    pub fn timeout(self, t: u64) -> Self {
        Settings { timeout: Some(t), ..self }
    }

    pub fn config_file(self, p: PathBuf) -> Self {
        Settings { config_file: Some(p), ..self }
    }

    pub fn download_dir(self, p: PathBuf) -> Self {
        Settings { download_dir: Some(p), ..self }
    }

    pub fn polling(self) -> Self {
        Settings { polling: true, ..self }
    }

    pub fn listen_socket(self, s: String) -> Self {
        Settings { listen_socket: s, ..self }
    }

    pub fn server_address(self, s: String) -> Self {
        Settings { server_address: s, ..self }
    }

    pub fn supported_install_modes(self, l: Vec<&'static str>) -> Self {
        Settings { install_modes: Some(l), ..self }
    }

    pub fn state_change_callback(self, s: &'static str) -> Self {
        Settings { state_change_callback: Some(s), ..self }
    }
}

pub fn get_output_server(
    handle: &mut rexpect::session::PtySession,
    stop_message: StopMessage,
) -> (String, String) {
    let stdout = handle
        .exp_regex(match stop_message {
            StopMessage::Custom(ref s) => s,
            StopMessage::Polling(Polling::Enable) => "\r\n.* TRCE delaying transition for:.*",
            StopMessage::Polling(Polling::Disable) => {
                "\r\n.* TRCE stopping transition until awoken.*"
            }
        })
        .expect("fail to match the required string")
        .0;

    rewrite_log_output(stdout)
}

pub fn run_client_probe(server: Server, listen_socket: &str) -> String {
    let cmd_string = format!(
        "{} client --listen-socket {} probe",
        cargo_bin("updatehub").to_string_lossy(),
        listen_socket
    );
    let cmd = match server {
        Server::Custom(server_address) => format!("{} --server {}", cmd_string, server_address),
        Server::Standard => cmd_string,
    };
    let mut handle = rexpect::spawn(&cmd, None).expect("fail to spawn probe command");
    handle.exp_eof().expect("fail to match the EOF for client")
}

pub fn run_client_log(listen_socket: &str) -> String {
    let cmd = format!(
        "{} client --listen-socket {} log",
        cargo_bin("updatehub").to_string_lossy(),
        listen_socket
    );
    let mut handle = rexpect::spawn(&cmd, None).expect("fail to spawn log command");
    let stdout = handle.exp_eof().expect("fail to match the EOF for client");
    rewrite_log_output(stdout).0
}

pub fn cargo_bin<S: AsRef<str>>(name: S) -> PathBuf {
    let mut target_dir = env::current_exe().expect("fail to get current binary name");

    target_dir.pop();
    if target_dir.ends_with("deps") {
        target_dir.pop();
    }

    target_dir.join(format!("{}{}", name.as_ref(), env::consts::EXE_SUFFIX))
}

pub fn create_mock_server(server: FakeServer) -> Vec<Mock> {
    use mockito::Matcher;

    let json_update = json!({
        "product": "0123456789",
        "version": "1.2",
        "supported-hardware": ["board"],
        "objects":
        [
            [
                {
                    "mode": "test",
                    "filename": "testfile",
                    "target": "/dev/device1",
                    "sha256sum": "23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4",
                    "size": 40960,
                    "force-check-requirements-fail": false
                }
            ],
            [
                {
                    "mode": "test",
                    "filename": "testfile",
                    "target": "/dev/device2",
                    "sha256sum": "23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4",
                    "size": 40960,
                    "force-check-requirements-fail": false
                }
            ]
        ]
    });

    let wrong_json_update = json!({
        "product": "0123456789",
        "version": "1.2",
        "supported-hardware": ["board"],
        "objects":
        [
            [
                {
                    "mode": "test",
                    "filename": "testfile",
                    "target": "/dev/device1",
                    "sha256sum": "23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4",
                    "size": 40960,
                    "force-check-requirements-fail": true
                }
            ],
            [
                {
                    "mode": "test",
                    "filename": "testfile",
                    "target": "/dev/device2",
                    "sha256sum": "23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4",
                    "size": 40960,
                    "force-check-requirements-fail": true
                }
            ]
        ]
    });

    let request_body = Matcher::Json(json!({
        "product-uid": "229ffd7e08721d716163fc81a2dbaf6c90d449f0a3b009b6a2defe8a0b0d7381",
        "version": "1.1",
        "hardware": "board",
        "device-identity": {
            "id1":"value1",
            "id2":"value2"
        },
        "device-attributes": {
            "attr1":"attrvalue1",
            "attr2":"attrvalue2"
        }
    }));

    match server {
        FakeServer::NoUpdate => vec![
            mock("POST", "/upgrades")
                .match_header("Content-Type", "application/json")
                .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
                .match_body(request_body)
                .expect_at_least(1)
                .with_status(404)
                .create(),
        ],
        FakeServer::HasUpdate(product_uid) => vec![
            mock("POST", "/upgrades")
                .match_header("Content-Type", "application/json")
                .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
                .match_body(request_body)
                .with_status(200)
                .with_header("UH-Signature", &openssl::base64::encode_block(b"some_signature"))
                .with_body(&json_update.to_string())
                .create(),
            mock(
                "GET",
                format!("/products/{}/packages/87effe73b80453f397cee4db3c3589a8630b220876dff8fb23447315037ff96d/objects/23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4", product_uid)
                    .as_str(),
            )
            .match_header("Content-Type", "application/json")
            .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
            .with_status(200)
                .with_body(std::iter::repeat(0xF).take(40960).collect::<Vec<_>>())
            .create(),
        ],
        FakeServer::CheckRequirementsTest(product_uid) => vec![
            mock("POST", "/upgrades")
                .match_header("Content-Type", "application/json")
                .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
                .match_body(request_body)
                .with_status(200)
                .with_header("UH-Signature", &openssl::base64::encode_block(b"some_signature"))
                .with_body(&wrong_json_update.to_string())
                .create(),
            mock(
                "GET",
                format!("/products/{}/packages/fb21b217cb83e8af368c773eb13bad0a94e1b0088c6bf561072decf3c1ae9df3/objects/23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4", product_uid)
                    .as_str(),
            )
            .match_header("Content-Type", "application/json")
            .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
            .with_status(200)
                .with_body(std::iter::repeat(0xF).take(40960).collect::<Vec<_>>())
            .create(),
        ],
    }
}

pub fn rewrite_log_output(s: String) -> (String, String) {
    let version_re = Regex::new(r"Agent .*").unwrap();
    let tmpfile_re = Regex::new(r#""/.*/.tmp.*""#).unwrap();
    let date_re = Regex::new(r"\b(?:Jan|...|Dec) (\d{2}) (\d{2}):(\d{2}):(\d{2}).(\d{3})").unwrap();
    let time_re = Regex::new(r#"(\d{5}) seconds"#).unwrap();
    let trce_re = Regex::new(r"<timestamp> TRCE.*").unwrap();
    let debg_re = Regex::new(r"<timestamp> DEBG.*").unwrap();
    let download_re = Regex::new(r"DEBG (\d{2})%").unwrap();

    let s = version_re.replace_all(&s, "Agent <version>");
    let s = tmpfile_re.replace_all(&s, r#""<file>""#);
    let s = date_re.replace_all(&s, "<timestamp>");
    let s = download_re.replace_all(&s, "DEBG <percentage>%");
    let s = time_re.replace_all(&s, r#"<time>"#);
    let s_trce = s.replace("\r\n", "\n").trim().to_owned();

    let s_info = trce_re.replace_all(&s_trce, "");
    let s_info = debg_re.replace_all(&s_info, "");
    let s_info = s_info
        .split('\n')
        .map(|s| s.trim())
        .filter(|s| !s.is_empty())
        .collect::<Vec<_>>()
        .join("\n");

    (s_trce, s_info)
}

pub fn remove_carriage_newline_characters(s: String) -> String {
    s.replace("\r\n", "\n")
}

fn listen_available_port() -> Option<u16> {
    (8080..65535).find(|port| TcpListener::bind(("localhost", *port)).is_ok())
}
<commit_msg>updatehub: tests: Fix commands used to run updatehub<commit_after>// Copyright (C) 2020 O.S. Systems Sofware LTDA
//
// SPDX-License-Identifier: Apache-2.0

use mockito::{mock, Mock};
use regex::Regex;
use serde_json::json;
use std::{env, net::TcpListener, path::PathBuf};

pub enum FakeServer {
    NoUpdate,
    HasUpdate(String),
    CheckRequirementsTest(String),
}

pub enum StopMessage {
    Custom(String),
    Polling(Polling),
}

pub enum Polling {
    Enable,
    Disable,
}

pub enum Server {
    Custom(String),
    Standard,
}

pub struct Settings {
    polling: bool,
    listen_socket: String,
    server_address: String,
    download_dir: Option<PathBuf>,
    config_file: Option<PathBuf>,
    timeout: Option<u64>,
    install_modes: Option<Vec<&'static str>>,
    state_change_callback: Option<&'static str>,
}

impl Default for Settings {
    fn default() -> Self {
        Settings {
            polling: false,
            listen_socket: format!(
                "localhost:{}",
                listen_available_port().expect("failed to bind a socket.")
            ),
            server_address: mockito::server_url(),
            download_dir: None,
            config_file: None,
            timeout: None,
            install_modes: None,
            state_change_callback: None,
        }
    }
}

impl Settings {
    pub fn init_server(self) -> (rexpect::session::PtySession, updatehub::tests::TestEnvironment) {
        let mut setup = updatehub::tests::TestEnvironment::build()
            .listen_socket(self.listen_socket)
            .server_address(self.server_address)
            .add_echo_binary("reboot");
        if let Some(s) = self.state_change_callback {
            setup = setup.state_change_callback(s.to_owned());
        }
        if let Some(l) = self.install_modes {
            setup = setup.supported_install_modes(l)
        }
        if !self.polling {
            setup = setup.disable_polling()
        }
        let mut setup = setup.finish();

        if let Some(download_dir) = self.download_dir {
            setup.settings.data.update.download_dir = download_dir;

            let content = toml::ser::to_string_pretty(&setup.settings.data.0)
                .expect("fail to convert the data to toml");
            std::fs::write(&setup.settings.stored_path, content)
                .expect("fail to write the content on settings file");
        }

        if let Some(config_file) = self.config_file {
            setup.settings.stored_path = config_file;
        }

        let cmd = format!(
            "{} daemon -v trace -c {}",
            cargo_bin("updatehub").to_string_lossy(),
            setup.settings.stored_path.to_string_lossy()
        );

        let handle = rexpect::spawn(&cmd, self.timeout).expect("fail to spawn server command");

        (handle, setup)
    }

    pub fn timeout(self, t: u64) -> Self {
        Settings { timeout: Some(t), ..self }
    }

    pub fn config_file(self, p: PathBuf) -> Self {
        Settings { config_file: Some(p), ..self }
    }

    pub fn download_dir(self, p: PathBuf) -> Self {
        Settings { download_dir: Some(p), ..self }
    }

    pub fn polling(self) -> Self {
        Settings { polling: true, ..self }
    }

    pub fn listen_socket(self, s: String) -> Self {
        Settings { listen_socket: s, ..self }
    }

    pub fn server_address(self, s: String) -> Self {
        Settings { server_address: s, ..self }
    }

    pub fn supported_install_modes(self, l: Vec<&'static str>) -> Self {
        Settings { install_modes: Some(l), ..self }
    }

    pub fn state_change_callback(self, s: &'static str) -> Self {
        Settings { state_change_callback: Some(s), ..self }
    }
}

pub fn get_output_server(
    handle: &mut rexpect::session::PtySession,
    stop_message: StopMessage,
) -> (String, String) {
    let stdout = handle
        .exp_regex(match stop_message {
            StopMessage::Custom(ref s) => s,
            StopMessage::Polling(Polling::Enable) => "\r\n.* TRCE delaying transition for:.*",
            StopMessage::Polling(Polling::Disable) => {
                "\r\n.* TRCE stopping transition until awoken.*"
            }
        })
        .expect("fail to match the required string")
        .0;

    rewrite_log_output(stdout)
}

pub fn run_client_probe(server: Server, daemon_address: &str) -> String {
    let cmd_string = format!(
        "{} client --daemon-address {} probe",
        cargo_bin("updatehub").to_string_lossy(),
        daemon_address
    );
    let cmd = match server {
        Server::Custom(server_address) => format!("{} --server {}", cmd_string, server_address),
        Server::Standard => cmd_string,
    };
    let mut handle = rexpect::spawn(&cmd, None).expect("fail to spawn probe command");
    handle.exp_eof().expect("fail to match the EOF for client")
}

pub fn run_client_log(daemon_address: &str) -> String {
    let cmd = format!(
        "{} client --daemon-address {} log",
        cargo_bin("updatehub").to_string_lossy(),
        daemon_address
    );
    let mut handle = rexpect::spawn(&cmd, None).expect("fail to spawn log command");
    let stdout = handle.exp_eof().expect("fail to match the EOF for client");
    rewrite_log_output(stdout).0
}

pub fn cargo_bin<S: AsRef<str>>(name: S) -> PathBuf {
    let mut target_dir = env::current_exe().expect("fail to get current binary name");

    target_dir.pop();
    if target_dir.ends_with("deps") {
        target_dir.pop();
    }

    target_dir.join(format!("{}{}", name.as_ref(), env::consts::EXE_SUFFIX))
}

pub fn create_mock_server(server: FakeServer) -> Vec<Mock> {
    use mockito::Matcher;

    let json_update = json!({
        "product": "0123456789",
        "version": "1.2",
        "supported-hardware": ["board"],
        "objects":
        [
            [
                {
                    "mode": "test",
                    "filename": "testfile",
                    "target": "/dev/device1",
                    "sha256sum": "23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4",
                    "size": 40960,
                    "force-check-requirements-fail": false
                }
            ],
            [
                {
                    "mode": "test",
                    "filename": "testfile",
                    "target": "/dev/device2",
                    "sha256sum": "23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4",
                    "size": 40960,
                    "force-check-requirements-fail": false
                }
            ]
        ]
    });

    let wrong_json_update = json!({
        "product": "0123456789",
        "version": "1.2",
        "supported-hardware": ["board"],
        "objects":
        [
            [
                {
                    "mode": "test",
                    "filename": "testfile",
                    "target": "/dev/device1",
                    "sha256sum": "23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4",
                    "size": 40960,
                    "force-check-requirements-fail": true
                }
            ],
            [
                {
                    "mode": "test",
                    "filename": "testfile",
                    "target": "/dev/device2",
                    "sha256sum": "23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4",
                    "size": 40960,
                    "force-check-requirements-fail": true
                }
            ]
        ]
    });

    let request_body = Matcher::Json(json!({
        "product-uid": "229ffd7e08721d716163fc81a2dbaf6c90d449f0a3b009b6a2defe8a0b0d7381",
        "version": "1.1",
        "hardware": "board",
        "device-identity": {
            "id1":"value1",
            "id2":"value2"
        },
        "device-attributes": {
            "attr1":"attrvalue1",
            "attr2":"attrvalue2"
        }
    }));

    match server {
        FakeServer::NoUpdate => vec![
            mock("POST", "/upgrades")
                .match_header("Content-Type", "application/json")
                .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
                .match_body(request_body)
                .expect_at_least(1)
                .with_status(404)
                .create(),
        ],
        FakeServer::HasUpdate(product_uid) => vec![
            mock("POST", "/upgrades")
                .match_header("Content-Type", "application/json")
                .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
                .match_body(request_body)
                .with_status(200)
                .with_header("UH-Signature", &openssl::base64::encode_block(b"some_signature"))
                .with_body(&json_update.to_string())
                .create(),
            mock(
                "GET",
                format!("/products/{}/packages/87effe73b80453f397cee4db3c3589a8630b220876dff8fb23447315037ff96d/objects/23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4", product_uid)
                    .as_str(),
            )
            .match_header("Content-Type", "application/json")
            .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
            .with_status(200)
                .with_body(std::iter::repeat(0xF).take(40960).collect::<Vec<_>>())
            .create(),
        ],
        FakeServer::CheckRequirementsTest(product_uid) => vec![
            mock("POST", "/upgrades")
                .match_header("Content-Type", "application/json")
                .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
                .match_body(request_body)
                .with_status(200)
                .with_header("UH-Signature", &openssl::base64::encode_block(b"some_signature"))
                .with_body(&wrong_json_update.to_string())
                .create(),
            mock(
                "GET",
                format!("/products/{}/packages/fb21b217cb83e8af368c773eb13bad0a94e1b0088c6bf561072decf3c1ae9df3/objects/23c3c412177bd37b9b61bf4738b18dc1fe003811c2583a14d2d9952d8b6a75b4", product_uid)
                    .as_str(),
            )
            .match_header("Content-Type", "application/json")
            .match_header("Api-Content-Type", "application/vnd.updatehub-v1+json")
            .with_status(200)
                .with_body(std::iter::repeat(0xF).take(40960).collect::<Vec<_>>())
            .create(),
        ],
    }
}

pub fn rewrite_log_output(s: String) -> (String, String) {
    let version_re = Regex::new(r"Agent .*").unwrap();
    let tmpfile_re = Regex::new(r#""/.*/.tmp.*""#).unwrap();
    let date_re = Regex::new(r"\b(?:Jan|...|Dec) (\d{2}) (\d{2}):(\d{2}):(\d{2}).(\d{3})").unwrap();
    let time_re = Regex::new(r#"(\d{5}) seconds"#).unwrap();
    let trce_re = Regex::new(r"<timestamp> TRCE.*").unwrap();
    let debg_re = Regex::new(r"<timestamp> DEBG.*").unwrap();
    let download_re = Regex::new(r"DEBG (\d{2})%").unwrap();

    let s = version_re.replace_all(&s, "Agent <version>");
    let s = tmpfile_re.replace_all(&s, r#""<file>""#);
    let s = date_re.replace_all(&s, "<timestamp>");
    let s = download_re.replace_all(&s, "DEBG <percentage>%");
    let s = time_re.replace_all(&s, r#"<time>"#);
    let s_trce = s.replace("\r\n", "\n").trim().to_owned();

    let s_info = trce_re.replace_all(&s_trce, "");
    let s_info = debg_re.replace_all(&s_info, "");
    let s_info = s_info
        .split('\n')
        .map(|s| s.trim())
        .filter(|s| !s.is_empty())
        .collect::<Vec<_>>()
        .join("\n");

    (s_trce, s_info)
}

pub fn remove_carriage_newline_characters(s: String) -> String {
    s.replace("\r\n", "\n")
}

fn listen_available_port() -> Option<u16> {
    (8080..65535).find(|port| TcpListener::bind(("localhost", *port)).is_ok())
}
<|endoftext|>


<commit_before>use linefeed::{Reader, Terminal};

use utils::environment::EnvironmentUtils;
use utils::file::{read_lines_from_file, write_file};

const HISTORY_SIZE: usize = 100;
const SECRET_DATA: [&str; 2] = ["seed", "key"];

pub fn load<T>(reader: &mut Reader<T>) -> Result<(), String> where T: Terminal {
    reader.set_history_size(HISTORY_SIZE);

    let path = EnvironmentUtils::history_file_path();

    for line in read_lines_from_file(path)? {
        if let Ok(line) = line {
            reader.add_history(line)
        }
    }
    Ok(())
}

pub fn persist<T>(reader: &Reader<T>) -> Result<(), String> where T: Terminal {
    let content =
        reader
            .history()
            .filter(|record|
                !SECRET_DATA.iter().any(|secret_word| record.contains(&format!("{}=", secret_word)))
            )
            .collect::<std::collections::HashSet<&str>>()
            .into_iter()
            .collect::<Vec<&str>>()
            .join("\n");

    let path = EnvironmentUtils::history_file_path();
    write_file(&path, &content)
}<commit_msg>IS-1307 Corrected secret data check<commit_after>use linefeed::{Reader, Terminal};

use utils::environment::EnvironmentUtils;
use utils::file::{read_lines_from_file, write_file};

const HISTORY_SIZE: usize = 100;
const SECRET_DATA: [&str; 2] = ["seed=", "key="];

pub fn load<T>(reader: &mut Reader<T>) -> Result<(), String> where T: Terminal {
    reader.set_history_size(HISTORY_SIZE);

    let path = EnvironmentUtils::history_file_path();

    for line in read_lines_from_file(path)? {
        if let Ok(line) = line {
            reader.add_history(line)
        }
    }
    Ok(())
}

pub fn persist<T>(reader: &Reader<T>) -> Result<(), String> where T: Terminal {
    let content =
        reader
            .history()
            .filter(|record|
                !SECRET_DATA.iter().any(|secret_word| record.contains(secret_word))
            )
            .collect::<std::collections::HashSet<&str>>()
            .into_iter()
            .collect::<Vec<&str>>()
            .join("\n");

    let path = EnvironmentUtils::history_file_path();
    write_file(&path, &content)
}<|endoftext|>