<commit_before>//! Test against the `sx` program, and itself
extern crate tempfile;
extern crate rand;
extern crate xmodem;

use std::process::{Command, Stdio, ChildStdin, ChildStdout};
use std::io::{self, Read, Write, Seek, ErrorKind};
use tempfile::NamedTempFile;
use rand::{Rng, thread_rng};
use xmodem::{Xmodem,Checksum};
use std::sync::mpsc::{channel,Sender,Receiver};

struct ChildStdInOut {
    stdin: ChildStdin,
    stdout: ChildStdout,
}

impl Read for ChildStdInOut {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        self.stdout.read(buf)
    }
}

impl Write for ChildStdInOut {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        self.stdin.write(buf)
    }

    fn flush(&mut self) -> io::Result<()> {
        self.stdin.flush()
    }
}

struct BidirectionalPipe {
    pin : Receiver<u8>,
    pout : Sender<u8>,
}

impl Read for BidirectionalPipe {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        for idx in 0..buf.len() {
            buf[idx] = match self.pin.recv() {
                Ok(v) => v,
                Err(e) => return Err(std::io::Error::new(ErrorKind::BrokenPipe,e)),
            }
        }
        Ok(buf.len())
    }
}

impl Write for BidirectionalPipe {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        for v in buf { self.pout.send(*v).unwrap(); }
        Ok(buf.len())
    }

    fn flush(&mut self) -> io::Result<()> {
        Ok(())
    }
}

fn loopback() -> (BidirectionalPipe, BidirectionalPipe) {
    let (s1,r1) = channel();
    let (s2,r2) = channel();
    (BidirectionalPipe{ pin : r1, pout : s2 }, BidirectionalPipe{ pin : r2, pout : s1 })
}

#[cfg(test)]
fn xmodem_recv(checksum_mode:Checksum) {
    let data_len = 2000;
    let mut data = vec![0; data_len];
    thread_rng().fill_bytes(&mut data);

    let mut send_file = NamedTempFile::new().unwrap();
    send_file.write_all(&data).unwrap();

    let send = Command::new("sb")
        .arg("--xmodem")
        .arg(send_file.path())
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::null())
        .spawn().unwrap();

    let tx_stream = send.stdin.unwrap();
    let rx_stream = send.stdout.unwrap();
    let mut serial_dev = ChildStdInOut { stdin: tx_stream, stdout: rx_stream };

    let mut xmodem = Xmodem::new();
    let mut recv_data = Vec::new();
    xmodem.recv(&mut serial_dev,&mut recv_data, checksum_mode).unwrap();

    let mut sent_data = Vec::new();
    send_file.seek(std::io::SeekFrom::Start(0)).unwrap();
    send_file.read_to_end(&mut sent_data).unwrap();
    let mut padded_data = sent_data.clone();
    for _ in 0..(128 - sent_data.len() % 128) {
         padded_data.push(0x1a);
    }
    assert_eq!(padded_data, recv_data);
}

#[cfg(test)]
fn xmodem_loopback(checksum_mode:Checksum) {
    let data_len=51200;
    let mut data_out = vec![0; data_len];
    thread_rng().fill_bytes(&mut data_out);
    let mut data_in = vec![0; data_len];
    let (mut p1, mut p2) = loopback();
    let handle = std::thread::spawn(move || {
        let mut xmodem = Xmodem::new();
        xmodem.send(&mut p1, &mut &data_out[..]).unwrap();
    });
    let handle2 = std::thread::spawn(move || {
        let mut xmodem = Xmodem::new();
        xmodem.recv(&mut p2, &mut data_in, checksum_mode).unwrap();
    });

    handle.join().unwrap();
    handle2.join().unwrap();
}

#[test]
fn xmodem_loopback_standard() {
    xmodem_loopback(Checksum::Standard);
}

#[test]
fn xmodem_recv_standard() {
    xmodem_recv(Checksum::Standard);
}

#[test]
fn xmodem_recv_crc() {
    xmodem_recv(Checksum::CRC16);
}

#[test]
fn xmodem_send_standard() {
    let data_len = 2000;
    let mut data = vec![0; data_len];
    thread_rng().fill_bytes(&mut data);

    let mut recv_file = NamedTempFile::new().unwrap();
    let recv = Command::new("rb")
                       .arg("--xmodem")
                       .arg(recv_file.path())
                       .stdin(Stdio::piped())
                       .stdout(Stdio::piped())
                       .stderr(Stdio::null())
                       .spawn().unwrap();

    let tx_stream = recv.stdin.unwrap();
    let rx_stream = recv.stdout.unwrap();
    let mut serial_dev = ChildStdInOut { stdin: tx_stream, stdout: rx_stream };

    let mut xmodem = Xmodem::new();
    xmodem.send(&mut serial_dev, &mut &data[..]).unwrap();

    let mut received_data = Vec::new();
    recv_file.read_to_end(&mut received_data).unwrap();
    let mut padded_data = data.clone();
    for _ in 0..(128 - data.len() % 128) {
        padded_data.push(0x1a);
    }
    assert_eq!(received_data, padded_data);
}

#[test]
fn xmodem_send_crc() {
    let data_len = 2000;
    let mut data = vec![0; data_len];
    thread_rng().fill_bytes(&mut data);

    let mut recv_file = NamedTempFile::new().unwrap();
    let recv = Command::new("rb")
                       .arg("--xmodem")
                       .arg("--with-crc")
                       .arg(recv_file.path())
                       .stdin(Stdio::piped())
                       .stdout(Stdio::piped())
                       .stderr(Stdio::null())
                       .spawn().unwrap();

    let tx_stream = recv.stdin.unwrap();
    let rx_stream = recv.stdout.unwrap();
    let mut serial_dev = ChildStdInOut { stdin: tx_stream, stdout: rx_stream };

    let mut xmodem = Xmodem::new();
    xmodem.send(&mut serial_dev, &mut &data[..]).unwrap();

    let mut received_data = Vec::new();
    recv_file.read_to_end(&mut received_data).unwrap();
    let mut padded_data = data.clone();
    for _ in 0..(128 - data.len() % 128) {
        padded_data.push(0x1a);
    }
    assert_eq!(received_data, padded_data);
}
<commit_msg>wrapped up testing for loopback (added long, crc modes)<commit_after>//! Test against the `sx` program, and itself
extern crate tempfile;
extern crate rand;
extern crate xmodem;

use std::process::{Command, Stdio, ChildStdin, ChildStdout};
use std::io::{self, Read, Write, Seek, ErrorKind};
use tempfile::NamedTempFile;
use rand::{Rng, thread_rng};
use xmodem::{Xmodem,Checksum};
use std::sync::mpsc::{channel,Sender,Receiver};

struct ChildStdInOut {
    stdin: ChildStdin,
    stdout: ChildStdout,
}

impl Read for ChildStdInOut {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        self.stdout.read(buf)
    }
}

impl Write for ChildStdInOut {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        self.stdin.write(buf)
    }

    fn flush(&mut self) -> io::Result<()> {
        self.stdin.flush()
    }
}

struct BidirectionalPipe {
    pin : Receiver<u8>,
    pout : Sender<u8>,
}

impl Read for BidirectionalPipe {
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        for idx in 0..buf.len() {
            buf[idx] = match self.pin.recv() {
                Ok(v) => v,
                Err(e) => return Err(std::io::Error::new(ErrorKind::BrokenPipe,e)),
            }
        }
        Ok(buf.len())
    }
}

impl Write for BidirectionalPipe {
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        for v in buf { self.pout.send(*v).unwrap(); }
        Ok(buf.len())
    }

    fn flush(&mut self) -> io::Result<()> {
        Ok(())
    }
}

fn loopback() -> (BidirectionalPipe, BidirectionalPipe) {
    let (s1,r1) = channel();
    let (s2,r2) = channel();
    (BidirectionalPipe{ pin : r1, pout : s2 }, BidirectionalPipe{ pin : r2, pout : s1 })
}

#[cfg(test)]
fn xmodem_recv(checksum_mode:Checksum) {
    let data_len = 2000;
    let mut data = vec![0; data_len];
    thread_rng().fill_bytes(&mut data);

    let mut send_file = NamedTempFile::new().unwrap();
    send_file.write_all(&data).unwrap();

    let send = Command::new("sb")
        .arg("--xmodem")
        .arg(send_file.path())
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::null())
        .spawn().unwrap();

    let tx_stream = send.stdin.unwrap();
    let rx_stream = send.stdout.unwrap();
    let mut serial_dev = ChildStdInOut { stdin: tx_stream, stdout: rx_stream };

    let mut xmodem = Xmodem::new();
    let mut recv_data = Vec::new();
    xmodem.recv(&mut serial_dev,&mut recv_data, checksum_mode).unwrap();

    let mut sent_data = Vec::new();
    send_file.seek(std::io::SeekFrom::Start(0)).unwrap();
    send_file.read_to_end(&mut sent_data).unwrap();
    let mut padded_data = sent_data.clone();
    for _ in 0..(128 - sent_data.len() % 128) {
         padded_data.push(0x1a);
    }
    assert_eq!(padded_data, recv_data);
}

#[cfg(test)]
fn xmodem_loopback(checksum_mode:Checksum, data_len : usize) {
    let mut data_out = vec![0; data_len];
    // We don't really need the rng here
    for idx in 0..data_len { data_out[idx] = ((idx+7) * 13) as u8; }
    let (mut p1, mut p2) = loopback();
    let handle = std::thread::spawn(move || {
        let mut xmodem = Xmodem::new();
        xmodem.send(&mut p1, &mut &data_out[..]).unwrap();
        data_out
    });
    let handle2 = std::thread::spawn(move || {
        let mut xmodem = Xmodem::new();
        let mut data_in= vec![0; 0];
        xmodem.recv(&mut p2, &mut data_in, checksum_mode).unwrap();
        data_in
    });

    let mut dato = handle.join().unwrap();
    // Pad output data to multiple of 128 for comparison
    for _ in 0..(128 - data_len % 128) { dato.push(0x1a); }
    let dati = handle2.join().unwrap();
    assert_eq!(dato.len(),dati.len());
    assert_eq!(dato,dati);
}

#[test]
fn xmodem_loopback_standard() {
    xmodem_loopback(Checksum::Standard,2000);
}

#[test]
fn xmodem_loopback_crc() {
    xmodem_loopback(Checksum::CRC16,2000);
}

#[test]
fn xmodem_loopback_long_crc() {
    // make sure we wrap block counter
    xmodem_loopback(Checksum::CRC16,50000);
}

#[test]
fn xmodem_recv_standard() {
    xmodem_recv(Checksum::Standard);
}

#[test]
fn xmodem_recv_crc() {
    xmodem_recv(Checksum::CRC16);
}

#[test]
fn xmodem_send_standard() {
    let data_len = 2000;
    let mut data = vec![0; data_len];
    thread_rng().fill_bytes(&mut data);

    let mut recv_file = NamedTempFile::new().unwrap();
    let recv = Command::new("rb")
                       .arg("--xmodem")
                       .arg(recv_file.path())
                       .stdin(Stdio::piped())
                       .stdout(Stdio::piped())
                       .stderr(Stdio::null())
                       .spawn().unwrap();

    let tx_stream = recv.stdin.unwrap();
    let rx_stream = recv.stdout.unwrap();
    let mut serial_dev = ChildStdInOut { stdin: tx_stream, stdout: rx_stream };

    let mut xmodem = Xmodem::new();
    xmodem.send(&mut serial_dev, &mut &data[..]).unwrap();

    let mut received_data = Vec::new();
    recv_file.read_to_end(&mut received_data).unwrap();
    let mut padded_data = data.clone();
    for _ in 0..(128 - data.len() % 128) {
        padded_data.push(0x1a);
    }
    assert_eq!(received_data, padded_data);
}

#[test]
fn xmodem_send_crc() {
    let data_len = 2000;
    let mut data = vec![0; data_len];
    thread_rng().fill_bytes(&mut data);

    let mut recv_file = NamedTempFile::new().unwrap();
    let recv = Command::new("rb")
                       .arg("--xmodem")
                       .arg("--with-crc")
                       .arg(recv_file.path())
                       .stdin(Stdio::piped())
                       .stdout(Stdio::piped())
                       .stderr(Stdio::null())
                       .spawn().unwrap();

    let tx_stream = recv.stdin.unwrap();
    let rx_stream = recv.stdout.unwrap();
    let mut serial_dev = ChildStdInOut { stdin: tx_stream, stdout: rx_stream };

    let mut xmodem = Xmodem::new();
    xmodem.send(&mut serial_dev, &mut &data[..]).unwrap();

    let mut received_data = Vec::new();
    recv_file.read_to_end(&mut received_data).unwrap();
    let mut padded_data = data.clone();
    for _ in 0..(128 - data.len() % 128) {
        padded_data.push(0x1a);
    }
    assert_eq!(received_data, padded_data);
}
<|endoftext|>


<commit_before>//! [PUT /_matrix/client/r0/rooms/{roomId}/send/{eventType}/{txnId}](https://matrix.org/docs/spec/client_server/r0.6.1#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid)

use ruma_api::ruma_api;
use ruma_events::{AnyMessageEventContent, EventContent as _};
use ruma_identifiers::{EventId, RoomId};
use ruma_serde::Raw;

ruma_api! {
    metadata: {
        description: "Send a message event to a room.",
        method: PUT,
        name: "create_message_event",
        path: "/_matrix/client/r0/rooms/:room_id/send/:event_type/:txn_id",
        rate_limited: false,
        authentication: AccessToken,
    }

    request: {
        /// The room to send the event to.
        #[ruma_api(path)]
        pub room_id: &'a RoomId,

        /// The type of event to send.
        #[ruma_api(path)]
        pub event_type: &'a str,

        /// The transaction ID for this event.
        ///
        /// Clients should generate an ID unique across requests with the
        /// same access token; it will be used by the server to ensure
        /// idempotency of requests.
        #[ruma_api(path)]
        pub txn_id: &'a str,

        /// The event content to send.
        #[ruma_api(body)]
        pub body: Raw<AnyMessageEventContent>,
    }

    response: {
        /// A unique identifier for the event.
        pub event_id: EventId,
    }

    error: crate::Error
}

impl<'a> Request<'a> {
    /// Creates a new `Request` with the given room id, transaction id and event content.
    pub fn new(room_id: &'a RoomId, txn_id: &'a str, content: &'a AnyMessageEventContent) -> Self {
        Self { room_id, txn_id, event_type: content.event_type(), body: content.into() }
    }

    /// Creates a new `Request` with the given room id, transaction id, event type and raw event
    /// content.
    pub fn new_raw(
        room_id: &'a RoomId,
        txn_id: &'a str,
        event_type: &'a str,
        body: Raw<AnyMessageEventContent>,
    ) -> Self {
        Self { room_id, txn_id, event_type, body }
    }
}

impl Response {
    /// Creates a new `Response` with the given event id.
    pub fn new(event_id: EventId) -> Self {
        Self { event_id }
    }
}
<commit_msg>client-api: Fix clippy lint<commit_after>//! [PUT /_matrix/client/r0/rooms/{roomId}/send/{eventType}/{txnId}](https://matrix.org/docs/spec/client_server/r0.6.1#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid)

use ruma_api::ruma_api;
use ruma_events::{AnyMessageEventContent, EventContent as _};
use ruma_identifiers::{EventId, RoomId};
use ruma_serde::Raw;

ruma_api! {
    metadata: {
        description: "Send a message event to a room.",
        method: PUT,
        name: "create_message_event",
        path: "/_matrix/client/r0/rooms/:room_id/send/:event_type/:txn_id",
        rate_limited: false,
        authentication: AccessToken,
    }

    request: {
        /// The room to send the event to.
        #[ruma_api(path)]
        pub room_id: &'a RoomId,

        /// The type of event to send.
        #[ruma_api(path)]
        pub event_type: &'a str,

        /// The transaction ID for this event.
        ///
        /// Clients should generate an ID unique across requests with the
        /// same access token; it will be used by the server to ensure
        /// idempotency of requests.
        #[ruma_api(path)]
        pub txn_id: &'a str,

        /// The event content to send.
        #[ruma_api(body)]
        pub body: Raw<AnyMessageEventContent>,
    }

    response: {
        /// A unique identifier for the event.
        pub event_id: EventId,
    }

    error: crate::Error
}

impl<'a> Request<'a> {
    /// Creates a new `Request` with the given room id, transaction id and event content.
    pub fn new(room_id: &'a RoomId, txn_id: &'a str, content: &'a AnyMessageEventContent) -> Self {
        Self { room_id, txn_id, event_type: content.event_type(), body: content.into() }
    }

    /// Creates a new `Request` with the given room id, transaction id, event type and raw event
    /// content.
    pub fn new_raw(
        room_id: &'a RoomId,
        txn_id: &'a str,
        event_type: &'a str,
        body: Raw<AnyMessageEventContent>,
    ) -> Self {
        Self { room_id, event_type, txn_id, body }
    }
}

impl Response {
    /// Creates a new `Response` with the given event id.
    pub fn new(event_id: EventId) -> Self {
        Self { event_id }
    }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::{
    cmp,
    fs::OpenOptions,
    panic,
    path::{Path, PathBuf},
};

use either::Either;
use serde_json::{from_reader, Value};
use uuid::Uuid;

use devicemapper::{
    devnode_to_devno, Bytes, Device, DmDevice, DmName, LinearDev, LinearDevTargetParams,
    LinearTargetParams, Sectors, TargetLine, IEC,
};

use crate::engine::strat_engine::{
    device::blkdev_size,
    dm::get_dm,
    tests::{logger::init_logger, util::clean_up},
    writing::wipe_sectors,
};

pub struct RealTestDev {
    dev: Either<PathBuf, LinearDev>,
}

impl RealTestDev {
    /// Construct a new test device for the given path.
    /// Wipe initial MiB to clear metadata.
    pub fn new(dev: Either<PathBuf, LinearDev>) -> RealTestDev {
        let test_dev = RealTestDev { dev };
        wipe_sectors(
            test_dev.as_path(),
            Sectors(0),
            Bytes::from(IEC::Mi).sectors(),
        )
        .unwrap();
        test_dev
    }

    /// Get the device node of the device.
    fn as_path(&self) -> PathBuf {
        self.dev.as_ref().either(|p| p.clone(), |l| l.devnode())
    }

    /// Teardown a real test dev
    fn teardown(self) {
        wipe_sectors(&self.as_path(), Sectors(0), Bytes::from(IEC::Mi).sectors()).unwrap();
        if let Some(mut ld) = self.dev.right() {
            ld.teardown(get_dm()).unwrap();
        }
    }
}

/// Shorthand for specifying devices to use for tests
/// For the minimum size, if None is given, the default 1 GiB is chosen.
pub enum DeviceLimits {
    /// Use exactly the number of devices specified
    /// The second argument is the minimum size for all devices.
    /// The third argument is the maximum size for all devices.
    Exactly(usize, Option<Sectors>, Option<Sectors>),
    /// Use at least the number of devices specified, but if there are more
    /// devices available, also use the maximum number of devices.
    /// The second argument is the minimum size for all devices.
    /// The third argument is the maximum size for all devices.
    AtLeast(usize, Option<Sectors>, Option<Sectors>),
    /// Use exactly the number of devices specified in the first argument,
    /// and the minimum of the number of devices specified and the number
    /// of devices available in the second argument.
    /// The third argument is the minimum size for all devices.
    /// The fourth argument is the maximum size for all devices.
    #[allow(dead_code)]
    Range(usize, usize, Option<Sectors>, Option<Sectors>),
}

/// Get a list of lists of devices to use for tests.
/// May return an empty list if the request is not satisfiable.
#[allow(clippy::type_complexity)]
fn get_device_runs<'a>(
    limits: &DeviceLimits,
    dev_sizes: &[(&'a Path, Sectors)],
) -> Vec<Vec<(&'a Path, Option<(Sectors, Sectors)>)>> {
    let (lower, maybe_upper, min_size, max_size) = match limits {
        DeviceLimits::Exactly(num, min_size, max_size) => (*num, Some(*num), *min_size, *max_size),
        DeviceLimits::AtLeast(num, min_size, max_size) => (*num, None, *min_size, *max_size),
        DeviceLimits::Range(lower, upper, min_size, max_size) => {
            assert!(lower < upper);
            (*lower, Some(*upper), *min_size, *max_size)
        }
    };

    let min_size = min_size.unwrap_or_else(|| Bytes::from(IEC::Gi).sectors());

    assert!(max_size.is_none() || Some(min_size) <= max_size);

    // Retain only devices that are larger than min_size
    let (matches, _): (Vec<(&Path, Sectors)>, Vec<(&Path, Sectors)>) =
        dev_sizes.iter().partition(|&&(_, s)| s >= min_size);

    // If there is not a sufficient number of devices large enough to match
    // the lower bound, return an empty vec. TODO: It would be possible to try
    // harder, by merging several devices into a single linear device. If this
    // turns out to be necessary, this is the correct place to do it.
    if lower > matches.len() {
        return vec![];
    }

    // Retain only devices that are less than max_size
    let (mut matches, mut too_large): (Vec<(&Path, Sectors)>, Vec<(&Path, Sectors)>) =
        if max_size.is_none() {
            (matches, vec![])
        } else {
            let max_size = max_size.expect("!max_size.is_none()");
            matches.iter().partition(|&&(_, s)| s <= max_size)
        };

    let avail = matches.len();
    let needed = maybe_upper.unwrap_or(lower);
    let must_generate = if avail > needed { 0 } else { needed - avail };
    let avail_specs = {
        let mut avail_specs = vec![];
        while avail_specs.len() < must_generate && (!too_large.is_empty() || !matches.is_empty()) {
            let (path, size) = if too_large.is_empty() {
                matches.pop().expect("!matches.is_empty()")
            } else {
                too_large.pop().expect("!too_large.is_empty()")
            };
            let mut new_pairs = vec![];
            for i in 0..size / min_size {
                new_pairs.push((path, Some((i * min_size, min_size))))
            }
            avail_specs.extend(new_pairs);
        }
        avail_specs.extend(matches.iter().map(|&(p, _)| (p, None)));
        avail_specs
    };

    let avail = avail_specs.len();

    // If there is not a sufficient number of devices small enough to match
    // the lower bound, return an empty vec.
    if lower > avail {
        return vec![];
    }

    let mut device_lists = vec![];

    device_lists.push(avail_specs.iter().take(lower).cloned().collect::<Vec<_>>());

    if lower != avail {
        match maybe_upper {
            None => device_lists.push(avail_specs.iter().take(avail).cloned().collect::<Vec<_>>()),
            Some(upper) => {
                if lower < upper {
                    device_lists.push(
                        avail_specs
                            .iter()
                            .take(cmp::min(upper, avail))
                            .cloned()
                            .collect::<Vec<_>>(),
                    )
                }
            }
        }
    }

    device_lists
}

/// Make a new LinearDev according to specs
fn make_linear_test_dev(devnode: &Path, start: Sectors, length: Sectors) -> LinearDev {
    let params = LinearTargetParams::new(
        Device::from(devnode_to_devno(devnode).unwrap().unwrap()),
        start,
    );
    let table = vec![TargetLine::new(
        Sectors(0),
        length,
        LinearDevTargetParams::Linear(params),
    )];
    LinearDev::setup(
        get_dm(),
        DmName::new(&format!("stratis_test_{}", Uuid::new_v4())).expect("valid format"),
        None,
        table,
    )
    .unwrap()
}

/// Run test on real devices, using given constraints. Constraints may result
/// in multiple invocations of the test, with differing numbers of block
/// devices.
pub fn test_with_spec<F>(limits: &DeviceLimits, test: F)
where
    F: Fn(&[&Path]) + panic::RefUnwindSafe,
{
    let file = OpenOptions::new()
        .read(true)
        .open("tests/test_config.json")
        .unwrap();
    let config: Value = from_reader(&file).unwrap();
    let devpaths: Vec<_> = config
        .get("ok_to_destroy_dev_array_key")
        .unwrap()
        .as_array()
        .unwrap()
        .iter()
        .map(|x| Path::new(x.as_str().unwrap()))
        .collect();

    let dev_sizes: Vec<(&Path, Sectors)> = devpaths
        .iter()
        .map(|p| {
            (
                *p,
                blkdev_size(&OpenOptions::new().read(true).open(p).unwrap())
                    .unwrap()
                    .sectors(),
            )
        })
        .collect();

    let runs = get_device_runs(limits, &dev_sizes);

    assert!(!runs.is_empty());

    init_logger();

    for run_paths in runs {
        clean_up().unwrap();

        let mut devices: Vec<_> = run_paths
            .iter()
            .map(|&(path, spec)| match spec {
                Some((start, length)) => {
                    RealTestDev::new(Either::Right(make_linear_test_dev(path, start, length)))
                }
                None => RealTestDev::new(Either::Left(path.to_path_buf())),
            })
            .collect();

        let paths: Vec<PathBuf> = devices.iter().map(|x| x.as_path()).collect();
        let paths: Vec<&Path> = paths.iter().map(|x| x.as_path()).collect();
        let result = panic::catch_unwind(|| test(&paths));
        let tear_down = clean_up();

        result.unwrap();
        tear_down.unwrap();

        for dev in devices.drain(..) {
            dev.teardown();
        }
    }
}
<commit_msg>initialize device_lists via vec! macro<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::{
    cmp,
    fs::OpenOptions,
    panic,
    path::{Path, PathBuf},
};

use either::Either;
use serde_json::{from_reader, Value};
use uuid::Uuid;

use devicemapper::{
    devnode_to_devno, Bytes, Device, DmDevice, DmName, LinearDev, LinearDevTargetParams,
    LinearTargetParams, Sectors, TargetLine, IEC,
};

use crate::engine::strat_engine::{
    device::blkdev_size,
    dm::get_dm,
    tests::{logger::init_logger, util::clean_up},
    writing::wipe_sectors,
};

pub struct RealTestDev {
    dev: Either<PathBuf, LinearDev>,
}

impl RealTestDev {
    /// Construct a new test device for the given path.
    /// Wipe initial MiB to clear metadata.
    pub fn new(dev: Either<PathBuf, LinearDev>) -> RealTestDev {
        let test_dev = RealTestDev { dev };
        wipe_sectors(
            test_dev.as_path(),
            Sectors(0),
            Bytes::from(IEC::Mi).sectors(),
        )
        .unwrap();
        test_dev
    }

    /// Get the device node of the device.
    fn as_path(&self) -> PathBuf {
        self.dev.as_ref().either(|p| p.clone(), |l| l.devnode())
    }

    /// Teardown a real test dev
    fn teardown(self) {
        wipe_sectors(&self.as_path(), Sectors(0), Bytes::from(IEC::Mi).sectors()).unwrap();
        if let Some(mut ld) = self.dev.right() {
            ld.teardown(get_dm()).unwrap();
        }
    }
}

/// Shorthand for specifying devices to use for tests
/// For the minimum size, if None is given, the default 1 GiB is chosen.
pub enum DeviceLimits {
    /// Use exactly the number of devices specified
    /// The second argument is the minimum size for all devices.
    /// The third argument is the maximum size for all devices.
    Exactly(usize, Option<Sectors>, Option<Sectors>),
    /// Use at least the number of devices specified, but if there are more
    /// devices available, also use the maximum number of devices.
    /// The second argument is the minimum size for all devices.
    /// The third argument is the maximum size for all devices.
    AtLeast(usize, Option<Sectors>, Option<Sectors>),
    /// Use exactly the number of devices specified in the first argument,
    /// and the minimum of the number of devices specified and the number
    /// of devices available in the second argument.
    /// The third argument is the minimum size for all devices.
    /// The fourth argument is the maximum size for all devices.
    #[allow(dead_code)]
    Range(usize, usize, Option<Sectors>, Option<Sectors>),
}

/// Get a list of lists of devices to use for tests.
/// May return an empty list if the request is not satisfiable.
#[allow(clippy::type_complexity)]
fn get_device_runs<'a>(
    limits: &DeviceLimits,
    dev_sizes: &[(&'a Path, Sectors)],
) -> Vec<Vec<(&'a Path, Option<(Sectors, Sectors)>)>> {
    let (lower, maybe_upper, min_size, max_size) = match limits {
        DeviceLimits::Exactly(num, min_size, max_size) => (*num, Some(*num), *min_size, *max_size),
        DeviceLimits::AtLeast(num, min_size, max_size) => (*num, None, *min_size, *max_size),
        DeviceLimits::Range(lower, upper, min_size, max_size) => {
            assert!(lower < upper);
            (*lower, Some(*upper), *min_size, *max_size)
        }
    };

    let min_size = min_size.unwrap_or_else(|| Bytes::from(IEC::Gi).sectors());

    assert!(max_size.is_none() || Some(min_size) <= max_size);

    // Retain only devices that are larger than min_size
    let (matches, _): (Vec<(&Path, Sectors)>, Vec<(&Path, Sectors)>) =
        dev_sizes.iter().partition(|&&(_, s)| s >= min_size);

    // If there is not a sufficient number of devices large enough to match
    // the lower bound, return an empty vec. TODO: It would be possible to try
    // harder, by merging several devices into a single linear device. If this
    // turns out to be necessary, this is the correct place to do it.
    if lower > matches.len() {
        return vec![];
    }

    // Retain only devices that are less than max_size
    let (mut matches, mut too_large): (Vec<(&Path, Sectors)>, Vec<(&Path, Sectors)>) =
        if max_size.is_none() {
            (matches, vec![])
        } else {
            let max_size = max_size.expect("!max_size.is_none()");
            matches.iter().partition(|&&(_, s)| s <= max_size)
        };

    let avail = matches.len();
    let needed = maybe_upper.unwrap_or(lower);
    let must_generate = if avail > needed { 0 } else { needed - avail };
    let avail_specs = {
        let mut avail_specs = vec![];
        while avail_specs.len() < must_generate && (!too_large.is_empty() || !matches.is_empty()) {
            let (path, size) = if too_large.is_empty() {
                matches.pop().expect("!matches.is_empty()")
            } else {
                too_large.pop().expect("!too_large.is_empty()")
            };
            let mut new_pairs = vec![];
            for i in 0..size / min_size {
                new_pairs.push((path, Some((i * min_size, min_size))))
            }
            avail_specs.extend(new_pairs);
        }
        avail_specs.extend(matches.iter().map(|&(p, _)| (p, None)));
        avail_specs
    };

    let avail = avail_specs.len();

    // If there is not a sufficient number of devices small enough to match
    // the lower bound, return an empty vec.
    if lower > avail {
        return vec![];
    }

    let mut device_lists = vec![avail_specs.iter().take(lower).cloned().collect::<Vec<_>>()];

    if lower != avail {
        match maybe_upper {
            None => device_lists.push(avail_specs.iter().take(avail).cloned().collect::<Vec<_>>()),
            Some(upper) => {
                if lower < upper {
                    device_lists.push(
                        avail_specs
                            .iter()
                            .take(cmp::min(upper, avail))
                            .cloned()
                            .collect::<Vec<_>>(),
                    )
                }
            }
        }
    }

    device_lists
}

/// Make a new LinearDev according to specs
fn make_linear_test_dev(devnode: &Path, start: Sectors, length: Sectors) -> LinearDev {
    let params = LinearTargetParams::new(
        Device::from(devnode_to_devno(devnode).unwrap().unwrap()),
        start,
    );
    let table = vec![TargetLine::new(
        Sectors(0),
        length,
        LinearDevTargetParams::Linear(params),
    )];
    LinearDev::setup(
        get_dm(),
        DmName::new(&format!("stratis_test_{}", Uuid::new_v4())).expect("valid format"),
        None,
        table,
    )
    .unwrap()
}

/// Run test on real devices, using given constraints. Constraints may result
/// in multiple invocations of the test, with differing numbers of block
/// devices.
pub fn test_with_spec<F>(limits: &DeviceLimits, test: F)
where
    F: Fn(&[&Path]) + panic::RefUnwindSafe,
{
    let file = OpenOptions::new()
        .read(true)
        .open("tests/test_config.json")
        .unwrap();
    let config: Value = from_reader(&file).unwrap();
    let devpaths: Vec<_> = config
        .get("ok_to_destroy_dev_array_key")
        .unwrap()
        .as_array()
        .unwrap()
        .iter()
        .map(|x| Path::new(x.as_str().unwrap()))
        .collect();

    let dev_sizes: Vec<(&Path, Sectors)> = devpaths
        .iter()
        .map(|p| {
            (
                *p,
                blkdev_size(&OpenOptions::new().read(true).open(p).unwrap())
                    .unwrap()
                    .sectors(),
            )
        })
        .collect();

    let runs = get_device_runs(limits, &dev_sizes);

    assert!(!runs.is_empty());

    init_logger();

    for run_paths in runs {
        clean_up().unwrap();

        let mut devices: Vec<_> = run_paths
            .iter()
            .map(|&(path, spec)| match spec {
                Some((start, length)) => {
                    RealTestDev::new(Either::Right(make_linear_test_dev(path, start, length)))
                }
                None => RealTestDev::new(Either::Left(path.to_path_buf())),
            })
            .collect();

        let paths: Vec<PathBuf> = devices.iter().map(|x| x.as_path()).collect();
        let paths: Vec<&Path> = paths.iter().map(|x| x.as_path()).collect();
        let result = panic::catch_unwind(|| test(&paths));
        let tear_down = clean_up();

        result.unwrap();
        tear_down.unwrap();

        for dev in devices.drain(..) {
            dev.teardown();
        }
    }
}
<|endoftext|>


<commit_before>//! Provides a timer which runs in the tail-end of each slot and maybe advances the state of the
//! head block forward a single slot.
//!
//! This provides an optimization with the following benefits:
//!
//! 1. Removes the burden of a single, mandatory `per_slot_processing` call from the leading-edge of
//!    block processing. This helps import blocks faster.
//! 2. Allows the node to learn of the shuffling for the next epoch, before the first block from
//!    that epoch has arrived. This helps reduce gossip block propagation times.
//!
//! The downsides to this optimization are:
//!
//! 1. We are required to store an additional `BeaconState` for the head block. This consumes
//!    memory.
//! 2. There's a possibility that the head block is never built upon, causing wasted CPU cycles.
use crate::validator_monitor::HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS;
use crate::{
    beacon_chain::BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT, snapshot_cache::StateAdvance, BeaconChain,
    BeaconChainError, BeaconChainTypes,
};
use slog::{debug, error, warn, Logger};
use slot_clock::SlotClock;
use state_processing::per_slot_processing;
use std::sync::{
    atomic::{AtomicBool, Ordering},
    Arc,
};
use task_executor::TaskExecutor;
use tokio::time::sleep;
use types::{EthSpec, Hash256, Slot};

/// If the head slot is more than `MAX_ADVANCE_DISTANCE` from the current slot, then don't perform
/// the state advancement.
///
/// This avoids doing unnecessary work whilst the node is syncing or has perhaps been put to sleep
/// for some period of time.
const MAX_ADVANCE_DISTANCE: u64 = 4;

#[derive(Debug)]
enum Error {
    BeaconChain(BeaconChainError),
    HeadMissingFromSnapshotCache(Hash256),
    MaxDistanceExceeded { current_slot: Slot, head_slot: Slot },
    StateAlreadyAdvanced { block_root: Hash256 },
    BadStateSlot { state_slot: Slot, block_slot: Slot },
}

impl From<BeaconChainError> for Error {
    fn from(e: BeaconChainError) -> Self {
        Self::BeaconChain(e)
    }
}

/// Provides a simple thread-safe lock to be used for task co-ordination. Practically equivalent to
/// `Mutex<()>`.
#[derive(Clone)]
struct Lock(Arc<AtomicBool>);

impl Lock {
    /// Instantiate an unlocked self.
    pub fn new() -> Self {
        Self(Arc::new(AtomicBool::new(false)))
    }

    /// Lock self, returning `true` if the lock was already set.
    pub fn lock(&self) -> bool {
        self.0.fetch_or(true, Ordering::SeqCst)
    }

    /// Unlock self.
    pub fn unlock(&self) {
        self.0.store(false, Ordering::SeqCst);
    }
}

/// Spawns the timer described in the module-level documentation.
pub fn spawn_state_advance_timer<T: BeaconChainTypes>(
    executor: TaskExecutor,
    beacon_chain: Arc<BeaconChain<T>>,
    log: Logger,
) {
    executor.spawn(
        state_advance_timer(executor.clone(), beacon_chain, log),
        "state_advance_timer",
    );
}

/// Provides the timer described in the module-level documentation.
async fn state_advance_timer<T: BeaconChainTypes>(
    executor: TaskExecutor,
    beacon_chain: Arc<BeaconChain<T>>,
    log: Logger,
) {
    let is_running = Lock::new();
    let slot_clock = &beacon_chain.slot_clock;
    let slot_duration = slot_clock.slot_duration();

    loop {
        match beacon_chain.slot_clock.duration_to_next_slot() {
            Some(duration) => sleep(duration + (slot_duration / 4) * 3).await,
            None => {
                error!(log, "Failed to read slot clock");
                // If we can't read the slot clock, just wait another slot.
                sleep(slot_duration).await;
                continue;
            }
        };

        // Only start spawn the state advance task if the lock was previously free.
        if !is_running.lock() {
            let log = log.clone();
            let beacon_chain = beacon_chain.clone();
            let is_running = is_running.clone();

            executor.spawn_blocking(
                move || {
                    match advance_head(&beacon_chain, &log) {
                        Ok(()) => (),
                        Err(Error::BeaconChain(e)) => error!(
                            log,
                            "Failed to advance head state";
                            "error" => ?e
                        ),
                        Err(Error::StateAlreadyAdvanced { block_root }) => debug!(
                            log,
                            "State already advanced on slot";
                            "block_root" => ?block_root
                        ),
                        Err(Error::MaxDistanceExceeded {
                            current_slot,
                            head_slot,
                        }) => debug!(
                            log,
                            "Refused to advance head state";
                            "head_slot" => head_slot,
                            "current_slot" => current_slot,
                        ),
                        other => warn!(
                            log,
                            "Did not advance head state";
                            "reason" => ?other
                        ),
                    };

                    // Permit this blocking task to spawn again, next time the timer fires.
                    is_running.unlock();
                },
                "state_advance_blocking",
            );
        } else {
            warn!(
                log,
                "State advance routine overloaded";
                "msg" => "system resources may be overloaded"
            )
        }
    }
}

/// Reads the `snapshot_cache` from the `beacon_chain` and attempts to take a clone of the
/// `BeaconState` of the head block. If it obtains this clone, the state will be advanced a single
/// slot then placed back in the `snapshot_cache` to be used for block verification.
///
/// See the module-level documentation for rationale.
fn advance_head<T: BeaconChainTypes>(
    beacon_chain: &BeaconChain<T>,
    log: &Logger,
) -> Result<(), Error> {
    let current_slot = beacon_chain.slot()?;

    // These brackets ensure that the `head_slot` value is dropped before we run fork choice and
    // potentially invalidate it.
    //
    // Fork-choice is not run *before* this function to avoid unnecessary calls whilst syncing.
    {
        let head_slot = beacon_chain.head_info()?.slot;

        // Don't run this when syncing or if lagging too far behind.
        if head_slot + MAX_ADVANCE_DISTANCE < current_slot {
            return Err(Error::MaxDistanceExceeded {
                current_slot,
                head_slot,
            });
        }
    }

    // Run fork choice so we get the latest view of the head.
    //
    // This is useful since it's quite likely that the last time we ran fork choice was shortly
    // after receiving the latest gossip block, but not necessarily after we've received the
    // majority of attestations.
    beacon_chain.fork_choice()?;

    let head_root = beacon_chain.head_info()?.block_root;

    let (head_slot, head_state_root, mut state) = match beacon_chain
        .snapshot_cache
        .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
        .ok_or(BeaconChainError::SnapshotCacheLockTimeout)?
        .get_for_state_advance(head_root)
    {
        StateAdvance::AlreadyAdvanced => {
            return Err(Error::StateAlreadyAdvanced {
                block_root: head_root,
            })
        }
        StateAdvance::BlockNotFound => return Err(Error::HeadMissingFromSnapshotCache(head_root)),
        StateAdvance::State {
            state,
            state_root,
            block_slot,
        } => (block_slot, state_root, *state),
    };

    let initial_slot = state.slot;
    let initial_epoch = state.current_epoch();

    let state_root = if state.slot == head_slot {
        Some(head_state_root)
    } else {
        // Protect against advancing a state more than a single slot.
        //
        // Advancing more than one slot without storing the intermediate state would corrupt the
        // database. Future works might store temporary, intermediate states inside this function.
        return Err(Error::BadStateSlot {
            block_slot: head_slot,
            state_slot: state.slot,
        });
    };

    // Advance the state a single slot.
    if let Some(summary) = per_slot_processing(&mut state, state_root, &beacon_chain.spec)
        .map_err(BeaconChainError::from)?
    {
        // Only notify the validator monitor for recent blocks.
        if state.current_epoch() + VALIDATOR_MONITOR_HISTORIC_EPOCHS as u64
            >= current_slot.epoch(T::EthSpec::slots_per_epoch())
        {
            // Potentially create logs/metrics for locally monitored validators.
            beacon_chain
                .validator_monitor
                .read()
                .process_validator_statuses(state.current_epoch(), &summary.statuses);
        }
    }

    debug!(
        log,
        "Advanced head state one slot";
        "head_root" => ?head_root,
        "state_slot" => state.slot,
        "current_slot" => current_slot,
    );

    // If the `pre_state` is in a later epoch than `state`, pre-emptively add the proposer
    // shuffling for the next epoch into the cache.
    if initial_epoch > state.current_epoch() {
        debug!(
            log,
            "Priming proposer cache";
            "head_root" => ?head_root,
            "state_epoch" => state.current_epoch(),
            "current_epoch" => current_slot.epoch(T::EthSpec::slots_per_epoch()),
        );
        beacon_chain
            .beacon_proposer_cache
            .lock()
            .insert(
                state.current_epoch(),
                head_root,
                state
                    .get_beacon_proposer_indices(&beacon_chain.spec)
                    .map_err(BeaconChainError::from)?,
                state.fork,
            )
            .map_err(BeaconChainError::from)?;
    }

    let final_slot = state.slot;

    // Insert the advanced state back into the snapshot cache.
    beacon_chain
        .snapshot_cache
        .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
        .ok_or(BeaconChainError::SnapshotCacheLockTimeout)?
        .update_pre_state(head_root, state)
        .ok_or(Error::HeadMissingFromSnapshotCache(head_root))?;

    let current_slot = beacon_chain.slot()?;
    if final_slot <= current_slot {
        warn!(
            log,
            "State advance too slow";
            "head_root" => %head_root,
            "advanced_slot" => final_slot,
            "current_slot" => current_slot,
            "initial_slot" => initial_slot,
            "msg" => "system resources may be overloaded",
        );
    }

    debug!(
        log,
        "Completed state advance";
        "head_root" => ?head_root,
        "advanced_slot" => final_slot,
        "initial_slot" => initial_slot,
    );

    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn lock() {
        let lock = Lock::new();
        assert_eq!(lock.lock(), false);
        assert_eq!(lock.lock(), true);
        assert_eq!(lock.lock(), true);
        lock.unlock();
        assert_eq!(lock.lock(), false);
        assert_eq!(lock.lock(), true);
    }
}
<commit_msg>Fix false positive "State advance too slow" logs (#2218)<commit_after>//! Provides a timer which runs in the tail-end of each slot and maybe advances the state of the
//! head block forward a single slot.
//!
//! This provides an optimization with the following benefits:
//!
//! 1. Removes the burden of a single, mandatory `per_slot_processing` call from the leading-edge of
//!    block processing. This helps import blocks faster.
//! 2. Allows the node to learn of the shuffling for the next epoch, before the first block from
//!    that epoch has arrived. This helps reduce gossip block propagation times.
//!
//! The downsides to this optimization are:
//!
//! 1. We are required to store an additional `BeaconState` for the head block. This consumes
//!    memory.
//! 2. There's a possibility that the head block is never built upon, causing wasted CPU cycles.
use crate::validator_monitor::HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS;
use crate::{
    beacon_chain::BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT, snapshot_cache::StateAdvance, BeaconChain,
    BeaconChainError, BeaconChainTypes,
};
use slog::{debug, error, warn, Logger};
use slot_clock::SlotClock;
use state_processing::per_slot_processing;
use std::sync::{
    atomic::{AtomicBool, Ordering},
    Arc,
};
use task_executor::TaskExecutor;
use tokio::time::sleep;
use types::{EthSpec, Hash256, Slot};

/// If the head slot is more than `MAX_ADVANCE_DISTANCE` from the current slot, then don't perform
/// the state advancement.
///
/// This avoids doing unnecessary work whilst the node is syncing or has perhaps been put to sleep
/// for some period of time.
const MAX_ADVANCE_DISTANCE: u64 = 4;

#[derive(Debug)]
enum Error {
    BeaconChain(BeaconChainError),
    HeadMissingFromSnapshotCache(Hash256),
    MaxDistanceExceeded { current_slot: Slot, head_slot: Slot },
    StateAlreadyAdvanced { block_root: Hash256 },
    BadStateSlot { state_slot: Slot, block_slot: Slot },
}

impl From<BeaconChainError> for Error {
    fn from(e: BeaconChainError) -> Self {
        Self::BeaconChain(e)
    }
}

/// Provides a simple thread-safe lock to be used for task co-ordination. Practically equivalent to
/// `Mutex<()>`.
#[derive(Clone)]
struct Lock(Arc<AtomicBool>);

impl Lock {
    /// Instantiate an unlocked self.
    pub fn new() -> Self {
        Self(Arc::new(AtomicBool::new(false)))
    }

    /// Lock self, returning `true` if the lock was already set.
    pub fn lock(&self) -> bool {
        self.0.fetch_or(true, Ordering::SeqCst)
    }

    /// Unlock self.
    pub fn unlock(&self) {
        self.0.store(false, Ordering::SeqCst);
    }
}

/// Spawns the timer described in the module-level documentation.
pub fn spawn_state_advance_timer<T: BeaconChainTypes>(
    executor: TaskExecutor,
    beacon_chain: Arc<BeaconChain<T>>,
    log: Logger,
) {
    executor.spawn(
        state_advance_timer(executor.clone(), beacon_chain, log),
        "state_advance_timer",
    );
}

/// Provides the timer described in the module-level documentation.
async fn state_advance_timer<T: BeaconChainTypes>(
    executor: TaskExecutor,
    beacon_chain: Arc<BeaconChain<T>>,
    log: Logger,
) {
    let is_running = Lock::new();
    let slot_clock = &beacon_chain.slot_clock;
    let slot_duration = slot_clock.slot_duration();

    loop {
        match beacon_chain.slot_clock.duration_to_next_slot() {
            Some(duration) => sleep(duration + (slot_duration / 4) * 3).await,
            None => {
                error!(log, "Failed to read slot clock");
                // If we can't read the slot clock, just wait another slot.
                sleep(slot_duration).await;
                continue;
            }
        };

        // Only start spawn the state advance task if the lock was previously free.
        if !is_running.lock() {
            let log = log.clone();
            let beacon_chain = beacon_chain.clone();
            let is_running = is_running.clone();

            executor.spawn_blocking(
                move || {
                    match advance_head(&beacon_chain, &log) {
                        Ok(()) => (),
                        Err(Error::BeaconChain(e)) => error!(
                            log,
                            "Failed to advance head state";
                            "error" => ?e
                        ),
                        Err(Error::StateAlreadyAdvanced { block_root }) => debug!(
                            log,
                            "State already advanced on slot";
                            "block_root" => ?block_root
                        ),
                        Err(Error::MaxDistanceExceeded {
                            current_slot,
                            head_slot,
                        }) => debug!(
                            log,
                            "Refused to advance head state";
                            "head_slot" => head_slot,
                            "current_slot" => current_slot,
                        ),
                        other => warn!(
                            log,
                            "Did not advance head state";
                            "reason" => ?other
                        ),
                    };

                    // Permit this blocking task to spawn again, next time the timer fires.
                    is_running.unlock();
                },
                "state_advance_blocking",
            );
        } else {
            warn!(
                log,
                "State advance routine overloaded";
                "msg" => "system resources may be overloaded"
            )
        }
    }
}

/// Reads the `snapshot_cache` from the `beacon_chain` and attempts to take a clone of the
/// `BeaconState` of the head block. If it obtains this clone, the state will be advanced a single
/// slot then placed back in the `snapshot_cache` to be used for block verification.
///
/// See the module-level documentation for rationale.
fn advance_head<T: BeaconChainTypes>(
    beacon_chain: &BeaconChain<T>,
    log: &Logger,
) -> Result<(), Error> {
    let current_slot = beacon_chain.slot()?;

    // These brackets ensure that the `head_slot` value is dropped before we run fork choice and
    // potentially invalidate it.
    //
    // Fork-choice is not run *before* this function to avoid unnecessary calls whilst syncing.
    {
        let head_slot = beacon_chain.head_info()?.slot;

        // Don't run this when syncing or if lagging too far behind.
        if head_slot + MAX_ADVANCE_DISTANCE < current_slot {
            return Err(Error::MaxDistanceExceeded {
                current_slot,
                head_slot,
            });
        }
    }

    // Run fork choice so we get the latest view of the head.
    //
    // This is useful since it's quite likely that the last time we ran fork choice was shortly
    // after receiving the latest gossip block, but not necessarily after we've received the
    // majority of attestations.
    beacon_chain.fork_choice()?;

    let head_root = beacon_chain.head_info()?.block_root;

    let (head_slot, head_state_root, mut state) = match beacon_chain
        .snapshot_cache
        .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
        .ok_or(BeaconChainError::SnapshotCacheLockTimeout)?
        .get_for_state_advance(head_root)
    {
        StateAdvance::AlreadyAdvanced => {
            return Err(Error::StateAlreadyAdvanced {
                block_root: head_root,
            })
        }
        StateAdvance::BlockNotFound => return Err(Error::HeadMissingFromSnapshotCache(head_root)),
        StateAdvance::State {
            state,
            state_root,
            block_slot,
        } => (block_slot, state_root, *state),
    };

    let initial_slot = state.slot;
    let initial_epoch = state.current_epoch();

    let state_root = if state.slot == head_slot {
        Some(head_state_root)
    } else {
        // Protect against advancing a state more than a single slot.
        //
        // Advancing more than one slot without storing the intermediate state would corrupt the
        // database. Future works might store temporary, intermediate states inside this function.
        return Err(Error::BadStateSlot {
            block_slot: head_slot,
            state_slot: state.slot,
        });
    };

    // Advance the state a single slot.
    if let Some(summary) = per_slot_processing(&mut state, state_root, &beacon_chain.spec)
        .map_err(BeaconChainError::from)?
    {
        // Only notify the validator monitor for recent blocks.
        if state.current_epoch() + VALIDATOR_MONITOR_HISTORIC_EPOCHS as u64
            >= current_slot.epoch(T::EthSpec::slots_per_epoch())
        {
            // Potentially create logs/metrics for locally monitored validators.
            beacon_chain
                .validator_monitor
                .read()
                .process_validator_statuses(state.current_epoch(), &summary.statuses);
        }
    }

    debug!(
        log,
        "Advanced head state one slot";
        "head_root" => ?head_root,
        "state_slot" => state.slot,
        "current_slot" => current_slot,
    );

    // If the `pre_state` is in a later epoch than `state`, pre-emptively add the proposer
    // shuffling for the next epoch into the cache.
    if initial_epoch > state.current_epoch() {
        debug!(
            log,
            "Priming proposer cache";
            "head_root" => ?head_root,
            "state_epoch" => state.current_epoch(),
            "current_epoch" => current_slot.epoch(T::EthSpec::slots_per_epoch()),
        );
        beacon_chain
            .beacon_proposer_cache
            .lock()
            .insert(
                state.current_epoch(),
                head_root,
                state
                    .get_beacon_proposer_indices(&beacon_chain.spec)
                    .map_err(BeaconChainError::from)?,
                state.fork,
            )
            .map_err(BeaconChainError::from)?;
    }

    let final_slot = state.slot;

    // Insert the advanced state back into the snapshot cache.
    beacon_chain
        .snapshot_cache
        .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT)
        .ok_or(BeaconChainError::SnapshotCacheLockTimeout)?
        .update_pre_state(head_root, state)
        .ok_or(Error::HeadMissingFromSnapshotCache(head_root))?;

    // If we have moved into the next slot whilst processing the state then this function is going
    // to become ineffective and likely become a hindrance as we're stealing the tree hash cache
    // from the snapshot cache (which may force the next block to rebuild a new one).
    //
    // If this warning occurs very frequently on well-resourced machines then we should consider
    // starting it earlier in the slot. Otherwise, it's a good indication that the machine is too
    // slow/overloaded and will be useful information for the user.
    let starting_slot = current_slot;
    let current_slot = beacon_chain.slot()?;
    if starting_slot < current_slot {
        warn!(
            log,
            "State advance too slow";
            "head_root" => %head_root,
            "advanced_slot" => final_slot,
            "current_slot" => current_slot,
            "starting_slot" => starting_slot,
            "msg" => "system resources may be overloaded",
        );
    }

    debug!(
        log,
        "Completed state advance";
        "head_root" => ?head_root,
        "advanced_slot" => final_slot,
        "initial_slot" => initial_slot,
    );

    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn lock() {
        let lock = Lock::new();
        assert_eq!(lock.lock(), false);
        assert_eq!(lock.lock(), true);
        assert_eq!(lock.lock(), true);
        lock.unlock();
        assert_eq!(lock.lock(), false);
        assert_eq!(lock.lock(), true);
    }
}
<|endoftext|>


<commit_before>#![crate_id="CSVProvider#0.1-pre"]
#![crate_type="dylib"]

// why doesn't "lib" work?

// rustc 0.11.0-pre (b47f2226a25654c5b781d27a91f2fa5274b3a347 2014-06-28 14:31:37 +0000)

#![feature(globs, macro_rules, quote, managed_boxes, plugin_registrar)]

#![allow(unused_imports)]
//#![allow(unused_variable)]
#![allow(uppercase_variables)]
#![allow(unnecessary_parens)]

extern crate syntax;
extern crate rustc;

use syntax::ast::{
   Name,
   TokenTree,
   Expr,
   ExprLit,
   ExprVec,
   LitStr,
   Ident,
   DUMMY_NODE_ID,
   MutImmutable
};
use syntax::codemap::Span;
use syntax::ext::base::{
   SyntaxExtension,
   BasicMacroExpander,
   ExtCtxt,
   MacResult,
   MacPat,
   MacItem,
   DummyResult,
   MacExpr,
   NormalTT
};
use std::gc::Gc;
use syntax::ext::quote::{
   //expand_parse_call
};
use syntax::parse;
use syntax::parse::token;
use syntax::parse::token::{InternedString, COMMA, EOF};

use std::io;
use std::io::fs;
use std::os;
use std::str;
use std::io::Process;
use syntax::ast;
use syntax::ast::Name;

use syntax::util::small_vector::SmallVector;

use std::collections::HashMap;
use std::gc::{Gc, GC};

use rustc::plugin::Registry;

#[plugin_registrar]
pub fn macro_registrar(reg: &mut Registry) {
   reg.register_macro("ProvideCSV_labels", provide_csv_given_labels);
}

#[deriving(Clone)]
struct Entry {
   str: InternedString,
   expr: Gc<Expr>
}

// see https://github.com/sfackler/syntax-ext-talk/blob/gh-pages/simple-ext/lib.rs
fn parse_entries(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Option<Vec<Entry>> {
   let mut parser = parse::new_parser_from_tts(
       cx.parse_sess(),
       cx.cfg(),
       tts.iter()
          .map(|x| (*x).clone())
          .collect()
   );

   let mut entries: Vec<Entry> = Vec::new();

   let mut error = false;
   while parser.token != EOF {
      let entry = parser.parse_expr();

      let entry_str = match entry.node {
         ExprLit(lit) => {
            match lit.node {
               LitStr(ref s, _) => s.clone(),
               _ => {
                  cx.span_err(entry.span, "expected string literal (1)");
                  error = true;
                  InternedString::new("")
               }
            }
         }
         _ => {
                cx.span_err(entry.span, "expected string literal (2)");
                error = true;
                InternedString::new("")
         }
      };

      entries.push(Entry { str: entry_str, expr: entry });

      if !parser.eat(&COMMA) && parser.token != EOF {
         cx.span_err(parser.span, "expected `,`");
         return None;
      }
    }

   if error {
      return None;
   }

   Some(entries)
}

pub struct MacItems {
   items: Vec<Gc<ast::Item>>
}
impl MacItems {
   pub fn new(items: Vec<Gc<ast::Item>>) -> Box<MacResult> {
      box MacItems { items: items } as Box<MacResult>
   }
}
impl MacResult for MacItems {
   fn make_items(&self) -> Option<SmallVector<Gc<ast::Item>>> {
      Some(SmallVector::many(self.items.clone()))
   }
}

fn parse_csv_row<'a>(raw: &'a str, expected_columns: Option<uint>) -> str::StrSplits<'a> {
   let mut it = raw.split_str(",");

   // take a pulse
   let _whatever = it.map(|xx| {
      println!("iterating over row elements: {}", xx.to_string());
      return xx;
   });

   match expected_columns {
      Some(count) => {
         if (count == it.count()) {
            println!("parse_csv_row: matching length expected");
         } else {
            println!("parse_csv_row: wrong length!");
         }
      },
      None => (),
   }

   return it;
}

fn provide_csv_given_labels(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult> {
   let mut entries = match parse_entries(cx, tts) {
      Some(entries) => entries,
      None => return DummyResult::expr(sp),
   };

   let raw_name   : Entry = entries.shift().expect("should be given a type name");
   let raw_path   : Entry = entries.shift().expect("should be given a CSV file path");
   let raw_labels : Entry = entries.shift().expect("should be given column labels");

   let name   : InternedString = raw_name.str;
   let path   : InternedString = raw_path.str;
   let labels : InternedString = raw_labels.str;

   println!("provide_csv_given_labels: name:   {}", name);
   println!("provide_csv_given_labels: path:   {}", path);
   println!("provide_csv_given_labels: labels: {}", labels);

   // PENDING
   // read the CSV and try to get the type of data in it
   // create a type with
   //  * this name,
   //  * these labeled columns,
   //  * discovered type of data
   //  * a constructor which reads the whole file

   fn interned_to_ident(xx: InternedString) -> Ident {
      let yy: String = format!("{}", xx);
      let zz: &str = yy.as_slice();
      return token::str_to_ident(zz);
   }

   fn interned_to_ident_with_suffix(xx: InternedString, suff: &str) -> Ident {
      let yy: String = format!("{}{}", xx, suff);
      let zz: &str = yy.as_slice();
      return token::str_to_ident(zz);
   }

   let MyCsv    = interned_to_ident(name.clone());
   let MyCsvRow = interned_to_ident_with_suffix(name, "Row");
   let col0     = interned_to_ident(labels.clone());

   println!("Parsing given labels...");

   let labels1 = format!("{}", labels);
   let labels_iter = parse_csv_row(labels1.as_slice(), None);

   println!("Defining items in the CSV provider...");

/*
   fn mut_ref_cx<'a>(context: &'a mut ExtCtxt) -> &'a mut ExtCtxt {
        let &ExtCtxt(ref mut context_prime) = context;
        return context_prime;
   }
*/

/*
   pub fn expand_quote_item(
      cx: &ExtCtxt,
      sp: Span,
      tts: &[ast::TokenTree])
      -> Box<base::MacResult>
   {
      let expanded = expand_parse_call(cx, sp, "parse_item_with_outer_attributes", vec!(), tts);
      base::MacExpr::new(expanded)
   }
*/

   let define_my_csv_row = |cx0 : &mut ExtCtxt| {

      let cx1 : &ExtCtxt = cx0;

      let col = quote_item!(cx1, pub $col0: String).expect("column parsing");

      return quote_item!(cx1,
         pub struct $MyCsvRow {
            $col,
         }
      );
   };

   let define_my_csv = |cx0 : &mut ExtCtxt| {
      let item1: Option<Gc<syntax::ast::Item>> = quote_item!(cx0,
         pub struct $MyCsv {
            pub data: Vec<$MyCsvRow>,
         }
      );
      return item1;
   };

   let item0: Option<Gc<syntax::ast::Item>> = define_my_csv_row(cx);
   let item1: Option<Gc<syntax::ast::Item>> = define_my_csv(cx);
   let item2: Option<Gc<syntax::ast::Item>> = quote_item!(cx,
      impl $MyCsv {
         pub fn new() -> $MyCsv {
            println!("HMMMMMM.");
            return $MyCsv {
               data: (vec![
                  $MyCsvRow { $col0: "zero".to_string() },
                  $MyCsvRow { $col0:  "one".to_string() },
                  $MyCsvRow { $col0:  "two".to_string() },
               ]),
            };
         }
      }
   );

   let items = vec![item0.expect("Should be able to make struct MyCsvRow"),
                    item1.expect("Should be able to make struct MyCsv."),
                    item2.expect("Should be able to implement MyCsv.")];

   println!("OK");

   return MacItems::new(items);
}

<commit_msg>yaaaaay<commit_after>#![crate_id="CSVProvider#0.1-pre"]
#![crate_type="dylib"]

// why doesn't "lib" work?

// rustc 0.11.0-pre (b47f2226a25654c5b781d27a91f2fa5274b3a347 2014-06-28 14:31:37 +0000)

#![feature(globs, macro_rules, quote, managed_boxes, plugin_registrar)]

#![allow(unused_imports)]
//#![allow(unused_variable)]
#![allow(uppercase_variables)]
#![allow(unnecessary_parens)]

extern crate syntax;
extern crate rustc;

use syntax::ast::{
   Name,
   TokenTree,
   Expr,
   ExprLit,
   ExprVec,
   LitStr,
   Ident,
   DUMMY_NODE_ID,
   MutImmutable
};
use syntax::codemap::Span;
use syntax::ext::base::{
   SyntaxExtension,
   BasicMacroExpander,
   ExtCtxt,
   MacResult,
   MacPat,
   MacItem,
   DummyResult,
   MacExpr,
   NormalTT
};
use std::gc::Gc;
use syntax::ext::quote::{
   //expand_parse_call
};
use syntax::parse;
use syntax::parse::token;
use syntax::parse::token::{InternedString, COMMA, EOF};

use std::io;
use std::io::fs;
use std::os;
use std::str;
use std::io::Process;
use syntax::ast;
use syntax::ast::Name;

use syntax::util::small_vector::SmallVector;

use std::collections::HashMap;
use std::gc::{Gc, GC};

use rustc::plugin::Registry;

#[plugin_registrar]
pub fn macro_registrar(reg: &mut Registry) {
   reg.register_macro("ProvideCSV_labels", provide_csv_given_labels);
}

#[deriving(Clone)]
struct Entry {
   str: InternedString,
   expr: Gc<Expr>
}

// see https://github.com/sfackler/syntax-ext-talk/blob/gh-pages/simple-ext/lib.rs
fn parse_entries(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Option<Vec<Entry>> {
   let mut parser = parse::new_parser_from_tts(
       cx.parse_sess(),
       cx.cfg(),
       tts.iter()
          .map(|x| (*x).clone())
          .collect()
   );

   let mut entries: Vec<Entry> = Vec::new();

   let mut error = false;
   while parser.token != EOF {
      let entry = parser.parse_expr();

      let entry_str = match entry.node {
         ExprLit(lit) => {
            match lit.node {
               LitStr(ref s, _) => s.clone(),
               _ => {
                  cx.span_err(entry.span, "expected string literal (1)");
                  error = true;
                  InternedString::new("")
               }
            }
         }
         _ => {
                cx.span_err(entry.span, "expected string literal (2)");
                error = true;
                InternedString::new("")
         }
      };

      entries.push(Entry { str: entry_str, expr: entry });

      if !parser.eat(&COMMA) && parser.token != EOF {
         cx.span_err(parser.span, "expected `,`");
         return None;
      }
    }

   if error {
      return None;
   }

   Some(entries)
}

pub struct MacItems {
   items: Vec<Gc<ast::Item>>
}
impl MacItems {
   pub fn new(items: Vec<Gc<ast::Item>>) -> Box<MacResult> {
      box MacItems { items: items } as Box<MacResult>
   }
}
impl MacResult for MacItems {
   fn make_items(&self) -> Option<SmallVector<Gc<ast::Item>>> {
      Some(SmallVector::many(self.items.clone()))
   }
}

fn parse_csv_row<'a>(raw: &'a str, expected_columns: Option<uint>) -> str::StrSplits<'a> {
   let mut it = raw.split_str(",");

   // take a pulse
   let _whatever = it.map(|xx| {
      println!("iterating over row elements: {}", xx.to_string());
      return xx;
   });

   match expected_columns {
      Some(count) => {
         if (count == it.count()) {
            println!("parse_csv_row: matching length expected");
         } else {
            println!("parse_csv_row: wrong length!");
         }
      },
      None => (),
   }

   return it;
}

fn provide_csv_given_labels(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult> {
   let mut entries = match parse_entries(cx, tts) {
      Some(entries) => entries,
      None => return DummyResult::expr(sp),
   };

   let raw_name   : Entry = entries.shift().expect("should be given a type name");
   let raw_path   : Entry = entries.shift().expect("should be given a CSV file path");
   let raw_labels : Entry = entries.shift().expect("should be given column labels");

   let name   : InternedString = raw_name.str;
   let path   : InternedString = raw_path.str;
   let labels : InternedString = raw_labels.str;

   println!("provide_csv_given_labels: name:   {}", name);
   println!("provide_csv_given_labels: path:   {}", path);
   println!("provide_csv_given_labels: labels: {}", labels);

   // PENDING
   // read the CSV and try to get the type of data in it
   // create a type with
   //  * this name,
   //  * these labeled columns,
   //  * discovered type of data
   //  * a constructor which reads the whole file

   fn interned_to_ident(xx: InternedString) -> Ident {
      let yy: String = format!("{}", xx);
      let zz: &str = yy.as_slice();
      return token::str_to_ident(zz);
   }

   fn interned_to_ident_with_suffix(xx: InternedString, suff: &str) -> Ident {
      let yy: String = format!("{}{}", xx, suff);
      let zz: &str = yy.as_slice();
      return token::str_to_ident(zz);
   }

   let MyCsv    = interned_to_ident(name.clone());
   let MyCsvRow = interned_to_ident_with_suffix(name, "Row");
   let col0     = interned_to_ident(labels.clone());

   println!("Parsing given labels...");

   let labels1 = format!("{}", labels);
   let labels_iter = parse_csv_row(labels1.as_slice(), None);

   println!("Defining items in the CSV provider...");

/*
   fn mut_ref_cx<'a>(context: &'a mut ExtCtxt) -> &'a mut ExtCtxt {
        let &ExtCtxt(ref mut context_prime) = context;
        return context_prime;
   }
*/

/*
   pub fn expand_quote_item(
      cx: &ExtCtxt,
      sp: Span,
      tts: &[ast::TokenTree])
      -> Box<base::MacResult>
   {
      let expanded = expand_parse_call(cx, sp, "parse_item_with_outer_attributes", vec!(), tts);
      base::MacExpr::new(expanded)
   }
*/

   let icx : &ExtCtxt = cx;  // an immutable borrow

   // TODO: iterate through call columns
   let col = quote_item!(icx, pub $col0: String).expect("column parsing");

   let item0 = quote_item!(icx,
      pub struct $MyCsvRow {
         $col,
      }
   );

   let item1: Option<Gc<syntax::ast::Item>> = quote_item!(icx,
      pub struct $MyCsv {
         pub data: Vec<$MyCsvRow>,
      }
   );

   let item2: Option<Gc<syntax::ast::Item>> = quote_item!(icx,
      impl $MyCsv {
         pub fn new() -> $MyCsv {
            println!("HMMMMMM.");
            return $MyCsv {
               data: (vec![
                  $MyCsvRow { $col0: "zero".to_string() },
                  $MyCsvRow { $col0:  "one".to_string() },
                  $MyCsvRow { $col0:  "two".to_string() },
               ]),
            };
         }
      }
   );

   let items = vec![item0.expect("Should be able to make struct MyCsvRow"),
                    item1.expect("Should be able to make struct MyCsv."),
                    item2.expect("Should be able to implement MyCsv.")];

   println!("OK");

   return MacItems::new(items);
}

<|endoftext|>


<commit_before>//! Crate to `#include` C++ headers in your Rust code, and generate
//! idiomatic bindings using `cxx`. See [include_cpp] for details.

// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::{
    fs::File,
    io::Write,
    os::unix::prelude::PermissionsExt,
    path::{Path, PathBuf},
    process::Command,
};

use autocxx_engine::preprocess;
use clap::{crate_authors, crate_version, App, Arg, ArgMatches};
use indoc::indoc;
use itertools::Itertools;
use tempfile::TempDir;

static LONG_HELP: &str = indoc! {"
Command line utility to minimize autocxx bug cases.

This is a wrapper for creduce.

Example command-line:
autocxx-reduce -I my-inc-dir -h my-header -d 'generate!(\"MyClass\")' -k -- --n 64 --remove-pass pass_line_markers
"};

fn main() {
    let matches = App::new("autocxx-reduce")
        .version(crate_version!())
        .author(crate_authors!())
        .about("Reduce a C++ test case")
        .long_about(LONG_HELP)
        .arg(
            Arg::with_name("inc")
                .short("I")
                .long("inc")
                .multiple(true)
                .number_of_values(1)
                .value_name("INCLUDE DIRS")
                .help("include path")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("define")
                .short("D")
                .long("define")
                .multiple(true)
                .number_of_values(1)
                .value_name("DEFINE")
                .help("macro definition")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("header")
                .short("h")
                .long("header")
                .multiple(true)
                .number_of_values(1)
                .required(true)
                .value_name("HEADER")
                .help("header file name")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("directive")
                .short("d")
                .long("directive")
                .multiple(true)
                .number_of_values(1)
                .value_name("DIRECTIVE")
                .help("directives to put within include_cpp!")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("problem")
                .short("p")
                .long("problem")
                .required(true)
                .value_name("PROBLEM")
                .help("problem string we're looking for")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("creduce")
                .long("creduce")
                .required(true)
                .value_name("PATH")
                .help("creduce binary location")
                .default_value("/usr/bin/creduce")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("output")
                .short("o")
                .long("output")
                .value_name("OUTPUT")
                .help("where to write minimized output")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("keep")
                .short("k")
                .long("keep-dir")
                .help("keep the temporary directory for debugging purposes"),
        )
        .arg(Arg::with_name("creduce-args").last(true).multiple(true))
        .get_matches();
    run(matches).unwrap();
}

fn run(matches: ArgMatches) -> Result<(), std::io::Error> {
    let keep_tmp = matches.is_present("keep");
    let tmp_dir = TempDir::new()?;
    let r = do_run(matches, &tmp_dir);
    if keep_tmp {
        println!(
            "Keeping temp dir created at: {}",
            tmp_dir.into_path().to_str().unwrap()
        );
    }
    r
}

fn do_run(matches: ArgMatches, tmp_dir: &TempDir) -> Result<(), std::io::Error> {
    let incs: Vec<_> = matches
        .values_of("inc")
        .unwrap_or_default()
        .map(PathBuf::from)
        .collect();
    let defs: Vec<_> = matches.values_of("define").unwrap_or_default().collect();
    let headers: Vec<_> = matches.values_of("header").unwrap_or_default().collect();
    let listing_path = tmp_dir.path().join("listing.h");
    create_concatenated_header(&headers, &listing_path)?;
    let concat_path = tmp_dir.path().join("concat.h");
    announce_progress(&format!(
        "Preprocessing {:?} to {:?}",
        listing_path, concat_path
    ));
    preprocess(&listing_path, &concat_path, &incs, &defs)?;
    let rs_path = tmp_dir.path().join("input.rs");
    let directives: Vec<_> = std::iter::once("#include \"concat.h\"\n".to_string())
        .chain(
            matches
                .values_of("directive")
                .unwrap_or_default()
                .map(|s| format!("{}\n", s)),
        )
        .collect();
    create_rs_file(&rs_path, &directives)?;
    run_sample_gen_cmd(&rs_path, &tmp_dir.path())?;
    let interestingness_test = tmp_dir.path().join("test.sh");
    create_interestingness_test(
        &interestingness_test,
        matches.value_of("problem").unwrap(),
        &rs_path,
    )?;
    run_interestingness_test(&interestingness_test);
    run_creduce(
        matches.value_of("creduce").unwrap(),
        &interestingness_test,
        &concat_path,
        matches.values_of("creduce-args").unwrap_or_default(),
    );
    let output_path = matches.value_of("output");
    match output_path {
        None => print_minimized_case(&concat_path)?,
        Some(output_path) => {
            std::fs::copy(&concat_path, &PathBuf::from(output_path))?;
        }
    };
    Ok(())
}

fn announce_progress(msg: &str) {
    println!("=== {} ===", msg);
}

fn print_minimized_case(concat_path: &Path) -> Result<(), std::io::Error> {
    announce_progress("Completed. Minimized test case:");
    let contents = std::fs::read_to_string(concat_path)?;
    println!("{}", contents);
    Ok(())
}

fn run_creduce<'a>(
    creduce_cmd: &str,
    interestingness_test: &Path,
    concat_path: &Path,
    creduce_args: impl Iterator<Item = &'a str>,
) {
    announce_progress("creduce");
    Command::new(creduce_cmd)
        .arg(interestingness_test.to_str().unwrap())
        .arg(concat_path.to_str().unwrap())
        .args(creduce_args)
        .status()
        .expect("failed to creduce");
}

fn run_sample_gen_cmd(rs_file: &Path, tmp_dir: &Path) -> Result<(), std::io::Error> {
    let (gen_cmd, args) = format_gen_cmd(rs_file, tmp_dir.to_str().unwrap())?;
    let args = args.collect::<Vec<_>>();
    let args_str = args.join(" ");
    announce_progress(&format!(
        "Running sample gen cmd: {} {}",
        gen_cmd.to_str().unwrap(),
        args_str
    ));
    Command::new(gen_cmd).args(args).status()?;
    Ok(())
}

fn format_gen_cmd(
    rs_file: &Path,
    dir: &str,
) -> Result<(PathBuf, impl Iterator<Item = String>), std::io::Error> {
    let me = std::env::current_exe()?;
    let gen = me.parent().unwrap().join("autocxx-gen");
    let args = [
        "-o".to_string(),
        dir.to_string(),
        "-I".to_string(),
        dir.to_string(),
        rs_file.to_str().unwrap().to_string(),
        "--gen-rs-complete".to_string(),
    ]
    .to_vec();
    Ok((gen, args.into_iter()))
}

fn create_interestingness_test(
    test_path: &Path,
    problem: &str,
    rs_file: &Path,
) -> Result<(), std::io::Error> {
    announce_progress("Creating interestingness test");
    // Ensure we refer to the input header by relative path
    // because creduce will invoke us in some other directory with
    // a copy thereof.
    let (gen_cmd, mut args) = format_gen_cmd(rs_file, "$(pwd)")?;
    let args = args.join(" ");
    let content = format!(
        indoc! {"
        #!/bin/sh
        {} {} 2>&1 | grep \"{}\"  >/dev/null 2>&1
    "},
        gen_cmd.to_str().unwrap(),
        args,
        problem
    );
    println!("Interestingness test:\n{}", content);
    {
        let mut file = File::create(test_path)?;
        file.write_all(content.as_bytes())?;
    }

    let mut perms = std::fs::metadata(&test_path)?.permissions();
    perms.set_mode(0o700);
    std::fs::set_permissions(&test_path, perms)?;
    Ok(())
}

fn run_interestingness_test(test_path: &Path) {
    announce_progress("Running interestingness test");
    let status = Command::new(test_path).status().unwrap();
    announce_progress(&format!(
        "Have run interestingness test - result is {:?}",
        status.code()
    ));
}

fn create_rs_file(rs_path: &Path, directives: &[String]) -> Result<(), std::io::Error> {
    announce_progress("Creating Rust input file");
    let mut file = File::create(rs_path)?;
    file.write_all("use autocxx::include_cpp;\ninclude_cpp! (\n".as_bytes())?;
    for directive in directives {
        file.write_all(directive.as_bytes())?;
    }
    file.write_all(");\n".as_bytes())?;
    Ok(())
}

fn create_concatenated_header(headers: &[&str], listing_path: &Path) -> Result<(), std::io::Error> {
    announce_progress("Creating preprocessed header");
    let mut file = File::create(listing_path)?;
    for header in headers {
        file.write_all(format!("#include \"{}\"\n", header).as_bytes())?;
    }
    Ok(())
}
<commit_msg>Remove troublesome reduction pass.<commit_after>//! Crate to `#include` C++ headers in your Rust code, and generate
//! idiomatic bindings using `cxx`. See [include_cpp] for details.

// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::{
    fs::File,
    io::Write,
    os::unix::prelude::PermissionsExt,
    path::{Path, PathBuf},
    process::Command,
};

use autocxx_engine::preprocess;
use clap::{crate_authors, crate_version, App, Arg, ArgMatches};
use indoc::indoc;
use itertools::Itertools;
use tempfile::TempDir;

static LONG_HELP: &str = indoc! {"
Command line utility to minimize autocxx bug cases.

This is a wrapper for creduce.

Example command-line:
autocxx-reduce -I my-inc-dir -h my-header -d 'generate!(\"MyClass\")' -k -- --n 64 --remove-pass pass_line_markers
"};

fn main() {
    let matches = App::new("autocxx-reduce")
        .version(crate_version!())
        .author(crate_authors!())
        .about("Reduce a C++ test case")
        .long_about(LONG_HELP)
        .arg(
            Arg::with_name("inc")
                .short("I")
                .long("inc")
                .multiple(true)
                .number_of_values(1)
                .value_name("INCLUDE DIRS")
                .help("include path")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("define")
                .short("D")
                .long("define")
                .multiple(true)
                .number_of_values(1)
                .value_name("DEFINE")
                .help("macro definition")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("header")
                .short("h")
                .long("header")
                .multiple(true)
                .number_of_values(1)
                .required(true)
                .value_name("HEADER")
                .help("header file name")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("directive")
                .short("d")
                .long("directive")
                .multiple(true)
                .number_of_values(1)
                .value_name("DIRECTIVE")
                .help("directives to put within include_cpp!")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("problem")
                .short("p")
                .long("problem")
                .required(true)
                .value_name("PROBLEM")
                .help("problem string we're looking for")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("creduce")
                .long("creduce")
                .required(true)
                .value_name("PATH")
                .help("creduce binary location")
                .default_value("/usr/bin/creduce")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("output")
                .short("o")
                .long("output")
                .value_name("OUTPUT")
                .help("where to write minimized output")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("keep")
                .short("k")
                .long("keep-dir")
                .help("keep the temporary directory for debugging purposes"),
        )
        .arg(Arg::with_name("creduce-args").last(true).multiple(true))
        .get_matches();
    run(matches).unwrap();
}

fn run(matches: ArgMatches) -> Result<(), std::io::Error> {
    let keep_tmp = matches.is_present("keep");
    let tmp_dir = TempDir::new()?;
    let r = do_run(matches, &tmp_dir);
    if keep_tmp {
        println!(
            "Keeping temp dir created at: {}",
            tmp_dir.into_path().to_str().unwrap()
        );
    }
    r
}

fn do_run(matches: ArgMatches, tmp_dir: &TempDir) -> Result<(), std::io::Error> {
    let incs: Vec<_> = matches
        .values_of("inc")
        .unwrap_or_default()
        .map(PathBuf::from)
        .collect();
    let defs: Vec<_> = matches.values_of("define").unwrap_or_default().collect();
    let headers: Vec<_> = matches.values_of("header").unwrap_or_default().collect();
    let listing_path = tmp_dir.path().join("listing.h");
    create_concatenated_header(&headers, &listing_path)?;
    let concat_path = tmp_dir.path().join("concat.h");
    announce_progress(&format!(
        "Preprocessing {:?} to {:?}",
        listing_path, concat_path
    ));
    preprocess(&listing_path, &concat_path, &incs, &defs)?;
    let rs_path = tmp_dir.path().join("input.rs");
    let directives: Vec<_> = std::iter::once("#include \"concat.h\"\n".to_string())
        .chain(
            matches
                .values_of("directive")
                .unwrap_or_default()
                .map(|s| format!("{}\n", s)),
        )
        .collect();
    create_rs_file(&rs_path, &directives)?;
    run_sample_gen_cmd(&rs_path, &tmp_dir.path())?;
    let interestingness_test = tmp_dir.path().join("test.sh");
    create_interestingness_test(
        &interestingness_test,
        matches.value_of("problem").unwrap(),
        &rs_path,
    )?;
    run_interestingness_test(&interestingness_test);
    run_creduce(
        matches.value_of("creduce").unwrap(),
        &interestingness_test,
        &concat_path,
        matches.values_of("creduce-args").unwrap_or_default(),
    );
    let output_path = matches.value_of("output");
    match output_path {
        None => print_minimized_case(&concat_path)?,
        Some(output_path) => {
            std::fs::copy(&concat_path, &PathBuf::from(output_path))?;
        }
    };
    Ok(())
}

fn announce_progress(msg: &str) {
    println!("=== {} ===", msg);
}

fn print_minimized_case(concat_path: &Path) -> Result<(), std::io::Error> {
    announce_progress("Completed. Minimized test case:");
    let contents = std::fs::read_to_string(concat_path)?;
    println!("{}", contents);
    Ok(())
}

/// Arguments we always pass to creduce. This pass always seems to cause a crash
/// as far as I can tell, so always exclude it. It may be environment-dependent,
/// of course, but as I'm the primary user of this tool I am ruthlessly removing it.
const CREDUCE_STANDARD_ARGS: &[&str] = &["--remove-pass", "pass_line_markers"];

fn run_creduce<'a>(
    creduce_cmd: &str,
    interestingness_test: &Path,
    concat_path: &Path,
    creduce_args: impl Iterator<Item = &'a str>,
) {
    announce_progress("creduce");
    Command::new(creduce_cmd)
        .arg(interestingness_test.to_str().unwrap())
        .arg(concat_path.to_str().unwrap())
        .args(CREDUCE_STANDARD_ARGS)
        .args(creduce_args)
        .status()
        .expect("failed to creduce");
}

fn run_sample_gen_cmd(rs_file: &Path, tmp_dir: &Path) -> Result<(), std::io::Error> {
    let (gen_cmd, args) = format_gen_cmd(rs_file, tmp_dir.to_str().unwrap())?;
    let args = args.collect::<Vec<_>>();
    let args_str = args.join(" ");
    announce_progress(&format!(
        "Running sample gen cmd: {} {}",
        gen_cmd.to_str().unwrap(),
        args_str
    ));
    Command::new(gen_cmd).args(args).status()?;
    Ok(())
}

fn format_gen_cmd(
    rs_file: &Path,
    dir: &str,
) -> Result<(PathBuf, impl Iterator<Item = String>), std::io::Error> {
    let me = std::env::current_exe()?;
    let gen = me.parent().unwrap().join("autocxx-gen");
    let args = [
        "-o".to_string(),
        dir.to_string(),
        "-I".to_string(),
        dir.to_string(),
        rs_file.to_str().unwrap().to_string(),
        "--gen-rs-complete".to_string(),
    ]
    .to_vec();
    Ok((gen, args.into_iter()))
}

fn create_interestingness_test(
    test_path: &Path,
    problem: &str,
    rs_file: &Path,
) -> Result<(), std::io::Error> {
    announce_progress("Creating interestingness test");
    // Ensure we refer to the input header by relative path
    // because creduce will invoke us in some other directory with
    // a copy thereof.
    let (gen_cmd, mut args) = format_gen_cmd(rs_file, "$(pwd)")?;
    let args = args.join(" ");
    let content = format!(
        indoc! {"
        #!/bin/sh
        {} {} 2>&1 | grep \"{}\"  >/dev/null 2>&1
    "},
        gen_cmd.to_str().unwrap(),
        args,
        problem
    );
    println!("Interestingness test:\n{}", content);
    {
        let mut file = File::create(test_path)?;
        file.write_all(content.as_bytes())?;
    }

    let mut perms = std::fs::metadata(&test_path)?.permissions();
    perms.set_mode(0o700);
    std::fs::set_permissions(&test_path, perms)?;
    Ok(())
}

fn run_interestingness_test(test_path: &Path) {
    announce_progress("Running interestingness test");
    let status = Command::new(test_path).status().unwrap();
    announce_progress(&format!(
        "Have run interestingness test - result is {:?}",
        status.code()
    ));
}

fn create_rs_file(rs_path: &Path, directives: &[String]) -> Result<(), std::io::Error> {
    announce_progress("Creating Rust input file");
    let mut file = File::create(rs_path)?;
    file.write_all("use autocxx::include_cpp;\ninclude_cpp! (\n".as_bytes())?;
    for directive in directives {
        file.write_all(directive.as_bytes())?;
    }
    file.write_all(");\n".as_bytes())?;
    Ok(())
}

fn create_concatenated_header(headers: &[&str], listing_path: &Path) -> Result<(), std::io::Error> {
    announce_progress("Creating preprocessed header");
    let mut file = File::create(listing_path)?;
    for header in headers {
        file.write_all(format!("#include \"{}\"\n", header).as_bytes())?;
    }
    Ok(())
}
<|endoftext|>