<commit_before>use crate::{
    aws::Aws,
    cluster::Cluster,
    health::{Commit, Event, ValidatorEvent},
    util::unix_timestamp_now,
};
use failure::{self, prelude::*};
use flate2::read::GzDecoder;
use regex::Regex;
use rusoto_kinesis::{GetRecordsInput, GetShardIteratorInput, Kinesis, Record};
use serde_json::{self, value as json};
use std::{
    collections::HashMap,
    env,
    fs::File,
    io::Write,
    sync::{
        atomic::{AtomicI64, Ordering},
        mpsc, Arc,
    },
    thread,
    time::{Duration, Instant},
};

pub struct AwsLogTail {
    pub event_receiver: mpsc::Receiver<ValidatorEvent>,
    pending_messages: Arc<AtomicI64>,
}

struct AwsLogThread {
    aws: Aws,
    kinesis_iterator: String,
    event_sender: mpsc::Sender<ValidatorEvent>,
    startup_sender: Option<mpsc::Sender<()>>,
    event_log_file: Option<File>,
    last_seen_ts_by_validator: HashMap<String, Duration>,
    expected_validators_count: usize,
    pending_messages: Arc<AtomicI64>,

    re_commit: Regex,
    re_validator: Regex,
    re_started: Regex,
}

struct LogEntry<'a> {
    validator: String,
    message: &'a str,
    timestamp: Duration,
}

impl AwsLogTail {
    pub fn spawn_new(aws: Aws, cluster: &Cluster) -> failure::Result<Self> {
        let kinesis_iterator = Self::make_kinesis_iterator(&aws)?;
        let (event_sender, event_receiver) = mpsc::channel();
        // We block this method until first request to kinesis completed
        let (startup_sender, startup_receiver) = mpsc::channel();
        let event_log_file = match env::var("EVENT_LOG") {
            Ok(f) => Some(File::create(f).expect("Can't create EVENT_LOG file")),
            Err(..) => None,
        };
        let pending_messages = Arc::new(AtomicI64::new(0));
        let aws_log_thread = AwsLogThread {
            aws,
            kinesis_iterator,
            event_sender,
            startup_sender: Some(startup_sender),
            event_log_file,
            last_seen_ts_by_validator: HashMap::new(),
            expected_validators_count: cluster.instances().len(),
            pending_messages: pending_messages.clone(),

            re_commit: Regex::new(
                r"Committed.+\[id: ([a-z0-9]+), round: ([a-z0-9]+), parent_id: ([a-z0-9]+)]",
            )
            .unwrap(),
            re_validator: Regex::new(r"^validator-([0-9a-z]+)/").unwrap(),
            re_started: Regex::new(r"Chained BFT SMR started.$").unwrap(),
        };
        let builder = thread::Builder::new();
        builder
            .name("aws-log-tail".into())
            .spawn(move || aws_log_thread.run())
            .unwrap();
        startup_receiver
            .recv()
            .expect("Aws log tail thread died after first request");
        Ok(AwsLogTail {
            event_receiver,
            pending_messages: pending_messages.clone(),
        })
    }

    pub fn recv_all_until_deadline(&self, deadline: Instant) -> Vec<ValidatorEvent> {
        let mut events = vec![];
        while Instant::now() < deadline {
            match self.event_receiver.try_recv() {
                Ok(event) => events.push(event),
                Err(..) => thread::sleep(Duration::from_millis(1)),
            }
        }
        let events_count = events.len() as i64;
        let prev = self
            .pending_messages
            .fetch_sub(events_count, Ordering::Relaxed);
        let pending = prev - events_count;
        let now = unix_timestamp_now();
        if let Some(last) = events.last() {
            println!(
                "{} Last event delay: {}, pending {}",
                now.as_millis(),
                (now - last.received_timestamp).as_millis(),
                pending
            );
        } else {
            println!("{} No events", now.as_millis());
        }
        events
    }

    pub fn recv_all(&self) -> Vec<ValidatorEvent> {
        let mut events = vec![];
        while let Ok(event) = self.event_receiver.try_recv() {
            self.pending_messages.fetch_sub(1, Ordering::Relaxed);
            events.push(event);
        }
        events
    }

    fn log_start_offset_sec() -> u64 {
        let default = 8;
        match env::var("LOG_START_OFFSET") {
            Ok(s) => s.parse().expect("LOG_START_OFFSET env is not a number"),
            Err(..) => default,
        }
    }

    fn make_kinesis_iterator(aws: &Aws) -> failure::Result<String> {
        let timestamp = unix_timestamp_now();
        let timestamp = timestamp - Duration::from_secs(Self::log_start_offset_sec());
        let stream_name = match env::var("KINESIS_STREAM") {
            Err(..) => format!("{}-RecipientStream", aws.workplace()),
            Ok(s) => s,
        };
        let response = aws
            .kc()
            .get_shard_iterator(GetShardIteratorInput {
                shard_id: "0".into(),
                shard_iterator_type: "AT_TIMESTAMP".into(),
                stream_name,
                starting_sequence_number: None,
                timestamp: Some(timestamp.as_secs() as f64),
            })
            .sync()?;
        if let Some(shard_iterator) = response.shard_iterator {
            Ok(shard_iterator)
        } else {
            Err(format_err!(
                "Can not make kinesis iterator - empty response"
            ))
        }
    }
}

impl AwsLogThread {
    fn run(mut self) {
        let startup_timeout_sec = match env::var("STARTUP_TIMEOUT") {
            Err(..) => 5u64,
            Ok(v) => v.parse().expect("Failed to parse STARTUP_TIMEOUT env"),
        };
        let startup_deadline = Instant::now() + Duration::from_secs(startup_timeout_sec);
        loop {
            let response = self
                .aws
                .kc()
                .get_records(GetRecordsInput {
                    shard_iterator: self.kinesis_iterator.clone(),
                    limit: Some(10000),
                })
                .sync()
                .expect("Request to kinesis failed");
            let next_iterator = response
                .next_shard_iterator
                .expect("Next iterator is expected for kinesis stream");
            let records = response.records;
            let millis_behind = response
                .millis_behind_latest
                .expect("no millis_behind_latest in kinesis response");
            self.kinesis_iterator = next_iterator;
            self.write_event_log(format!(
                "Kinesis response {} records, millis behind: {}",
                records.len(),
                millis_behind,
            ));
            for record in records {
                self.handle_kinesis_record(record)
                    .expect("Failed to process aws record");
            }
            if self.startup_sender.is_some() && millis_behind == 0 {
                if self.last_seen_ts_by_validator.len() >= self.expected_validators_count {
                    println!("Received events from all validators");
                    self.startup_sender
                        .take()
                        .unwrap()
                        .send(())
                        .expect("Startup receiver dropped");
                } else if Instant::now() > startup_deadline {
                    println!("Aws log startup deadline reached");
                    self.startup_sender
                        .take()
                        .unwrap()
                        .send(())
                        .expect("Startup receiver dropped");
                }
            }
            thread::sleep(Duration::from_millis(300))
        }
    }

    fn handle_kinesis_record(&mut self, record: Record) -> failure::Result<()> {
        let decoder = GzDecoder::new(&record.data[..]);
        let json: json::Value = serde_json::from_reader(decoder)?;
        let log_stream = json
            .get("logStream")
            .expect("No logStream in kinesis event")
            .as_str()
            .expect("logStream in kinesis event is not a string");
        let validator = self.log_stream_to_validator_short_str(log_stream);
        let validator = if let Some(validator) = validator {
            validator
        } else {
            return Ok(());
        };
        let events = json
            .get("logEvents")
            .expect("No logEvents in kinesis event");
        let events = events
            .as_array()
            .expect("logEvents in kinesis event is not array");
        self.write_event_log(format!(
            "Kinesis record for {}, {} events",
            validator,
            events.len()
        ));
        let now = unix_timestamp_now();
        let mut behind_max = 0u128;
        let mut behind_sum = 0u128;
        let mut count = 0u128;
        for event in events {
            let message = event.get("message").expect("No message in kinesis event");
            let message = message
                .as_str()
                .expect("message in kinesis event is not a string");
            let timestamp = event
                .get("timestamp")
                .expect("No timestamp in kinesis event");
            let timestamp = timestamp
                .as_u64()
                .expect("timestamp in kinesis event is not a u64");
            let timestamp = Duration::from_millis(timestamp);
            if now > timestamp {
                let behind_millis = (now - timestamp).as_millis();
                if behind_millis > behind_max {
                    behind_max = behind_millis;
                }
                behind_sum += behind_millis;
            }
            count += 1;
            let log_entry = LogEntry {
                validator: validator.clone(),
                message,
                timestamp,
            };
            self.handle_log_entry(log_entry);
        }
        if behind_max > 15_000 {
            println!(
                "Logs behind avg={} ms, max={} ms; {} entries",
                behind_sum / count,
                behind_max,
                count
            );
        }
        Ok(())
    }

    fn write_event_log(&mut self, s: String) {
        let event_log_file = if let Some(ref mut event_log_file) = self.event_log_file {
            event_log_file
        } else {
            return;
        };
        let now = unix_timestamp_now().as_millis();
        writeln!(event_log_file, "{} {}", now, s).expect("Can't write to EVENT_LOG file");
        event_log_file.flush().expect("Can't flush EVENT_LOG file");
    }

    fn handle_log_entry(&mut self, e: LogEntry) {
        let received_timestamp = unix_timestamp_now();
        let event = self.parse_log_entry(&e);
        if let Some(event) = event {
            let ve = ValidatorEvent {
                received_timestamp,
                validator: e.validator,
                timestamp: e.timestamp,
                event,
            };
            // In rare cases Kinesis delivers messages out of order
            // We are ignoring them so that invariant in CommitHistoryHealthCheck does not break
            let skip = if let Some(last) = self.last_seen_ts_by_validator.get(&ve.validator) {
                *last > ve.timestamp
            } else {
                false
            };
            let skip_msg = if skip { "; [OUT_OF_ORDER]" } else { "" };
            self.write_event_log(format!("{:?}{}", ve, skip_msg));
            if !skip {
                self.last_seen_ts_by_validator
                    .insert(ve.validator.clone(), ve.timestamp);
                self.pending_messages.fetch_add(1, Ordering::Relaxed);
                let _ignore = self.event_sender.send(ve);
            }
        }
    }

    fn parse_log_entry(&self, e: &LogEntry) -> Option<Event> {
        let commit = self.parse_commit_log_entry(&e);
        if let Some(commit) = commit {
            return Some(Event::Commit(commit));
        }
        let consensus_started = self.parse_consensus_started_log_entry(&e);
        if consensus_started.is_some() {
            return Some(Event::ConsensusStarted);
        }
        None
    }

    fn log_stream_to_validator_short_str(&self, s: &str) -> Option<String> {
        let cap = self.re_validator.captures(s);
        if let Some(cap) = cap {
            Some(cap[1].into())
        } else {
            None
        }
    }

    fn parse_commit_log_entry(&self, e: &LogEntry) -> Option<Commit> {
        let cap = match self.re_commit.captures(e.message) {
            Some(cap) => cap,
            None => return None,
        };
        let commit = &cap[1];
        let round = &cap[2];
        let parent = &cap[3];
        let round = match round.parse::<u64>() {
            Ok(round) => round,
            Err(..) => return None,
        };
        Some(Commit {
            commit: commit.into(),
            round,
            parent: parent.into(),
        })
    }

    fn parse_consensus_started_log_entry(&self, e: &LogEntry) -> Option<()> {
        if self.re_started.is_match(e.message) {
            Some(())
        } else {
            None
        }
    }
}
<commit_msg>[cluster_test] Retry kinesis failures<commit_after>use crate::{
    aws::Aws,
    cluster::Cluster,
    health::{Commit, Event, ValidatorEvent},
    util::unix_timestamp_now,
};
use failure::{self, prelude::*};
use flate2::read::GzDecoder;
use regex::Regex;
use rusoto_kinesis::{GetRecordsInput, GetShardIteratorInput, Kinesis, Record};
use serde_json::{self, value as json};
use std::{
    collections::HashMap,
    env,
    fs::File,
    io::Write,
    sync::{
        atomic::{AtomicI64, Ordering},
        mpsc, Arc,
    },
    thread,
    time::{Duration, Instant},
};

pub struct AwsLogTail {
    pub event_receiver: mpsc::Receiver<ValidatorEvent>,
    pending_messages: Arc<AtomicI64>,
}

struct AwsLogThread {
    aws: Aws,
    kinesis_iterator: String,
    event_sender: mpsc::Sender<ValidatorEvent>,
    startup_sender: Option<mpsc::Sender<()>>,
    event_log_file: Option<File>,
    last_seen_ts_by_validator: HashMap<String, Duration>,
    expected_validators_count: usize,
    pending_messages: Arc<AtomicI64>,

    re_commit: Regex,
    re_validator: Regex,
    re_started: Regex,
}

struct LogEntry<'a> {
    validator: String,
    message: &'a str,
    timestamp: Duration,
}

impl AwsLogTail {
    pub fn spawn_new(aws: Aws, cluster: &Cluster) -> failure::Result<Self> {
        let kinesis_iterator = Self::make_kinesis_iterator(&aws)?;
        let (event_sender, event_receiver) = mpsc::channel();
        // We block this method until first request to kinesis completed
        let (startup_sender, startup_receiver) = mpsc::channel();
        let event_log_file = match env::var("EVENT_LOG") {
            Ok(f) => Some(File::create(f).expect("Can't create EVENT_LOG file")),
            Err(..) => None,
        };
        let pending_messages = Arc::new(AtomicI64::new(0));
        let aws_log_thread = AwsLogThread {
            aws,
            kinesis_iterator,
            event_sender,
            startup_sender: Some(startup_sender),
            event_log_file,
            last_seen_ts_by_validator: HashMap::new(),
            expected_validators_count: cluster.instances().len(),
            pending_messages: pending_messages.clone(),

            re_commit: Regex::new(
                r"Committed.+\[id: ([a-z0-9]+), round: ([a-z0-9]+), parent_id: ([a-z0-9]+)]",
            )
            .unwrap(),
            re_validator: Regex::new(r"^validator-([0-9a-z]+)/").unwrap(),
            re_started: Regex::new(r"Chained BFT SMR started.$").unwrap(),
        };
        let builder = thread::Builder::new();
        builder
            .name("aws-log-tail".into())
            .spawn(move || aws_log_thread.run())
            .unwrap();
        startup_receiver
            .recv()
            .expect("Aws log tail thread died after first request");
        Ok(AwsLogTail {
            event_receiver,
            pending_messages: pending_messages.clone(),
        })
    }

    pub fn recv_all_until_deadline(&self, deadline: Instant) -> Vec<ValidatorEvent> {
        let mut events = vec![];
        while Instant::now() < deadline {
            match self.event_receiver.try_recv() {
                Ok(event) => events.push(event),
                Err(..) => thread::sleep(Duration::from_millis(1)),
            }
        }
        let events_count = events.len() as i64;
        let prev = self
            .pending_messages
            .fetch_sub(events_count, Ordering::Relaxed);
        let pending = prev - events_count;
        let now = unix_timestamp_now();
        if let Some(last) = events.last() {
            println!(
                "{} Last event delay: {}, pending {}",
                now.as_millis(),
                (now - last.received_timestamp).as_millis(),
                pending
            );
        } else {
            println!("{} No events", now.as_millis());
        }
        events
    }

    pub fn recv_all(&self) -> Vec<ValidatorEvent> {
        let mut events = vec![];
        while let Ok(event) = self.event_receiver.try_recv() {
            self.pending_messages.fetch_sub(1, Ordering::Relaxed);
            events.push(event);
        }
        events
    }

    fn log_start_offset_sec() -> u64 {
        let default = 8;
        match env::var("LOG_START_OFFSET") {
            Ok(s) => s.parse().expect("LOG_START_OFFSET env is not a number"),
            Err(..) => default,
        }
    }

    fn make_kinesis_iterator(aws: &Aws) -> failure::Result<String> {
        let timestamp = unix_timestamp_now();
        let timestamp = timestamp - Duration::from_secs(Self::log_start_offset_sec());
        let stream_name = match env::var("KINESIS_STREAM") {
            Err(..) => format!("{}-RecipientStream", aws.workplace()),
            Ok(s) => s,
        };
        let response = aws
            .kc()
            .get_shard_iterator(GetShardIteratorInput {
                shard_id: "0".into(),
                shard_iterator_type: "AT_TIMESTAMP".into(),
                stream_name,
                starting_sequence_number: None,
                timestamp: Some(timestamp.as_secs() as f64),
            })
            .sync()?;
        if let Some(shard_iterator) = response.shard_iterator {
            Ok(shard_iterator)
        } else {
            Err(format_err!(
                "Can not make kinesis iterator - empty response"
            ))
        }
    }
}

impl AwsLogThread {
    fn run(mut self) {
        let startup_timeout_sec = match env::var("STARTUP_TIMEOUT") {
            Err(..) => 5u64,
            Ok(v) => v.parse().expect("Failed to parse STARTUP_TIMEOUT env"),
        };
        let startup_deadline = Instant::now() + Duration::from_secs(startup_timeout_sec);
        let mut kinesis_fail_retries = 0usize;
        loop {
            let response = self
                .aws
                .kc()
                .get_records(GetRecordsInput {
                    shard_iterator: self.kinesis_iterator.clone(),
                    limit: Some(10000),
                })
                .sync();
            let response = match response {
                Err(e) => {
                    println!("Kinesis failure: {:?}, retry {}", e, kinesis_fail_retries);
                    kinesis_fail_retries += 1;
                    if kinesis_fail_retries > 10 {
                        panic!("Too many kinesis failures");
                    }
                    thread::sleep(Duration::from_secs(1));
                    continue;
                }
                Ok(r) => r,
            };
            kinesis_fail_retries = 0;
            let next_iterator = response
                .next_shard_iterator
                .expect("Next iterator is expected for kinesis stream");
            let records = response.records;
            let millis_behind = response
                .millis_behind_latest
                .expect("no millis_behind_latest in kinesis response");
            self.kinesis_iterator = next_iterator;
            self.write_event_log(format!(
                "Kinesis response {} records, millis behind: {}",
                records.len(),
                millis_behind,
            ));
            for record in records {
                self.handle_kinesis_record(record)
                    .expect("Failed to process aws record");
            }
            if self.startup_sender.is_some() && millis_behind == 0 {
                if self.last_seen_ts_by_validator.len() >= self.expected_validators_count {
                    println!("Received events from all validators");
                    self.startup_sender
                        .take()
                        .unwrap()
                        .send(())
                        .expect("Startup receiver dropped");
                } else if Instant::now() > startup_deadline {
                    println!("Aws log startup deadline reached");
                    self.startup_sender
                        .take()
                        .unwrap()
                        .send(())
                        .expect("Startup receiver dropped");
                }
            }
            thread::sleep(Duration::from_millis(300))
        }
    }

    fn handle_kinesis_record(&mut self, record: Record) -> failure::Result<()> {
        let decoder = GzDecoder::new(&record.data[..]);
        let json: json::Value = serde_json::from_reader(decoder)?;
        let log_stream = json
            .get("logStream")
            .expect("No logStream in kinesis event")
            .as_str()
            .expect("logStream in kinesis event is not a string");
        let validator = self.log_stream_to_validator_short_str(log_stream);
        let validator = if let Some(validator) = validator {
            validator
        } else {
            return Ok(());
        };
        let events = json
            .get("logEvents")
            .expect("No logEvents in kinesis event");
        let events = events
            .as_array()
            .expect("logEvents in kinesis event is not array");
        self.write_event_log(format!(
            "Kinesis record for {}, {} events",
            validator,
            events.len()
        ));
        let now = unix_timestamp_now();
        let mut behind_max = 0u128;
        let mut behind_sum = 0u128;
        let mut count = 0u128;
        for event in events {
            let message = event.get("message").expect("No message in kinesis event");
            let message = message
                .as_str()
                .expect("message in kinesis event is not a string");
            let timestamp = event
                .get("timestamp")
                .expect("No timestamp in kinesis event");
            let timestamp = timestamp
                .as_u64()
                .expect("timestamp in kinesis event is not a u64");
            let timestamp = Duration::from_millis(timestamp);
            if now > timestamp {
                let behind_millis = (now - timestamp).as_millis();
                if behind_millis > behind_max {
                    behind_max = behind_millis;
                }
                behind_sum += behind_millis;
            }
            count += 1;
            let log_entry = LogEntry {
                validator: validator.clone(),
                message,
                timestamp,
            };
            self.handle_log_entry(log_entry);
        }
        if behind_max > 15_000 {
            println!(
                "Logs behind avg={} ms, max={} ms; {} entries",
                behind_sum / count,
                behind_max,
                count
            );
        }
        Ok(())
    }

    fn write_event_log(&mut self, s: String) {
        let event_log_file = if let Some(ref mut event_log_file) = self.event_log_file {
            event_log_file
        } else {
            return;
        };
        let now = unix_timestamp_now().as_millis();
        writeln!(event_log_file, "{} {}", now, s).expect("Can't write to EVENT_LOG file");
        event_log_file.flush().expect("Can't flush EVENT_LOG file");
    }

    fn handle_log_entry(&mut self, e: LogEntry) {
        let received_timestamp = unix_timestamp_now();
        let event = self.parse_log_entry(&e);
        if let Some(event) = event {
            let ve = ValidatorEvent {
                received_timestamp,
                validator: e.validator,
                timestamp: e.timestamp,
                event,
            };
            // In rare cases Kinesis delivers messages out of order
            // We are ignoring them so that invariant in CommitHistoryHealthCheck does not break
            let skip = if let Some(last) = self.last_seen_ts_by_validator.get(&ve.validator) {
                *last > ve.timestamp
            } else {
                false
            };
            let skip_msg = if skip { "; [OUT_OF_ORDER]" } else { "" };
            self.write_event_log(format!("{:?}{}", ve, skip_msg));
            if !skip {
                self.last_seen_ts_by_validator
                    .insert(ve.validator.clone(), ve.timestamp);
                self.pending_messages.fetch_add(1, Ordering::Relaxed);
                let _ignore = self.event_sender.send(ve);
            }
        }
    }

    fn parse_log_entry(&self, e: &LogEntry) -> Option<Event> {
        let commit = self.parse_commit_log_entry(&e);
        if let Some(commit) = commit {
            return Some(Event::Commit(commit));
        }
        let consensus_started = self.parse_consensus_started_log_entry(&e);
        if consensus_started.is_some() {
            return Some(Event::ConsensusStarted);
        }
        None
    }

    fn log_stream_to_validator_short_str(&self, s: &str) -> Option<String> {
        let cap = self.re_validator.captures(s);
        if let Some(cap) = cap {
            Some(cap[1].into())
        } else {
            None
        }
    }

    fn parse_commit_log_entry(&self, e: &LogEntry) -> Option<Commit> {
        let cap = match self.re_commit.captures(e.message) {
            Some(cap) => cap,
            None => return None,
        };
        let commit = &cap[1];
        let round = &cap[2];
        let parent = &cap[3];
        let round = match round.parse::<u64>() {
            Ok(round) => round,
            Err(..) => return None,
        };
        Some(Commit {
            commit: commit.into(),
            round,
            parent: parent.into(),
        })
    }

    fn parse_consensus_started_log_entry(&self, e: &LogEntry) -> Option<()> {
        if self.re_started.is_match(e.message) {
            Some(())
        } else {
            None
        }
    }
}
<|endoftext|>


<commit_before>use std::fs::OpenOptions;
use std::io::{Write, BufReader, BufRead, Seek, SeekFrom};
use std::path::PathBuf;

use errors::DecodeError;
use crc32;
use constants::{NUL, CR, LF, ESCAPE, SPACE, DEFAULT_LINE_SIZE};

#[derive(Default,Debug)]
struct MetaData {
    name: Option<String>,
    line_length: Option<u16>,
    size: Option<usize>,
    crc32: Option<u32>,
    part: Option<u32>,
    begin: Option<usize>,
    end: Option<usize>,
}

/// Decodes the input file in a new output file.
///
/// If ok, returns the path of the decoded file.
///
/// # Example
/// ```rust,no_run
/// yenc::ydecode_file("test2.bin.yenc", "test2.bin");
/// ```
/// # Errors
/// - when the output file already exists
///
pub fn ydecode_file(input_filename: &str, output_path: &str) -> Result<String, DecodeError> {
    let input_file = OpenOptions::new().read(true).open(input_filename)?;
    let mut output_pathbuf = PathBuf::new();
    output_pathbuf.push(output_path);
    let mut rdr = BufReader::new(input_file);
    let mut line_buf = Vec::<u8>::with_capacity(2 * DEFAULT_LINE_SIZE as usize);
    let mut checksum = crc32::Crc32::new();
    let mut yenc_block_found = false;

    while !yenc_block_found {
        line_buf.clear();
        let length = rdr.read_until(LF, &mut line_buf)?;
        if length == 0 {
            break;
        }
        if line_buf.starts_with(b"=ybegin ") {
            yenc_block_found = true;
        }
    }

    if yenc_block_found {
        // parse header line and determine output filename
        let mut metadata = parse_header_line(&line_buf, 8)?;
        output_pathbuf.push(metadata.name.unwrap().to_string().trim());
        let mut output_file =
            OpenOptions::new().create(true).write(true).open(output_pathbuf.as_path())?;

        let mut footer_found = false;
        while !footer_found {
            line_buf.clear();
            let length = rdr.read_until(LF, &mut line_buf)?;
            if length == 0 {
                break;
            }
            if line_buf.starts_with(b"=ypart ") {
                let part_metadata = parse_header_line(&line_buf, 7)?;
                metadata.part = part_metadata.part;
                metadata.begin = part_metadata.begin;
                metadata.end = part_metadata.end;
                if let Some(begin) = metadata.begin {
                    output_file.seek(SeekFrom::Start((begin - 1) as u64))?;
                }
            } else if line_buf.starts_with(b"=yend ") {
                footer_found = true;
            } else {
                let decoded = ydecode_buffer(&line_buf[0..length])?;
                checksum.update_with_slice(decoded.as_slice());
                output_file.write_all(decoded.as_slice())?;
            }
        }
        if footer_found {
            let metadata = parse_header_line(&line_buf, 6)?;
            println!("{:?}", metadata);
            if let Some(expected_size) = metadata.size {
                if expected_size != checksum.num_bytes {
                    return Err(DecodeError::IncompleteData {
                        expected_size: expected_size,
                        actual_size: checksum.num_bytes,
                    });
                }
            }
            if let Some(expected_crc) = metadata.crc32 {
                if expected_crc != checksum.crc {
                    return Err(DecodeError::InvalidChecksum);
                }
            }
        } else if let Some(expected_size) = metadata.size {
            if expected_size != checksum.num_bytes {
                return Err(DecodeError::IncompleteData {
                    expected_size: expected_size,
                    actual_size: checksum.num_bytes,
                });
            }
        }

    }
    Ok(output_pathbuf.to_str().unwrap().to_string())
}

/// Decode the yEncoded byte slice into a vector of bytes.
/// Carriage Return (CR) and Line Feed (LF) are ignored.
pub fn ydecode_buffer(input: &[u8]) -> Result<Vec<u8>, DecodeError> {
    // TODO remove heap allocation
    let mut output = Vec::<u8>::with_capacity((input.len() as f64 * 1.02) as usize);
    let mut iter = input.iter();
    while let Some(byte) = iter.next() {
        let mut byte = *byte;
        match byte {
            NUL | CR | LF => {
                // for now, just continue
                continue;
            }
            ESCAPE => {
                match iter.next() {
                    Some(b) => {
                        byte = b.overflowing_sub(64).0;
                    }
                    None => {
                        // for now, just continue, the only
                        continue;
                    }
                }
            }
            _ => {}
        }
        output.push(byte.overflowing_sub(42).0);
    }
    Ok(output)
}

fn parse_header_line(line_buf: &[u8], offset: usize) -> Result<MetaData, DecodeError> {
    #[derive(Debug)]
    enum State {
        Keyword,
        Value,
        End,
    };

    let mut metadata: MetaData = Default::default();
    let mut state = State::Keyword;
    let mut keyword = Vec::<u8>::with_capacity(6);
    let mut value = Vec::<u8>::with_capacity(96);

    let header_line = String::from_utf8_lossy(line_buf).to_string();
    for (i, &c) in line_buf[offset..].iter().enumerate() {
        let position = i + offset;
        match state {
            State::End => unreachable!(),
            State::Keyword => {
                match c {
                    b'a'...b'z' | b'0'...b'9' => keyword.push(c),
                    b'=' => {
                        if keyword.is_empty() || !is_known_keyword(&keyword) {
                            return Err(DecodeError::InvalidHeader {
                                line: header_line,
                                position: position,
                            });
                        } else {
                            state = State::Value;
                        }
                    }
                    CR | LF => {}
                    _ => {
                        return Err(DecodeError::InvalidHeader {
                            line: header_line,
                            position: position,
                        });
                    }
                }
            }
            State::Value => {
                match keyword.as_slice() {
                    b"name" => {
                        match c {
                            CR => {}
                            LF => {
                                state = State::End;
                                metadata.name = Some(String::from_utf8_lossy(&value).to_string());
                            }
                            _ => value.push(c),
                        }
                    }
                    b"size" => {
                        match c {
                            b'0'...b'9' => value.push(c),
                            SPACE => {
                                metadata.size =
                                    Some(usize::from_str_radix(&String::from_utf8_lossy(&value),
                                                               10)
                                        .unwrap());
                                state = State::Keyword;
                                keyword.clear();
                                value.clear();
                            }
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    b"begin" => {
                        match c {
                            b'0'...b'9' => value.push(c),
                            SPACE => {
                                metadata.begin =
                                    Some(usize::from_str_radix(&String::from_utf8_lossy(&value),
                                                               10)
                                        .unwrap());
                                state = State::Keyword;
                                keyword.clear();
                                value.clear();
                            }
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    b"end" => {
                        match c {
                            b'0'...b'9' => value.push(c),
                            SPACE | LF | CR => {
                                metadata.end =
                                    Some(usize::from_str_radix(&String::from_utf8_lossy(&value),
                                                               10)
                                        .unwrap());
                                state = State::Keyword;
                                keyword.clear();
                                value.clear();
                            }
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    b"line" => {
                        match c {
                            b'0'...b'9' => value.push(c),
                            SPACE => {
                                metadata.line_length =
                                    Some(u16::from_str_radix(&String::from_utf8_lossy(&value), 10)
                                        .unwrap());
                                state = State::Keyword;
                                keyword.clear();
                                value.clear();
                            }
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    b"part" => {
                        match c {
                            b'0'...b'9' => value.push(c),
                            SPACE => {
                                metadata.part =
                                    Some(u32::from_str_radix(&String::from_utf8_lossy(&value), 10)
                                        .unwrap());
                                state = State::Keyword;
                                keyword.clear();
                                value.clear();
                            }
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    b"crc32" | b"pcrc32" => {
                        match c {
                            b'0'...b'9' | b'A'...b'F' | b'a'...b'f' => value.push(c),
                            SPACE => {
                                state = State::Keyword;
                                metadata.crc32 =
                                    Some(u32::from_str_radix(&String::from_utf8_lossy(&value), 16)
                                        .unwrap());
                                keyword.clear();
                                value.clear();
                            }
                            LF => {
                                state = State::End;
                                metadata.crc32 =
                                    Some(u32::from_str_radix(&String::from_utf8_lossy(&value), 16)
                                        .unwrap());
                                keyword.clear();
                                value.clear();
                            }
                            CR => {}
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    _ => unreachable!(),
                }
            }
        };
    }
    Ok(metadata)
}

fn is_known_keyword(keyword_slice: &[u8]) -> bool {
    keyword_slice == b"name" || keyword_slice == b"line" || keyword_slice == b"size" ||
    keyword_slice == b"part" || keyword_slice == b"begin" ||
    keyword_slice == b"end" || keyword_slice == b"pcrc32" ||
    keyword_slice == b"crc32"
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn decode_invalid() {
        assert!(ydecode_buffer(&[b'=']).unwrap().is_empty());
    }

    #[test]
    fn decode_valid_ff() {
        assert_eq!(&vec![0xff - 0x2A], &ydecode_buffer(&[0xff]).unwrap());
    }

    #[test]
    fn decode_valid_01() {
        assert_eq!(&vec![0xff - 0x28], &ydecode_buffer(&[0x01]).unwrap());
    }

    #[test]
    fn decode_valid_esc_ff() {
        assert_eq!(&vec![0xff - 0x40 - 0x2A],
                   &ydecode_buffer(&[b'=', 0xff]).unwrap());
    }

    #[test]
    fn decode_valid_esc_01() {
        assert_eq!(&vec![0xff - 0x40 - 0x2A + 2],
                   &ydecode_buffer(&[b'=', 0x01]).unwrap());
    }
}
<commit_msg>Remove debug printing<commit_after>use std::fs::OpenOptions;
use std::io::{Write, BufReader, BufRead, Seek, SeekFrom};
use std::path::PathBuf;

use errors::DecodeError;
use crc32;
use constants::{NUL, CR, LF, ESCAPE, SPACE, DEFAULT_LINE_SIZE};

#[derive(Default,Debug)]
struct MetaData {
    name: Option<String>,
    line_length: Option<u16>,
    size: Option<usize>,
    crc32: Option<u32>,
    part: Option<u32>,
    begin: Option<usize>,
    end: Option<usize>,
}

/// Decodes the input file in a new output file.
///
/// If ok, returns the path of the decoded file.
///
/// # Example
/// ```rust,no_run
/// yenc::ydecode_file("test2.bin.yenc", "test2.bin");
/// ```
/// # Errors
/// - when the output file already exists
///
pub fn ydecode_file(input_filename: &str, output_path: &str) -> Result<String, DecodeError> {
    let input_file = OpenOptions::new().read(true).open(input_filename)?;
    let mut output_pathbuf = PathBuf::new();
    output_pathbuf.push(output_path);
    let mut rdr = BufReader::new(input_file);
    let mut line_buf = Vec::<u8>::with_capacity(2 * DEFAULT_LINE_SIZE as usize);
    let mut checksum = crc32::Crc32::new();
    let mut yenc_block_found = false;

    while !yenc_block_found {
        line_buf.clear();
        let length = rdr.read_until(LF, &mut line_buf)?;
        if length == 0 {
            break;
        }
        if line_buf.starts_with(b"=ybegin ") {
            yenc_block_found = true;
        }
    }

    if yenc_block_found {
        // parse header line and determine output filename
        let mut metadata = parse_header_line(&line_buf, 8)?;
        output_pathbuf.push(metadata.name.unwrap().to_string().trim());
        let mut output_file =
            OpenOptions::new().create(true).write(true).open(output_pathbuf.as_path())?;

        let mut footer_found = false;
        while !footer_found {
            line_buf.clear();
            let length = rdr.read_until(LF, &mut line_buf)?;
            if length == 0 {
                break;
            }
            if line_buf.starts_with(b"=ypart ") {
                let part_metadata = parse_header_line(&line_buf, 7)?;
                metadata.part = part_metadata.part;
                metadata.begin = part_metadata.begin;
                metadata.end = part_metadata.end;
                if let Some(begin) = metadata.begin {
                    output_file.seek(SeekFrom::Start((begin - 1) as u64))?;
                }
            } else if line_buf.starts_with(b"=yend ") {
                footer_found = true;
            } else {
                let decoded = ydecode_buffer(&line_buf[0..length])?;
                checksum.update_with_slice(decoded.as_slice());
                output_file.write_all(decoded.as_slice())?;
            }
        }
        if footer_found {
            let metadata = parse_header_line(&line_buf, 6)?;
            if let Some(expected_size) = metadata.size {
                if expected_size != checksum.num_bytes {
                    return Err(DecodeError::IncompleteData {
                        expected_size: expected_size,
                        actual_size: checksum.num_bytes,
                    });
                }
            }
            if let Some(expected_crc) = metadata.crc32 {
                if expected_crc != checksum.crc {
                    return Err(DecodeError::InvalidChecksum);
                }
            }
        } else if let Some(expected_size) = metadata.size {
            if expected_size != checksum.num_bytes {
                return Err(DecodeError::IncompleteData {
                    expected_size: expected_size,
                    actual_size: checksum.num_bytes,
                });
            }
        }

    }
    Ok(output_pathbuf.to_str().unwrap().to_string())
}

/// Decode the yEncoded byte slice into a vector of bytes.
/// Carriage Return (CR) and Line Feed (LF) are ignored.
pub fn ydecode_buffer(input: &[u8]) -> Result<Vec<u8>, DecodeError> {
    // TODO remove heap allocation
    let mut output = Vec::<u8>::with_capacity((input.len() as f64 * 1.02) as usize);
    let mut iter = input.iter();
    while let Some(byte) = iter.next() {
        let mut byte = *byte;
        match byte {
            NUL | CR | LF => {
                // for now, just continue
                continue;
            }
            ESCAPE => {
                match iter.next() {
                    Some(b) => {
                        byte = b.overflowing_sub(64).0;
                    }
                    None => {
                        // for now, just continue, the only
                        continue;
                    }
                }
            }
            _ => {}
        }
        output.push(byte.overflowing_sub(42).0);
    }
    Ok(output)
}

fn parse_header_line(line_buf: &[u8], offset: usize) -> Result<MetaData, DecodeError> {
    #[derive(Debug)]
    enum State {
        Keyword,
        Value,
        End,
    };

    let mut metadata: MetaData = Default::default();
    let mut state = State::Keyword;
    let mut keyword = Vec::<u8>::with_capacity(6);
    let mut value = Vec::<u8>::with_capacity(96);

    let header_line = String::from_utf8_lossy(line_buf).to_string();
    for (i, &c) in line_buf[offset..].iter().enumerate() {
        let position = i + offset;
        match state {
            State::End => unreachable!(),
            State::Keyword => {
                match c {
                    b'a'...b'z' | b'0'...b'9' => keyword.push(c),
                    b'=' => {
                        if keyword.is_empty() || !is_known_keyword(&keyword) {
                            return Err(DecodeError::InvalidHeader {
                                line: header_line,
                                position: position,
                            });
                        } else {
                            state = State::Value;
                        }
                    }
                    CR | LF => {}
                    _ => {
                        return Err(DecodeError::InvalidHeader {
                            line: header_line,
                            position: position,
                        });
                    }
                }
            }
            State::Value => {
                match keyword.as_slice() {
                    b"name" => {
                        match c {
                            CR => {}
                            LF => {
                                state = State::End;
                                metadata.name = Some(String::from_utf8_lossy(&value).to_string());
                            }
                            _ => value.push(c),
                        }
                    }
                    b"size" => {
                        match c {
                            b'0'...b'9' => value.push(c),
                            SPACE => {
                                metadata.size =
                                    Some(usize::from_str_radix(&String::from_utf8_lossy(&value),
                                                               10)
                                        .unwrap());
                                state = State::Keyword;
                                keyword.clear();
                                value.clear();
                            }
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    b"begin" => {
                        match c {
                            b'0'...b'9' => value.push(c),
                            SPACE => {
                                metadata.begin =
                                    Some(usize::from_str_radix(&String::from_utf8_lossy(&value),
                                                               10)
                                        .unwrap());
                                state = State::Keyword;
                                keyword.clear();
                                value.clear();
                            }
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    b"end" => {
                        match c {
                            b'0'...b'9' => value.push(c),
                            SPACE | LF | CR => {
                                metadata.end =
                                    Some(usize::from_str_radix(&String::from_utf8_lossy(&value),
                                                               10)
                                        .unwrap());
                                state = State::Keyword;
                                keyword.clear();
                                value.clear();
                            }
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    b"line" => {
                        match c {
                            b'0'...b'9' => value.push(c),
                            SPACE => {
                                metadata.line_length =
                                    Some(u16::from_str_radix(&String::from_utf8_lossy(&value), 10)
                                        .unwrap());
                                state = State::Keyword;
                                keyword.clear();
                                value.clear();
                            }
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    b"part" => {
                        match c {
                            b'0'...b'9' => value.push(c),
                            SPACE => {
                                metadata.part =
                                    Some(u32::from_str_radix(&String::from_utf8_lossy(&value), 10)
                                        .unwrap());
                                state = State::Keyword;
                                keyword.clear();
                                value.clear();
                            }
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    b"crc32" | b"pcrc32" => {
                        match c {
                            b'0'...b'9' | b'A'...b'F' | b'a'...b'f' => value.push(c),
                            SPACE => {
                                state = State::Keyword;
                                metadata.crc32 =
                                    Some(u32::from_str_radix(&String::from_utf8_lossy(&value), 16)
                                        .unwrap());
                                keyword.clear();
                                value.clear();
                            }
                            LF => {
                                state = State::End;
                                metadata.crc32 =
                                    Some(u32::from_str_radix(&String::from_utf8_lossy(&value), 16)
                                        .unwrap());
                                keyword.clear();
                                value.clear();
                            }
                            CR => {}
                            _ => {
                                return Err(DecodeError::InvalidHeader {
                                    line: header_line,
                                    position: position,
                                });
                            }
                        }
                    }
                    _ => unreachable!(),
                }
            }
        };
    }
    Ok(metadata)
}

fn is_known_keyword(keyword_slice: &[u8]) -> bool {
    keyword_slice == b"name" || keyword_slice == b"line" || keyword_slice == b"size" ||
    keyword_slice == b"part" || keyword_slice == b"begin" ||
    keyword_slice == b"end" || keyword_slice == b"pcrc32" ||
    keyword_slice == b"crc32"
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn decode_invalid() {
        assert!(ydecode_buffer(&[b'=']).unwrap().is_empty());
    }

    #[test]
    fn decode_valid_ff() {
        assert_eq!(&vec![0xff - 0x2A], &ydecode_buffer(&[0xff]).unwrap());
    }

    #[test]
    fn decode_valid_01() {
        assert_eq!(&vec![0xff - 0x28], &ydecode_buffer(&[0x01]).unwrap());
    }

    #[test]
    fn decode_valid_esc_ff() {
        assert_eq!(&vec![0xff - 0x40 - 0x2A],
                   &ydecode_buffer(&[b'=', 0xff]).unwrap());
    }

    #[test]
    fn decode_valid_esc_01() {
        assert_eq!(&vec![0xff - 0x40 - 0x2A + 2],
                   &ydecode_buffer(&[b'=', 0x01]).unwrap());
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

//! Cache the pipeline objects to disk for faster reloads.
//!
//! A pipeline cache is an opaque type that allow you to cache your graphics and compute
//! pipelines on the disk.
//!
//! You can create either an empty cache or a cache from some initial data. Whenever you create a
//! graphics or compute pipeline, you have the possibility to pass a reference to that cache.
//! The Vulkan implementation will then look in the cache for an existing entry, or add one if it
//! doesn't exist.
//!
//! Once that is done, you can extract the data from the cache and store it.
//!
use std::mem;
use std::ptr;
use std::sync::Arc;

use device::Device;

use OomError;
use VulkanObject;
use VulkanPointers;
use check_errors;
use vk;

/// Opaque cache that contains pipeline objects.
pub struct PipelineCache {
    device: Arc<Device>,
    cache: vk::PipelineCache,
}

impl PipelineCache {
    /// Builds a new pipeline cache from existing data.
    ///
    /// The data must have been previously obtained with `get_data`.
    // TODO: is that unsafe? is it safe to pass garbage data?
    pub unsafe fn with_data(device: &Arc<Device>, initial_data: &[u8])
                            -> Result<Arc<PipelineCache>, OomError>
    {
        PipelineCache::new_impl(device, Some(initial_data))
    }

    /// Builds a new empty pipeline cache.
    pub fn empty(device: &Arc<Device>) -> Result<Arc<PipelineCache>, OomError> {
        unsafe { PipelineCache::new_impl(device, None) }
    }

    unsafe fn new_impl(device: &Arc<Device>, initial_data: Option<&[u8]>)
                       -> Result<Arc<PipelineCache>, OomError>
    {
        let vk = device.pointers();

        let cache = {
            let infos = vk::PipelineCacheCreateInfo {
                sType: vk::STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO,
                pNext: ptr::null(),
                flags: 0,   // reserved
                initialDataSize: initial_data.map(|d| d.len()).unwrap_or(0),
                pInitialData: initial_data.map(|d| d.as_ptr() as *const _).unwrap_or(ptr::null()),
            };

            let mut output = mem::uninitialized();
            try!(check_errors(vk.CreatePipelineCache(device.internal_object(), &infos,
                                                     ptr::null(), &mut output)));
            output
        };

        Ok(Arc::new(PipelineCache {
            device: device.clone(),
            cache: cache,
        }))
    }

    /// Merges other pipeline caches into this one.
    ///
    /// # Panic
    ///
    /// - Panics if `self` is included in the list of other pipelines.
    ///
    // FIXME: vkMergePipelineCaches is not thread safe for the destination cache
    pub fn merge<'a, I>(&self, pipelines: I) -> Result<(), OomError>
        where I: IntoIterator<Item = &'a &'a Arc<PipelineCache>>
    {
        unsafe {
            let vk = self.device.pointers();

            let pipelines = pipelines.into_iter().map(|pipeline| {
                assert!(&***pipeline as *const _ != &*self as *const _);
                pipeline.cache
            }).collect::<Vec<_>>();

            try!(check_errors(vk.MergePipelineCaches(self.device.internal_object(), self.cache,
                                                     pipelines.len() as u32, pipelines.as_ptr())));

            Ok(())
        }
    }

    /// Obtains the data from the cache.
    ///
    /// This data can be stored and then reloaded and passed to `PipelineCache::new`.
    pub fn get_data(&self) -> Result<Vec<u8>, OomError> {
        unsafe {
            let vk = self.device.pointers();

            let mut num = 0;
            try!(check_errors(vk.GetPipelineCacheData(self.device.internal_object(), self.cache,
                                                      &mut num, ptr::null_mut())));

            let mut data: Vec<u8> = Vec::with_capacity(num as usize);
            try!(check_errors(vk.GetPipelineCacheData(self.device.internal_object(), self.cache,
                                                      &mut num, data.as_mut_ptr() as *mut _)));
            data.set_len(num as usize);

            Ok(data)
        }
    }
}

unsafe impl VulkanObject for PipelineCache {
    type Object = vk::PipelineCache;

    #[inline]
    fn internal_object(&self) -> vk::PipelineCache {
        self.cache
    }
}

impl Drop for PipelineCache {
    #[inline]
    fn drop(&mut self) {
        unsafe {
            let vk = self.device.pointers();
            vk.DestroyPipelineCache(self.device.internal_object(), self.cache, ptr::null());
        }
    }
}

#[cfg(test)]
mod tests {
    use pipeline::cache::PipelineCache;

    #[test]
    #[should_panic]
    fn merge_self_forbidden() {
        let (device, queue) = gfx_dev_and_queue!();
        let pipeline = PipelineCache::empty(&device).unwrap();
        pipeline.merge(&[&pipeline]).unwrap();
    }
}
<commit_msg>Better documentation for pipeline caches<commit_after>// Copyright (c) 2016 The vulkano developers
// Licensed under the Apache License, Version 2.0
// <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT
// license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
// at your option. All files in the project carrying such
// notice may not be copied, modified, or distributed except
// according to those terms.

//! Cache the pipeline objects to disk for faster reloads.
//!
//! A pipeline cache is an opaque type that allow you to cache your graphics and compute
//! pipelines on the disk.
//!
//! You can create either an empty cache or a cache from some initial data. Whenever you create a
//! graphics or compute pipeline, you have the possibility to pass a reference to that cache.
//! TODO: ^ that's not the case yet
//! The Vulkan implementation will then look in the cache for an existing entry, or add one if it
//! doesn't exist.
//!
//! Once that is done, you can extract the data from the cache and store it. See the documentation
//! of [`get_data`](struct.PipelineCache.html#method.get_data) for example of how to store the data
//! on the disk, and [`with_data`](struct.PipelineCache.html#method.with_data) for how to reload it.
//!
use std::mem;
use std::ptr;
use std::sync::Arc;

use device::Device;

use OomError;
use VulkanObject;
use VulkanPointers;
use check_errors;
use vk;

/// Opaque cache that contains pipeline objects.
///
/// See [the documentation of the module](index.html) for more info.
pub struct PipelineCache {
    device: Arc<Device>,
    cache: vk::PipelineCache,
}

impl PipelineCache {
    /// Builds a new pipeline cache from existing data. The data must have been previously obtained
    /// with [`get_data`](#method.get_data).
    ///
    /// The data passed to this function will most likely be blindly trusted by the Vulkan
    /// implementation. Therefore you can easily crash your application or the system by passing
    /// wrong data. Hence why this function is unsafe.
    ///
    /// # Example
    ///
    /// This example loads a cache from a file, if it exists.
    /// See [`get_data`](#method.get_data) for how to store the data in a file.
    /// TODO: there's a header in the cached data that must be checked ; talk about this
    ///
    /// ```
    /// # use std::sync::Arc;
    /// # use vulkano::device::Device;
    /// use std::fs::File;
    /// use std::io::Read;
    /// use vulkano::pipeline::cache::PipelineCache;
    /// # let device: Arc<Device> = return;
    ///
    /// let data = {
    ///     let file = File::open("pipeline_cache.bin");
    ///     if let Ok(mut file) = file {
    ///         let mut data = Vec::new();
    ///         if let Ok(_) = file.read_to_end(&mut data) {
    ///             Some(data)
    ///         } else { None }
    ///     } else { None }
    /// };
    ///
    /// let cache = if let Some(data) = data {
    ///     // This is unsafe because there is no way to be sure that the file contains valid data.
    ///     unsafe { PipelineCache::with_data(&device, &data).unwrap() }
    /// } else {
    ///     PipelineCache::empty(&device).unwrap()
    /// };
    /// ```
    #[inline]
    pub unsafe fn with_data(device: &Arc<Device>, initial_data: &[u8])
                            -> Result<Arc<PipelineCache>, OomError>
    {
        PipelineCache::new_impl(device, Some(initial_data))
    }

    /// Builds a new empty pipeline cache.
    ///
    /// # Example
    ///
    /// ```
    /// # use std::sync::Arc;
    /// # use vulkano::device::Device;
    /// use vulkano::pipeline::cache::PipelineCache;
    /// # let device: Arc<Device> = return;
    /// let cache = PipelineCache::empty(&device).unwrap();
    /// ```
    #[inline]
    pub fn empty(device: &Arc<Device>) -> Result<Arc<PipelineCache>, OomError> {
        unsafe { PipelineCache::new_impl(device, None) }
    }

    // Actual implementation of the constructor.
    unsafe fn new_impl(device: &Arc<Device>, initial_data: Option<&[u8]>)
                       -> Result<Arc<PipelineCache>, OomError>
    {
        let vk = device.pointers();

        let cache = {
            let infos = vk::PipelineCacheCreateInfo {
                sType: vk::STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO,
                pNext: ptr::null(),
                flags: 0,   // reserved
                initialDataSize: initial_data.map(|d| d.len()).unwrap_or(0),
                pInitialData: initial_data.map(|d| d.as_ptr() as *const _).unwrap_or(ptr::null()),
            };

            let mut output = mem::uninitialized();
            try!(check_errors(vk.CreatePipelineCache(device.internal_object(), &infos,
                                                     ptr::null(), &mut output)));
            output
        };

        Ok(Arc::new(PipelineCache {
            device: device.clone(),
            cache: cache,
        }))
    }

    /// Merges other pipeline caches into this one.
    ///
    /// It is `self` that is modified here. The pipeline caches passed as parameter are untouched.
    ///
    /// # Panic
    ///
    /// - Panics if `self` is included in the list of other pipelines.
    ///
    // FIXME: vkMergePipelineCaches is not thread safe for the destination cache
    // TODO: write example
    pub fn merge<'a, I>(&self, pipelines: I) -> Result<(), OomError>
        where I: IntoIterator<Item = &'a &'a Arc<PipelineCache>>
    {
        unsafe {
            let vk = self.device.pointers();

            let pipelines = pipelines.into_iter().map(|pipeline| {
                assert!(&***pipeline as *const _ != &*self as *const _);
                pipeline.cache
            }).collect::<Vec<_>>();

            try!(check_errors(vk.MergePipelineCaches(self.device.internal_object(), self.cache,
                                                     pipelines.len() as u32, pipelines.as_ptr())));

            Ok(())
        }
    }

    /// Obtains the data from the cache.
    ///
    /// This data can be stored and then reloaded and passed to `PipelineCache::with_data`.
    ///
    /// # Example
    ///
    /// This example stores the data of a pipeline cache on the disk.
    /// See [`with_data`](#method.with_data) for how to reload it.
    ///
    /// ```
    /// use std::fs;
    /// use std::fs::File;
    /// use std::io::Write;
    /// # use std::sync::Arc;
    /// # use vulkano::pipeline::cache::PipelineCache;
    ///
    /// # let cache: Arc<PipelineCache> = return;
    /// // If an error happens (eg. no permission for the file) we simply skip storing the cache.
    /// if let Ok(data) = cache.get_data() {
    ///     if let Ok(mut file) = File::create("pipeline_cache.bin.tmp") {
    ///         if let Ok(_) = file.write_all(&data) {
    ///             let _ = fs::rename("pipeline_cache.bin.tmp", "pipeline_cache.bin");
    ///         } else {
    ///             let _ = fs::remove_file("pipeline_cache.bin.tmp");
    ///         }
    ///     }
    /// }
    /// ```
    pub fn get_data(&self) -> Result<Vec<u8>, OomError> {
        unsafe {
            let vk = self.device.pointers();

            let mut num = 0;
            try!(check_errors(vk.GetPipelineCacheData(self.device.internal_object(), self.cache,
                                                      &mut num, ptr::null_mut())));

            let mut data: Vec<u8> = Vec::with_capacity(num as usize);
            try!(check_errors(vk.GetPipelineCacheData(self.device.internal_object(), self.cache,
                                                      &mut num, data.as_mut_ptr() as *mut _)));
            data.set_len(num as usize);

            Ok(data)
        }
    }
}

unsafe impl VulkanObject for PipelineCache {
    type Object = vk::PipelineCache;

    #[inline]
    fn internal_object(&self) -> vk::PipelineCache {
        self.cache
    }
}

impl Drop for PipelineCache {
    #[inline]
    fn drop(&mut self) {
        unsafe {
            let vk = self.device.pointers();
            vk.DestroyPipelineCache(self.device.internal_object(), self.cache, ptr::null());
        }
    }
}

#[cfg(test)]
mod tests {
    use pipeline::cache::PipelineCache;

    #[test]
    #[should_panic]
    fn merge_self_forbidden() {
        let (device, queue) = gfx_dev_and_queue!();
        let pipeline = PipelineCache::empty(&device).unwrap();
        pipeline.merge(&[&pipeline]).unwrap();
    }
}
<|endoftext|>


<commit_before>//!
//! The main `Desync` struct
//!

pub use crate::desync_borrow_future::*;
use super::scheduler::*;

use std::sync::{Arc};
use std::marker::{PhantomData};
use futures::prelude::*;
use futures::channel::oneshot;
use futures::future::{Future, BoxFuture};

use std::cell::{UnsafeCell};
use std::mem;
use std::panic::{UnwindSafe};
use std::result::{Result};

///
/// A data storage structure used to govern synchronous and asynchronous access to an underlying object.
///
pub struct Desync<T: Send> {
    /// Queue used for scheduling runtime for this object
    queue:   Arc<JobQueue>,

    /// Data for this object. Boxed so the pointer remains the same through the lifetime of the object.
    data:    *mut T,

    /// Mark this type as !RefUnwindSafe.
    _marker: PhantomData<UnsafeCell<T>>
}

unsafe impl<T: Send> Send for Desync<T> {}

// True iff queue: Sync
unsafe impl<T: Send> Sync for Desync<T> {}

impl<T: Send+UnwindSafe> UnwindSafe for Desync<T> {}

///
/// Used for passing the data pointer through to the queue
///
/// 'Safe' because the queue is synchronised during drop, so we can never use the pointer
/// if the object does not exist.
///
struct DataRef<T: Send>(*mut T);
unsafe impl<T: Send> Send for DataRef<T> {}

// TODO: we can change DataRef to Shared (https://doc.rust-lang.org/std/ptr/struct.Shared.html in the future)

// TODO: T does not need to be static as we know that its lifetime is at least the lifetime of Desync<T> and hence the queue
impl<T: 'static+Send> Desync<T> {
    ///
    /// Creates a new Desync object
    ///
    pub fn new(data: T) -> Desync<T> {
        let queue = queue();

        Desync {
            queue:   queue,
            data:    Box::into_raw(Box::new(data)),
            _marker: PhantomData
        }
    }

    ///
    /// Performs an operation asynchronously on this item. This function will return
    /// immediately and the job will happen on a separate thread at some time in the
    /// future (generally fairly soon).
    ///
    /// Jobs are always performed in the order that they are queued and are always
    /// performed synchronously with respect to this object.
    ///
    #[inline]
    #[deprecated(since="0.3.0", note="please use `desync` instead")]
    pub fn r#async<TFn>(&self, job: TFn)
    where TFn: 'static+Send+FnOnce(&mut T) -> () {
        self.desync(job)
    }

    ///
    /// Performs an operation asynchronously on this item. This function will return
    /// immediately and the job will happen on a separate thread at some time in the
    /// future (generally fairly soon).
    ///
    /// Jobs are always performed in the order that they are queued and are always
    /// performed synchronously with respect to this object.
    ///
    pub fn desync<TFn>(&self, job: TFn)
    where TFn: 'static+Send+FnOnce(&mut T) -> () {
        // As drop() is the last thing called, we know that this object will still exist at the point where the queue makes the asynchronous callback
        let data = DataRef::<T>(self.data);

        desync(&self.queue, move || {
            let data = data.0;
            job(unsafe { &mut *data });
        })
    }

    ///
    /// Performs an operation synchronously on this item. This will be queued with any other
    /// jobs that this item may be performing, and this function will not return until the
    /// job is complete and the result is available.
    ///
    pub fn sync<TFn, Result>(&self, job: TFn) -> Result
    where TFn: Send+FnOnce(&mut T) -> Result, Result: Send {
        let result = {
            // As drop() is the last thing called, we know that this object will still exist at the point where the callback occurs
            // Exclusivity is guaranteed because the queue executes only one task at a time
            let data = DataRef::<T>(self.data);

            sync(&self.queue, move || {
                let data = data.0;
                job(unsafe { &mut *data })
            })
        };

        result
    }

    ///
    /// Performs an operation synchronously on this item,
    ///
    pub fn try_sync<TFn, FnResult>(&self, job: TFn) -> Result<FnResult, TrySyncError>
    where TFn: Send+FnOnce(&mut T) -> FnResult, FnResult: Send {
        let result = {
            // As drop() is the last thing called, we know that this object will still exist at the point where the callback occurs
            let data = DataRef::<T>(self.data);

            try_sync(&self.queue, move || {
                let data = data.0;
                job(unsafe { &mut *data })
            })
        };

        result
    }

    ///
    /// Deprecated name for `future_desync`
    ///
    #[inline]
    #[deprecated(since="0.7.0", note="please use either `future_desync` or `future_sync` to schedule futures")]
    pub fn future<TFn, TOutput>(&self, job: TFn) -> impl Future<Output=Result<TOutput, oneshot::Canceled>>+Send
    where   TFn:        'static+Send+for<'a> FnOnce(&'a mut T) -> BoxFuture<'a, TOutput>,
            TOutput:    'static+Send {
        self.future_desync(job)
    }

    ///
    /// Performs an operation asynchronously on the contents of this item, returning the
    /// result via a future.
    ///
    /// The future will be scheduled in the background, so it will make progress even if the current scheduler is
    /// blocked for any reason. Additionally, it's not necessary to await the returned future, which can be discarded
    /// if necessary. As soon as this function returns, the order of the operations in the future are fixed: that is,
    /// any other operation queued on the same `Desync` will be performed after the future completes.
    ///
    /// Call `detach()` on the returned future to fully put it in the background. If awaited the scheduler may try to
    /// 'steal' runtime on the current thread to complete the future, but it will also run the future in the background
    /// if the current thread is not available or a background thread is available faster. The future here has static
    /// lifetime so can be passed around in case something needs to monitor when it completes.
    ///
    /// This can also be used to run the future synchronously on the current thread, useful for cases where an async
    /// function needs to be called from a non-async context. Call `sync()` on the returned future to wait for its
    /// result synchronously.
    ///
    /// Note that this has a companion function, `future_sync()` which is more useful for cases where the future is
    /// awaited, as the 'inner' future is able to borrow values from the 'outer' future due to the shorter lifetime
    /// of the returned value.
    ///
    /// The normal `sync()` and `desync()` methods will schedule their operations in order around any `future_sync()`
    /// or `future_desync()` calls, so it's possible to easily mix async and traditional threaded code in one program.
    ///
    pub fn future_desync<'a, TFn, TOutput>(&self, job: TFn) -> SchedulerFuture<TOutput>
    where
        TFn:        'static + Send + for<'borrow> DesyncBorrowStaticFuture<'borrow, T, TOutput>,
        TOutput:    'static + Send,
    {
        // The future will have a lifetime shorter than the lifetime of this structure, and exclusivity is guaranteed
        // because queues only execute one task at a time
        let data = DataRef::<T>(self.data);

        scheduler().future_desync(&self.queue, move || {
            let data        = data.0;
            let job         = job.make_future_box(unsafe { &mut *data });

            async {
                job.await
            }
        })
    }

    ///
    /// Performs an operation asynchronously on the contents of this item, returning a future that must be awaited
    /// before it is dropped.
    ///
    /// The future will be scheduled in the current execution context, so it will only make progress if the current
    /// scheduler is running. The task will be cancelled and will not complete execution if the future is dropped before
    /// completion, so it's usually necessary to await the result of this function for the task to behave correctly.
    ///
    /// This is most useful for 'foreground' futures: note that the future can safely borrow values from the outer context
    /// to use while it's running (in much the same way `sync()` can in synchronous contexts). The need to dispatch to
    /// the awaiting context can make this future slightly slower than `future_desync()`
    ///
    /// Desync has a feature that makes it possible to wait synchronously for futures to complete: slightly
    /// counter-intuitively this is found in the future returned by `future_desync()` function rather than this function,
    /// which is synchronous with the current async context instead of with the current thread.
    ///
    /// The normal `sync()` and `desync()` methods will schedule their operations in order around any `future_sync()`
    /// or `future_desync()` calls, so it's possible to easily mix async and traditional threaded code in one program.
    ///
    pub fn future_sync<'a, TFn, TOutput>(&'a self, job: TFn) -> impl 'a + Future<Output=Result<TOutput, oneshot::Canceled>> + Send
    where
        TFn:        'a + Send + for<'borrow> DesyncBorrowFuture<'borrow, T, TOutput>,
        TOutput:    'a + Send,
    {
        // The future will have a lifetime shorter than the lifetime of this structure
        let data = DataRef::<T>(self.data);

        scheduler().future_sync(&self.queue, move || {
            let data        = data.0;
            let job         = job.make_future_box(unsafe { &mut *data });

            async {
                job.await
            }
        })
    }

    ///
    /// After the pending operations for this item are performed, waits for the
    /// supplied future to complete and then calls the specified function
    ///
    pub fn after<'a, TFn, Res, Fut>(&self, after: Fut, job: TFn) -> impl 'static + Future<Output=Result<Res, oneshot::Canceled>> + Send
    where
        Res: 'static + Send,
        Fut: 'static + Future + Send,
        TFn: 'static + Send + FnOnce(&mut T, Fut::Output) -> Res
    {
        self.future_desync(move |data| {
            async move {
                let future_result = after.await;
                job(data, future_result)
            }
        })
    }
}

impl<T: Send> Drop for Desync<T> {
    fn drop(&mut self) {
        use std::thread;

        // Take the data we're about to drop from the object
        let data = DataRef::<T>(self.data);

        // Ensure that everything on the queue has committed by queueing a last synchronous event
        // (Not synchronising the queue would make this unsafe as we would hold on to a pointer to
        // the internal data structure)
        if thread::panicking() {
            // If the thread is already panicking when we're dropped, do not panic again
            scheduler().sync_no_panic(&self.queue, move || {
                let data = data.0;
                mem::drop(unsafe { Box::from_raw(data) });
            });
        } else {
            // Thread is not panicking
            sync(&self.queue, move || {
                let data = data.0;
                mem::drop(unsafe { Box::from_raw(data) });
            });
        }
    }
}
<commit_msg>Move back to the old version that just used the FnOnce -> BoxFuture syntax<commit_after>//!
//! The main `Desync` struct
//!

pub use crate::desync_borrow_future::*;
use super::scheduler::*;

use std::sync::{Arc};
use std::marker::{PhantomData};
use futures::prelude::*;
use futures::channel::oneshot;
use futures::future::{Future, BoxFuture};

use std::cell::{UnsafeCell};
use std::mem;
use std::panic::{UnwindSafe};
use std::result::{Result};

///
/// A data storage structure used to govern synchronous and asynchronous access to an underlying object.
///
pub struct Desync<T: Send> {
    /// Queue used for scheduling runtime for this object
    queue:   Arc<JobQueue>,

    /// Data for this object. Boxed so the pointer remains the same through the lifetime of the object.
    data:    *mut T,

    /// Mark this type as !RefUnwindSafe.
    _marker: PhantomData<UnsafeCell<T>>
}

unsafe impl<T: Send> Send for Desync<T> {}

// True iff queue: Sync
unsafe impl<T: Send> Sync for Desync<T> {}

impl<T: Send+UnwindSafe> UnwindSafe for Desync<T> {}

///
/// Used for passing the data pointer through to the queue
///
/// 'Safe' because the queue is synchronised during drop, so we can never use the pointer
/// if the object does not exist.
///
struct DataRef<T: Send>(*mut T);
unsafe impl<T: Send> Send for DataRef<T> {}

// TODO: we can change DataRef to Shared (https://doc.rust-lang.org/std/ptr/struct.Shared.html in the future)

// TODO: T does not need to be static as we know that its lifetime is at least the lifetime of Desync<T> and hence the queue
impl<T: 'static+Send> Desync<T> {
    ///
    /// Creates a new Desync object
    ///
    pub fn new(data: T) -> Desync<T> {
        let queue = queue();

        Desync {
            queue:   queue,
            data:    Box::into_raw(Box::new(data)),
            _marker: PhantomData
        }
    }

    ///
    /// Performs an operation asynchronously on this item. This function will return
    /// immediately and the job will happen on a separate thread at some time in the
    /// future (generally fairly soon).
    ///
    /// Jobs are always performed in the order that they are queued and are always
    /// performed synchronously with respect to this object.
    ///
    #[inline]
    #[deprecated(since="0.3.0", note="please use `desync` instead")]
    pub fn r#async<TFn>(&self, job: TFn)
    where TFn: 'static+Send+FnOnce(&mut T) -> () {
        self.desync(job)
    }

    ///
    /// Performs an operation asynchronously on this item. This function will return
    /// immediately and the job will happen on a separate thread at some time in the
    /// future (generally fairly soon).
    ///
    /// Jobs are always performed in the order that they are queued and are always
    /// performed synchronously with respect to this object.
    ///
    pub fn desync<TFn>(&self, job: TFn)
    where TFn: 'static+Send+FnOnce(&mut T) -> () {
        // As drop() is the last thing called, we know that this object will still exist at the point where the queue makes the asynchronous callback
        let data = DataRef::<T>(self.data);

        desync(&self.queue, move || {
            let data = data.0;
            job(unsafe { &mut *data });
        })
    }

    ///
    /// Performs an operation synchronously on this item. This will be queued with any other
    /// jobs that this item may be performing, and this function will not return until the
    /// job is complete and the result is available.
    ///
    pub fn sync<TFn, Result>(&self, job: TFn) -> Result
    where TFn: Send+FnOnce(&mut T) -> Result, Result: Send {
        let result = {
            // As drop() is the last thing called, we know that this object will still exist at the point where the callback occurs
            // Exclusivity is guaranteed because the queue executes only one task at a time
            let data = DataRef::<T>(self.data);

            sync(&self.queue, move || {
                let data = data.0;
                job(unsafe { &mut *data })
            })
        };

        result
    }

    ///
    /// Performs an operation synchronously on this item,
    ///
    pub fn try_sync<TFn, FnResult>(&self, job: TFn) -> Result<FnResult, TrySyncError>
    where TFn: Send+FnOnce(&mut T) -> FnResult, FnResult: Send {
        let result = {
            // As drop() is the last thing called, we know that this object will still exist at the point where the callback occurs
            let data = DataRef::<T>(self.data);

            try_sync(&self.queue, move || {
                let data = data.0;
                job(unsafe { &mut *data })
            })
        };

        result
    }

    ///
    /// Deprecated name for `future_desync`
    ///
    #[inline]
    #[deprecated(since="0.7.0", note="please use either `future_desync` or `future_sync` to schedule futures")]
    pub fn future<TFn, TOutput>(&self, job: TFn) -> impl Future<Output=Result<TOutput, oneshot::Canceled>>+Send
    where   TFn:        'static+Send+for<'a> FnOnce(&'a mut T) -> BoxFuture<'a, TOutput>,
            TOutput:    'static+Send {
        self.future_desync(job)
    }

    ///
    /// Performs an operation asynchronously on the contents of this item, returning the
    /// result via a future.
    ///
    /// The future will be scheduled in the background, so it will make progress even if the current scheduler is
    /// blocked for any reason. Additionally, it's not necessary to await the returned future, which can be discarded
    /// if necessary. As soon as this function returns, the order of the operations in the future are fixed: that is,
    /// any other operation queued on the same `Desync` will be performed after the future completes.
    ///
    /// Call `detach()` on the returned future to fully put it in the background. If awaited the scheduler may try to
    /// 'steal' runtime on the current thread to complete the future, but it will also run the future in the background
    /// if the current thread is not available or a background thread is available faster. The future here has static
    /// lifetime so can be passed around in case something needs to monitor when it completes.
    ///
    /// This can also be used to run the future synchronously on the current thread, useful for cases where an async
    /// function needs to be called from a non-async context. Call `sync()` on the returned future to wait for its
    /// result synchronously.
    ///
    /// Note that this has a companion function, `future_sync()` which is more useful for cases where the future is
    /// awaited, as the 'inner' future is able to borrow values from the 'outer' future due to the shorter lifetime
    /// of the returned value.
    ///
    /// The normal `sync()` and `desync()` methods will schedule their operations in order around any `future_sync()`
    /// or `future_desync()` calls, so it's possible to easily mix async and traditional threaded code in one program.
    ///
    pub fn future_desync<'a, TFn, TOutput>(&self, job: TFn) -> SchedulerFuture<TOutput>
    where
        TFn:        'static + Send + for<'borrow> FnOnce(&'borrow mut T) -> BoxFuture<'borrow, TOutput>,
        TOutput:    'static + Send,
    {
        // The future will have a lifetime shorter than the lifetime of this structure, and exclusivity is guaranteed
        // because queues only execute one task at a time
        let data = DataRef::<T>(self.data);

        scheduler().future_desync(&self.queue, move || {
            let data        = data.0;
            let job         = job(unsafe { &mut *data });

            async {
                job.await
            }
        })
    }

    ///
    /// Performs an operation asynchronously on the contents of this item, returning a future that must be awaited
    /// before it is dropped.
    ///
    /// The future will be scheduled in the current execution context, so it will only make progress if the current
    /// scheduler is running. The task will be cancelled and will not complete execution if the future is dropped before
    /// completion, so it's usually necessary to await the result of this function for the task to behave correctly.
    ///
    /// This is most useful for 'foreground' futures: note that the future can safely borrow values from the outer context
    /// to use while it's running (in much the same way `sync()` can in synchronous contexts). The need to dispatch to
    /// the awaiting context can make this future slightly slower than `future_desync()`
    ///
    /// Desync has a feature that makes it possible to wait synchronously for futures to complete: slightly
    /// counter-intuitively this is found in the future returned by `future_desync()` function rather than this function,
    /// which is synchronous with the current async context instead of with the current thread.
    ///
    /// The normal `sync()` and `desync()` methods will schedule their operations in order around any `future_sync()`
    /// or `future_desync()` calls, so it's possible to easily mix async and traditional threaded code in one program.
    ///
    pub fn future_sync<'a, TFn, TOutput>(&'a self, job: TFn) -> impl 'a + Future<Output=Result<TOutput, oneshot::Canceled>> + Send
    where
        TFn:        'a + Send + for<'borrow> FnOnce(&'borrow mut T) -> BoxFuture<'borrow, TOutput>,
        TOutput:    'a + Send,
    {
        // The future will have a lifetime shorter than the lifetime of this structure
        let data = DataRef::<T>(self.data);

        scheduler().future_sync(&self.queue, move || {
            let data        = data.0;
            let job         = job(unsafe { &mut *data });

            async {
                job.await
            }
        })
    }

    ///
    /// After the pending operations for this item are performed, waits for the
    /// supplied future to complete and then calls the specified function
    ///
    pub fn after<'a, TFn, Res, Fut>(&self, after: Fut, job: TFn) -> impl 'static + Future<Output=Result<Res, oneshot::Canceled>> + Send
    where
        Res: 'static + Send,
        Fut: 'static + Future + Send,
        TFn: 'static + Send + FnOnce(&mut T, Fut::Output) -> Res
    {
        self.future_desync(move |data| {
            async move {
                let future_result = after.await;
                job(data, future_result)
            }
        })
    }
}

impl<T: Send> Drop for Desync<T> {
    fn drop(&mut self) {
        use std::thread;

        // Take the data we're about to drop from the object
        let data = DataRef::<T>(self.data);

        // Ensure that everything on the queue has committed by queueing a last synchronous event
        // (Not synchronising the queue would make this unsafe as we would hold on to a pointer to
        // the internal data structure)
        if thread::panicking() {
            // If the thread is already panicking when we're dropped, do not panic again
            scheduler().sync_no_panic(&self.queue, move || {
                let data = data.0;
                mem::drop(unsafe { Box::from_raw(data) });
            });
        } else {
            // Thread is not panicking
            sync(&self.queue, move || {
                let data = data.0;
                mem::drop(unsafe { Box::from_raw(data) });
            });
        }
    }
}
<|endoftext|>