<commit_before>use rustup::dist::Notification as In;
use rustup::utils::tty;
use rustup::utils::Notification as Un;
use rustup::Notification;
use std::collections::VecDeque;
use std::fmt;
use time::precise_time_s;

/// Keep track of this many past download amounts
const DOWNLOAD_TRACK_COUNT: usize = 5;

/// Tracks download progress and displays information about it to a terminal.
pub struct DownloadTracker {
    /// Content-Length of the to-be downloaded object.
    content_len: Option<u64>,
    /// Total data downloaded in bytes.
    total_downloaded: usize,
    /// Data downloaded this second.
    downloaded_this_sec: usize,
    /// Keeps track of amount of data downloaded every last few secs.
    /// Used for averaging the download speed.
    downloaded_last_few_secs: VecDeque<usize>,
    /// Time stamp of the last second
    last_sec: Option<f64>,
    /// How many seconds have elapsed since the download started
    seconds_elapsed: u32,
    /// The terminal we write the information to.
    term: Option<Box<term::StdoutTerminal>>,
    /// Whether we displayed progress for the download or not.
    ///
    /// If the download is quick enough, we don't have time to
    /// display the progress info.
    /// In that case, we do not want to do some cleanup stuff we normally do.
    displayed_progress: bool,
}

impl DownloadTracker {
    /// Creates a new DownloadTracker.
    pub fn new() -> Self {
        DownloadTracker {
            content_len: None,
            total_downloaded: 0,
            downloaded_this_sec: 0,
            downloaded_last_few_secs: VecDeque::with_capacity(DOWNLOAD_TRACK_COUNT),
            seconds_elapsed: 0,
            last_sec: None,
            term: term::stdout(),
            displayed_progress: false,
        }
    }

    pub fn handle_notification(&mut self, n: &Notification<'_>) -> bool {
        match *n {
            Notification::Install(In::Utils(Un::DownloadContentLengthReceived(content_len))) => {
                self.content_length_received(content_len);

                true
            }
            Notification::Install(In::Utils(Un::DownloadDataReceived(data))) => {
                if tty::stdout_isatty() && self.term.is_some() {
                    self.data_received(data.len());
                }
                true
            }
            Notification::Install(In::Utils(Un::DownloadFinished)) => {
                self.download_finished();
                true
            }
            _ => false,
        }
    }

    /// Notifies self that Content-Length information has been received.
    pub fn content_length_received(&mut self, content_len: u64) {
        self.content_len = Some(content_len);
    }
    /// Notifies self that data of size `len` has been received.
    pub fn data_received(&mut self, len: usize) {
        self.total_downloaded += len;
        self.downloaded_this_sec += len;

        let current_time = precise_time_s();

        match self.last_sec {
            None => self.last_sec = Some(current_time),
            Some(start) => {
                let elapsed = current_time - start;
                if elapsed >= 1.0 {
                    self.seconds_elapsed += 1;

                    self.display();
                    self.last_sec = Some(current_time);
                    if self.downloaded_last_few_secs.len() == DOWNLOAD_TRACK_COUNT {
                        self.downloaded_last_few_secs.pop_back();
                    }
                    self.downloaded_last_few_secs
                        .push_front(self.downloaded_this_sec);
                    self.downloaded_this_sec = 0;
                }
            }
        }
    }
    /// Notifies self that the download has finished.
    pub fn download_finished(&mut self) {
        if self.displayed_progress {
            // Display the finished state
            self.display();
            let _ = writeln!(self.term.as_mut().unwrap());
        }
        self.prepare_for_new_download();
    }
    /// Resets the state to be ready for a new download.
    fn prepare_for_new_download(&mut self) {
        self.content_len = None;
        self.total_downloaded = 0;
        self.downloaded_this_sec = 0;
        self.downloaded_last_few_secs.clear();
        self.seconds_elapsed = 0;
        self.last_sec = None;
        self.displayed_progress = false;
    }
    /// Display the tracked download information to the terminal.
    fn display(&mut self) {
        let total_h = HumanReadable(self.total_downloaded as f64);
        let sum = self
            .downloaded_last_few_secs
            .iter()
            .fold(0., |a, &v| a + v as f64);
        let len = self.downloaded_last_few_secs.len();
        let speed = if len > 0 { sum / len as f64 } else { 0. };
        let speed_h = HumanReadable(speed);

        // First, move to the start of the current line and clear it.
        let _ = self.term.as_mut().unwrap().carriage_return();
        let _ = self.term.as_mut().unwrap().delete_line();

        match self.content_len {
            Some(content_len) => {
                let content_len = content_len as f64;
                let percent = (self.total_downloaded as f64 / content_len) * 100.;
                let content_len_h = HumanReadable(content_len);
                let remaining = content_len - self.total_downloaded as f64;
                let eta_h = HumanReadable(remaining / speed);
                let _ = write!(
                    self.term.as_mut().unwrap(),
                    "{} / {} ({:3.0} %) {}/s ETA: {:#}",
                    total_h,
                    content_len_h,
                    percent,
                    speed_h,
                    eta_h
                );
            }
            None => {
                let _ = write!(
                    self.term.as_mut().unwrap(),
                    "Total: {} Speed: {}/s",
                    total_h,
                    speed_h
                );
            }
        }

        // Since stdout is typically line-buffered and we don't print a newline, we manually flush.
        let _ = self.term.as_mut().unwrap().flush();
        self.displayed_progress = true;
    }
}

/// Human readable representation of data size in bytes
struct HumanReadable(f64);

impl fmt::Display for HumanReadable {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        if f.alternate() {
            // repurposing the alternate mode for ETA
            let sec = self.0;

            if sec.is_infinite() {
                write!(f, "Unknown")
            } else {
                // we're doing modular arithmetic, treat as integer
                let sec = sec as u32;
                if sec > 48 * 3600 {
                    let d = sec / (24 * 3600);
                    let h = sec % (24 * 3600);
                    let min = sec % 3600;
                    let sec = sec % 60;

                    write!(f, "{:3} days {:2} h {:2} min {:2} s", d, h, min, sec) // XYZ days PQ h RS min TU s
                } else if sec > 6_000 {
                    let h = sec / 3600;
                    let min = sec % 3600;
                    let sec = sec % 60;

                    write!(f, "{:3} h {:2} min {:2} s", h, min, sec) // XYZ h PQ min RS s
                } else if sec > 100 {
                    let min = sec / 60;
                    let sec = sec % 60;

                    write!(f, "{:3} min {:2} s", min, sec) // XYZ min PQ s
                } else {
                    write!(f, "{:3.0} s", self.0) // XYZ s
                }
            }
        } else {
            const KIB: f64 = 1024.0;
            const MIB: f64 = KIB * KIB;
            let size = self.0;

            if size >= MIB {
                write!(f, "{:5.1} MiB", size / MIB) // XYZ.P MiB
            } else if size >= KIB {
                write!(f, "{:5.1} KiB", size / KIB)
            } else {
                write!(f, "{:3.0} B", size)
            }
        }
    }
}
<commit_msg>download_tracker.rs: Erase line more robustly<commit_after>use rustup::dist::Notification as In;
use rustup::utils::tty;
use rustup::utils::Notification as Un;
use rustup::Notification;
use std::collections::VecDeque;
use std::fmt;
use time::precise_time_s;

/// Keep track of this many past download amounts
const DOWNLOAD_TRACK_COUNT: usize = 5;

/// Tracks download progress and displays information about it to a terminal.
pub struct DownloadTracker {
    /// Content-Length of the to-be downloaded object.
    content_len: Option<u64>,
    /// Total data downloaded in bytes.
    total_downloaded: usize,
    /// Data downloaded this second.
    downloaded_this_sec: usize,
    /// Keeps track of amount of data downloaded every last few secs.
    /// Used for averaging the download speed.
    downloaded_last_few_secs: VecDeque<usize>,
    /// Time stamp of the last second
    last_sec: Option<f64>,
    /// How many seconds have elapsed since the download started
    seconds_elapsed: u32,
    /// The terminal we write the information to.
    term: Option<Box<term::StdoutTerminal>>,
    /// Whether we displayed progress for the download or not.
    ///
    /// If the download is quick enough, we don't have time to
    /// display the progress info.
    /// In that case, we do not want to do some cleanup stuff we normally do.
    ///
    /// If we have displayed progress, this is the number of characters we
    /// rendered, so we can erase it cleanly.
    displayed_charcount: Option<usize>,
}

impl DownloadTracker {
    /// Creates a new DownloadTracker.
    pub fn new() -> Self {
        DownloadTracker {
            content_len: None,
            total_downloaded: 0,
            downloaded_this_sec: 0,
            downloaded_last_few_secs: VecDeque::with_capacity(DOWNLOAD_TRACK_COUNT),
            seconds_elapsed: 0,
            last_sec: None,
            term: term::stdout(),
            displayed_charcount: None,
        }
    }

    pub fn handle_notification(&mut self, n: &Notification<'_>) -> bool {
        match *n {
            Notification::Install(In::Utils(Un::DownloadContentLengthReceived(content_len))) => {
                self.content_length_received(content_len);

                true
            }
            Notification::Install(In::Utils(Un::DownloadDataReceived(data))) => {
                if tty::stdout_isatty() && self.term.is_some() {
                    self.data_received(data.len());
                }
                true
            }
            Notification::Install(In::Utils(Un::DownloadFinished)) => {
                self.download_finished();
                true
            }
            _ => false,
        }
    }

    /// Notifies self that Content-Length information has been received.
    pub fn content_length_received(&mut self, content_len: u64) {
        self.content_len = Some(content_len);
    }
    /// Notifies self that data of size `len` has been received.
    pub fn data_received(&mut self, len: usize) {
        self.total_downloaded += len;
        self.downloaded_this_sec += len;

        let current_time = precise_time_s();

        match self.last_sec {
            None => self.last_sec = Some(current_time),
            Some(start) => {
                let elapsed = current_time - start;
                if elapsed >= 1.0 {
                    self.seconds_elapsed += 1;

                    self.display();
                    self.last_sec = Some(current_time);
                    if self.downloaded_last_few_secs.len() == DOWNLOAD_TRACK_COUNT {
                        self.downloaded_last_few_secs.pop_back();
                    }
                    self.downloaded_last_few_secs
                        .push_front(self.downloaded_this_sec);
                    self.downloaded_this_sec = 0;
                }
            }
        }
    }
    /// Notifies self that the download has finished.
    pub fn download_finished(&mut self) {
        if self.displayed_charcount.is_some() {
            // Display the finished state
            self.display();
            let _ = writeln!(self.term.as_mut().unwrap());
        }
        self.prepare_for_new_download();
    }
    /// Resets the state to be ready for a new download.
    fn prepare_for_new_download(&mut self) {
        self.content_len = None;
        self.total_downloaded = 0;
        self.downloaded_this_sec = 0;
        self.downloaded_last_few_secs.clear();
        self.seconds_elapsed = 0;
        self.last_sec = None;
        self.displayed_charcount = None;
    }
    /// Display the tracked download information to the terminal.
    fn display(&mut self) {
        let total_h = HumanReadable(self.total_downloaded as f64);
        let sum = self
            .downloaded_last_few_secs
            .iter()
            .fold(0., |a, &v| a + v as f64);
        let len = self.downloaded_last_few_secs.len();
        let speed = if len > 0 { sum / len as f64 } else { 0. };
        let speed_h = HumanReadable(speed);

        // First, move to the start of the current line and clear it.
        let _ = self.term.as_mut().unwrap().carriage_return();
        // We'd prefer to use delete_line() but on Windows it seems to
        // sometimes do unusual things
        // let _ = self.term.as_mut().unwrap().delete_line();
        // So instead we do:
        if let Some(n) = self.displayed_charcount {
            // This is not ideal as very narrow terminals might mess up,
            // but it is more likely to succeed until term's windows console
            // fixes whatever's up with delete_line().
            let _ = write!(self.term.as_mut().unwrap(), "{}", " ".repeat(n));
            let _ = self.term.as_mut().unwrap().flush();
            let _ = self.term.as_mut().unwrap().carriage_return();
        }

        let output = match self.content_len {
            Some(content_len) => {
                let content_len = content_len as f64;
                let percent = (self.total_downloaded as f64 / content_len) * 100.;
                let content_len_h = HumanReadable(content_len);
                let remaining = content_len - self.total_downloaded as f64;
                let eta_h = HumanReadable(remaining / speed);
                format!(
                    "{} / {} ({:3.0} %) {}/s ETA: {:#}",
                    total_h, content_len_h, percent, speed_h, eta_h
                )
            }
            None => format!("Total: {} Speed: {}/s", total_h, speed_h),
        };

        let _ = write!(self.term.as_mut().unwrap(), "{}", output);
        // Since stdout is typically line-buffered and we don't print a newline, we manually flush.
        let _ = self.term.as_mut().unwrap().flush();
        self.displayed_charcount = Some(output.chars().count());
    }
}

/// Human readable representation of data size in bytes
struct HumanReadable(f64);

impl fmt::Display for HumanReadable {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        if f.alternate() {
            // repurposing the alternate mode for ETA
            let sec = self.0;

            if sec.is_infinite() {
                write!(f, "Unknown")
            } else {
                // we're doing modular arithmetic, treat as integer
                let sec = sec as u32;
                if sec > 48 * 3600 {
                    let d = sec / (24 * 3600);
                    let h = sec % (24 * 3600);
                    let min = sec % 3600;
                    let sec = sec % 60;

                    write!(f, "{:3} days {:2} h {:2} min {:2} s", d, h, min, sec) // XYZ days PQ h RS min TU s
                } else if sec > 6_000 {
                    let h = sec / 3600;
                    let min = sec % 3600;
                    let sec = sec % 60;

                    write!(f, "{:3} h {:2} min {:2} s", h, min, sec) // XYZ h PQ min RS s
                } else if sec > 100 {
                    let min = sec / 60;
                    let sec = sec % 60;

                    write!(f, "{:3} min {:2} s", min, sec) // XYZ min PQ s
                } else {
                    write!(f, "{:3.0} s", self.0) // XYZ s
                }
            }
        } else {
            const KIB: f64 = 1024.0;
            const MIB: f64 = KIB * KIB;
            let size = self.0;

            if size >= MIB {
                write!(f, "{:5.1} MiB", size / MIB) // XYZ.P MiB
            } else if size >= KIB {
                write!(f, "{:5.1} KiB", size / KIB)
            } else {
                write!(f, "{:3.0} B", size)
            }
        }
    }
}
<|endoftext|>


<commit_before>use serde::de::{Deserialize, Deserializer};

use types::*;

#[derive(Debug, Clone, PartialEq, PartialOrd)]
pub struct Update {
    pub id: Integer,
    pub kind: UpdateKind,
}

#[derive(Debug, Clone, PartialEq, PartialOrd)]
pub enum UpdateKind {
    Message(Message),
    EditedMessage(Message),
    ChannelPost(Message),
    EditedChannelPost(Message),
//    InlineQuery(InlineQuery),
//    ChosenInlineResult(ChosenInlineResult),
//    CallbackQuery(OptionCallbackQuery),
    #[doc(hidden)]
    Unknown(RawUpdate)
}

impl Deserialize for Update {
    fn deserialize<D>(deserializer: D) -> Result<Update, D::Error> where D: Deserializer {
        let raw: RawUpdate = Deserialize::deserialize(deserializer)?;
        macro_rules! maybe_field {
            ($name:ident, $variant:ident) => {{
                if let Some(val) = raw.$name {
                    return Ok(Update {
                        id: raw.update_id,
                        kind: UpdateKind::$variant(val),
                    })
                }
            }}
        }

        maybe_field!(message, Message);
        maybe_field!(edited_message, EditedMessage);
        maybe_field!(channel_post, ChannelPost);
        maybe_field!(edited_channel_post, EditedChannelPost);
//        maybe_field!(inline_query, InlineQuery);
//        maybe_field!(chosen_inline_result, ChosenInlineResult);
//        maybe_field!(callback_query, CallbackQuery);

        Ok(Update {
            id: raw.update_id,
            kind: UpdateKind::Unknown(raw),
        })
    }
}

#[derive(Debug, Clone, PartialEq, PartialOrd, Deserialize)]
pub struct RawUpdate {
    pub update_id: Integer,
    pub message: Option<Message>,
    pub edited_message: Option<Message>,
    pub channel_post: Option<Message>,
    pub edited_channel_post: Option<Message>,
//    pub inline_query: Option<InlineQuery>,
//    pub chosen_inline_result: Option<ChosenInlineResult>,
//    pub callback_query: Option<CallbackQuery>,
}
<commit_msg>docs: document update types<commit_after>use serde::de::{Deserialize, Deserializer};

use types::*;

/// This object represents an incoming update.
#[derive(Debug, Clone, PartialEq, PartialOrd)]
pub struct Update {
    /// The update‘s unique identifier. Update identifiers start from a certain
    /// positive number and increase sequentially.
    pub id: Integer,
    pub kind: UpdateKind,
}

/// Kind of incoming update.
#[derive(Debug, Clone, PartialEq, PartialOrd)]
pub enum UpdateKind {
    /// New incoming message of any kind — text, photo, sticker, etc.
    Message(Message),
    /// New version of a message that is known to the bot and was edited
    EditedMessage(Message),
    /// New incoming channel post of any kind — text, photo, sticker, etc.
    ChannelPost(Message),
    /// New version of a channel post that is known to the bot and was edited
    EditedChannelPost(Message),
//    InlineQuery(InlineQuery),
//    ChosenInlineResult(ChosenInlineResult),
//    CallbackQuery(OptionCallbackQuery),
    #[doc(hidden)]
    Unknown(RawUpdate)
}

impl Deserialize for Update {
    fn deserialize<D>(deserializer: D) -> Result<Update, D::Error> where D: Deserializer {
        let raw: RawUpdate = Deserialize::deserialize(deserializer)?;
        macro_rules! maybe_field {
            ($name:ident, $variant:ident) => {{
                if let Some(val) = raw.$name {
                    return Ok(Update {
                        id: raw.update_id,
                        kind: UpdateKind::$variant(val),
                    })
                }
            }}
        }

        maybe_field!(message, Message);
        maybe_field!(edited_message, EditedMessage);
        maybe_field!(channel_post, ChannelPost);
        maybe_field!(edited_channel_post, EditedChannelPost);
//        maybe_field!(inline_query, InlineQuery);
//        maybe_field!(chosen_inline_result, ChosenInlineResult);
//        maybe_field!(callback_query, CallbackQuery);

        Ok(Update {
            id: raw.update_id,
            kind: UpdateKind::Unknown(raw),
        })
    }
}

#[derive(Debug, Clone, PartialEq, PartialOrd, Deserialize)]
pub struct RawUpdate {
    /// The update‘s unique identifier. Update identifiers start from a certain
    /// positive number and increase sequentially.
    pub update_id: Integer,
    /// New incoming message of any kind — text, photo, sticker, etc.
    pub message: Option<Message>,
    /// New version of a message that is known to the bot and was edited
    pub edited_message: Option<Message>,
    /// New incoming channel post of any kind — text, photo, sticker, etc.
    pub channel_post: Option<Message>,
    /// New version of a channel post that is known to the bot and was edited
    pub edited_channel_post: Option<Message>,
//    pub inline_query: Option<InlineQuery>,
//    pub chosen_inline_result: Option<ChosenInlineResult>,
//    pub callback_query: Option<CallbackQuery>,
}
<|endoftext|>


<commit_before>// Copyright (c) 2020 DDN. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

use futures::{StreamExt, TryStreamExt};
use iml_command_utils::{wait_for_cmds_success, CmdUtilError};
use iml_graphql_queries::snapshot as snapshot_queries;
use iml_manager_client::{get_influx, graphql, Client, ImlManagerClientError};
use iml_manager_env::get_pool_limit;
use iml_postgres::{get_db_pool, sqlx, PgPool};
use iml_service_queue::service_queue::consume_data;
use iml_tracing::tracing;
use iml_wire_types::snapshot;
use std::{collections::HashMap, fmt::Debug};
use thiserror::Error;
use tokio::{
    time::Instant,
    time::{interval, Duration},
};

// Default pool limit if not overridden by POOL_LIMIT
const DEFAULT_POOL_LIMIT: u32 = 2;

#[derive(Error, Debug)]
enum Error {
    #[error(transparent)]
    CmdUtilError(#[from] CmdUtilError),
    #[error(transparent)]
    ImlManagerClientError(#[from] ImlManagerClientError),
    #[error(transparent)]
    ImlGraphqlQueriesError(#[from] iml_graphql_queries::Errors),
    #[error(transparent)]
    ImlPostgresError(#[from] iml_postgres::sqlx::Error),
}

#[derive(Debug, PartialEq, Eq, Hash, Clone)]
enum State {
    Monitoring(u64),
    CountingDown(Instant),
}

async fn tick(snapshot_client_counts: &mut HashMap<i32, State>, pool: PgPool) -> Result<(), Error> {
    let client: Client = iml_manager_client::get_client()?;
    let client_2 = client.clone();

    let query = iml_influx::filesystems::query();
    let stats_fut =
        get_influx::<iml_influx::filesystems::InfluxResponse>(client, "iml_stats", query.as_str());

    let influx_resp = stats_fut.await?;
    let stats = iml_influx::filesystems::Response::from(influx_resp);

    tracing::debug!("Influx stats: {:?}", stats);

    let snapshots =
        sqlx::query!("SELECT id, filesystem_name, snapshot_name, snapshot_fsname FROM snapshot")
            .fetch_all(&pool)
            .await?;
    tracing::debug!("Fetched {} snapshots from DB", snapshots.len());

    for snapshot in snapshots {
        let snapshot_id = snapshot.id;
        let snapshot_stats = match stats.get(&snapshot.snapshot_fsname) {
            Some(x) => x,
            None => continue,
        };

        let state = snapshot_client_counts.get_mut(&snapshot_id);
        let clients = snapshot_stats.clients.unwrap_or(0);

        match state {
            Some(State::Monitoring(prev_clients)) => {
                tracing::debug!(
                    "Monitoring. Snapshot {}: {} clients (previously {} clients)",
                    &snapshot.snapshot_fsname,
                    clients,
                    prev_clients,
                );
                if *prev_clients > 0 && clients == 0 {
                    tracing::trace!("counting down for job");
                    let instant = Instant::now() + Duration::from_secs(5 * 60);
                    state.map(|s| *s = State::CountingDown(instant));
                } else {
                    *prev_clients = clients;
                }
            }
            Some(State::CountingDown(when)) => {
                tracing::debug!(
                    "Counting down. Snapshot {}: 0 clients (previously {} clients)",
                    &snapshot.snapshot_fsname,
                    clients
                );
                if clients > 0 {
                    tracing::trace!("changing state");
                    state.map(|s| *s = State::Monitoring(clients));
                } else if Instant::now() >= *when {
                    tracing::trace!("running the job");
                    state.map(|s| *s = State::Monitoring(0));

                    let query = snapshot_queries::unmount::build(
                        &snapshot.filesystem_name,
                        &snapshot.snapshot_name,
                    );
                    let resp: iml_graphql_queries::Response<snapshot_queries::unmount::Resp> =
                        graphql(client_2.clone(), query).await?;
                    let command = Result::from(resp)?.data.unmount_snapshot;
                    wait_for_cmds_success(&[command], None).await?;
                }
            }
            None => {
                tracing::debug!(
                    "Just learnt about this snapshot. Snapshot {}: {} clients",
                    &snapshot.snapshot_fsname,
                    clients,
                );

                snapshot_client_counts.insert(snapshot_id, State::Monitoring(clients));
            }
        }
    }

    Ok(())
}

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    iml_tracing::init();

    let pool = iml_rabbit::connect_to_rabbit(1);

    let conn = iml_rabbit::get_conn(pool).await?;

    let ch = iml_rabbit::create_channel(&conn).await?;

    let mut s = consume_data::<Vec<snapshot::Snapshot>>(&ch, "rust_agent_snapshot_rx");

    let pool = get_db_pool(get_pool_limit().unwrap_or(DEFAULT_POOL_LIMIT)).await?;
    let pool_2 = pool.clone();
    sqlx::migrate!("../../migrations").run(&pool).await?;

    tokio::spawn(async move {
        let mut interval = interval(Duration::from_secs(60));
        let mut snapshot_client_counts: HashMap<i32, State> = HashMap::new();

        while let Some(_) = interval.next().await {
            let tick_result = tick(&mut snapshot_client_counts, pool_2.clone()).await;
            if let Err(e) = tick_result {
                tracing::error!("Error during handling snapshot autounmount: {}", e);
            }
        }
    });

    while let Some((fqdn, snapshots)) = s.try_next().await? {
        tracing::debug!("snapshots from {}: {:?}", fqdn, snapshots);

        let snaps = snapshots.into_iter().fold(
            (vec![], vec![], vec![], vec![], vec![], vec![], vec![]),
            |mut acc, s| {
                acc.0.push(s.filesystem_name);
                acc.1.push(s.snapshot_name);
                acc.2.push(s.create_time.naive_utc());
                acc.3.push(s.modify_time.naive_utc());
                acc.4.push(s.snapshot_fsname.clone());
                acc.5.push(s.mounted);
                acc.6.push(s.comment);

                acc
            },
        );

        sqlx::query!(
            r#"
            DELETE FROM snapshot
            WHERE (filesystem_name, snapshot_name) NOT IN (SELECT * FROM UNNEST ($1::text[], $2::text[]))
            "#,
            &snaps.0,
            &snaps.1,
        )
        .execute(&pool)
        .await?;

        sqlx::query!(
            r#"
            INSERT INTO snapshot (filesystem_name, snapshot_name, create_time, modify_time, snapshot_fsname, mounted, comment)
            SELECT * FROM
            UNNEST (
                $1::text[],
                $2::text[],
                $3::timestamp[],
                $4::timestamp[],
                $5::text[],
                $6::bool[],
                $7::text[]
            )
            ON CONFLICT (filesystem_name, snapshot_name) DO UPDATE
            SET
                create_time = excluded.create_time,
                modify_time = excluded.modify_time,
                snapshot_fsname = excluded.snapshot_fsname,
                mounted = excluded.mounted,
                comment = excluded.comment
            "#,
            &snaps.0,
            &snaps.1,
            &snaps.2,
            &snaps.3,
            &snaps.4,
            &snaps.5 as &[Option<bool>],
            &snaps.6 as &[Option<String>],
        )
        .execute(&pool)
        .await?;
    }

    Ok(())
}
<commit_msg>Use transaction for table update (#2277)<commit_after>// Copyright (c) 2020 DDN. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

use futures::{StreamExt, TryStreamExt};
use iml_command_utils::{wait_for_cmds_success, CmdUtilError};
use iml_graphql_queries::snapshot as snapshot_queries;
use iml_manager_client::{get_influx, graphql, Client, ImlManagerClientError};
use iml_manager_env::get_pool_limit;
use iml_postgres::{get_db_pool, sqlx, PgPool};
use iml_service_queue::service_queue::consume_data;
use iml_tracing::tracing;
use iml_wire_types::snapshot;
use std::{collections::HashMap, fmt::Debug};
use thiserror::Error;
use tokio::{
    time::Instant,
    time::{interval, Duration},
};

// Default pool limit if not overridden by POOL_LIMIT
const DEFAULT_POOL_LIMIT: u32 = 2;

#[derive(Error, Debug)]
enum Error {
    #[error(transparent)]
    CmdUtilError(#[from] CmdUtilError),
    #[error(transparent)]
    ImlManagerClientError(#[from] ImlManagerClientError),
    #[error(transparent)]
    ImlGraphqlQueriesError(#[from] iml_graphql_queries::Errors),
    #[error(transparent)]
    ImlPostgresError(#[from] iml_postgres::sqlx::Error),
}

#[derive(Debug, PartialEq, Eq, Hash, Clone)]
enum State {
    Monitoring(u64),
    CountingDown(Instant),
}

async fn tick(snapshot_client_counts: &mut HashMap<i32, State>, pool: PgPool) -> Result<(), Error> {
    let client: Client = iml_manager_client::get_client()?;
    let client_2 = client.clone();

    let query = iml_influx::filesystems::query();
    let stats_fut =
        get_influx::<iml_influx::filesystems::InfluxResponse>(client, "iml_stats", query.as_str());

    let influx_resp = stats_fut.await?;
    let stats = iml_influx::filesystems::Response::from(influx_resp);

    tracing::debug!("Influx stats: {:?}", stats);

    let snapshots =
        sqlx::query!("SELECT id, filesystem_name, snapshot_name, snapshot_fsname FROM snapshot")
            .fetch_all(&pool)
            .await?;
    tracing::debug!("Fetched {} snapshots from DB", snapshots.len());

    for snapshot in snapshots {
        let snapshot_id = snapshot.id;
        let snapshot_stats = match stats.get(&snapshot.snapshot_fsname) {
            Some(x) => x,
            None => continue,
        };

        let state = snapshot_client_counts.get_mut(&snapshot_id);
        let clients = snapshot_stats.clients.unwrap_or(0);

        match state {
            Some(State::Monitoring(prev_clients)) => {
                tracing::debug!(
                    "Monitoring. Snapshot {}: {} clients (previously {} clients)",
                    &snapshot.snapshot_fsname,
                    clients,
                    prev_clients,
                );
                if *prev_clients > 0 && clients == 0 {
                    tracing::trace!("counting down for job");
                    let instant = Instant::now() + Duration::from_secs(5 * 60);
                    state.map(|s| *s = State::CountingDown(instant));
                } else {
                    *prev_clients = clients;
                }
            }
            Some(State::CountingDown(when)) => {
                tracing::debug!(
                    "Counting down. Snapshot {}: 0 clients (previously {} clients)",
                    &snapshot.snapshot_fsname,
                    clients
                );
                if clients > 0 {
                    tracing::trace!("changing state");
                    state.map(|s| *s = State::Monitoring(clients));
                } else if Instant::now() >= *when {
                    tracing::trace!("running the job");
                    state.map(|s| *s = State::Monitoring(0));

                    let query = snapshot_queries::unmount::build(
                        &snapshot.filesystem_name,
                        &snapshot.snapshot_name,
                    );
                    let resp: iml_graphql_queries::Response<snapshot_queries::unmount::Resp> =
                        graphql(client_2.clone(), query).await?;
                    let command = Result::from(resp)?.data.unmount_snapshot;
                    wait_for_cmds_success(&[command], None).await?;
                }
            }
            None => {
                tracing::debug!(
                    "Just learnt about this snapshot. Snapshot {}: {} clients",
                    &snapshot.snapshot_fsname,
                    clients,
                );

                snapshot_client_counts.insert(snapshot_id, State::Monitoring(clients));
            }
        }
    }

    Ok(())
}

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    iml_tracing::init();

    let pool = iml_rabbit::connect_to_rabbit(1);

    let conn = iml_rabbit::get_conn(pool).await?;

    let ch = iml_rabbit::create_channel(&conn).await?;

    let mut s = consume_data::<Vec<snapshot::Snapshot>>(&ch, "rust_agent_snapshot_rx");

    let pool = get_db_pool(get_pool_limit().unwrap_or(DEFAULT_POOL_LIMIT)).await?;
    let pool_2 = pool.clone();
    sqlx::migrate!("../../migrations").run(&pool).await?;

    tokio::spawn(async move {
        let mut interval = interval(Duration::from_secs(60));
        let mut snapshot_client_counts: HashMap<i32, State> = HashMap::new();

        while let Some(_) = interval.next().await {
            let tick_result = tick(&mut snapshot_client_counts, pool_2.clone()).await;
            if let Err(e) = tick_result {
                tracing::error!("Error during handling snapshot autounmount: {}", e);
            }
        }
    });

    while let Some((fqdn, snapshots)) = s.try_next().await? {
        tracing::debug!("snapshots from {}: {:?}", fqdn, snapshots);

        let snaps = snapshots.into_iter().fold(
            (vec![], vec![], vec![], vec![], vec![], vec![], vec![]),
            |mut acc, s| {
                acc.0.push(s.filesystem_name);
                acc.1.push(s.snapshot_name);
                acc.2.push(s.create_time.naive_utc());
                acc.3.push(s.modify_time.naive_utc());
                acc.4.push(s.snapshot_fsname.clone());
                acc.5.push(s.mounted);
                acc.6.push(s.comment);

                acc
            },
        );

        let mut transaction = pool.begin().await?;

        sqlx::query!(
            r#"
            DELETE FROM snapshot
            WHERE (filesystem_name, snapshot_name) NOT IN (SELECT * FROM UNNEST ($1::text[], $2::text[]))
            "#,
            &snaps.0,
            &snaps.1,
        )
        .execute(&mut transaction)
        .await?;

        sqlx::query!(
            r#"
            INSERT INTO snapshot (filesystem_name, snapshot_name, create_time, modify_time, snapshot_fsname, mounted, comment)
            SELECT * FROM
            UNNEST (
                $1::text[],
                $2::text[],
                $3::timestamp[],
                $4::timestamp[],
                $5::text[],
                $6::bool[],
                $7::text[]
            )
            ON CONFLICT (filesystem_name, snapshot_name) DO UPDATE
            SET
                create_time = excluded.create_time,
                modify_time = excluded.modify_time,
                snapshot_fsname = excluded.snapshot_fsname,
                mounted = excluded.mounted,
                comment = excluded.comment
            "#,
            &snaps.0,
            &snaps.1,
            &snaps.2,
            &snaps.3,
            &snaps.4,
            &snaps.5 as &[Option<bool>],
            &snaps.6 as &[Option<String>],
        )
        .execute(&mut transaction)
        .await?;

        transaction.commit().await?;
    }

    Ok(())
}
<|endoftext|>


<commit_before>//! Defines database & queries for macro expansion.

use std::sync::Arc;

use mbe::{ExpandResult, MacroRules};
use ra_db::{salsa, SourceDatabase};
use ra_parser::FragmentKind;
use ra_prof::profile;
use ra_syntax::{algo::diff, AstNode, Parse, SyntaxKind::*, SyntaxNode};

use crate::{
    ast_id_map::AstIdMap, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallLoc, EagerMacroId,
    HirFileId, HirFileIdRepr, LazyMacroId, MacroCallId, MacroCallLoc, MacroDefId, MacroDefKind,
    MacroFile,
};

#[derive(Debug, Clone, Eq, PartialEq)]
pub enum TokenExpander {
    MacroRules(mbe::MacroRules),
    Builtin(BuiltinFnLikeExpander),
    BuiltinDerive(BuiltinDeriveExpander),
}

impl TokenExpander {
    pub fn expand(
        &self,
        db: &dyn AstDatabase,
        id: LazyMacroId,
        tt: &tt::Subtree,
    ) -> mbe::ExpandResult<tt::Subtree> {
        match self {
            TokenExpander::MacroRules(it) => it.expand(tt),
            // FIXME switch these to ExpandResult as well
            TokenExpander::Builtin(it) => it.expand(db, id, tt).into(),
            TokenExpander::BuiltinDerive(it) => it.expand(db, id, tt).into(),
        }
    }

    pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId {
        match self {
            TokenExpander::MacroRules(it) => it.map_id_down(id),
            TokenExpander::Builtin(..) => id,
            TokenExpander::BuiltinDerive(..) => id,
        }
    }

    pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, mbe::Origin) {
        match self {
            TokenExpander::MacroRules(it) => it.map_id_up(id),
            TokenExpander::Builtin(..) => (id, mbe::Origin::Call),
            TokenExpander::BuiltinDerive(..) => (id, mbe::Origin::Call),
        }
    }
}

// FIXME: rename to ExpandDatabase
#[salsa::query_group(AstDatabaseStorage)]
pub trait AstDatabase: SourceDatabase {
    fn ast_id_map(&self, file_id: HirFileId) -> Arc<AstIdMap>;

    #[salsa::transparent]
    fn parse_or_expand(&self, file_id: HirFileId) -> Option<SyntaxNode>;

    #[salsa::interned]
    fn intern_macro(&self, macro_call: MacroCallLoc) -> LazyMacroId;
    fn macro_arg(&self, id: MacroCallId) -> Option<Arc<(tt::Subtree, mbe::TokenMap)>>;
    fn macro_def(&self, id: MacroDefId) -> Option<Arc<(TokenExpander, mbe::TokenMap)>>;
    fn parse_macro(&self, macro_file: MacroFile)
        -> Option<(Parse<SyntaxNode>, Arc<mbe::TokenMap>)>;
    fn macro_expand(&self, macro_call: MacroCallId) -> (Option<Arc<tt::Subtree>>, Option<String>);

    #[salsa::interned]
    fn intern_eager_expansion(&self, eager: EagerCallLoc) -> EagerMacroId;
}

/// This expands the given macro call, but with different arguments. This is
/// used for completion, where we want to see what 'would happen' if we insert a
/// token. The `token_to_map` mapped down into the expansion, with the mapped
/// token returned.
pub fn expand_hypothetical(
    db: &dyn AstDatabase,
    actual_macro_call: MacroCallId,
    hypothetical_args: &ra_syntax::ast::TokenTree,
    token_to_map: ra_syntax::SyntaxToken,
) -> Option<(SyntaxNode, ra_syntax::SyntaxToken)> {
    let macro_file = MacroFile { macro_call_id: actual_macro_call };
    let (tt, tmap_1) = mbe::syntax_node_to_token_tree(hypothetical_args.syntax()).unwrap();
    let range =
        token_to_map.text_range().checked_sub(hypothetical_args.syntax().text_range().start())?;
    let token_id = tmap_1.token_by_range(range)?;
    let macro_def = expander(db, actual_macro_call)?;
    let (node, tmap_2) =
        parse_macro_with_arg(db, macro_file, Some(std::sync::Arc::new((tt, tmap_1))))?;
    let token_id = macro_def.0.map_id_down(token_id);
    let range = tmap_2.range_by_token(token_id)?.by_kind(token_to_map.kind())?;
    let token = ra_syntax::algo::find_covering_element(&node.syntax_node(), range).into_token()?;
    Some((node.syntax_node(), token))
}

pub(crate) fn ast_id_map(db: &dyn AstDatabase, file_id: HirFileId) -> Arc<AstIdMap> {
    let map =
        db.parse_or_expand(file_id).map_or_else(AstIdMap::default, |it| AstIdMap::from_source(&it));
    Arc::new(map)
}

pub(crate) fn macro_def(
    db: &dyn AstDatabase,
    id: MacroDefId,
) -> Option<Arc<(TokenExpander, mbe::TokenMap)>> {
    match id.kind {
        MacroDefKind::Declarative => {
            let macro_call = id.ast_id?.to_node(db);
            let arg = macro_call.token_tree()?;
            let (tt, tmap) = mbe::ast_to_token_tree(&arg).or_else(|| {
                log::warn!("fail on macro_def to token tree: {:#?}", arg);
                None
            })?;
            let rules = match MacroRules::parse(&tt) {
                Ok(it) => it,
                Err(err) => {
                    log::warn!("fail on macro_def parse: error: {:#?} {:#?}", err, tt);
                    return None;
                }
            };
            Some(Arc::new((TokenExpander::MacroRules(rules), tmap)))
        }
        MacroDefKind::BuiltIn(expander) => {
            Some(Arc::new((TokenExpander::Builtin(expander), mbe::TokenMap::default())))
        }
        MacroDefKind::BuiltInDerive(expander) => {
            Some(Arc::new((TokenExpander::BuiltinDerive(expander), mbe::TokenMap::default())))
        }
        MacroDefKind::BuiltInEager(_expander) => None,
    }
}

pub(crate) fn macro_arg(
    db: &dyn AstDatabase,
    id: MacroCallId,
) -> Option<Arc<(tt::Subtree, mbe::TokenMap)>> {
    let id = match id {
        MacroCallId::LazyMacro(id) => id,
        MacroCallId::EagerMacro(_id) => {
            // FIXME: support macro_arg for eager macro
            return None;
        }
    };
    let loc = db.lookup_intern_macro(id);
    let arg = loc.kind.arg(db)?;
    let (tt, tmap) = mbe::syntax_node_to_token_tree(&arg)?;
    Some(Arc::new((tt, tmap)))
}

pub(crate) fn macro_expand(
    db: &dyn AstDatabase,
    id: MacroCallId,
) -> (Option<Arc<tt::Subtree>>, Option<String>) {
    macro_expand_with_arg(db, id, None)
}

fn expander(db: &dyn AstDatabase, id: MacroCallId) -> Option<Arc<(TokenExpander, mbe::TokenMap)>> {
    let lazy_id = match id {
        MacroCallId::LazyMacro(id) => id,
        MacroCallId::EagerMacro(_id) => {
            return None;
        }
    };

    let loc = db.lookup_intern_macro(lazy_id);
    let macro_rules = db.macro_def(loc.def)?;
    Some(macro_rules)
}

fn macro_expand_with_arg(
    db: &dyn AstDatabase,
    id: MacroCallId,
    arg: Option<Arc<(tt::Subtree, mbe::TokenMap)>>,
) -> (Option<Arc<tt::Subtree>>, Option<String>) {
    let lazy_id = match id {
        MacroCallId::LazyMacro(id) => id,
        MacroCallId::EagerMacro(id) => {
            if arg.is_some() {
                return (
                    None,
                    Some("hypothetical macro expansion not implemented for eager macro".to_owned()),
                );
            } else {
                return (Some(db.lookup_intern_eager_expansion(id).subtree), None);
            }
        }
    };

    let loc = db.lookup_intern_macro(lazy_id);
    let macro_arg = match arg.or_else(|| db.macro_arg(id)) {
        Some(it) => it,
        None => return (None, Some("Fail to args in to tt::TokenTree".into())),
    };

    let macro_rules = match db.macro_def(loc.def) {
        Some(it) => it,
        None => return (None, Some("Fail to find macro definition".into())),
    };
    let ExpandResult(tt, err) = macro_rules.0.expand(db, lazy_id, &macro_arg.0);
    // Set a hard limit for the expanded tt
    let count = tt.count();
    if count > 65536 {
        return (None, Some(format!("Total tokens count exceed limit : count = {}", count)));
    }
    (Some(Arc::new(tt)), err.map(|e| format!("{:?}", e)))
}

pub(crate) fn parse_or_expand(db: &dyn AstDatabase, file_id: HirFileId) -> Option<SyntaxNode> {
    match file_id.0 {
        HirFileIdRepr::FileId(file_id) => Some(db.parse(file_id).tree().syntax().clone()),
        HirFileIdRepr::MacroFile(macro_file) => {
            db.parse_macro(macro_file).map(|(it, _)| it.syntax_node())
        }
    }
}

pub(crate) fn parse_macro(
    db: &dyn AstDatabase,
    macro_file: MacroFile,
) -> Option<(Parse<SyntaxNode>, Arc<mbe::TokenMap>)> {
    parse_macro_with_arg(db, macro_file, None)
}

pub fn parse_macro_with_arg(
    db: &dyn AstDatabase,
    macro_file: MacroFile,
    arg: Option<Arc<(tt::Subtree, mbe::TokenMap)>>,
) -> Option<(Parse<SyntaxNode>, Arc<mbe::TokenMap>)> {
    let _p = profile("parse_macro_query");

    let macro_call_id = macro_file.macro_call_id;
    let (tt, err) = if let Some(arg) = arg {
        macro_expand_with_arg(db, macro_call_id, Some(arg))
    } else {
        db.macro_expand(macro_call_id)
    };
    if let Some(err) = &err {
        // Note:
        // The final goal we would like to make all parse_macro success,
        // such that the following log will not call anyway.
        match macro_call_id {
            MacroCallId::LazyMacro(id) => {
                let loc: MacroCallLoc = db.lookup_intern_macro(id);
                let node = loc.kind.node(db);

                // collect parent information for warning log
                let parents = std::iter::successors(loc.kind.file_id().call_node(db), |it| {
                    it.file_id.call_node(db)
                })
                .map(|n| format!("{:#}", n.value))
                .collect::<Vec<_>>()
                .join("\n");

                log::warn!(
                    "fail on macro_parse: (reason: {} macro_call: {:#}) parents: {}",
                    err,
                    node.value,
                    parents
                );
            }
            _ => {
                log::warn!("fail on macro_parse: (reason: {})", err);
            }
        }
    };
    let tt = tt?;

    let fragment_kind = to_fragment_kind(db, macro_call_id);

    let (parse, rev_token_map) = mbe::token_tree_to_syntax_node(&tt, fragment_kind).ok()?;

    if err.is_none() {
        Some((parse, Arc::new(rev_token_map)))
    } else {
        // FIXME:
        // In future, we should proprate the actual error with recovery information
        // instead of ignore the error here.

        // Safe check for recurisve identity macro
        let node = parse.syntax_node();
        let file: HirFileId = macro_file.into();
        let call_node = file.call_node(db)?;

        if !diff(&node, &call_node.value).is_empty() {
            Some((parse, Arc::new(rev_token_map)))
        } else {
            None
        }
    }
}

/// Given a `MacroCallId`, return what `FragmentKind` it belongs to.
/// FIXME: Not completed
fn to_fragment_kind(db: &dyn AstDatabase, id: MacroCallId) -> FragmentKind {
    let lazy_id = match id {
        MacroCallId::LazyMacro(id) => id,
        MacroCallId::EagerMacro(id) => {
            return db.lookup_intern_eager_expansion(id).fragment;
        }
    };
    let syn = db.lookup_intern_macro(lazy_id).kind.node(db).value;

    let parent = match syn.parent() {
        Some(it) => it,
        None => {
            // FIXME:
            // If it is root, which means the parent HirFile
            // MacroKindFile must be non-items
            // return expr now.
            return FragmentKind::Expr;
        }
    };

    match parent.kind() {
        MACRO_ITEMS | SOURCE_FILE => FragmentKind::Items,
        ITEM_LIST => FragmentKind::Items,
        LET_STMT => {
            // FIXME: Handle Pattern
            FragmentKind::Expr
        }
        // FIXME: Expand to statements in appropriate positions; HIR lowering needs to handle that
        EXPR_STMT | BLOCK => FragmentKind::Expr,
        ARG_LIST => FragmentKind::Expr,
        TRY_EXPR => FragmentKind::Expr,
        TUPLE_EXPR => FragmentKind::Expr,
        PAREN_EXPR => FragmentKind::Expr,

        FOR_EXPR => FragmentKind::Expr,
        PATH_EXPR => FragmentKind::Expr,
        LAMBDA_EXPR => FragmentKind::Expr,
        CONDITION => FragmentKind::Expr,
        BREAK_EXPR => FragmentKind::Expr,
        RETURN_EXPR => FragmentKind::Expr,
        BLOCK_EXPR => FragmentKind::Expr,
        MATCH_EXPR => FragmentKind::Expr,
        MATCH_ARM => FragmentKind::Expr,
        MATCH_GUARD => FragmentKind::Expr,
        RECORD_FIELD => FragmentKind::Expr,
        CALL_EXPR => FragmentKind::Expr,
        INDEX_EXPR => FragmentKind::Expr,
        METHOD_CALL_EXPR => FragmentKind::Expr,
        AWAIT_EXPR => FragmentKind::Expr,
        CAST_EXPR => FragmentKind::Expr,
        REF_EXPR => FragmentKind::Expr,
        PREFIX_EXPR => FragmentKind::Expr,
        RANGE_EXPR => FragmentKind::Expr,
        BIN_EXPR => FragmentKind::Expr,
        _ => {
            // Unknown , Just guess it is `Items`
            FragmentKind::Items
        }
    }
}
<commit_msg>Fix typo<commit_after>//! Defines database & queries for macro expansion.

use std::sync::Arc;

use mbe::{ExpandResult, MacroRules};
use ra_db::{salsa, SourceDatabase};
use ra_parser::FragmentKind;
use ra_prof::profile;
use ra_syntax::{algo::diff, AstNode, Parse, SyntaxKind::*, SyntaxNode};

use crate::{
    ast_id_map::AstIdMap, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallLoc, EagerMacroId,
    HirFileId, HirFileIdRepr, LazyMacroId, MacroCallId, MacroCallLoc, MacroDefId, MacroDefKind,
    MacroFile,
};

#[derive(Debug, Clone, Eq, PartialEq)]
pub enum TokenExpander {
    MacroRules(mbe::MacroRules),
    Builtin(BuiltinFnLikeExpander),
    BuiltinDerive(BuiltinDeriveExpander),
}

impl TokenExpander {
    pub fn expand(
        &self,
        db: &dyn AstDatabase,
        id: LazyMacroId,
        tt: &tt::Subtree,
    ) -> mbe::ExpandResult<tt::Subtree> {
        match self {
            TokenExpander::MacroRules(it) => it.expand(tt),
            // FIXME switch these to ExpandResult as well
            TokenExpander::Builtin(it) => it.expand(db, id, tt).into(),
            TokenExpander::BuiltinDerive(it) => it.expand(db, id, tt).into(),
        }
    }

    pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId {
        match self {
            TokenExpander::MacroRules(it) => it.map_id_down(id),
            TokenExpander::Builtin(..) => id,
            TokenExpander::BuiltinDerive(..) => id,
        }
    }

    pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, mbe::Origin) {
        match self {
            TokenExpander::MacroRules(it) => it.map_id_up(id),
            TokenExpander::Builtin(..) => (id, mbe::Origin::Call),
            TokenExpander::BuiltinDerive(..) => (id, mbe::Origin::Call),
        }
    }
}

// FIXME: rename to ExpandDatabase
#[salsa::query_group(AstDatabaseStorage)]
pub trait AstDatabase: SourceDatabase {
    fn ast_id_map(&self, file_id: HirFileId) -> Arc<AstIdMap>;

    #[salsa::transparent]
    fn parse_or_expand(&self, file_id: HirFileId) -> Option<SyntaxNode>;

    #[salsa::interned]
    fn intern_macro(&self, macro_call: MacroCallLoc) -> LazyMacroId;
    fn macro_arg(&self, id: MacroCallId) -> Option<Arc<(tt::Subtree, mbe::TokenMap)>>;
    fn macro_def(&self, id: MacroDefId) -> Option<Arc<(TokenExpander, mbe::TokenMap)>>;
    fn parse_macro(&self, macro_file: MacroFile)
        -> Option<(Parse<SyntaxNode>, Arc<mbe::TokenMap>)>;
    fn macro_expand(&self, macro_call: MacroCallId) -> (Option<Arc<tt::Subtree>>, Option<String>);

    #[salsa::interned]
    fn intern_eager_expansion(&self, eager: EagerCallLoc) -> EagerMacroId;
}

/// This expands the given macro call, but with different arguments. This is
/// used for completion, where we want to see what 'would happen' if we insert a
/// token. The `token_to_map` mapped down into the expansion, with the mapped
/// token returned.
pub fn expand_hypothetical(
    db: &dyn AstDatabase,
    actual_macro_call: MacroCallId,
    hypothetical_args: &ra_syntax::ast::TokenTree,
    token_to_map: ra_syntax::SyntaxToken,
) -> Option<(SyntaxNode, ra_syntax::SyntaxToken)> {
    let macro_file = MacroFile { macro_call_id: actual_macro_call };
    let (tt, tmap_1) = mbe::syntax_node_to_token_tree(hypothetical_args.syntax()).unwrap();
    let range =
        token_to_map.text_range().checked_sub(hypothetical_args.syntax().text_range().start())?;
    let token_id = tmap_1.token_by_range(range)?;
    let macro_def = expander(db, actual_macro_call)?;
    let (node, tmap_2) =
        parse_macro_with_arg(db, macro_file, Some(std::sync::Arc::new((tt, tmap_1))))?;
    let token_id = macro_def.0.map_id_down(token_id);
    let range = tmap_2.range_by_token(token_id)?.by_kind(token_to_map.kind())?;
    let token = ra_syntax::algo::find_covering_element(&node.syntax_node(), range).into_token()?;
    Some((node.syntax_node(), token))
}

pub(crate) fn ast_id_map(db: &dyn AstDatabase, file_id: HirFileId) -> Arc<AstIdMap> {
    let map =
        db.parse_or_expand(file_id).map_or_else(AstIdMap::default, |it| AstIdMap::from_source(&it));
    Arc::new(map)
}

pub(crate) fn macro_def(
    db: &dyn AstDatabase,
    id: MacroDefId,
) -> Option<Arc<(TokenExpander, mbe::TokenMap)>> {
    match id.kind {
        MacroDefKind::Declarative => {
            let macro_call = id.ast_id?.to_node(db);
            let arg = macro_call.token_tree()?;
            let (tt, tmap) = mbe::ast_to_token_tree(&arg).or_else(|| {
                log::warn!("fail on macro_def to token tree: {:#?}", arg);
                None
            })?;
            let rules = match MacroRules::parse(&tt) {
                Ok(it) => it,
                Err(err) => {
                    log::warn!("fail on macro_def parse: error: {:#?} {:#?}", err, tt);
                    return None;
                }
            };
            Some(Arc::new((TokenExpander::MacroRules(rules), tmap)))
        }
        MacroDefKind::BuiltIn(expander) => {
            Some(Arc::new((TokenExpander::Builtin(expander), mbe::TokenMap::default())))
        }
        MacroDefKind::BuiltInDerive(expander) => {
            Some(Arc::new((TokenExpander::BuiltinDerive(expander), mbe::TokenMap::default())))
        }
        MacroDefKind::BuiltInEager(_expander) => None,
    }
}

pub(crate) fn macro_arg(
    db: &dyn AstDatabase,
    id: MacroCallId,
) -> Option<Arc<(tt::Subtree, mbe::TokenMap)>> {
    let id = match id {
        MacroCallId::LazyMacro(id) => id,
        MacroCallId::EagerMacro(_id) => {
            // FIXME: support macro_arg for eager macro
            return None;
        }
    };
    let loc = db.lookup_intern_macro(id);
    let arg = loc.kind.arg(db)?;
    let (tt, tmap) = mbe::syntax_node_to_token_tree(&arg)?;
    Some(Arc::new((tt, tmap)))
}

pub(crate) fn macro_expand(
    db: &dyn AstDatabase,
    id: MacroCallId,
) -> (Option<Arc<tt::Subtree>>, Option<String>) {
    macro_expand_with_arg(db, id, None)
}

fn expander(db: &dyn AstDatabase, id: MacroCallId) -> Option<Arc<(TokenExpander, mbe::TokenMap)>> {
    let lazy_id = match id {
        MacroCallId::LazyMacro(id) => id,
        MacroCallId::EagerMacro(_id) => {
            return None;
        }
    };

    let loc = db.lookup_intern_macro(lazy_id);
    let macro_rules = db.macro_def(loc.def)?;
    Some(macro_rules)
}

fn macro_expand_with_arg(
    db: &dyn AstDatabase,
    id: MacroCallId,
    arg: Option<Arc<(tt::Subtree, mbe::TokenMap)>>,
) -> (Option<Arc<tt::Subtree>>, Option<String>) {
    let lazy_id = match id {
        MacroCallId::LazyMacro(id) => id,
        MacroCallId::EagerMacro(id) => {
            if arg.is_some() {
                return (
                    None,
                    Some("hypothetical macro expansion not implemented for eager macro".to_owned()),
                );
            } else {
                return (Some(db.lookup_intern_eager_expansion(id).subtree), None);
            }
        }
    };

    let loc = db.lookup_intern_macro(lazy_id);
    let macro_arg = match arg.or_else(|| db.macro_arg(id)) {
        Some(it) => it,
        None => return (None, Some("Fail to args in to tt::TokenTree".into())),
    };

    let macro_rules = match db.macro_def(loc.def) {
        Some(it) => it,
        None => return (None, Some("Fail to find macro definition".into())),
    };
    let ExpandResult(tt, err) = macro_rules.0.expand(db, lazy_id, &macro_arg.0);
    // Set a hard limit for the expanded tt
    let count = tt.count();
    if count > 65536 {
        return (None, Some(format!("Total tokens count exceed limit : count = {}", count)));
    }
    (Some(Arc::new(tt)), err.map(|e| format!("{:?}", e)))
}

pub(crate) fn parse_or_expand(db: &dyn AstDatabase, file_id: HirFileId) -> Option<SyntaxNode> {
    match file_id.0 {
        HirFileIdRepr::FileId(file_id) => Some(db.parse(file_id).tree().syntax().clone()),
        HirFileIdRepr::MacroFile(macro_file) => {
            db.parse_macro(macro_file).map(|(it, _)| it.syntax_node())
        }
    }
}

pub(crate) fn parse_macro(
    db: &dyn AstDatabase,
    macro_file: MacroFile,
) -> Option<(Parse<SyntaxNode>, Arc<mbe::TokenMap>)> {
    parse_macro_with_arg(db, macro_file, None)
}

pub fn parse_macro_with_arg(
    db: &dyn AstDatabase,
    macro_file: MacroFile,
    arg: Option<Arc<(tt::Subtree, mbe::TokenMap)>>,
) -> Option<(Parse<SyntaxNode>, Arc<mbe::TokenMap>)> {
    let _p = profile("parse_macro_query");

    let macro_call_id = macro_file.macro_call_id;
    let (tt, err) = if let Some(arg) = arg {
        macro_expand_with_arg(db, macro_call_id, Some(arg))
    } else {
        db.macro_expand(macro_call_id)
    };
    if let Some(err) = &err {
        // Note:
        // The final goal we would like to make all parse_macro success,
        // such that the following log will not call anyway.
        match macro_call_id {
            MacroCallId::LazyMacro(id) => {
                let loc: MacroCallLoc = db.lookup_intern_macro(id);
                let node = loc.kind.node(db);

                // collect parent information for warning log
                let parents = std::iter::successors(loc.kind.file_id().call_node(db), |it| {
                    it.file_id.call_node(db)
                })
                .map(|n| format!("{:#}", n.value))
                .collect::<Vec<_>>()
                .join("\n");

                log::warn!(
                    "fail on macro_parse: (reason: {} macro_call: {:#}) parents: {}",
                    err,
                    node.value,
                    parents
                );
            }
            _ => {
                log::warn!("fail on macro_parse: (reason: {})", err);
            }
        }
    };
    let tt = tt?;

    let fragment_kind = to_fragment_kind(db, macro_call_id);

    let (parse, rev_token_map) = mbe::token_tree_to_syntax_node(&tt, fragment_kind).ok()?;

    if err.is_none() {
        Some((parse, Arc::new(rev_token_map)))
    } else {
        // FIXME:
        // In future, we should propagate the actual error with recovery information
        // instead of ignore the error here.

        // Safe check for recurisve identity macro
        let node = parse.syntax_node();
        let file: HirFileId = macro_file.into();
        let call_node = file.call_node(db)?;

        if !diff(&node, &call_node.value).is_empty() {
            Some((parse, Arc::new(rev_token_map)))
        } else {
            None
        }
    }
}

/// Given a `MacroCallId`, return what `FragmentKind` it belongs to.
/// FIXME: Not completed
fn to_fragment_kind(db: &dyn AstDatabase, id: MacroCallId) -> FragmentKind {
    let lazy_id = match id {
        MacroCallId::LazyMacro(id) => id,
        MacroCallId::EagerMacro(id) => {
            return db.lookup_intern_eager_expansion(id).fragment;
        }
    };
    let syn = db.lookup_intern_macro(lazy_id).kind.node(db).value;

    let parent = match syn.parent() {
        Some(it) => it,
        None => {
            // FIXME:
            // If it is root, which means the parent HirFile
            // MacroKindFile must be non-items
            // return expr now.
            return FragmentKind::Expr;
        }
    };

    match parent.kind() {
        MACRO_ITEMS | SOURCE_FILE => FragmentKind::Items,
        ITEM_LIST => FragmentKind::Items,
        LET_STMT => {
            // FIXME: Handle Pattern
            FragmentKind::Expr
        }
        // FIXME: Expand to statements in appropriate positions; HIR lowering needs to handle that
        EXPR_STMT | BLOCK => FragmentKind::Expr,
        ARG_LIST => FragmentKind::Expr,
        TRY_EXPR => FragmentKind::Expr,
        TUPLE_EXPR => FragmentKind::Expr,
        PAREN_EXPR => FragmentKind::Expr,

        FOR_EXPR => FragmentKind::Expr,
        PATH_EXPR => FragmentKind::Expr,
        LAMBDA_EXPR => FragmentKind::Expr,
        CONDITION => FragmentKind::Expr,
        BREAK_EXPR => FragmentKind::Expr,
        RETURN_EXPR => FragmentKind::Expr,
        BLOCK_EXPR => FragmentKind::Expr,
        MATCH_EXPR => FragmentKind::Expr,
        MATCH_ARM => FragmentKind::Expr,
        MATCH_GUARD => FragmentKind::Expr,
        RECORD_FIELD => FragmentKind::Expr,
        CALL_EXPR => FragmentKind::Expr,
        INDEX_EXPR => FragmentKind::Expr,
        METHOD_CALL_EXPR => FragmentKind::Expr,
        AWAIT_EXPR => FragmentKind::Expr,
        CAST_EXPR => FragmentKind::Expr,
        REF_EXPR => FragmentKind::Expr,
        PREFIX_EXPR => FragmentKind::Expr,
        RANGE_EXPR => FragmentKind::Expr,
        BIN_EXPR => FragmentKind::Expr,
        _ => {
            // Unknown , Just guess it is `Items`
            FragmentKind::Items
        }
    }
}
<|endoftext|>


<commit_before>use collections::HashMap;
use std::io::{Open, Read, Write, TempDir};
use std::num::Zero;
use std::comm::{Data, Empty, Disconnected};
use sync::{Arc, RWLock};
use sync::Future;
use rand::task_rng;
use rand::distributions::{IndependentSample, Range};
use rand::distributions::range::SampleRange;
use native::io::file;
use native::io::file::FileDesc;
use std::rt::rtio::RtioFileStream;

/* Linux seems to use 4K segments, that's a good bet */
pub static PAGE_SIZE: uint = 4 * 1024;
/*
 * how much of the page_id should be reserved for pages?
 * the rest goes for segments. The bigger this is, the larger the segment files are
 * and the more pages they contain. (2^PAGE_BITS)*PAGE_SIZE gives you segment size.
 *
 * Adjust as required.
 */
static PAGE_BITS: uint = 48;

pub struct BufferManager {
	size: uint,
	entries: HashMap<u64, BufferEntry>,
	path: Path,
}

struct BufferEntry {
	frame: Arc<RWLock<BufferFrame>>,
}

#[deriving(Eq)]
enum Status {
	Free,
	Fixed(uint),
}

struct BufferFrame {
	page_id: u64,
	data: Vec<u8>,
	fixed: Status,
}

/* Destructor trait implementation */
impl Drop for BufferManager {
	fn drop(&mut self) {
		// TODO
		info!("Dropping");
	}
}

impl BufferManager {
	pub fn new(size: uint, path: Path) -> BufferManager {
		let h = HashMap::with_capacity(size);
		BufferManager {size: size, entries: h, path: path}
	}

	fn open_or_create(&self, page_id: u64) -> FileDesc {
		let (segment, _) = split_segment(page_id);
		let file_path = self.path.join(segment.to_str());
		match file::open(&file_path.to_c_str(), Open, Read) {
			Ok(f) => return f,
			Err(_) => self.create(page_id),
		};
		match file::open(&file_path.to_c_str(), Open, Read) {
			Ok(f) => f,
			Err(e) => fail!("Failed reopening written file: {}", e),
		}
	}

	fn create(&self, page_id: u64) {
		let (segment, offset) = split_segment(page_id);
		let path = self.path.join(segment.to_str());

		match file::open(&path.to_c_str(), Open, Write) {
			Ok(mut f) => {
				match f.pwrite([0_u8, ..PAGE_SIZE], offset * PAGE_SIZE as u64) {
					Ok(_) => (),
					//Err(e) => fail!("Failed pwriting to file {}: {}", &path.to_c_str().as_str(), e),
					// in this case, err means everything is fine, since there is a bug in Rust' libnative
					Err(_) => (),
				};
			},
			Err(e) => fail!("Failed opening file for write: {}", e),
		};
	}

	/*
	 * returns true if page could be loaded
	 */
	fn load_page(&mut self, page_id: u64) -> bool {
		if self.entries.len() == self.size {
			if !self.evict_page() {
				return false;
			}
		}
		let (_, offset) = split_segment(page_id);

		let mut file_handle = self.open_or_create(page_id);
		let mut buf = [0_u8, ..PAGE_SIZE];
		let content = match file_handle.pread(buf, offset * PAGE_SIZE as u64) {
			Ok(_) => {
				Vec::from_slice(buf)
				/*
				if n == PAGE_SIZE as int {
					Vec::from_slice(buf)
				} else {
					fail!("pread failed: wanted {}, got {}",
						PAGE_SIZE, n)
				}*/
			},
			Err(e) => fail!("Couldn't read from page: {}", e),
		};

		let frame = BufferFrame {data: content, page_id: page_id, fixed: Free};
		let entry = BufferEntry {frame: Arc::new(RWLock::new(frame))};
		self.entries.insert(page_id, entry);
		true
	}

	/*
	 * return false if no page could be evicted
	 */
	fn evict_page(&mut self) -> bool {
		let random_key = {
			let mut free_keys = self.entries.iter().filter_map(|e| {
				// iterate over entries, check if they are free
				// and return the keys of the free entries
				let (k,v) = e;
				let frame = v.frame.read();
				if frame.fixed == Free {
					Some(k)
				} else {
					None
				}
			});
			// pick a random key
			sample(&mut free_keys).map(|v| v.clone())
		};

		match random_key {
			None => false,
			Some(key) => self.entries.remove(&key),
		}
	}

	pub fn fix_page(&mut self, page_id: u64) -> Option<Arc<RWLock<BufferFrame>>> {
		if !self.entries.contains_key(&page_id) {
			if !self.load_page(page_id) {
				return None;
			}
		}
		let entry = self.entries.get(&page_id);
		{
			let mut f = entry.frame.write();
			f.fixed = match f.fixed {
				Free => Fixed(1),
				Fixed(n) => Fixed(n+1),
			};
		}
		// Arcs can be cloned and they will all point to the same RWLock
		Some(entry.frame.clone())
	}

	pub fn unfix_page(&mut self, frame: Arc<RWLock<BufferFrame>>, is_dirty: bool) {
		{
			let mut frame = frame.write();
			frame.fixed = match frame.fixed {
				Fixed(1) => Free,
				Fixed(n) => Fixed(n-1),
				Free => fail!("Unfixing unfixed page"),
			};
		}

		if !is_dirty {
			return;
		}
		let frame = frame.read();
		if frame.fixed == Free {
			//info!("writing back page \\#{}", frame.page_id);
			self.write_page(frame.page_id, frame.get_data());
		}
	}

	fn write_page(&mut self, page_id: u64, data: &[u8]) {
		let (segment, offset) = split_segment(page_id);
		let file_path = self.path.join(segment.to_str());
		let mut handle = match file::open(&file_path.to_c_str(), Open, Write) {
			Ok(handle) => handle,
			Err(e) => fail!("Opening file for writing failed: {}", e),
		};

		info!("Writing to segment {}, offset {}", segment, offset);
		match handle.pwrite(data, offset * PAGE_SIZE as u64) {
			Ok(_) => (),
			//Err(e) => fail!("Writing to file failed: {}", e),
			// again Rust 0.10 pwrite error
			Err(_) => (),
		};
	}
}

/*
 * A buffer frame, the unit that gets the data
 */
impl BufferFrame {
	/*
	 * get_mut_data returns a borrowed reference to a mutable slice to the vector
	 * contents.
	 */
	pub fn get_mut_data<'a>(&'a mut self) -> &'a mut [u8] {
		self.data.as_mut_slice()
	}
	pub fn get_data<'a>(&'a self) -> &'a [u8] {
		self.data.as_slice()
	}
}

fn sample<'a, T, I:Iterator<T>>(from: &'a mut I) -> Option<T> {
	let from: ~[T] = from.collect();
	let l = from.len();
	if l == 0 {
		return None;
	}
	let index = randrange(l);
	Some(from[index])
}

fn randrange<X: SampleRange + Ord + Zero>(high: X) -> X {
	let between: Range<X> = Range::new(Zero::zero(), high);
	let mut rng = task_rng();
	between.ind_sample(&mut rng)
}

fn split_segment(num: u64) -> (u64, u64) {
	let high = num >> PAGE_BITS;
	let low = num & ((1 << PAGE_BITS) - 1);
	(high, low)
}

#[test]
fn test_create() {
	let dir = match TempDir::new("buffermanager") {
		Some(temp_dir) => temp_dir,
		None => fail!("creation of temporary directory"),
	};

	let mut bm = BufferManager::new(16, dir.path().clone());
	let pageref = match bm.fix_page(42) {
		Some(p) => p,
		None => fail!("Getting page failed"),
	};
	{
		let mut page = pageref.write();
		let data = page.get_mut_data();
		data[0] = 42;
	}
	bm.unfix_page(pageref, true);
}

#[test]
fn test_threads() {
	use rand::random;
	use std::task::spawn;
	use std::os;

	let pages_in_ram: uint = match os::getenv("PAGES_IN_RAM") {
		Some(v) => from_str(v).expect("PAGES_IN_RAM expects integer"),
		None => 1,
	};
	let pages_on_disk: u64 = match os::getenv("PAGES_ON_DISK") {
		Some(v) => from_str(v).expect("PAGES_ON_DISK expects integer"),
		None => 20,
	};
	let thread_count: uint = match os::getenv("THREADS") {
		Some(v) => from_str(v).expect("THREADS expects integer"),
		None => 3,
	};
	let dir = match TempDir::new("buffermanager") {
		Some(temp_dir) => temp_dir,
		None => fail!("creation of temporary directory"),
	};
	let p = dir.path();
	//let p = Path::new(".");

	let mut buffermanager = BufferManager::new(pages_in_ram, p.clone());

	for i in range(0, pages_on_disk) {
		let bf = match buffermanager.fix_page(i) {
			Some(frame) => frame,
			None => fail!("Couldn't fix page {}", i),
		};
		{
			let mut lock = bf.write();
			lock.get_mut_data()[0] = 0;
		}
		buffermanager.unfix_page(bf, true);
	}
	let bm = Arc::new(RWLock::new(buffermanager));

	// start scan thread
	let (tx, rx) = channel();
	let bm_scan = bm.clone();
	let mut scan = Future::spawn(proc() {
		let mut counters = Vec::from_elem(pages_on_disk as uint, 0_u8);
		loop {
			info!("Running scan thread...");
			match rx.try_recv() {
				Empty => {
					let page_number = randrange(pages_on_disk);
					let mut bm = bm_scan.write();
					let bf = match bm.fix_page(page_number) {
						Some(frame) => frame,
						None => fail!("Couldn't scan/fix page"),
					};
					let current_val = {
						let lock = bf.read();
						lock.get_data()[0]
					};
					bm.unfix_page(bf, false);
					bm.downgrade();
					// check if the value is going up
					assert!(&current_val >= counters.get(page_number as uint));
					// set to the new value
					let v = counters.get_mut(page_number as uint);
					*v = current_val;

				},
				/*
				 * terminate on both disconnect and when any kind of
				 * data arrives
				 */
				Disconnected => break,
				Data(_) => break,
			};
		}
	});

	// start read/write threads
	let mut rw_threads: Vec<Future<int>> = Vec::new();
	for _ in range(0, thread_count) {
		let bm = bm.clone();
		rw_threads.push(Future::spawn(proc() {
			let is_write = random::<bool>();
			info!("Creating new {} task",
				if is_write {"write"} else {"read"});
			let page_number = randrange(pages_on_disk);
			let mut bm = bm.write();
			if is_write {
				let bf = match bm.fix_page(page_number) {
					Some(frame) => frame,
					None => fail!("Couldn't fix page"),
				};
				{
					let mut lock = bf.write();
					let data = lock.get_mut_data();
					data[0] = data[0] + 1;
					info!("Wrote to page {}", page_number);
					debug!("data: {}", Vec::from_slice(data));
				}
				bm.unfix_page(bf, is_write);
			} else {
				let bf = match bm.fix_page(page_number) {
					Some(frame) => frame,
					None => fail!("Couldn't fix page"),
				};
				bm.unfix_page(bf, is_write);
			}
			// return whether we wrote (1) or read (0) as future
			if is_write {1} else {0}
		}));
	}

	// Rust does not have join, but we can wait on Futures which does the same
	let total_count = rw_threads.mut_iter().fold(0, |acc, val| acc + val.get());
	info!("Total count in RAM: {}", total_count);

	// terminate the scan thread and wait until it has completed
	tx.send("terminate");
	scan.get();

	// re-open the pages and check whether all numbers got saved
	let mut bm = BufferManager::new(pages_in_ram, p.clone());
	let mut total_count_on_disk = 0;
	for i in range(0, pages_on_disk) {
		let bf = match bm.fix_page(i) {
			Some(frame) => frame,
			None => fail!("Couldn't fix page")
		};
		let value = {
			let lock = bf.read();
			let data = lock.get_data();
			data[0]
		};
		bm.unfix_page(bf, false);
		// cast up from u8 to int
		total_count_on_disk += value as int;
	}
	info!("Total count on disk: {}", total_count_on_disk);
	assert_eq!(total_count, total_count_on_disk);
}
<commit_msg>Buffer now only writes to disc on eviction and drop<commit_after>use collections::HashMap;
use std::io::{Open, Read, Write, TempDir};
use std::num::Zero;
use std::comm::{Data, Empty, Disconnected};
use sync::{Arc, RWLock};
use sync::Future;
use rand::task_rng;
use rand::distributions::{IndependentSample, Range};
use rand::distributions::range::SampleRange;
use native::io::file;
use native::io::file::FileDesc;
use std::rt::rtio::RtioFileStream;

/* Linux seems to use 4K segments, that's a good bet */
pub static PAGE_SIZE: uint = 4 * 1024;
/*
 * how much of the page_id should be reserved for pages?
 * the rest goes for segments. The bigger this is, the larger the segment files are
 * and the more pages they contain. (2^PAGE_BITS)*PAGE_SIZE gives you segment size.
 *
 * Adjust as required.
 */
static PAGE_BITS: uint = 48;

pub struct BufferManager {
	size: uint,
	entries: HashMap<u64, BufferEntry>,
	path: Path,
}

struct BufferEntry {
	frame: Arc<RWLock<BufferFrame>>,
	written: Cleanliness,
}

#[deriving(Eq)]
enum Status {
	Free,
	Fixed(uint),
}

#[deriving(Eq)]
enum Cleanliness {
	Clean,
	Dirty,
}

struct BufferFrame {
	page_id: u64,
	data: Vec<u8>,
	fixed: Status,
}

/* Destructor trait implementation */
impl Drop for BufferManager {
	fn drop(&mut self) {
		for (page, entry) in self.entries.iter() {
			info!("Drop k == {}, dirty? {:?}", page, entry.written);
			match entry.written {
				Clean => (),
				Dirty => {
					let frame = entry.frame.read();
					self.write_page(frame.page_id, frame.get_data());
				},
			}
		}
	}
}

impl BufferManager {
	pub fn new(size: uint, path: Path) -> BufferManager {
		let h = HashMap::with_capacity(size);
		BufferManager {size: size, entries: h, path: path}
	}

	fn open_or_create(&self, page_id: u64) -> FileDesc {
		let (segment, _) = split_segment(page_id);
		let file_path = self.path.join(segment.to_str());
		match file::open(&file_path.to_c_str(), Open, Read) {
			Ok(f) => return f,
			Err(_) => self.create(page_id),
		};
		match file::open(&file_path.to_c_str(), Open, Read) {
			Ok(f) => f,
			Err(e) => fail!("Failed reopening written file: {}", e),
		}
	}

	fn create(&self, page_id: u64) {
		let (segment, offset) = split_segment(page_id);
		let path = self.path.join(segment.to_str());

		match file::open(&path.to_c_str(), Open, Write) {
			Ok(mut f) => {
				match f.pwrite([0_u8, ..PAGE_SIZE], offset * PAGE_SIZE as u64) {
					Ok(_) => (),
					//Err(e) => fail!("Failed pwriting to file {}: {}", &path.to_c_str().as_str(), e),
					// in this case, err means everything is fine, since there is a bug in Rust' libnative
					Err(_) => (),
				};
			},
			Err(e) => fail!("Failed opening file for write: {}", e),
		};
	}

	/*
	 * returns true if page could be loaded
	 */
	fn load_page(&mut self, page_id: u64) -> bool {
		if self.entries.len() == self.size {
			if !self.evict_page() {
				return false;
			}
		}
		let (_, offset) = split_segment(page_id);

		let mut file_handle = self.open_or_create(page_id);
		let mut buf = [0_u8, ..PAGE_SIZE];
		let content = match file_handle.pread(buf, offset * PAGE_SIZE as u64) {
			Ok(_) => {
				Vec::from_slice(buf)
				/*
				if n == PAGE_SIZE as int {
					Vec::from_slice(buf)
				} else {
					fail!("pread failed: wanted {}, got {}",
						PAGE_SIZE, n)
				}*/
			},
			Err(e) => fail!("Couldn't read from page: {}", e),
		};

		let frame = BufferFrame {data: content, page_id: page_id, fixed: Free};
		let entry = BufferEntry {frame: Arc::new(RWLock::new(frame)), written: Clean};
		self.entries.insert(page_id, entry);
		true
	}

	/*
	 * return false if no page could be evicted
	 */
	fn evict_page(&mut self) -> bool {
		let random_key = {
			let mut free_keys = self.entries.iter().filter_map(|e| {
				// iterate over entries, check if they are free
				// and return the keys of the free entries
				let (k,v) = e;
				let frame = v.frame.read();
				if frame.fixed == Free {
					Some(k)
				} else {
					None
				}
			});
			// pick a random key
			sample(&mut free_keys).map(|v| v.clone())
		};

		match random_key {
			None => false,
			Some(key) => {
				match self.entries.pop(&key) {
					None => false,
					Some(entry) => {
						info!("Evicting entry: {:?}", entry);
						if entry.written == Dirty {
							let frame = entry.frame.read();
							self.write_page(frame.page_id, frame.get_data());
						}
						true
					},
				}
			},
		}
	}

	pub fn fix_page(&mut self, page_id: u64) -> Option<Arc<RWLock<BufferFrame>>> {
		if !self.entries.contains_key(&page_id) {
			if !self.load_page(page_id) {
				return None;
			}
		}
		let entry = self.entries.get(&page_id);
		{
			let mut f = entry.frame.write();
			f.fixed = match f.fixed {
				Free => Fixed(1),
				Fixed(n) => Fixed(n+1),
			};
		}
		// Arcs can be cloned and they will all point to the same RWLock
		Some(entry.frame.clone())
	}

	pub fn unfix_page(&mut self, frame: Arc<RWLock<BufferFrame>>, is_dirty: bool) {
		{
			let mut frame = frame.write();
			frame.fixed = match frame.fixed {
				Fixed(1) => Free,
				Fixed(n) => Fixed(n-1),
				Free => fail!("Unfixing unfixed page"),
			};
		}

		if is_dirty {
			let frame = frame.read();
			let mut entry = self.entries.get_mut(&frame.page_id);
			entry.written = Dirty;
		}
	}

	fn write_page(&self, page_id: u64, data: &[u8]) {
		let (segment, offset) = split_segment(page_id);
		let file_path = self.path.join(segment.to_str());
		let mut handle = match file::open(&file_path.to_c_str(), Open, Write) {
			Ok(handle) => handle,
			Err(e) => fail!("Opening file for writing failed: {}", e),
		};

		info!("Writing to segment {}, offset {}", segment, offset);
		match handle.pwrite(data, offset * PAGE_SIZE as u64) {
			Ok(_) => (),
			//Err(e) => fail!("Writing to file failed: {}", e),
			// again Rust 0.10 pwrite error
			Err(_) => (),
		};
	}
}

/*
 * A buffer frame, the unit that gets the data
 */
impl BufferFrame {
	/*
	 * get_mut_data returns a borrowed reference to a mutable slice to the vector
	 * contents.
	 */
	pub fn get_mut_data<'a>(&'a mut self) -> &'a mut [u8] {
		self.data.as_mut_slice()
	}
	pub fn get_data<'a>(&'a self) -> &'a [u8] {
		self.data.as_slice()
	}
}

fn sample<'a, T, I:Iterator<T>>(from: &'a mut I) -> Option<T> {
	let from: ~[T] = from.collect();
	let l = from.len();
	if l == 0 {
		return None;
	}
	let index = randrange(l);
	Some(from[index])
}

fn randrange<X: SampleRange + Ord + Zero>(high: X) -> X {
	let between: Range<X> = Range::new(Zero::zero(), high);
	let mut rng = task_rng();
	between.ind_sample(&mut rng)
}

fn split_segment(num: u64) -> (u64, u64) {
	let high = num >> PAGE_BITS;
	let low = num & ((1 << PAGE_BITS) - 1);
	(high, low)
}

#[test]
fn test_create() {
	let dir = match TempDir::new("buffermanager") {
		Some(temp_dir) => temp_dir,
		None => fail!("creation of temporary directory"),
	};

	let mut bm = BufferManager::new(16, dir.path().clone());
	let pageref = match bm.fix_page(42) {
		Some(p) => p,
		None => fail!("Getting page failed"),
	};
	{
		let mut page = pageref.write();
		let data = page.get_mut_data();
		data[0] = 42;
	}
	bm.unfix_page(pageref, true);
}

#[test]
fn test_threads() {
	use rand::random;
	use std::task::spawn;
	use std::os;

	let pages_in_ram: uint = match os::getenv("PAGES_IN_RAM") {
		Some(v) => from_str(v).expect("PAGES_IN_RAM expects integer"),
		None => 1,
	};
	let pages_on_disk: u64 = match os::getenv("PAGES_ON_DISK") {
		Some(v) => from_str(v).expect("PAGES_ON_DISK expects integer"),
		None => 20,
	};
	let thread_count: uint = match os::getenv("THREADS") {
		Some(v) => from_str(v).expect("THREADS expects integer"),
		None => 3,
	};
	let dir = match TempDir::new("buffermanager") {
		Some(temp_dir) => temp_dir,
		None => fail!("creation of temporary directory"),
	};
	let p = dir.path();
	//let p = Path::new(".");

	let mut buffermanager = BufferManager::new(pages_in_ram, p.clone());

	for i in range(0, pages_on_disk) {
		let bf = match buffermanager.fix_page(i) {
			Some(frame) => frame,
			None => fail!("Couldn't fix page {}", i),
		};
		{
			let mut lock = bf.write();
			lock.get_mut_data()[0] = 0;
		}
		buffermanager.unfix_page(bf, true);
	}
	let bm = Arc::new(RWLock::new(buffermanager));

	// start scan thread
	let (tx, rx) = channel();
	let bm_scan = bm.clone();
	let mut scan = Future::spawn(proc() {
		let mut counters = Vec::from_elem(pages_on_disk as uint, 0_u8);
		loop {
			info!("Running scan thread...");
			match rx.try_recv() {
				Empty => {
					let page_number = randrange(pages_on_disk);
					let mut bm = bm_scan.write();
					let bf = match bm.fix_page(page_number) {
						Some(frame) => frame,
						None => fail!("Couldn't scan/fix page"),
					};
					let current_val = {
						let lock = bf.read();
						lock.get_data()[0]
					};
					bm.unfix_page(bf, false);
					bm.downgrade();
					// check if the value is going up
					assert!(&current_val >= counters.get(page_number as uint));
					// set to the new value
					let v = counters.get_mut(page_number as uint);
					*v = current_val;

				},
				/*
				 * terminate on both disconnect and when any kind of
				 * data arrives
				 */
				Disconnected => break,
				Data(_) => break,
			};
		}
	});

	// start read/write threads
	let mut rw_threads: Vec<Future<int>> = Vec::new();
	for _ in range(0, thread_count) {
		let bm = bm.clone();
		rw_threads.push(Future::spawn(proc() {
			let is_write = random::<bool>();
			info!("Creating new {} task",
				if is_write {"write"} else {"read"});
			let page_number = randrange(pages_on_disk);
			let mut bm = bm.write();
			if is_write {
				let bf = match bm.fix_page(page_number) {
					Some(frame) => frame,
					None => fail!("Couldn't fix page"),
				};
				{
					let mut lock = bf.write();
					let data = lock.get_mut_data();
					data[0] = data[0] + 1;
					info!("Wrote to page {}", page_number);
					debug!("data: {}", Vec::from_slice(data));
				}
				bm.unfix_page(bf, is_write);
			} else {
				let bf = match bm.fix_page(page_number) {
					Some(frame) => frame,
					None => fail!("Couldn't fix page"),
				};
				bm.unfix_page(bf, is_write);
			}
			// return whether we wrote (1) or read (0) as future
			if is_write {1} else {0}
		}));
	}

	// Rust does not have join, but we can wait on Futures which does the same
	let total_count = rw_threads.mut_iter().fold(0, |acc, val| acc + val.get());
	info!("Total count in RAM: {}", total_count);

	// terminate the scan thread and wait until it has completed
	tx.send("terminate");
	scan.get();

	// re-open the pages and check whether all numbers got saved
	let mut bm = BufferManager::new(pages_in_ram, p.clone());
	let mut total_count_on_disk = 0;
	for i in range(0, pages_on_disk) {
		let bf = match bm.fix_page(i) {
			Some(frame) => frame,
			None => fail!("Couldn't fix page")
		};
		let value = {
			let lock = bf.read();
			let data = lock.get_data();
			data[0]
		};
		bm.unfix_page(bf, false);
		// cast up from u8 to int
		total_count_on_disk += value as int;
	}
	info!("Total count on disk: {}", total_count_on_disk);
	assert_eq!(total_count, total_count_on_disk);
}
<|endoftext|>


<commit_before>import std::map;
import std::vec;
import std::str;

type crate_metadata = rec(str name, vec[u8] data);

type cstore = @rec(map::hashmap[int, crate_metadata] metas,
                   mutable vec[str] used_crate_files,
                   mutable vec[str] used_libraries,
                   mutable vec[str] used_link_args);

fn mk_cstore() -> cstore {
    auto meta_cache = map::new_int_hash[crate_metadata]();
    ret @rec(metas = meta_cache,
             mutable used_crate_files = [],
             mutable used_libraries = [],
             mutable used_link_args = []);
}

fn get_crate_data(&cstore cstore, int cnum) -> crate_metadata {
    ret cstore.metas.get(cnum);
}

fn set_crate_data(&cstore cstore, int cnum, &crate_metadata data) {
    cstore.metas.insert(cnum, data);
}

fn have_crate_data(&cstore cstore, int cnum) -> bool {
    ret cstore.metas.contains_key(cnum);
}

fn add_used_crate_file(&cstore cstore, &str lib) {
    if (!vec::member(lib, cstore.used_crate_files)) {
        cstore.used_crate_files += [lib];
    }
}

fn get_used_crate_files(&cstore cstore) -> vec[str] {
    ret cstore.used_crate_files;
}

fn add_used_library(&cstore cstore, &str lib) -> bool {
    if (lib == "") { ret false; }

    if (vec::member(lib, cstore.used_libraries)) {
        ret false;
    }

    cstore.used_libraries += [lib];
    ret true;
}

fn get_used_libraries(&cstore cstore) -> vec[str] {
    ret cstore.used_libraries;
}

fn add_used_link_args(&cstore cstore, &str args) {
    cstore.used_link_args += str::split(args, ' ' as u8);
}

fn get_used_link_args(&cstore cstore) -> vec[str] {
    ret cstore.used_link_args;
}

// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// compile-command: "make -k -C $RBUILD 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
// End:
<commit_msg>Comment cstore<commit_after>// The crate store - a central repo for information collected about external
// crates and libraries

import std::map;
import std::vec;
import std::str;

type crate_metadata = rec(str name, vec[u8] data);

type cstore = @rec(map::hashmap[int, crate_metadata] metas,
                   mutable vec[str] used_crate_files,
                   mutable vec[str] used_libraries,
                   mutable vec[str] used_link_args);

fn mk_cstore() -> cstore {
    auto meta_cache = map::new_int_hash[crate_metadata]();
    ret @rec(metas = meta_cache,
             mutable used_crate_files = [],
             mutable used_libraries = [],
             mutable used_link_args = []);
}

fn get_crate_data(&cstore cstore, int cnum) -> crate_metadata {
    ret cstore.metas.get(cnum);
}

fn set_crate_data(&cstore cstore, int cnum, &crate_metadata data) {
    cstore.metas.insert(cnum, data);
}

fn have_crate_data(&cstore cstore, int cnum) -> bool {
    ret cstore.metas.contains_key(cnum);
}

fn add_used_crate_file(&cstore cstore, &str lib) {
    if (!vec::member(lib, cstore.used_crate_files)) {
        cstore.used_crate_files += [lib];
    }
}

fn get_used_crate_files(&cstore cstore) -> vec[str] {
    ret cstore.used_crate_files;
}

fn add_used_library(&cstore cstore, &str lib) -> bool {
    if (lib == "") { ret false; }

    if (vec::member(lib, cstore.used_libraries)) {
        ret false;
    }

    cstore.used_libraries += [lib];
    ret true;
}

fn get_used_libraries(&cstore cstore) -> vec[str] {
    ret cstore.used_libraries;
}

fn add_used_link_args(&cstore cstore, &str args) {
    cstore.used_link_args += str::split(args, ' ' as u8);
}

fn get_used_link_args(&cstore cstore) -> vec[str] {
    ret cstore.used_link_args;
}

// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// compile-command: "make -k -C $RBUILD 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
// End:
<|endoftext|>


<commit_before>
//! A 3D camera.

use vecmath::{
    Vector3,
    Matrix4,
    vec3_normalized_sub,
    vec3_cross,
    vec3_dot,
};
use vecmath::col_mat4_mul as mul;
use vecmath::traits::*;
use quaternion::{Quaternion, rotate_vector};

/// Computes a model view projection matrix.
pub fn model_view_projection<T: Float + Copy>(
    model: Matrix4<T>,
    view: Matrix4<T>,
    projection: Matrix4<T>
) -> Matrix4<T> {
    mul(mul(projection, view), model)
}

/// Models a camera with position and directions.
pub struct Camera<T=f32> {
    /// The camera position.
    pub position: Vector3<T>,
    /// The up direction.
    pub up: Vector3<T>,
    /// The right direction.
    pub right: Vector3<T>,
    /// The forward direction.
    pub forward: Vector3<T>
}

/// Models camera perspective settings.
pub struct CameraPerspective<T=f32> {
    /// Field of view (in degrees).
    pub fov: T,
    /// The near clip distance.
    pub near_clip: T,
    /// The far clip distance.
    pub far_clip: T,
    /// The aspect ratio, usually set to 1.0.
    pub aspect_ratio: T,
}

impl<T: Float + Copy> Camera<T> {
    /// Constructs a new camera.
    ///
    /// Places the camera at [x, y, z], looking towards pozitive z.
    pub fn new(position: Vector3<T>) -> Camera<T> {
        let _0 = Zero::zero();
        let _1 = One::one();
        Camera {
            position: position,
            right:   [_1, _0, _0],
            up:      [_0, _1, _0],
            forward: [_0, _0, _1]
        }
    }

    /// Computes an orthogonal matrix for the camera.
    ///
    /// This matrix can be used to transform coordinates to the screen.
    pub fn orthogonal(&self) -> Matrix4<T> {
        let p = self.position;
        let r = self.right;
        let u = self.up;
        let f = self.forward;
        let _0 = Zero::zero();
        [
            [r[0], u[0], f[0], _0],
            [r[1], u[1], f[1], _0],
            [r[2], u[2], f[2], _0],
            [-vec3_dot(r, p), -vec3_dot(u, p), -vec3_dot(f, p), One::one()]
        ]
    }

    /// Orients the camera to look at a point.
    pub fn look_at(&mut self, point: Vector3<T>) {
        self.forward = vec3_normalized_sub(self.position, point);
        self.update_right();
    }

    /// Sets yaw and pitch angle of camera in radians.
    pub fn set_yaw_pitch(&mut self, yaw: T, pitch: T) {
        let (y_s, y_c, p_s, p_c) = (yaw.sin(), yaw.cos(), pitch.sin(), pitch.cos());
        self.forward = [y_s * p_c, p_s, y_c * p_c];
        self.up = [y_s * -p_s, p_c, y_c * -p_s];
        self.update_right();
    }

    /// Sets forward, up, and right vectors from a Quaternion rotation
    /// relative to the positive z-axis
    pub fn set_rotation(&mut self, rotation: Quaternion<T>)
    {
        let _0: T = Zero::zero();
        let _1: T = One::one();
        let forward: Vector3<T> = [_0, _0, _1];
        let up: Vector3<T> = [_0, _1, _0];
        self.forward = rotate_vector(rotation, forward);
        self.up = rotate_vector(rotation, up);
        self.update_right();
    }

    fn update_right(&mut self) {
        self.right = vec3_cross(self.up, self.forward);
    }
}

impl<T: Float> CameraPerspective<T>
    where f64: Cast<T>
{
    /// Computes a projection matrix for the camera perspective.
    pub fn projection(&self) -> Matrix4<T> {
        let _0: T = Zero::zero();
        let _1: T = One::one();
        let _2: T = _1 + _1;
        let pi: T = Radians::_180();
        let _360: T = Cast::cast(360.0f64);
        let f = _1 / (self.fov * (pi / _360)).tan();
        let (far, near) = (self.far_clip, self.near_clip);
        [
            [f / self.aspect_ratio, _0, _0, _0],
            [_0, f, _0, _0],
            [_0, _0, (far + near) / (near - far), -_1],
            [_0, _0, (_2 * far * near) / (near - far), _0]
        ]
    }
}
<commit_msg>Derive common traits<commit_after>
//! A 3D camera.

use vecmath::{
    Vector3,
    Matrix4,
    vec3_normalized_sub,
    vec3_cross,
    vec3_dot,
};
use vecmath::col_mat4_mul as mul;
use vecmath::traits::*;
use quaternion::{Quaternion, rotate_vector};

/// Computes a model view projection matrix.
pub fn model_view_projection<T: Float + Copy>(
    model: Matrix4<T>,
    view: Matrix4<T>,
    projection: Matrix4<T>
) -> Matrix4<T> {
    mul(mul(projection, view), model)
}

/// Models a camera with position and directions.
#[derive(Clone, Debug, PartialEq, PartialOrd, Hash)]
pub struct Camera<T=f32> {
    /// The camera position.
    pub position: Vector3<T>,
    /// The up direction.
    pub up: Vector3<T>,
    /// The right direction.
    pub right: Vector3<T>,
    /// The forward direction.
    pub forward: Vector3<T>
}

/// Models camera perspective settings.
#[derive(Clone, Debug, PartialEq, PartialOrd, Hash)]
pub struct CameraPerspective<T=f32> {
    /// Field of view (in degrees).
    pub fov: T,
    /// The near clip distance.
    pub near_clip: T,
    /// The far clip distance.
    pub far_clip: T,
    /// The aspect ratio, usually set to 1.0.
    pub aspect_ratio: T,
}

impl<T: Float + Copy> Camera<T> {
    /// Constructs a new camera.
    ///
    /// Places the camera at [x, y, z], looking towards pozitive z.
    pub fn new(position: Vector3<T>) -> Camera<T> {
        let _0 = Zero::zero();
        let _1 = One::one();
        Camera {
            position: position,
            right:   [_1, _0, _0],
            up:      [_0, _1, _0],
            forward: [_0, _0, _1]
        }
    }

    /// Computes an orthogonal matrix for the camera.
    ///
    /// This matrix can be used to transform coordinates to the screen.
    pub fn orthogonal(&self) -> Matrix4<T> {
        let p = self.position;
        let r = self.right;
        let u = self.up;
        let f = self.forward;
        let _0 = Zero::zero();
        [
            [r[0], u[0], f[0], _0],
            [r[1], u[1], f[1], _0],
            [r[2], u[2], f[2], _0],
            [-vec3_dot(r, p), -vec3_dot(u, p), -vec3_dot(f, p), One::one()]
        ]
    }

    /// Orients the camera to look at a point.
    pub fn look_at(&mut self, point: Vector3<T>) {
        self.forward = vec3_normalized_sub(self.position, point);
        self.update_right();
    }

    /// Sets yaw and pitch angle of camera in radians.
    pub fn set_yaw_pitch(&mut self, yaw: T, pitch: T) {
        let (y_s, y_c, p_s, p_c) = (yaw.sin(), yaw.cos(), pitch.sin(), pitch.cos());
        self.forward = [y_s * p_c, p_s, y_c * p_c];
        self.up = [y_s * -p_s, p_c, y_c * -p_s];
        self.update_right();
    }

    /// Sets forward, up, and right vectors from a Quaternion rotation
    /// relative to the positive z-axis
    pub fn set_rotation(&mut self, rotation: Quaternion<T>)
    {
        let _0: T = Zero::zero();
        let _1: T = One::one();
        let forward: Vector3<T> = [_0, _0, _1];
        let up: Vector3<T> = [_0, _1, _0];
        self.forward = rotate_vector(rotation, forward);
        self.up = rotate_vector(rotation, up);
        self.update_right();
    }

    fn update_right(&mut self) {
        self.right = vec3_cross(self.up, self.forward);
    }
}

impl<T: Float> CameraPerspective<T>
    where f64: Cast<T>
{
    /// Computes a projection matrix for the camera perspective.
    pub fn projection(&self) -> Matrix4<T> {
        let _0: T = Zero::zero();
        let _1: T = One::one();
        let _2: T = _1 + _1;
        let pi: T = Radians::_180();
        let _360: T = Cast::cast(360.0f64);
        let f = _1 / (self.fov * (pi / _360)).tan();
        let (far, near) = (self.far_clip, self.near_clip);
        [
            [f / self.aspect_ratio, _0, _0, _0],
            [_0, f, _0, _0],
            [_0, _0, (far + near) / (near - far), -_1],
            [_0, _0, (_2 * far * near) / (near - far), _0]
        ]
    }
}
<|endoftext|>