<commit_before>use chrono::prelude::*;
use moka::future::Cache;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::convert::TryInto;
use std::ffi::OsStr;
use std::fs::{read_dir, File, OpenOptions};
use std::io::{BufRead, BufReader, Write};
use std::path::PathBuf;
use std::thread;
use std::time;
use std::time::Duration;
use thread_priority::{set_current_thread_priority, ThreadPriority};
use wildmatch::WildMatch;

use crate::common::{
    get_stem_from_path, TIMESTAMP_DAYS_FORMAT, TIMESTAMP_MINUTES_FORMAT, TIMESTAMP_SECONDS_FORMAT,
};

// This is the number of characters taken from timestamp to create new metrics file. E.g.
// from "2022-05-30_17.06.03", 13 means "2022-05-30.metrics".
const DEFAULT_METRICS_FILE_PRECISION: u8 = 10;

// The default time to sleep when trying to refresh metrics
const DEFAULT_METRICS_INTERVAL_SECONDS: u64 = 10;

// Show 30 by default, maps to showing one month with DEFAULT_METRICS_FILE_PRECISION
const DEFAULT_METRICS_LIMIT: usize = 30;

pub fn process_metrics(metrics_dir: PathBuf, metrics_precision: Option<u8>, log_dir: PathBuf) {
    thread::spawn(move || {
        set_current_thread_priority(ThreadPriority::Min).unwrap();
        let metrics_precision = metrics_precision.unwrap_or(DEFAULT_METRICS_FILE_PRECISION);
        let sleep_duration = time::Duration::from_secs(DEFAULT_METRICS_INTERVAL_SECONDS);

        // Read in the last file checked
        let metrics_state_file_path = metrics_dir.join(".state");

        loop {
            log::debug!(
                "Attempting metrics collection to {}",
                metrics_dir.to_str().unwrap()
            );

            let metrics_state_file: Option<File> = OpenOptions::new()
                .read(true)
                .open(&metrics_state_file_path)
                .map_or(None, |file| Some(file));
            let state: (String, usize) = if let Some(metrics_state_file) = metrics_state_file {
                let mut state_reader = BufReader::new(metrics_state_file);
                let mut state = String::new();
                state_reader.read_line(&mut state).unwrap();
                let split = state.split(" ");
                let state_split: Vec<&str> = split.collect();
                let state_path = state_split[0].to_string();
                let state_count_str = trim_newline(state_split[1]);
                (state_path, state_count_str.parse::<usize>().unwrap())
            } else {
                ("".to_string(), 0)
            };
            let mut paths = read_dir(log_dir.to_str().unwrap())
                .unwrap()
                .map(|res| res.map(|e| e.path()))
                .filter(|path| {
                    if let Ok(path) = path {
                        return path.is_file()
                            && path.extension() == Some(OsStr::new("log"))
                            && path.to_str().unwrap().to_string() >= state.0;
                    }
                    false
                })
                .collect::<Result<Vec<_>, std::io::Error>>()
                .unwrap();
            paths.sort();

            for path in paths {
                let stem: String = get_stem_from_path(&path);
                if stem.len() <= metrics_precision.into() {
                    log::warn!(
                        "Log file {} has too low precision, needs at least {}",
                        path.to_str().unwrap(),
                        metrics_precision + 1
                    );
                    continue;
                }

                let start_line_index: usize = if path.to_str().unwrap() == state.0 {
                    state.1 + 1
                } else {
                    0
                };
                let mut log_map: HashMap<String, usize> = HashMap::new();

                let log_file = File::open(&path).unwrap();
                let log_reader = BufReader::new(log_file);
                let mut last_processed_index: usize = 0;
                for (i, line) in log_reader.lines().enumerate() {
                    let line = line.unwrap();
                    if i >= start_line_index {
                        let split = line.split(" ");
                        let log: Vec<&str> = split.collect();
                        let key = format!("{} {}", log[1], trim_newline(log[2]));
                        if log.len() < 3 {
                            // If the line is shorter, it might be that it is being written ATM
                            log::warn!("Malformed log line: {}", &line);
                        } else if log_map.contains_key(&key) {
                            let new_count: usize = log_map.get(&key).unwrap() + 1;
                            log_map.insert(key, new_count);
                        } else {
                            log_map.insert(key, 1);
                        }
                        last_processed_index = i;
                    }
                }

                if !log_map.is_empty() {
                    log::debug!(
                        "Found (partially) unprocessed log file {}",
                        path.to_str().unwrap(),
                    );
                    let metrics_file_name: String =
                        stem[0..metrics_precision.into()].to_string() + ".metrics";
                    let metrics_file_tmp_name: String = format!("{}.tmp", &metrics_file_name);
                    let metrics_file_path = metrics_dir.join(&metrics_file_name);
                    let metrics_file_tmp_path = metrics_dir.join(metrics_file_tmp_name);
                    let metrics_file: Option<File> = OpenOptions::new()
                        .read(true)
                        .open(&metrics_file_path)
                        .map_or(None, |file| Some(file));

                    let mut metrics_file_tmp: File = OpenOptions::new()
                        .create(true)
                        .append(true)
                        .open(&metrics_file_tmp_path)
                        .unwrap();

                    // Read in existing and append to them
                    if let Some(metrics_reader) =
                        metrics_file.as_ref().map(|file| BufReader::new(file))
                    {
                        for line in metrics_reader.lines() {
                            let line = line.unwrap();
                            let split = line.split(" ");
                            let metric: Vec<&str> = split.collect();
                            if metric.len() < 3 {
                                log::warn!(
                                    "Malformed metric line: '{}' in file {}",
                                    &line,
                                    &metrics_file_name
                                );
                                continue;
                            }
                            let key = format!("{} {}", metric[0], metric[1]);
                            let existing_count_str = trim_newline(metric[2]);
                            let existing_count = existing_count_str.parse::<usize>().unwrap();

                            if let Some(additional_count) = log_map.get(&key) {
                                let new_line =
                                    format!("{} {}\n", key, existing_count + additional_count);
                                metrics_file_tmp.write_all(new_line.as_bytes()).unwrap();
                                log_map.remove(&key);
                            } else {
                                metrics_file_tmp
                                    .write_all((line + "\n").as_bytes())
                                    .unwrap();
                            }
                        }
                    }

                    // Write out new lines
                    for (key, count) in log_map {
                        let new_line = format!("{} {}\n", key, count);
                        metrics_file_tmp.write_all(new_line.as_bytes()).unwrap();
                    }

                    // Finally delete old metrics file, rename tmp, and change state
                    if metrics_file.is_some() {
                        std::fs::remove_file(&metrics_file_path).unwrap();
                    };
                    std::fs::rename(metrics_file_tmp_path, metrics_file_path).unwrap();
                    let mut metrics_state_file = std::fs::OpenOptions::new()
                        .create(true)
                        .write(true)
                        .truncate(true)
                        .open(&metrics_state_file_path)
                        .unwrap();
                    metrics_state_file
                        .write_all(
                            format!("{} {}\n", &path.to_str().unwrap(), last_processed_index)
                                .as_bytes(),
                        )
                        .unwrap();
                }
            }
            thread::sleep(sleep_duration);
        }
    });
}

#[derive(Serialize, Clone)]
struct MetricsResponse {
    start: String,
    end: String,
    metrics: Vec<MetricsRange>,
}

#[derive(Serialize, Clone)]
struct MetricsRange {
    start: String,
    end: String,
    entries: Vec<MetricsEntry>,
}

#[derive(Serialize, Clone)]
struct MetricsEntry {
    path: String,
    status: usize,
    count: usize,
    immutable: bool,
}

#[derive(Clone)]
pub struct ProduceMetrics {
    metrics_dir: PathBuf,
    metrics_secret: Option<String>,
    immutable_path_wildmatch: Option<Vec<WildMatch>>,
    cache: Cache<String, MetricsResponse>,
}

impl ProduceMetrics {
    pub fn new(
        metrics_endpoint: String,
        metrics_dir: PathBuf,
        metrics_secret: Option<String>,
        immutable_path: Option<Vec<String>>,
    ) -> Self {
        let cache_ttl_sec = DEFAULT_METRICS_INTERVAL_SECONDS;
        log::info!(
            "Setting up cache for metrics endpoint at {} with time-to-live: {}s",
            &metrics_endpoint,
            &cache_ttl_sec,
        );
        let cache = Cache::builder()
            .time_to_live(Duration::from_secs(cache_ttl_sec))
            .build();

        let immutable_path_wildmatch = match immutable_path {
            Some(immutable_path) => Some(
                immutable_path
                    .iter()
                    .map(|path| WildMatch::new(path))
                    .collect(),
            ),
            None => None,
        };

        Self {
            metrics_dir,
            metrics_secret,
            immutable_path_wildmatch,
            cache,
        }
    }

    fn get_metrics_response_from_cache(&self, query: String) -> Option<MetricsResponse> {
        self.cache.get(&query)
    }

    async fn insert_metrics_response_to_cache(
        &self,
        query: String,
        metrics_response: MetricsResponse,
    ) {
        self.cache.insert(query, metrics_response).await
    }
}

#[derive(Deserialize, Debug, Clone)]
struct MetricsParams {
    limit: Option<usize>,
    secret: Option<String>,
}

impl Default for MetricsParams {
    fn default() -> Self {
        MetricsParams {
            limit: None,
            secret: None,
        }
    }
}

#[async_trait::async_trait]
impl<State> tide::Endpoint<State> for ProduceMetrics
where
    State: Clone + Send + Sync + 'static,
{
    async fn call(&self, req: tide::Request<State>) -> tide::Result {
        let MetricsParams { limit, secret } = req.query().unwrap_or_default();
        let correct_secret = if let Some(metrics_secret) = &self.metrics_secret {
            if let Some(secret) = secret {
                secret.eq(metrics_secret)
            } else {
                false
            }
        } else {
            false
        };
        let limit: usize = if let Some(limit) = limit {
            if correct_secret {
                limit
            } else {
                DEFAULT_METRICS_LIMIT
            }
        } else {
            DEFAULT_METRICS_LIMIT
        };
        let query = format!("limit={}&correct_secret={}", limit, correct_secret);
        let url_path = &req.url().path();
        let cached_metrics_response = self.get_metrics_response_from_cache(query.clone());
        return if let Some(cached_metrics_response) = cached_metrics_response {
            // The body for the URL was found from the cache, return it without any file IO
            log::debug!("Metrics cache hit: {}", &url_path);
            let body = tide::Body::from_json(&cached_metrics_response).unwrap();
            Ok(tide::Response::builder(200).body(body).build())
        } else {
            // Read the file from the file system
            log::debug!("Metrics cache miss: {}", &url_path);

            let mut paths = read_dir(self.metrics_dir.to_str().unwrap())
                .unwrap()
                .map(|res| res.map(|e| e.path()))
                .filter(|path| {
                    if let Ok(path) = path {
                        return path.is_file() && path.extension() == Some(OsStr::new("metrics"));
                    }
                    false
                })
                .collect::<Result<Vec<_>, std::io::Error>>()
                .unwrap();
            paths.sort();

            let end_timestamp = get_timestamp_diff(&get_stem_from_path(&paths[paths.len() - 1]), 1);
            let limit: i64 = limit.try_into().unwrap();
            let start_timestamp = get_timestamp_diff(&end_timestamp, limit * -1);

            let mut metrics: Vec<MetricsRange> = vec![];
            for path in paths {
                let stem = get_stem_from_path(&path);
                if stem >= start_timestamp {
                    let mut entries: Vec<MetricsEntry> = vec![];
                    let metrics_file = File::open(&path).unwrap();
                    let metrics_reader = BufReader::new(metrics_file);
                    for line in metrics_reader.lines() {
                        let line = line.unwrap();
                        let split = line.split(" ");
                        let metric: Vec<&str> = split.collect();
                        let status = metric[0].parse::<usize>().unwrap();
                        let url_path: String = metric[1].to_string();
                        let count = trim_newline(metric[2]).parse::<usize>().unwrap();
                        let immutable: bool = match &self.immutable_path_wildmatch {
                            Some(immutable_path) => immutable_path
                                .iter()
                                .find(|wildmatch_path| wildmatch_path.matches(&url_path))
                                .is_some(),
                            None => false,
                        };
                        if correct_secret || (!immutable && status == 200) {
                            entries.push(MetricsEntry {
                                path: url_path,
                                status,
                                count,
                                immutable,
                            })
                        }
                    }
                    metrics.push(MetricsRange {
                        end: get_timestamp_diff(&stem, 1),
                        start: stem,
                        entries,
                    })
                }
            }

            let metrics_response: MetricsResponse = MetricsResponse {
                start: start_timestamp,
                end: end_timestamp,
                metrics,
            };

            self.insert_metrics_response_to_cache(query, metrics_response.clone())
                .await;

            let body = tide::Body::from_json(&metrics_response).unwrap();
            Ok(tide::Response::builder(200).body(body).build())
        };
    }
}

fn get_timestamp_diff(timestamp: &str, delta: i64) -> String {
    match timestamp.len() {
        10 => {
            let diff_date = Utc
                .datetime_from_str(
                    format!("{}_00.00.00", &timestamp).as_str(),
                    TIMESTAMP_SECONDS_FORMAT,
                )
                .unwrap();
            let diff = diff_date + chrono::Duration::days(delta);
            diff.format(TIMESTAMP_DAYS_FORMAT).to_string()
        }
        16 => {
            let diff_date = Utc
                .datetime_from_str(
                    format!("{}.00", &timestamp).as_str(),
                    TIMESTAMP_SECONDS_FORMAT,
                )
                .unwrap();
            let diff = diff_date + chrono::Duration::minutes(delta);
            diff.format(TIMESTAMP_MINUTES_FORMAT).to_string()
        }
        // TODO: Rest of the accepted precisions
        _ => "".to_string(),
    }
}

fn trim_newline(s: &str) -> String {
    if s.ends_with('\n') {
        s[0..s.len() - 1].to_string()
    } else {
        s.to_string()
    }
}
<commit_msg>Immutable file responses are fine to show<commit_after>use chrono::prelude::*;
use moka::future::Cache;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::convert::TryInto;
use std::ffi::OsStr;
use std::fs::{read_dir, File, OpenOptions};
use std::io::{BufRead, BufReader, Write};
use std::path::PathBuf;
use std::thread;
use std::time;
use std::time::Duration;
use thread_priority::{set_current_thread_priority, ThreadPriority};
use wildmatch::WildMatch;

use crate::common::{
    get_stem_from_path, TIMESTAMP_DAYS_FORMAT, TIMESTAMP_MINUTES_FORMAT, TIMESTAMP_SECONDS_FORMAT,
};

// This is the number of characters taken from timestamp to create new metrics file. E.g.
// from "2022-05-30_17.06.03", 13 means "2022-05-30.metrics".
const DEFAULT_METRICS_FILE_PRECISION: u8 = 10;

// The default time to sleep when trying to refresh metrics
const DEFAULT_METRICS_INTERVAL_SECONDS: u64 = 10;

// Show 30 by default, maps to showing one month with DEFAULT_METRICS_FILE_PRECISION
const DEFAULT_METRICS_LIMIT: usize = 30;

pub fn process_metrics(metrics_dir: PathBuf, metrics_precision: Option<u8>, log_dir: PathBuf) {
    thread::spawn(move || {
        set_current_thread_priority(ThreadPriority::Min).unwrap();
        let metrics_precision = metrics_precision.unwrap_or(DEFAULT_METRICS_FILE_PRECISION);
        let sleep_duration = time::Duration::from_secs(DEFAULT_METRICS_INTERVAL_SECONDS);

        // Read in the last file checked
        let metrics_state_file_path = metrics_dir.join(".state");

        loop {
            log::debug!(
                "Attempting metrics collection to {}",
                metrics_dir.to_str().unwrap()
            );

            let metrics_state_file: Option<File> = OpenOptions::new()
                .read(true)
                .open(&metrics_state_file_path)
                .map_or(None, |file| Some(file));
            let state: (String, usize) = if let Some(metrics_state_file) = metrics_state_file {
                let mut state_reader = BufReader::new(metrics_state_file);
                let mut state = String::new();
                state_reader.read_line(&mut state).unwrap();
                let split = state.split(" ");
                let state_split: Vec<&str> = split.collect();
                let state_path = state_split[0].to_string();
                let state_count_str = trim_newline(state_split[1]);
                (state_path, state_count_str.parse::<usize>().unwrap())
            } else {
                ("".to_string(), 0)
            };
            let mut paths = read_dir(log_dir.to_str().unwrap())
                .unwrap()
                .map(|res| res.map(|e| e.path()))
                .filter(|path| {
                    if let Ok(path) = path {
                        return path.is_file()
                            && path.extension() == Some(OsStr::new("log"))
                            && path.to_str().unwrap().to_string() >= state.0;
                    }
                    false
                })
                .collect::<Result<Vec<_>, std::io::Error>>()
                .unwrap();
            paths.sort();

            for path in paths {
                let stem: String = get_stem_from_path(&path);
                if stem.len() <= metrics_precision.into() {
                    log::warn!(
                        "Log file {} has too low precision, needs at least {}",
                        path.to_str().unwrap(),
                        metrics_precision + 1
                    );
                    continue;
                }

                let start_line_index: usize = if path.to_str().unwrap() == state.0 {
                    state.1 + 1
                } else {
                    0
                };
                let mut log_map: HashMap<String, usize> = HashMap::new();

                let log_file = File::open(&path).unwrap();
                let log_reader = BufReader::new(log_file);
                let mut last_processed_index: usize = 0;
                for (i, line) in log_reader.lines().enumerate() {
                    let line = line.unwrap();
                    if i >= start_line_index {
                        let split = line.split(" ");
                        let log: Vec<&str> = split.collect();
                        let key = format!("{} {}", log[1], trim_newline(log[2]));
                        if log.len() < 3 {
                            // If the line is shorter, it might be that it is being written ATM
                            log::warn!("Malformed log line: {}", &line);
                        } else if log_map.contains_key(&key) {
                            let new_count: usize = log_map.get(&key).unwrap() + 1;
                            log_map.insert(key, new_count);
                        } else {
                            log_map.insert(key, 1);
                        }
                        last_processed_index = i;
                    }
                }

                if !log_map.is_empty() {
                    log::debug!(
                        "Found (partially) unprocessed log file {}",
                        path.to_str().unwrap(),
                    );
                    let metrics_file_name: String =
                        stem[0..metrics_precision.into()].to_string() + ".metrics";
                    let metrics_file_tmp_name: String = format!("{}.tmp", &metrics_file_name);
                    let metrics_file_path = metrics_dir.join(&metrics_file_name);
                    let metrics_file_tmp_path = metrics_dir.join(metrics_file_tmp_name);
                    let metrics_file: Option<File> = OpenOptions::new()
                        .read(true)
                        .open(&metrics_file_path)
                        .map_or(None, |file| Some(file));

                    let mut metrics_file_tmp: File = OpenOptions::new()
                        .create(true)
                        .append(true)
                        .open(&metrics_file_tmp_path)
                        .unwrap();

                    // Read in existing and append to them
                    if let Some(metrics_reader) =
                        metrics_file.as_ref().map(|file| BufReader::new(file))
                    {
                        for line in metrics_reader.lines() {
                            let line = line.unwrap();
                            let split = line.split(" ");
                            let metric: Vec<&str> = split.collect();
                            if metric.len() < 3 {
                                log::warn!(
                                    "Malformed metric line: '{}' in file {}",
                                    &line,
                                    &metrics_file_name
                                );
                                continue;
                            }
                            let key = format!("{} {}", metric[0], metric[1]);
                            let existing_count_str = trim_newline(metric[2]);
                            let existing_count = existing_count_str.parse::<usize>().unwrap();

                            if let Some(additional_count) = log_map.get(&key) {
                                let new_line =
                                    format!("{} {}\n", key, existing_count + additional_count);
                                metrics_file_tmp.write_all(new_line.as_bytes()).unwrap();
                                log_map.remove(&key);
                            } else {
                                metrics_file_tmp
                                    .write_all((line + "\n").as_bytes())
                                    .unwrap();
                            }
                        }
                    }

                    // Write out new lines
                    for (key, count) in log_map {
                        let new_line = format!("{} {}\n", key, count);
                        metrics_file_tmp.write_all(new_line.as_bytes()).unwrap();
                    }

                    // Finally delete old metrics file, rename tmp, and change state
                    if metrics_file.is_some() {
                        std::fs::remove_file(&metrics_file_path).unwrap();
                    };
                    std::fs::rename(metrics_file_tmp_path, metrics_file_path).unwrap();
                    let mut metrics_state_file = std::fs::OpenOptions::new()
                        .create(true)
                        .write(true)
                        .truncate(true)
                        .open(&metrics_state_file_path)
                        .unwrap();
                    metrics_state_file
                        .write_all(
                            format!("{} {}\n", &path.to_str().unwrap(), last_processed_index)
                                .as_bytes(),
                        )
                        .unwrap();
                }
            }
            thread::sleep(sleep_duration);
        }
    });
}

#[derive(Serialize, Clone)]
struct MetricsResponse {
    start: String,
    end: String,
    metrics: Vec<MetricsRange>,
}

#[derive(Serialize, Clone)]
struct MetricsRange {
    start: String,
    end: String,
    entries: Vec<MetricsEntry>,
}

#[derive(Serialize, Clone)]
struct MetricsEntry {
    path: String,
    status: usize,
    count: usize,
    immutable: bool,
}

#[derive(Clone)]
pub struct ProduceMetrics {
    metrics_dir: PathBuf,
    metrics_secret: Option<String>,
    immutable_path_wildmatch: Option<Vec<WildMatch>>,
    cache: Cache<String, MetricsResponse>,
}

impl ProduceMetrics {
    pub fn new(
        metrics_endpoint: String,
        metrics_dir: PathBuf,
        metrics_secret: Option<String>,
        immutable_path: Option<Vec<String>>,
    ) -> Self {
        let cache_ttl_sec = DEFAULT_METRICS_INTERVAL_SECONDS;
        log::info!(
            "Setting up cache for metrics endpoint at {} with time-to-live: {}s",
            &metrics_endpoint,
            &cache_ttl_sec,
        );
        let cache = Cache::builder()
            .time_to_live(Duration::from_secs(cache_ttl_sec))
            .build();

        let immutable_path_wildmatch = match immutable_path {
            Some(immutable_path) => Some(
                immutable_path
                    .iter()
                    .map(|path| WildMatch::new(path))
                    .collect(),
            ),
            None => None,
        };

        Self {
            metrics_dir,
            metrics_secret,
            immutable_path_wildmatch,
            cache,
        }
    }

    fn get_metrics_response_from_cache(&self, query: String) -> Option<MetricsResponse> {
        self.cache.get(&query)
    }

    async fn insert_metrics_response_to_cache(
        &self,
        query: String,
        metrics_response: MetricsResponse,
    ) {
        self.cache.insert(query, metrics_response).await
    }
}

#[derive(Deserialize, Debug, Clone)]
struct MetricsParams {
    limit: Option<usize>,
    secret: Option<String>,
}

impl Default for MetricsParams {
    fn default() -> Self {
        MetricsParams {
            limit: None,
            secret: None,
        }
    }
}

#[async_trait::async_trait]
impl<State> tide::Endpoint<State> for ProduceMetrics
where
    State: Clone + Send + Sync + 'static,
{
    async fn call(&self, req: tide::Request<State>) -> tide::Result {
        let MetricsParams { limit, secret } = req.query().unwrap_or_default();
        let correct_secret = if let Some(metrics_secret) = &self.metrics_secret {
            if let Some(secret) = secret {
                secret.eq(metrics_secret)
            } else {
                false
            }
        } else {
            false
        };
        let limit: usize = if let Some(limit) = limit {
            if correct_secret {
                limit
            } else {
                DEFAULT_METRICS_LIMIT
            }
        } else {
            DEFAULT_METRICS_LIMIT
        };
        let query = format!("limit={}&correct_secret={}", limit, correct_secret);
        let url_path = &req.url().path();
        let cached_metrics_response = self.get_metrics_response_from_cache(query.clone());
        return if let Some(cached_metrics_response) = cached_metrics_response {
            // The body for the URL was found from the cache, return it without any file IO
            log::debug!("Metrics cache hit: {}", &url_path);
            let body = tide::Body::from_json(&cached_metrics_response).unwrap();
            Ok(tide::Response::builder(200).body(body).build())
        } else {
            // Read the file from the file system
            log::debug!("Metrics cache miss: {}", &url_path);

            let mut paths = read_dir(self.metrics_dir.to_str().unwrap())
                .unwrap()
                .map(|res| res.map(|e| e.path()))
                .filter(|path| {
                    if let Ok(path) = path {
                        return path.is_file() && path.extension() == Some(OsStr::new("metrics"));
                    }
                    false
                })
                .collect::<Result<Vec<_>, std::io::Error>>()
                .unwrap();
            paths.sort();

            let end_timestamp = get_timestamp_diff(&get_stem_from_path(&paths[paths.len() - 1]), 1);
            let limit: i64 = limit.try_into().unwrap();
            let start_timestamp = get_timestamp_diff(&end_timestamp, limit * -1);

            let mut metrics: Vec<MetricsRange> = vec![];
            for path in paths {
                let stem = get_stem_from_path(&path);
                if stem >= start_timestamp {
                    let mut entries: Vec<MetricsEntry> = vec![];
                    let metrics_file = File::open(&path).unwrap();
                    let metrics_reader = BufReader::new(metrics_file);
                    for line in metrics_reader.lines() {
                        let line = line.unwrap();
                        let split = line.split(" ");
                        let metric: Vec<&str> = split.collect();
                        let status = metric[0].parse::<usize>().unwrap();
                        let url_path: String = metric[1].to_string();
                        let count = trim_newline(metric[2]).parse::<usize>().unwrap();
                        let immutable: bool = match &self.immutable_path_wildmatch {
                            Some(immutable_path) => immutable_path
                                .iter()
                                .find(|wildmatch_path| wildmatch_path.matches(&url_path))
                                .is_some(),
                            None => false,
                        };
                        if correct_secret || status == 200 {
                            entries.push(MetricsEntry {
                                path: url_path,
                                status,
                                count,
                                immutable,
                            })
                        }
                    }
                    metrics.push(MetricsRange {
                        end: get_timestamp_diff(&stem, 1),
                        start: stem,
                        entries,
                    })
                }
            }

            let metrics_response: MetricsResponse = MetricsResponse {
                start: start_timestamp,
                end: end_timestamp,
                metrics,
            };

            self.insert_metrics_response_to_cache(query, metrics_response.clone())
                .await;

            let body = tide::Body::from_json(&metrics_response).unwrap();
            Ok(tide::Response::builder(200).body(body).build())
        };
    }
}

fn get_timestamp_diff(timestamp: &str, delta: i64) -> String {
    match timestamp.len() {
        10 => {
            let diff_date = Utc
                .datetime_from_str(
                    format!("{}_00.00.00", &timestamp).as_str(),
                    TIMESTAMP_SECONDS_FORMAT,
                )
                .unwrap();
            let diff = diff_date + chrono::Duration::days(delta);
            diff.format(TIMESTAMP_DAYS_FORMAT).to_string()
        }
        16 => {
            let diff_date = Utc
                .datetime_from_str(
                    format!("{}.00", &timestamp).as_str(),
                    TIMESTAMP_SECONDS_FORMAT,
                )
                .unwrap();
            let diff = diff_date + chrono::Duration::minutes(delta);
            diff.format(TIMESTAMP_MINUTES_FORMAT).to_string()
        }
        // TODO: Rest of the accepted precisions
        _ => "".to_string(),
    }
}

fn trim_newline(s: &str) -> String {
    if s.ends_with('\n') {
        s[0..s.len() - 1].to_string()
    } else {
        s.to_string()
    }
}
<|endoftext|>


<commit_before>use std::io::Write;

use log::{error, warn};

use crate::task_handler::TaskHandler;

impl TaskHandler {
    /// Send some input to a child process' stdin.
    pub fn send(&mut self, task_id: usize, input: String) {
        let child = match self.children.get_child_mut(task_id) {
            Some(child) => child,
            None => {
                warn!("Task {task_id} finished before input could be sent");
                return;
            }
        };
        {
            let child_stdin = child.stdin.as_mut().unwrap();
            if let Err(err) = child_stdin.write_all(&input.clone().into_bytes()) {
                error!("Failed to send input to task {task_id} with err {err:?}");
            };
        }
    }
}
<commit_msg>fix: Clippy issue<commit_after>use std::io::Write;

use log::{error, warn};

use crate::task_handler::TaskHandler;

impl TaskHandler {
    /// Send some input to a child process' stdin.
    pub fn send(&mut self, task_id: usize, input: String) {
        let child = match self.children.get_child_mut(task_id) {
            Some(child) => child,
            None => {
                warn!("Task {task_id} finished before input could be sent");
                return;
            }
        };
        {
            let child_stdin = child.stdin.as_mut().unwrap();
            if let Err(err) = child_stdin.write_all(&input.into_bytes()) {
                error!("Failed to send input to task {task_id} with err {err:?}");
            };
        }
    }
}
<|endoftext|>


<commit_before><commit_msg>rust: add `to_le_bytes!` macro for generating tensor_content fields (#4739)<commit_after><|endoftext|>


<commit_before>#![feature(
    i128_type,
    rustc_private,
    conservative_impl_trait,
    catch_expr,
    inclusive_range_fields
)]

#[macro_use]
extern crate log;

// From rustc.
#[macro_use]
extern crate rustc;
extern crate rustc_mir;
extern crate rustc_data_structures;
extern crate syntax;
extern crate regex;
#[macro_use]
extern crate lazy_static;

use rustc::ty::{self, TyCtxt};
use rustc::ty::layout::{TyLayout, LayoutOf};
use rustc::hir::def_id::DefId;
use rustc::mir;

use syntax::ast::Mutability;
use syntax::codemap::Span;

use std::collections::{HashMap, BTreeMap};

pub use rustc::mir::interpret::*;
pub use rustc_mir::interpret::*;

mod fn_call;
mod operator;
mod intrinsic;
mod helpers;
mod memory;
mod tls;
mod locks;
mod range_map;
mod validation;

use fn_call::EvalContextExt as MissingFnsEvalContextExt;
use operator::EvalContextExt as OperatorEvalContextExt;
use intrinsic::EvalContextExt as IntrinsicEvalContextExt;
use tls::EvalContextExt as TlsEvalContextExt;
use locks::LockInfo;
use locks::MemoryExt as LockMemoryExt;
use validation::EvalContextExt as ValidationEvalContextExt;
use range_map::RangeMap;
use validation::{ValidationQuery, AbsPlace};

pub fn eval_main<'a, 'tcx: 'a>(
    tcx: TyCtxt<'a, 'tcx, 'tcx>,
    main_id: DefId,
    start_wrapper: Option<DefId>,
) {
    fn run_main<'a, 'mir: 'a, 'tcx: 'mir>(
        ecx: &mut rustc_mir::interpret::EvalContext<'a, 'mir, 'tcx, Evaluator<'tcx>>,
        main_id: DefId,
        start_wrapper: Option<DefId>,
    ) -> EvalResult<'tcx> {
        let main_instance = ty::Instance::mono(ecx.tcx.tcx, main_id);
        let main_mir = ecx.load_mir(main_instance.def)?;
        let mut cleanup_ptr = None; // Pointer to be deallocated when we are done

        if !main_mir.return_ty().is_nil() || main_mir.arg_count != 0 {
            return err!(Unimplemented(
                "miri does not support main functions without `fn()` type signatures"
                    .to_owned(),
            ));
        }

        if let Some(start_id) = start_wrapper {
            let main_ret_ty = ecx.tcx.fn_sig(main_id).output();
            let main_ret_ty = main_ret_ty.no_late_bound_regions().unwrap();
            let start_instance = ty::Instance::resolve(
                ecx.tcx.tcx,
                ty::ParamEnv::reveal_all(),
                start_id,
                ecx.tcx.mk_substs(
                    ::std::iter::once(ty::subst::Kind::from(main_ret_ty)))).unwrap();
            let start_mir = ecx.load_mir(start_instance.def)?;

            if start_mir.arg_count != 3 {
                return err!(AbiViolation(format!(
                    "'start' lang item should have three arguments, but has {}",
                    start_mir.arg_count
                )));
            }

            // Return value
            let size = ecx.tcx.data_layout.pointer_size.bytes();
            let align = ecx.tcx.data_layout.pointer_align;
            let ret_ptr = ecx.memory_mut().allocate(size, align, Some(MemoryKind::Stack))?;
            cleanup_ptr = Some(ret_ptr);

            // Push our stack frame
            ecx.push_stack_frame(
                start_instance,
                start_mir.span,
                start_mir,
                Place::from_ptr(ret_ptr, align),
                StackPopCleanup::None,
            )?;

            let mut args = ecx.frame().mir.args_iter();

            // First argument: pointer to main()
            let main_ptr = ecx.memory_mut().create_fn_alloc(main_instance);
            let dest = ecx.eval_place(&mir::Place::Local(args.next().unwrap()))?;
            let main_ty = main_instance.ty(ecx.tcx.tcx);
            let main_ptr_ty = ecx.tcx.mk_fn_ptr(main_ty.fn_sig(ecx.tcx.tcx));
            ecx.write_value(
                ValTy {
                    value: Value::ByVal(PrimVal::Ptr(main_ptr)),
                    ty: main_ptr_ty,
                },
                dest,
            )?;

            // Second argument (argc): 1
            let dest = ecx.eval_place(&mir::Place::Local(args.next().unwrap()))?;
            let ty = ecx.tcx.types.isize;
            ecx.write_primval(dest, PrimVal::Bytes(1), ty)?;

            // FIXME: extract main source file path
            // Third argument (argv): &[b"foo"]
            let dest = ecx.eval_place(&mir::Place::Local(args.next().unwrap()))?;
            let ty = ecx.tcx.mk_imm_ptr(ecx.tcx.mk_imm_ptr(ecx.tcx.types.u8));
            let foo = ecx.memory.allocate_cached(b"foo\0");
            let ptr_size = ecx.memory.pointer_size();
            let ptr_align = ecx.tcx.data_layout.pointer_align;
            let foo_ptr = ecx.memory.allocate(ptr_size, ptr_align, None)?;
            ecx.memory.write_primval(foo_ptr, ptr_align, PrimVal::Ptr(foo.into()), ptr_size, false)?;
            ecx.memory.mark_static_initialized(foo_ptr.alloc_id, Mutability::Immutable)?;
            ecx.write_ptr(dest, foo_ptr.into(), ty)?;

            assert!(args.next().is_none(), "start lang item has more arguments than expected");
        } else {
            ecx.push_stack_frame(
                main_instance,
                main_mir.span,
                main_mir,
                Place::undef(),
                StackPopCleanup::None,
            )?;

            // No arguments
            let mut args = ecx.frame().mir.args_iter();
            assert!(args.next().is_none(), "main function must not have arguments");
        }

        while ecx.step()? {}
        ecx.run_tls_dtors()?;
        if let Some(cleanup_ptr) = cleanup_ptr {
            ecx.memory_mut().deallocate(
                cleanup_ptr,
                None,
                MemoryKind::Stack,
            )?;
        }
        Ok(())
    }

    let mut ecx = EvalContext::new(tcx.at(syntax::codemap::DUMMY_SP), ty::ParamEnv::reveal_all(), Default::default(), Default::default());
    match run_main(&mut ecx, main_id, start_wrapper) {
        Ok(()) => {
            let leaks = ecx.memory().leak_report();
            if leaks != 0 {
                tcx.sess.err("the evaluated program leaked memory");
            }
        }
        Err(mut e) => {
            ecx.report(&mut e, true, None);
        }
    }
}

#[derive(Default)]
pub struct Evaluator<'tcx> {
    /// Environment variables set by `setenv`
    /// Miri does not expose env vars from the host to the emulated program
    pub(crate) env_vars: HashMap<Vec<u8>, MemoryPointer>,

    /// Places that were suspended by the validation subsystem, and will be recovered later
    pub(crate) suspended: HashMap<DynamicLifetime, Vec<ValidationQuery<'tcx>>>,
}

pub type TlsKey = usize;

#[derive(Copy, Clone, Debug)]
pub struct TlsEntry<'tcx> {
    data: Pointer, // Will eventually become a map from thread IDs to `Pointer`s, if we ever support more than one thread.
    dtor: Option<ty::Instance<'tcx>>,
}

#[derive(Default)]
pub struct MemoryData<'tcx> {
    /// The Key to use for the next thread-local allocation.
    next_thread_local: TlsKey,

    /// pthreads-style thread-local storage.
    thread_local: BTreeMap<TlsKey, TlsEntry<'tcx>>,

    /// Memory regions that are locked by some function
    ///
    /// Only mutable (static mut, heap, stack) allocations have an entry in this map.
    /// The entry is created when allocating the memory and deleted after deallocation.
    locks: HashMap<AllocId, RangeMap<LockInfo<'tcx>>>,
}

impl<'mir, 'tcx: 'mir> Machine<'mir, 'tcx> for Evaluator<'tcx> {
    type MemoryData = MemoryData<'tcx>;
    type MemoryKinds = memory::MemoryKind;

    /// Returns Ok() when the function was handled, fail otherwise
    fn eval_fn_call<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        instance: ty::Instance<'tcx>,
        destination: Option<(Place, mir::BasicBlock)>,
        args: &[ValTy<'tcx>],
        span: Span,
        sig: ty::FnSig<'tcx>,
    ) -> EvalResult<'tcx, bool> {
        ecx.eval_fn_call(instance, destination, args, span, sig)
    }

    fn call_intrinsic<'a>(
        ecx: &mut rustc_mir::interpret::EvalContext<'a, 'mir, 'tcx, Self>,
        instance: ty::Instance<'tcx>,
        args: &[ValTy<'tcx>],
        dest: Place,
        dest_layout: TyLayout<'tcx>,
        target: mir::BasicBlock,
    ) -> EvalResult<'tcx> {
        ecx.call_intrinsic(instance, args, dest, dest_layout, target)
    }

    fn try_ptr_op<'a>(
        ecx: &rustc_mir::interpret::EvalContext<'a, 'mir, 'tcx, Self>,
        bin_op: mir::BinOp,
        left: PrimVal,
        left_ty: ty::Ty<'tcx>,
        right: PrimVal,
        right_ty: ty::Ty<'tcx>,
    ) -> EvalResult<'tcx, Option<(PrimVal, bool)>> {
        ecx.ptr_op(bin_op, left, left_ty, right, right_ty)
    }

    fn mark_static_initialized<'a>(
        _mem: &mut Memory<'a, 'mir, 'tcx, Self>,
        _id: AllocId,
        _mutability: Mutability,
    ) -> EvalResult<'tcx, bool> {
        /*use memory::MemoryKind::*;
        match m {
            // FIXME: This could be allowed, but not for env vars set during miri execution
            Env => err!(Unimplemented("statics can't refer to env vars".to_owned())),
            _ => Ok(false), // TODO: What does the bool mean?
        }*/
        Ok(false)
    }

    fn init_static<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        cid: GlobalId<'tcx>,
    ) -> EvalResult<'tcx, AllocId> {
        let def_id = cid.instance.def_id();
        let ty = ecx.tcx.type_of(def_id);
        let layout = ecx.tcx.layout_of(ty::ParamEnvAnd {
            param_env: ty::ParamEnv::reveal_all(),
            value: ty
        }).expect("Couldn't compute layout for the type of a static");
        ecx.memory.allocate(layout.size.bytes(), layout.align, None).map(|mptr|mptr.alloc_id)
    }

    fn box_alloc<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        ty: ty::Ty<'tcx>,
        dest: Place,
    ) -> EvalResult<'tcx> {
        let layout = ecx.layout_of(ty)?;

        // Call the `exchange_malloc` lang item
        let malloc = ecx.tcx.lang_items().exchange_malloc_fn().unwrap();
        let malloc = ty::Instance::mono(ecx.tcx.tcx, malloc);
        let malloc_mir = ecx.load_mir(malloc.def)?;
        ecx.push_stack_frame(
            malloc,
            malloc_mir.span,
            malloc_mir,
            dest,
            // Don't do anything when we are done.  The statement() function will increment
            // the old stack frame's stmt counter to the next statement, which means that when
            // exchange_malloc returns, we go on evaluating exactly where we want to be.
            StackPopCleanup::None,
        )?;

        let mut args = ecx.frame().mir.args_iter();
        let usize = ecx.tcx.types.usize;

        // First argument: size
        let dest = ecx.eval_place(&mir::Place::Local(args.next().unwrap()))?;
        ecx.write_value(
            ValTy {
                value: Value::ByVal(PrimVal::Bytes(layout.size.bytes().into())),
                ty: usize,
            },
            dest,
        )?;

        // Second argument: align
        let dest = ecx.eval_place(&mir::Place::Local(args.next().unwrap()))?;
        ecx.write_value(
            ValTy {
                value: Value::ByVal(PrimVal::Bytes(layout.align.abi().into())),
                ty: usize,
            },
            dest,
        )?;

        // No more arguments
        assert!(args.next().is_none(), "exchange_malloc lang item has more arguments than expected");
        Ok(())
    }

    fn global_item_with_linkage<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        instance: ty::Instance<'tcx>,
        mutability: Mutability,
    ) -> EvalResult<'tcx> {
        // FIXME: check that it's `#[linkage = "extern_weak"]`
        trace!("Initializing an extern global with NULL");
        let ptr_size = ecx.memory.pointer_size();
        let ptr_align = ecx.tcx.data_layout.pointer_align;
        let ptr = ecx.memory.allocate(
            ptr_size,
            ptr_align,
            None,
        )?;
        ecx.memory.write_ptr_sized_unsigned(ptr, ptr_align, PrimVal::Bytes(0))?;
        ecx.memory.mark_static_initialized(ptr.alloc_id, mutability)?;
        ecx.tcx.interpret_interner.cache(
            instance.def_id(),
            ptr.alloc_id,
        );
        Ok(())
    }

    fn check_locks<'a>(
        mem: &Memory<'a, 'mir, 'tcx, Self>,
        ptr: MemoryPointer,
        size: u64,
        access: AccessKind,
    ) -> EvalResult<'tcx> {
        mem.check_locks(ptr, size, access)
    }

    fn add_lock<'a>(
        mem: &mut Memory<'a, 'mir, 'tcx, Self>,
        id: AllocId,
    ) {
        mem.data.locks.insert(id, RangeMap::new());
    }

    fn free_lock<'a>(
        mem: &mut Memory<'a, 'mir, 'tcx, Self>,
        id: AllocId,
        len: u64,
    ) -> EvalResult<'tcx> {
        mem.data.locks
            .remove(&id)
            .expect("allocation has no corresponding locks")
            .check(
                Some(mem.cur_frame),
                0,
                len,
                AccessKind::Read,
            )
            .map_err(|lock| {
                EvalErrorKind::DeallocatedLockedMemory {
                    //ptr, FIXME
                    ptr: MemoryPointer {
                        alloc_id: AllocId(0),
                        offset: 0,
                    },
                    lock: lock.active,
                }.into()
            })
    }

    fn end_region<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        reg: Option<::rustc::middle::region::Scope>,
    ) -> EvalResult<'tcx> {
        ecx.end_region(reg)
    }

    fn validation_op<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        op: ::rustc::mir::ValidationOp,
        operand: &::rustc::mir::ValidationOperand<'tcx, ::rustc::mir::Place<'tcx>>,
    ) -> EvalResult<'tcx> {
        ecx.validation_op(op, operand)
    }
}
<commit_msg>Hack: copy init_static from rustc CompileTimeEvaluator to try to fix uninitialized static error<commit_after>#![feature(
    i128_type,
    rustc_private,
    conservative_impl_trait,
    catch_expr,
    inclusive_range_fields
)]

#[macro_use]
extern crate log;

// From rustc.
#[macro_use]
extern crate rustc;
extern crate rustc_mir;
extern crate rustc_data_structures;
extern crate syntax;
extern crate regex;
#[macro_use]
extern crate lazy_static;

use rustc::ty::{self, TyCtxt};
use rustc::ty::layout::{TyLayout, LayoutOf};
use rustc::hir::def_id::DefId;
use rustc::mir;

use syntax::ast::Mutability;
use syntax::codemap::Span;

use std::collections::{HashMap, BTreeMap};

pub use rustc::mir::interpret::*;
pub use rustc_mir::interpret::*;

mod fn_call;
mod operator;
mod intrinsic;
mod helpers;
mod memory;
mod tls;
mod locks;
mod range_map;
mod validation;

use fn_call::EvalContextExt as MissingFnsEvalContextExt;
use operator::EvalContextExt as OperatorEvalContextExt;
use intrinsic::EvalContextExt as IntrinsicEvalContextExt;
use tls::EvalContextExt as TlsEvalContextExt;
use locks::LockInfo;
use locks::MemoryExt as LockMemoryExt;
use validation::EvalContextExt as ValidationEvalContextExt;
use range_map::RangeMap;
use validation::{ValidationQuery, AbsPlace};

pub fn eval_main<'a, 'tcx: 'a>(
    tcx: TyCtxt<'a, 'tcx, 'tcx>,
    main_id: DefId,
    start_wrapper: Option<DefId>,
) {
    fn run_main<'a, 'mir: 'a, 'tcx: 'mir>(
        ecx: &mut rustc_mir::interpret::EvalContext<'a, 'mir, 'tcx, Evaluator<'tcx>>,
        main_id: DefId,
        start_wrapper: Option<DefId>,
    ) -> EvalResult<'tcx> {
        let main_instance = ty::Instance::mono(ecx.tcx.tcx, main_id);
        let main_mir = ecx.load_mir(main_instance.def)?;
        let mut cleanup_ptr = None; // Pointer to be deallocated when we are done

        if !main_mir.return_ty().is_nil() || main_mir.arg_count != 0 {
            return err!(Unimplemented(
                "miri does not support main functions without `fn()` type signatures"
                    .to_owned(),
            ));
        }

        if let Some(start_id) = start_wrapper {
            let main_ret_ty = ecx.tcx.fn_sig(main_id).output();
            let main_ret_ty = main_ret_ty.no_late_bound_regions().unwrap();
            let start_instance = ty::Instance::resolve(
                ecx.tcx.tcx,
                ty::ParamEnv::reveal_all(),
                start_id,
                ecx.tcx.mk_substs(
                    ::std::iter::once(ty::subst::Kind::from(main_ret_ty)))).unwrap();
            let start_mir = ecx.load_mir(start_instance.def)?;

            if start_mir.arg_count != 3 {
                return err!(AbiViolation(format!(
                    "'start' lang item should have three arguments, but has {}",
                    start_mir.arg_count
                )));
            }

            // Return value
            let size = ecx.tcx.data_layout.pointer_size.bytes();
            let align = ecx.tcx.data_layout.pointer_align;
            let ret_ptr = ecx.memory_mut().allocate(size, align, Some(MemoryKind::Stack))?;
            cleanup_ptr = Some(ret_ptr);

            // Push our stack frame
            ecx.push_stack_frame(
                start_instance,
                start_mir.span,
                start_mir,
                Place::from_ptr(ret_ptr, align),
                StackPopCleanup::None,
            )?;

            let mut args = ecx.frame().mir.args_iter();

            // First argument: pointer to main()
            let main_ptr = ecx.memory_mut().create_fn_alloc(main_instance);
            let dest = ecx.eval_place(&mir::Place::Local(args.next().unwrap()))?;
            let main_ty = main_instance.ty(ecx.tcx.tcx);
            let main_ptr_ty = ecx.tcx.mk_fn_ptr(main_ty.fn_sig(ecx.tcx.tcx));
            ecx.write_value(
                ValTy {
                    value: Value::ByVal(PrimVal::Ptr(main_ptr)),
                    ty: main_ptr_ty,
                },
                dest,
            )?;

            // Second argument (argc): 1
            let dest = ecx.eval_place(&mir::Place::Local(args.next().unwrap()))?;
            let ty = ecx.tcx.types.isize;
            ecx.write_primval(dest, PrimVal::Bytes(1), ty)?;

            // FIXME: extract main source file path
            // Third argument (argv): &[b"foo"]
            let dest = ecx.eval_place(&mir::Place::Local(args.next().unwrap()))?;
            let ty = ecx.tcx.mk_imm_ptr(ecx.tcx.mk_imm_ptr(ecx.tcx.types.u8));
            let foo = ecx.memory.allocate_cached(b"foo\0");
            let ptr_size = ecx.memory.pointer_size();
            let ptr_align = ecx.tcx.data_layout.pointer_align;
            let foo_ptr = ecx.memory.allocate(ptr_size, ptr_align, None)?;
            ecx.memory.write_primval(foo_ptr, ptr_align, PrimVal::Ptr(foo.into()), ptr_size, false)?;
            ecx.memory.mark_static_initialized(foo_ptr.alloc_id, Mutability::Immutable)?;
            ecx.write_ptr(dest, foo_ptr.into(), ty)?;

            assert!(args.next().is_none(), "start lang item has more arguments than expected");
        } else {
            ecx.push_stack_frame(
                main_instance,
                main_mir.span,
                main_mir,
                Place::undef(),
                StackPopCleanup::None,
            )?;

            // No arguments
            let mut args = ecx.frame().mir.args_iter();
            assert!(args.next().is_none(), "main function must not have arguments");
        }

        while ecx.step()? {}
        ecx.run_tls_dtors()?;
        if let Some(cleanup_ptr) = cleanup_ptr {
            ecx.memory_mut().deallocate(
                cleanup_ptr,
                None,
                MemoryKind::Stack,
            )?;
        }
        Ok(())
    }

    let mut ecx = EvalContext::new(tcx.at(syntax::codemap::DUMMY_SP), ty::ParamEnv::reveal_all(), Default::default(), Default::default());
    match run_main(&mut ecx, main_id, start_wrapper) {
        Ok(()) => {
            let leaks = ecx.memory().leak_report();
            if leaks != 0 {
                tcx.sess.err("the evaluated program leaked memory");
            }
        }
        Err(mut e) => {
            ecx.report(&mut e, true, None);
        }
    }
}

#[derive(Default)]
pub struct Evaluator<'tcx> {
    /// Environment variables set by `setenv`
    /// Miri does not expose env vars from the host to the emulated program
    pub(crate) env_vars: HashMap<Vec<u8>, MemoryPointer>,

    /// Places that were suspended by the validation subsystem, and will be recovered later
    pub(crate) suspended: HashMap<DynamicLifetime, Vec<ValidationQuery<'tcx>>>,
}

pub type TlsKey = usize;

#[derive(Copy, Clone, Debug)]
pub struct TlsEntry<'tcx> {
    data: Pointer, // Will eventually become a map from thread IDs to `Pointer`s, if we ever support more than one thread.
    dtor: Option<ty::Instance<'tcx>>,
}

#[derive(Default)]
pub struct MemoryData<'tcx> {
    /// The Key to use for the next thread-local allocation.
    next_thread_local: TlsKey,

    /// pthreads-style thread-local storage.
    thread_local: BTreeMap<TlsKey, TlsEntry<'tcx>>,

    /// Memory regions that are locked by some function
    ///
    /// Only mutable (static mut, heap, stack) allocations have an entry in this map.
    /// The entry is created when allocating the memory and deleted after deallocation.
    locks: HashMap<AllocId, RangeMap<LockInfo<'tcx>>>,
}

impl<'mir, 'tcx: 'mir> Machine<'mir, 'tcx> for Evaluator<'tcx> {
    type MemoryData = MemoryData<'tcx>;
    type MemoryKinds = memory::MemoryKind;

    /// Returns Ok() when the function was handled, fail otherwise
    fn eval_fn_call<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        instance: ty::Instance<'tcx>,
        destination: Option<(Place, mir::BasicBlock)>,
        args: &[ValTy<'tcx>],
        span: Span,
        sig: ty::FnSig<'tcx>,
    ) -> EvalResult<'tcx, bool> {
        ecx.eval_fn_call(instance, destination, args, span, sig)
    }

    fn call_intrinsic<'a>(
        ecx: &mut rustc_mir::interpret::EvalContext<'a, 'mir, 'tcx, Self>,
        instance: ty::Instance<'tcx>,
        args: &[ValTy<'tcx>],
        dest: Place,
        dest_layout: TyLayout<'tcx>,
        target: mir::BasicBlock,
    ) -> EvalResult<'tcx> {
        ecx.call_intrinsic(instance, args, dest, dest_layout, target)
    }

    fn try_ptr_op<'a>(
        ecx: &rustc_mir::interpret::EvalContext<'a, 'mir, 'tcx, Self>,
        bin_op: mir::BinOp,
        left: PrimVal,
        left_ty: ty::Ty<'tcx>,
        right: PrimVal,
        right_ty: ty::Ty<'tcx>,
    ) -> EvalResult<'tcx, Option<(PrimVal, bool)>> {
        ecx.ptr_op(bin_op, left, left_ty, right, right_ty)
    }

    fn mark_static_initialized<'a>(
        _mem: &mut Memory<'a, 'mir, 'tcx, Self>,
        _id: AllocId,
        _mutability: Mutability,
    ) -> EvalResult<'tcx, bool> {
        /*use memory::MemoryKind::*;
        match m {
            // FIXME: This could be allowed, but not for env vars set during miri execution
            Env => err!(Unimplemented("statics can't refer to env vars".to_owned())),
            _ => Ok(false), // TODO: What does the bool mean?
        }*/
        Ok(false)
    }

    fn init_static<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        cid: GlobalId<'tcx>,
    ) -> EvalResult<'tcx, AllocId> {
ecx.const_eval(cid)?;
        return Ok(ecx
            .tcx
            .interpret_interner
            .get_cached(cid.instance.def_id())
            .expect("uncached static"));
        let def_id = cid.instance.def_id();
        let ty = ecx.tcx.type_of(def_id);
        let layout = ecx.tcx.layout_of(ty::ParamEnvAnd {
            param_env: ty::ParamEnv::reveal_all(),
            value: ty
        }).expect("Couldn't compute layout for the type of a static");
        ecx.memory.allocate(layout.size.bytes(), layout.align, None).map(|mptr|mptr.alloc_id)
    }

    fn box_alloc<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        ty: ty::Ty<'tcx>,
        dest: Place,
    ) -> EvalResult<'tcx> {
        let layout = ecx.layout_of(ty)?;

        // Call the `exchange_malloc` lang item
        let malloc = ecx.tcx.lang_items().exchange_malloc_fn().unwrap();
        let malloc = ty::Instance::mono(ecx.tcx.tcx, malloc);
        let malloc_mir = ecx.load_mir(malloc.def)?;
        ecx.push_stack_frame(
            malloc,
            malloc_mir.span,
            malloc_mir,
            dest,
            // Don't do anything when we are done.  The statement() function will increment
            // the old stack frame's stmt counter to the next statement, which means that when
            // exchange_malloc returns, we go on evaluating exactly where we want to be.
            StackPopCleanup::None,
        )?;

        let mut args = ecx.frame().mir.args_iter();
        let usize = ecx.tcx.types.usize;

        // First argument: size
        let dest = ecx.eval_place(&mir::Place::Local(args.next().unwrap()))?;
        ecx.write_value(
            ValTy {
                value: Value::ByVal(PrimVal::Bytes(layout.size.bytes().into())),
                ty: usize,
            },
            dest,
        )?;

        // Second argument: align
        let dest = ecx.eval_place(&mir::Place::Local(args.next().unwrap()))?;
        ecx.write_value(
            ValTy {
                value: Value::ByVal(PrimVal::Bytes(layout.align.abi().into())),
                ty: usize,
            },
            dest,
        )?;

        // No more arguments
        assert!(args.next().is_none(), "exchange_malloc lang item has more arguments than expected");
        Ok(())
    }

    fn global_item_with_linkage<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        instance: ty::Instance<'tcx>,
        mutability: Mutability,
    ) -> EvalResult<'tcx> {
        // FIXME: check that it's `#[linkage = "extern_weak"]`
        trace!("Initializing an extern global with NULL");
        let ptr_size = ecx.memory.pointer_size();
        let ptr_align = ecx.tcx.data_layout.pointer_align;
        let ptr = ecx.memory.allocate(
            ptr_size,
            ptr_align,
            None,
        )?;
        ecx.memory.write_ptr_sized_unsigned(ptr, ptr_align, PrimVal::Bytes(0))?;
        ecx.memory.mark_static_initialized(ptr.alloc_id, mutability)?;
        ecx.tcx.interpret_interner.cache(
            instance.def_id(),
            ptr.alloc_id,
        );
        Ok(())
    }

    fn check_locks<'a>(
        mem: &Memory<'a, 'mir, 'tcx, Self>,
        ptr: MemoryPointer,
        size: u64,
        access: AccessKind,
    ) -> EvalResult<'tcx> {
        mem.check_locks(ptr, size, access)
    }

    fn add_lock<'a>(
        mem: &mut Memory<'a, 'mir, 'tcx, Self>,
        id: AllocId,
    ) {
        mem.data.locks.insert(id, RangeMap::new());
    }

    fn free_lock<'a>(
        mem: &mut Memory<'a, 'mir, 'tcx, Self>,
        id: AllocId,
        len: u64,
    ) -> EvalResult<'tcx> {
        mem.data.locks
            .remove(&id)
            .expect("allocation has no corresponding locks")
            .check(
                Some(mem.cur_frame),
                0,
                len,
                AccessKind::Read,
            )
            .map_err(|lock| {
                EvalErrorKind::DeallocatedLockedMemory {
                    //ptr, FIXME
                    ptr: MemoryPointer {
                        alloc_id: AllocId(0),
                        offset: 0,
                    },
                    lock: lock.active,
                }.into()
            })
    }

    fn end_region<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        reg: Option<::rustc::middle::region::Scope>,
    ) -> EvalResult<'tcx> {
        ecx.end_region(reg)
    }

    fn validation_op<'a>(
        ecx: &mut EvalContext<'a, 'mir, 'tcx, Self>,
        op: ::rustc::mir::ValidationOp,
        operand: &::rustc::mir::ValidationOperand<'tcx, ::rustc::mir::Place<'tcx>>,
    ) -> EvalResult<'tcx> {
        ecx.validation_op(op, operand)
    }
}
<|endoftext|>


<commit_before>use std::io::Write;

use clap::{App, ArgMatches, Shell};

pub trait CliSpec<'a> {
    fn name(&self) -> &str;
    fn matches(self) -> ArgMatches<'a>;
    fn completions<W: Write, S: Into<String>>(&mut self, _: S, _: Shell, _: &mut W) {}
}

impl<'a> CliSpec<'a> for App<'a, 'a> {
    fn name(&self) -> &str {
        self.get_name()
    }

    fn matches(self) -> ArgMatches<'a> {
        self.get_matches()
    }

    fn completions<W: Write, S: Into<String>>(&mut self,
                                              bin_name: S,
                                              for_shell: Shell,
                                              buf: &mut W) {

        self.gen_completions_to(bin_name, for_shell, buf);
    }
}
<commit_msg>Add documentation for `CliSpec`<commit_after>use std::io::Write;

use clap::{App, ArgMatches, Shell};

/// An abstraction over `clap::App` functionality needed for initializing `Runtime`. Different
/// implementations can be used for testing `imag` binaries without running them as separate
/// processes.
pub trait CliSpec<'a> {
    fn name(&self) -> &str;
    fn matches(self) -> ArgMatches<'a>;
    fn completions<W: Write, S: Into<String>>(&mut self, _: S, _: Shell, _: &mut W) {}
}

impl<'a> CliSpec<'a> for App<'a, 'a> {
    fn name(&self) -> &str {
        self.get_name()
    }

    fn matches(self) -> ArgMatches<'a> {
        self.get_matches()
    }

    fn completions<W: Write, S: Into<String>>(&mut self,
                                              bin_name: S,
                                              for_shell: Shell,
                                              buf: &mut W) {

        self.gen_completions_to(bin_name, for_shell, buf);
    }
}
<|endoftext|>


<commit_before>#[doc = "Converts the Rust AST to the rustdoc document model"];

import rustc::syntax::ast;

export from_srv, extract;

fn from_srv(
    srv: astsrv::srv,
    default_name: str
) -> doc::cratedoc {

    #[doc = "Use the AST service to create a document tree"];

    astsrv::exec(srv) {|ctxt|
        extract(ctxt.ast, default_name)
    }
}

fn extract(
    crate: @ast::crate,
    default_name: str
) -> doc::cratedoc {
    {
        topmod: top_moddoc_from_crate(crate, default_name),
    }
}

fn top_moddoc_from_crate(
    crate: @ast::crate,
    default_name: str
) -> doc::moddoc {
    moddoc_from_mod(crate.node.module, default_name, ast::crate_node_id)
}

fn moddoc_from_mod(
    module: ast::_mod,
    name: ast::ident,
    id: ast::node_id
) -> doc::moddoc {
    {
        id: id,
        name: name,
        path: [],
        brief: none,
        desc: none,
        items: ~vec::filter_map(module.items) {|item|
            alt item.node {
              ast::item_mod(m) {
                some(doc::modtag(
                    moddoc_from_mod(m, item.ident, item.id)
                ))
              }
              ast::item_fn(decl, _, _) {
                some(doc::fntag(
                    fndoc_from_fn(decl, item.ident, item.id)
                ))
              }
              ast::item_const(_, _) {
                some(doc::consttag(
                    constdoc_from_const(item.ident, item.id)
                ))
              }
              ast::item_enum(variants, _) {
                some(doc::enumtag(
                    enumdoc_from_enum(item.ident, item.id, variants)
                ))
              }
              ast::item_res(decl, _, _, _, _) {
                some(doc::restag(
                    resdoc_from_resource(decl, item.ident, item.id)
                ))
              }
              _ {
                none
              }
            }
        }
    }
}

fn fndoc_from_fn(
    decl: ast::fn_decl,
    name: ast::ident,
    id: ast::node_id
) -> doc::fndoc {
    {
        id: id,
        name: name,
        brief: none,
        desc: none,
        args: argdocs_from_args(decl.inputs),
        return: {
            desc: none,
            ty: none
        },
        failure: none,
        sig: none
    }
}

#[test]
fn should_extract_fn_args() {
    let source = "fn a(b: int, c: int) { }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    let fn_ = doc.topmod.fns()[0];
    assert fn_.args[0].name == "b";
    assert fn_.args[1].name == "c";
}

fn argdocs_from_args(args: [ast::arg]) -> [doc::argdoc] {
    vec::map(args, argdoc_from_arg)
}

fn argdoc_from_arg(arg: ast::arg) -> doc::argdoc {
    {
        name: arg.ident,
        desc: none,
        ty: none
    }
}

fn constdoc_from_const(
    name: ast::ident,
    id: ast::node_id
) -> doc::constdoc {
    {
        id: id,
        name: name,
        brief: none,
        desc: none,
        ty: none
    }
}

#[test]
fn should_extract_const_name_and_id() {
    let source = "const a: int = 0;";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.consts()[0].id != 0;
    assert doc.topmod.consts()[0].name == "a";
}

fn enumdoc_from_enum(
    name: ast::ident,
    id: ast::node_id,
    variants: [ast::variant]
) -> doc::enumdoc {
    {
        id: id,
        name: name,
        brief: none,
        desc: none,
        variants: variantdocs_from_variants(variants)
    }
}

fn variantdocs_from_variants(
    variants: [ast::variant]
) -> [doc::variantdoc] {
    vec::map(variants, variantdoc_from_variant)
}

fn variantdoc_from_variant(variant: ast::variant) -> doc::variantdoc {
    {
        name: variant.node.name,
        desc: none,
        sig: none
    }
}

#[test]
fn should_extract_enums() {
    let source = "enum e { v }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.enums()[0].id != 0;
    assert doc.topmod.enums()[0].name == "e";
}

#[test]
fn should_extract_enum_variants() {
    let source = "enum e { v }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.enums()[0].variants[0].name == "v";
}

fn resdoc_from_resource(
    decl: ast::fn_decl,
    name: str,
    id: ast::node_id
) -> doc::resdoc {
    {
        id: id,
        name: name,
        brief: none,
        desc: none,
        args: argdocs_from_args(decl.inputs),
        sig: none
    }
}

#[test]
fn should_extract_resources() {
    let source = "resource r(b: bool) { }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.resources()[0].id != 0;
    assert doc.topmod.resources()[0].name == "r";
}

#[test]
fn should_extract_resource_args() {
    let source = "resource r(b: bool) { }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.resources()[0].args[0].name == "b";
}

#[cfg(test)]
mod tests {

    #[test]
    fn extract_empty_crate() {
        let source = ""; // empty crate
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert vec::is_empty(doc.topmod.mods());
        assert vec::is_empty(doc.topmod.fns());
    }

    #[test]
    fn extract_mods() {
        let source = "mod a { mod b { } mod c { } }";
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert doc.topmod.mods()[0].name == "a";
        assert doc.topmod.mods()[0].mods()[0].name == "b";
        assert doc.topmod.mods()[0].mods()[1].name == "c";
    }

    #[test]
    fn extract_mods_deep() {
        let source = "mod a { mod b { mod c { } } }";
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert doc.topmod.mods()[0].mods()[0].mods()[0].name == "c";
    }

    #[test]
    fn extract_should_set_mod_ast_id() {
        let source = "mod a { }";
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert doc.topmod.mods()[0].id != 0;
    }

    #[test]
    fn extract_fns() {
        let source =
            "fn a() { } \
             mod b { fn c() { } }";
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert doc.topmod.fns()[0].name == "a";
        assert doc.topmod.mods()[0].fns()[0].name == "c";
    }

    #[test]
    fn extract_should_set_fn_ast_id() {
        let source = "fn a() { }";
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert doc.topmod.fns()[0].id != 0;
    }

    #[test]
    fn extract_should_use_default_crate_name() {
        let source = "";
        let ast = parse::from_str(source);
        let doc = extract(ast, "burp");
        assert doc.topmod.name == "burp";
    }

    #[test]
    fn extract_from_seq_srv() {
        let source = "";
        let srv = astsrv::mk_srv_from_str(source);
        let doc = from_srv(srv, "name");
        assert doc.topmod.name == "name";
    }
}<commit_msg>rustdoc: Extract iface doc nodes from the AST<commit_after>#[doc = "Converts the Rust AST to the rustdoc document model"];

import rustc::syntax::ast;

export from_srv, extract;

fn from_srv(
    srv: astsrv::srv,
    default_name: str
) -> doc::cratedoc {

    #[doc = "Use the AST service to create a document tree"];

    astsrv::exec(srv) {|ctxt|
        extract(ctxt.ast, default_name)
    }
}

fn extract(
    crate: @ast::crate,
    default_name: str
) -> doc::cratedoc {
    {
        topmod: top_moddoc_from_crate(crate, default_name),
    }
}

fn top_moddoc_from_crate(
    crate: @ast::crate,
    default_name: str
) -> doc::moddoc {
    moddoc_from_mod(crate.node.module, default_name, ast::crate_node_id)
}

fn moddoc_from_mod(
    module: ast::_mod,
    name: ast::ident,
    id: ast::node_id
) -> doc::moddoc {
    {
        id: id,
        name: name,
        path: [],
        brief: none,
        desc: none,
        items: ~vec::filter_map(module.items) {|item|
            alt item.node {
              ast::item_mod(m) {
                some(doc::modtag(
                    moddoc_from_mod(m, item.ident, item.id)
                ))
              }
              ast::item_fn(decl, _, _) {
                some(doc::fntag(
                    fndoc_from_fn(decl, item.ident, item.id)
                ))
              }
              ast::item_const(_, _) {
                some(doc::consttag(
                    constdoc_from_const(item.ident, item.id)
                ))
              }
              ast::item_enum(variants, _) {
                some(doc::enumtag(
                    enumdoc_from_enum(item.ident, item.id, variants)
                ))
              }
              ast::item_res(decl, _, _, _, _) {
                some(doc::restag(
                    resdoc_from_resource(decl, item.ident, item.id)
                ))
              }
              ast::item_iface(_, methods) {
                some(doc::ifacetag(
                    ifacedoc_from_iface(methods, item.ident, item.id)
                ))
              }
              _ {
                none
              }
            }
        }
    }
}

fn fndoc_from_fn(
    decl: ast::fn_decl,
    name: ast::ident,
    id: ast::node_id
) -> doc::fndoc {
    {
        id: id,
        name: name,
        brief: none,
        desc: none,
        args: argdocs_from_args(decl.inputs),
        return: {
            desc: none,
            ty: none
        },
        failure: none,
        sig: none
    }
}

#[test]
fn should_extract_fn_args() {
    let source = "fn a(b: int, c: int) { }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    let fn_ = doc.topmod.fns()[0];
    assert fn_.args[0].name == "b";
    assert fn_.args[1].name == "c";
}

fn argdocs_from_args(args: [ast::arg]) -> [doc::argdoc] {
    vec::map(args, argdoc_from_arg)
}

fn argdoc_from_arg(arg: ast::arg) -> doc::argdoc {
    {
        name: arg.ident,
        desc: none,
        ty: none
    }
}

fn constdoc_from_const(
    name: ast::ident,
    id: ast::node_id
) -> doc::constdoc {
    {
        id: id,
        name: name,
        brief: none,
        desc: none,
        ty: none
    }
}

#[test]
fn should_extract_const_name_and_id() {
    let source = "const a: int = 0;";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.consts()[0].id != 0;
    assert doc.topmod.consts()[0].name == "a";
}

fn enumdoc_from_enum(
    name: ast::ident,
    id: ast::node_id,
    variants: [ast::variant]
) -> doc::enumdoc {
    {
        id: id,
        name: name,
        brief: none,
        desc: none,
        variants: variantdocs_from_variants(variants)
    }
}

fn variantdocs_from_variants(
    variants: [ast::variant]
) -> [doc::variantdoc] {
    vec::map(variants, variantdoc_from_variant)
}

fn variantdoc_from_variant(variant: ast::variant) -> doc::variantdoc {
    {
        name: variant.node.name,
        desc: none,
        sig: none
    }
}

#[test]
fn should_extract_enums() {
    let source = "enum e { v }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.enums()[0].id != 0;
    assert doc.topmod.enums()[0].name == "e";
}

#[test]
fn should_extract_enum_variants() {
    let source = "enum e { v }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.enums()[0].variants[0].name == "v";
}

fn resdoc_from_resource(
    decl: ast::fn_decl,
    name: str,
    id: ast::node_id
) -> doc::resdoc {
    {
        id: id,
        name: name,
        brief: none,
        desc: none,
        args: argdocs_from_args(decl.inputs),
        sig: none
    }
}

#[test]
fn should_extract_resources() {
    let source = "resource r(b: bool) { }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.resources()[0].id != 0;
    assert doc.topmod.resources()[0].name == "r";
}

#[test]
fn should_extract_resource_args() {
    let source = "resource r(b: bool) { }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.resources()[0].args[0].name == "b";
}

fn ifacedoc_from_iface(
    methods: [ast::ty_method],
    name: str,
    id: ast::node_id
) -> doc::ifacedoc {
    {
        id: id,
        name: name,
        brief: none,
        desc: none,
        methods: vec::map(methods) {|method|
            {
                name: method.ident,
                brief: none,
                desc: none,
                args: argdocs_from_args(method.decl.inputs),
                return: {
                    desc: none,
                    ty: none
                },
                failure: none,
                sig: none
            }
        }
    }
}

#[test]
fn should_extract_ifaces() {
    let source = "iface i { fn f(); }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.ifaces()[0].name == "i";
}

#[test]
fn should_extract_iface_methods() {
    let source = "iface i { fn f(); }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.ifaces()[0].methods[0].name == "f";
}

#[test]
fn should_extract_iface_method_args() {
    let source = "iface i { fn f(a: bool); }";
    let ast = parse::from_str(source);
    let doc = extract(ast, "");
    assert doc.topmod.ifaces()[0].methods[0].args[0].name == "a";
}

#[cfg(test)]
mod tests {

    #[test]
    fn extract_empty_crate() {
        let source = ""; // empty crate
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert vec::is_empty(doc.topmod.mods());
        assert vec::is_empty(doc.topmod.fns());
    }

    #[test]
    fn extract_mods() {
        let source = "mod a { mod b { } mod c { } }";
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert doc.topmod.mods()[0].name == "a";
        assert doc.topmod.mods()[0].mods()[0].name == "b";
        assert doc.topmod.mods()[0].mods()[1].name == "c";
    }

    #[test]
    fn extract_mods_deep() {
        let source = "mod a { mod b { mod c { } } }";
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert doc.topmod.mods()[0].mods()[0].mods()[0].name == "c";
    }

    #[test]
    fn extract_should_set_mod_ast_id() {
        let source = "mod a { }";
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert doc.topmod.mods()[0].id != 0;
    }

    #[test]
    fn extract_fns() {
        let source =
            "fn a() { } \
             mod b { fn c() { } }";
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert doc.topmod.fns()[0].name == "a";
        assert doc.topmod.mods()[0].fns()[0].name == "c";
    }

    #[test]
    fn extract_should_set_fn_ast_id() {
        let source = "fn a() { }";
        let ast = parse::from_str(source);
        let doc = extract(ast, "");
        assert doc.topmod.fns()[0].id != 0;
    }

    #[test]
    fn extract_should_use_default_crate_name() {
        let source = "";
        let ast = parse::from_str(source);
        let doc = extract(ast, "burp");
        assert doc.topmod.name == "burp";
    }

    #[test]
    fn extract_from_seq_srv() {
        let source = "";
        let srv = astsrv::mk_srv_from_str(source);
        let doc = from_srv(srv, "name");
        assert doc.topmod.name == "name";
    }
}<|endoftext|>


<commit_before>use crate::ram::types::Id;
use crate::server::NebServer;
use crate::{
    index::builder::IndexBuilder,
    ram::cell::{CellHeader, OwnedCell, ReadError, WriteError},
};
use bifrost::rpc::*;
use dovahkiin::expr::serde::Expr;
use dovahkiin::expr::symbols::utils::is_true;
use dovahkiin::expr::SExpr;
use dovahkiin::integrated::lisp;
use dovahkiin::types::{OwnedValue, SharedMap, SharedValue};
use futures::future::BoxFuture;
use futures::prelude::*;

use bifrost_plugins::hash_ident;
use itertools::Itertools;

pub static DEFAULT_SERVICE_ID: u64 = hash_ident!(NEB_CELL_RPC_SERVICE) as u64;

service! {
    rpc read_cell(key: Id) -> Result<OwnedCell, ReadError>;
    rpc read_all_cells(keys: Vec<Id>) -> Vec<Result<OwnedCell, ReadError>>;
    rpc read_all_cells_selected(keys: Vec<Id>, colums: Vec<u64>) -> Vec<Result<OwnedCell, ReadError>>;
    rpc read_all_cells_proced(keys: Vec<Id>, colums: Vec<u64>, filter: Expr, proc: Expr) -> Vec<Result<OwnedCell, ReadError>>;
    rpc write_cell(cell:OwnedCell) -> Result<CellHeader, WriteError>;
    rpc update_cell(cell: OwnedCell) -> Result<CellHeader, WriteError>;
    rpc upsert_cell(cell: OwnedCell) -> Result<CellHeader, WriteError>;
    rpc remove_cell(key: Id) -> Result<(), WriteError>;
    rpc count() -> u64;
}

pub struct NebRPCService {
    server: Arc<NebServer>,
}

impl Service for NebRPCService {
    fn read_cell(&self, key: Id) -> BoxFuture<Result<OwnedCell, ReadError>> {
        future::ready(self.server.chunks.read_cell(&key).map(|c| c.to_owned())).boxed()
    }
    fn read_all_cells(&self, keys: Vec<Id>) -> BoxFuture<Vec<Result<OwnedCell, ReadError>>> {
        future::ready(
            keys.into_iter()
                .map(|id| self.server.chunks.read_cell(&id).map(|c| c.to_owned()))
                .collect(),
        )
        .boxed()
    }
    fn read_all_cells_selected(
        &self,
        keys: Vec<Id>,
        colums: Vec<u64>,
    ) -> BoxFuture<Vec<Result<OwnedCell, ReadError>>> {
        future::ready(
            keys.into_iter()
                .map(|id| {
                    self.server
                        .chunks
                        .read_selected(&id, colums.as_slice())
                        .map(|c| c.to_owned())
                })
                .collect(),
        )
        .boxed()
    }
    fn write_cell(&self, mut cell: OwnedCell) -> BoxFuture<Result<CellHeader, WriteError>> {
        self.with_indices_ensured(self.server.chunks.write_cell(&mut cell))
    }

    fn update_cell(&self, mut cell: OwnedCell) -> BoxFuture<Result<CellHeader, WriteError>> {
        self.with_indices_ensured(self.server.chunks.update_cell(&mut cell))
    }
    fn remove_cell(&self, key: Id) -> BoxFuture<Result<(), WriteError>> {
        self.with_indices_ensured(self.server.chunks.remove_cell(&key))
    }
    fn upsert_cell(&self, mut cell: OwnedCell) -> BoxFuture<Result<CellHeader, WriteError>> {
        self.with_indices_ensured(self.server.chunks.upsert_cell(&mut cell))
    }
    fn count(&self) -> BoxFuture<u64> {
        future::ready(self.server.chunks.count() as u64).boxed()
    }

    fn read_all_cells_proced(
        &self,
        keys: Vec<Id>,
        colums: Vec<u64>,
        filter: Expr,
        proc: Expr,
    ) -> BoxFuture<Vec<Result<OwnedCell, ReadError>>> {
        let filter_empty = filter.is_empty();
        let proc_empty = proc.is_empty();
        if filter_empty && proc_empty {
            // Fast path for naive query
            if colums.is_empty() {
                let res = keys
                    .iter()
                    .map(|id| self.server.chunks.read_cell(id).map(|c| c.to_owned()))
                    .collect_vec();
                return future::ready(res).boxed();
            } else {
                let res = keys
                    .iter()
                    .map(|id| {
                        self.server
                            .chunks
                            .read_selected(id, colums.as_slice())
                            .map(|c| c.to_owned())
                    })
                    .collect_vec();
                return future::ready(res).boxed();
            }
        };
        let mut rows = if colums.is_empty() {
            keys.into_iter()
                .map(|id| self.server.chunks.read_cell(&id))
                .map(|cell| {
                    cell.map(|cell| {
                        let mut interpreter = lisp::get_interpreter();
                        if let &SharedValue::Map(ref map) = &cell.data {
                            for (id, val) in &map.map {
                                interpreter.bind_by_id(*id, SExpr::shared_value(val.clone()));
                            }
                        }
                        interpreter.bind("data", SExpr::shared_value(cell.data.clone()));
                        (cell, interpreter)
                    })
                })
                .collect_vec()
        } else {
            keys.into_iter()
                .map(|id| self.server.chunks.read_selected(&id, colums.as_slice()))
                .map(|fields| {
                    fields.map(|fields| {
                        if let &SharedValue::Array(ref arr) = &fields.data {
                            let mut interpreter = lisp::get_interpreter();
                            debug_assert_eq!(arr.len(), colums.len());
                            for (i, col) in colums.iter().enumerate() {
                                interpreter.bind_by_id(*col, SExpr::shared_value(arr[i].clone()));
                            }
                            interpreter.bind("data", SExpr::shared_value(fields.data.clone()));
                            return (fields, interpreter);
                        } else {
                            unreachable!()
                        }
                    })
                })
                .collect_vec()
        };
        if !filter_empty {
            let filter = filter.to_sexpr();
            rows = rows
                .into_iter()
                .map(|row| {
                    if let Ok((cell, mut exec)) = row {
                        let check_res = filter.clone().eval(exec.get_env());
                        match check_res {
                            Ok(sexp) => {
                                if is_true(sexp) {
                                    return Ok((cell, exec));
                                } else {
                                    return Err(ReadError::NotMatch);
                                }
                            }
                            Err(e) => return Err(ReadError::ExecError(e)),
                        }
                    } else {
                        return row;
                    }
                })
                .collect()
        }
        if !proc_empty {
            let proc = proc.to_sexpr();
            let res = rows
                .into_iter()
                .map(|row| match row {
                    Ok((cell, mut exec)) => {
                        let proc_res = proc.clone().eval(exec.get_env());
                        match proc_res {
                            Ok(sexp) => {
                                let val = sexp.owned_val().unwrap_or(OwnedValue::NA);
                                return Ok(OwnedCell {
                                    header: cell.header,
                                    data: val,
                                });
                            }
                            Err(e) => return Err(ReadError::ExecError(e)),
                        }
                    }
                    Err(e) => Err(e),
                })
                .collect();
            return future::ready(res).boxed();
        } else {
            let res: Vec<_> = rows
                .into_iter()
                .map(|row| row.map(|(cell, _)| cell.to_owned()))
                .collect();
            return future::ready(res).boxed();
        }
    }
}

dispatch_rpc_service_functions!(NebRPCService);

impl NebRPCService {
    pub fn new(server: &Arc<NebServer>) -> Arc<NebRPCService> {
        Arc::new(NebRPCService {
            server: server.clone(),
        })
    }
    fn with_indices_ensured<'a, R>(&'a self, res: R) -> BoxFuture<R>
    where
        R: Send + 'a,
    {
        if self.server.indexer.is_some() {
            IndexBuilder::await_indices().map(|_| res).boxed()
        } else {
            future::ready(res).boxed()
        }
    }
}
<commit_msg>refactoring read all processed<commit_after>use crate::ram::types::Id;
use crate::server::NebServer;
use crate::{
    index::builder::IndexBuilder,
    ram::cell::{CellHeader, OwnedCell, ReadError, WriteError},
};
use bifrost::rpc::*;
use dovahkiin::expr::serde::Expr;
use dovahkiin::expr::symbols::utils::is_true;
use dovahkiin::expr::SExpr;
use dovahkiin::integrated::lisp;
use dovahkiin::types::{OwnedValue, SharedMap, SharedValue};
use futures::future::BoxFuture;
use futures::prelude::*;

use bifrost_plugins::hash_ident;
use itertools::Itertools;

pub static DEFAULT_SERVICE_ID: u64 = hash_ident!(NEB_CELL_RPC_SERVICE) as u64;

service! {
    rpc read_cell(key: Id) -> Result<OwnedCell, ReadError>;
    rpc read_all_cells(keys: Vec<Id>) -> Vec<Result<OwnedCell, ReadError>>;
    rpc read_all_cells_selected(keys: Vec<Id>, colums: Vec<u64>) -> Vec<Result<OwnedCell, ReadError>>;
    rpc read_all_cells_proced(keys: Vec<Id>, colums: Vec<u64>, filter: Expr, proc: Expr) -> Vec<Result<OwnedCell, ReadError>>;
    rpc write_cell(cell:OwnedCell) -> Result<CellHeader, WriteError>;
    rpc update_cell(cell: OwnedCell) -> Result<CellHeader, WriteError>;
    rpc upsert_cell(cell: OwnedCell) -> Result<CellHeader, WriteError>;
    rpc remove_cell(key: Id) -> Result<(), WriteError>;
    rpc count() -> u64;
}

pub struct NebRPCService {
    server: Arc<NebServer>,
}

impl Service for NebRPCService {
    fn read_cell(&self, key: Id) -> BoxFuture<Result<OwnedCell, ReadError>> {
        future::ready(self.server.chunks.read_cell(&key).map(|c| c.to_owned())).boxed()
    }
    fn read_all_cells(&self, keys: Vec<Id>) -> BoxFuture<Vec<Result<OwnedCell, ReadError>>> {
        future::ready(
            keys.into_iter()
                .map(|id| self.server.chunks.read_cell(&id).map(|c| c.to_owned()))
                .collect(),
        )
        .boxed()
    }
    fn read_all_cells_selected(
        &self,
        keys: Vec<Id>,
        colums: Vec<u64>,
    ) -> BoxFuture<Vec<Result<OwnedCell, ReadError>>> {
        future::ready(
            keys.into_iter()
                .map(|id| {
                    self.server
                        .chunks
                        .read_selected(&id, colums.as_slice())
                        .map(|c| c.to_owned())
                })
                .collect(),
        )
        .boxed()
    }
    fn write_cell(&self, mut cell: OwnedCell) -> BoxFuture<Result<CellHeader, WriteError>> {
        self.with_indices_ensured(self.server.chunks.write_cell(&mut cell))
    }

    fn update_cell(&self, mut cell: OwnedCell) -> BoxFuture<Result<CellHeader, WriteError>> {
        self.with_indices_ensured(self.server.chunks.update_cell(&mut cell))
    }
    fn remove_cell(&self, key: Id) -> BoxFuture<Result<(), WriteError>> {
        self.with_indices_ensured(self.server.chunks.remove_cell(&key))
    }
    fn upsert_cell(&self, mut cell: OwnedCell) -> BoxFuture<Result<CellHeader, WriteError>> {
        self.with_indices_ensured(self.server.chunks.upsert_cell(&mut cell))
    }
    fn count(&self) -> BoxFuture<u64> {
        future::ready(self.server.chunks.count() as u64).boxed()
    }

    fn read_all_cells_proced(
        &self,
        keys: Vec<Id>,
        colums: Vec<u64>,
        filter: Expr,
        proc: Expr,
    ) -> BoxFuture<Vec<Result<OwnedCell, ReadError>>> {
        let filter_empty = filter.is_empty();
        let proc_empty = proc.is_empty();
        let mut cells = if colums.is_empty() {
            keys.iter()
                .map(|id| self.server.chunks.read_cell(id))
                .collect_vec()
        } else {
            keys.iter()
                .map(|id| self.server.chunks.read_selected(id, colums.as_slice()))
                .collect_vec()
        };
        if (!filter_empty) | (!proc_empty) {
            let mut interpreters = Vec::with_capacity(cells.capacity());
            cells.iter().for_each(|cell_res| {
                interpreters.push(cell_res.as_ref().ok().map(|cell| {
                    let mut interpreter = lisp::get_interpreter();
                    if let &SharedValue::Map(ref map) = &cell.data {
                        for (id, val) in &map.map {
                            interpreter.bind_by_id(*id, SExpr::shared_value(val.clone()));
                        }
                    }
                    interpreter.bind("data", SExpr::shared_value(cell.data.clone()));
                    interpreter
                }))
            });
            if !filter_empty {
                let filter = filter.to_sexpr();
                cells = cells
                    .into_iter()
                    .enumerate()
                    .map(|(i, cell_res)| {
                        cell_res.and_then(|cell| {
                            let exec = interpreters[i].as_mut().unwrap();
                            let check_res = filter.clone().eval(exec.get_env());
                            match check_res {
                                Ok(sexp) => {
                                    if is_true(sexp) {
                                        Ok(cell)
                                    } else {
                                        Err(ReadError::NotMatch)
                                    }
                                }
                                Err(e) => return Err(ReadError::ExecError(e)),
                            }
                        })
                    })
                    .collect();
            }
            if !proc_empty {
                let proc = proc.to_sexpr();
                let cells = cells
                    .into_iter()
                    .enumerate()
                    .map(|(i, cell_res)| {
                        let exec = interpreters[i].as_mut().unwrap();
                        cell_res.and_then(|cell| {
                            let proc_res = proc.clone().eval(exec.get_env());
                            match proc_res {
                                Ok(sexp) => {
                                    let val = sexp.owned_val().unwrap_or(OwnedValue::NA);
                                    Ok(OwnedCell {
                                        header: cell.header,
                                        data: val,
                                    })
                                }
                                Err(e) => Err(ReadError::ExecError(e)),
                            }
                        })
                    })
                    .collect();
                return future::ready(cells).boxed();
            }
        }
        let res: Vec<_> = cells
            .into_iter()
            .map(|row| row.map(|cell| cell.to_owned()))
            .collect();
        return future::ready(res).boxed();
    }
}

dispatch_rpc_service_functions!(NebRPCService);

impl NebRPCService {
    pub fn new(server: &Arc<NebServer>) -> Arc<NebRPCService> {
        Arc::new(NebRPCService {
            server: server.clone(),
        })
    }
    fn with_indices_ensured<'a, R>(&'a self, res: R) -> BoxFuture<R>
    where
        R: Send + 'a,
    {
        if self.server.indexer.is_some() {
            IndexBuilder::await_indices().map(|_| res).boxed()
        } else {
            future::ready(res).boxed()
        }
    }
}
<|endoftext|>