<commit_before>use cgmath::EuclideanSpace;
use gilrs::{Axis, Button, Gilrs};
use std::{f64::consts::PI, path::PathBuf};
use structopt::StructOpt;
use winit::{
    event,
    event_loop::{ControlFlow, EventLoop},
};

#[derive(Debug, StructOpt)]
struct Opt {
    #[structopt(short, long, default_value = "8FH495PF+29")]
    plus: String,
    #[structopt(short, long, default_value = "0")]
    heading: f64,
    #[structopt(short, long, default_value = "200000")]
    elevation: f64,
    #[structopt(long)]
    generate: Option<PathBuf>,
}

fn compute_projection_matrix(width: f32, height: f32) -> cgmath::Matrix4<f32> {
    let aspect = width / height;
    let f = 1.0 / (45.0f32.to_radians() / aspect).tan();
    let near = 0.1;

    #[cfg_attr(rustfmt, rustfmt_skip)]
    cgmath::Matrix4::new(
        f/aspect,  0.0,  0.0,   0.0,
        0.0,       f,    0.0,   0.0,
        0.0,       0.0,  0.0,  -1.0,
        0.0,       0.0,  near,  0.0)
}

fn make_swapchain(
    device: &wgpu::Device,
    surface: &wgpu::Surface,
    width: u32,
    height: u32,
) -> wgpu::SwapChain {
    device.create_swap_chain(
        &surface,
        &wgpu::SwapChainDescriptor {
            usage: wgpu::TextureUsage::RENDER_ATTACHMENT,
            format: wgpu::TextureFormat::Bgra8UnormSrgb,
            width,
            height,
            present_mode: wgpu::PresentMode::Fifo, // disable vsync by switching to Mailbox,
        },
    )
}
fn make_depth_buffer(device: &wgpu::Device, width: u32, height: u32) -> wgpu::TextureView {
    device
        .create_texture(&wgpu::TextureDescriptor {
            size: wgpu::Extent3d { width, height, depth: 1 },
            mip_level_count: 1,
            sample_count: 1,
            dimension: wgpu::TextureDimension::D2,
            format: wgpu::TextureFormat::Depth32Float,
            usage: wgpu::TextureUsage::RENDER_ATTACHMENT,
            label: None,
        })
        .create_view(&Default::default())
}

fn main() {
    // env_logger::init();

    let runtime = tokio::runtime::Runtime::new().unwrap();

    let trace_path: Option<&std::path::Path> = if cfg!(feature = "trace") {
        std::fs::create_dir_all("trace").unwrap();
        Some(std::path::Path::new("trace"))
    } else {
        None
    };

    let event_loop = EventLoop::new();
    let monitor = event_loop
        .available_monitors()
        .find(|monitor| monitor.video_modes().any(|mode| mode.size().width == 1920));
    let window = winit::window::WindowBuilder::new()
        .with_visible(false)
        .with_fullscreen(Some(winit::window::Fullscreen::Borderless(monitor)))
        .build(&event_loop)
        .unwrap();

    let instance = wgpu::Instance::new(wgpu::BackendBit::PRIMARY);
    let surface = unsafe { instance.create_surface(&window) };
    let adapter = runtime
        .block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
            power_preference: wgpu::PowerPreference::HighPerformance,
            compatible_surface: Some(&surface),
        }))
        .expect("Unable to create compatible wgpu adapter");

    // Terra requires support for BC texture compression.
    assert!(adapter.features().contains(wgpu::Features::TEXTURE_COMPRESSION_BC));

    let features = if !adapter.features().contains(wgpu::Features::SHADER_FLOAT64)
        || cfg!(feature = "soft-float64")
    {
        wgpu::Features::TEXTURE_COMPRESSION_BC
    } else {
        wgpu::Features::TEXTURE_COMPRESSION_BC | wgpu::Features::SHADER_FLOAT64
    };

    let (device, mut queue) = runtime
        .block_on(adapter.request_device(
            &wgpu::DeviceDescriptor { features, limits: wgpu::Limits::default(), label: None },
            trace_path,
        ))
        .expect("Unable to create compatible wgpu device");

    let mut size = window.inner_size();
    let mut swap_chain = make_swapchain(&device, &surface, size.width, size.height);
    let mut depth_buffer = make_depth_buffer(&device, size.width, size.height);
    let mut proj = compute_projection_matrix(size.width as f32, size.height as f32);

    let mut gilrs = Gilrs::new().unwrap();
    let mut current_gamepad = None;
    for (_id, gamepad) in gilrs.gamepads() {
        current_gamepad = Some(gamepad.id());
    }

    let opt = Opt::from_args();
    let plus_center =
        open_location_code::decode(&opt.plus).expect("Failed to parse plus code").center;

    let planet_radius = 6371000.0;
    let mut angle = opt.heading.to_radians();
    let mut lat = plus_center.y().to_radians();
    let mut long = plus_center.x().to_radians();
    let mut altitude = opt.elevation;

    let mut terrain = terra::Terrain::new(&device, &mut queue).unwrap();

    if let Some(dataset_directory) = opt.generate {
        let pb = indicatif::ProgressBar::new(100);
        pb.set_style(
            indicatif::ProgressStyle::default_bar()
                .template("{msg} {pos}/{len} [{wide_bar}] {percent}% {per_sec} {eta}")
                .progress_chars("=> "),
        );
        let mut last_message = None;
        let mut progress_callback = |l: &str, i: usize, total: usize| {
            if last_message.is_none() || l != last_message.as_ref().unwrap() {
                pb.set_message(l);
                pb.reset_eta();
                last_message = Some(l.to_string());
            }
            pb.set_length(total as u64);
            pb.set_position(i as u64);
        };

        runtime
            .block_on(terrain.generate_heightmaps(
                dataset_directory.join("ETOPO1_Ice_c_geotiff.zip"),
                dataset_directory.join("strm3"),
                &mut progress_callback,
            ))
            .unwrap();
        runtime
            .block_on(
                terrain
                    .generate_albedos(dataset_directory.join("bluemarble"), &mut progress_callback),
            )
            .unwrap();
        runtime.block_on(terrain.generate_roughness(&mut progress_callback)).unwrap();
    }

    {
        let r = altitude + planet_radius;
        let eye = cgmath::Point3::new(
            r * lat.cos() * long.cos(),
            r * lat.cos() * long.sin(),
            r * lat.sin(),
        );
        while terrain.poll_loading_status(&device, &mut queue, eye.into()) {
            std::thread::sleep(std::time::Duration::from_millis(10));
        }
    }

    let mut set_visible = false;
    event_loop.run(move |event, _, control_flow| {
        *control_flow = if cfg!(feature = "metal-auto-capture") {
            ControlFlow::Exit
        } else {
            ControlFlow::Poll
        };
        match event {
            event::Event::WindowEvent { event, .. } => match event {
                event::WindowEvent::CloseRequested => {
                    *control_flow = ControlFlow::Exit;
                }
                event::WindowEvent::KeyboardInput {
                    input:
                        event::KeyboardInput {
                            virtual_keycode: Some(keycode),
                            state: event::ElementState::Pressed,
                            ..
                        },
                    ..
                } => match keycode {
                    event::VirtualKeyCode::Escape => *control_flow = ControlFlow::Exit,
                    event::VirtualKeyCode::Space => altitude += 0.1 * altitude,
                    event::VirtualKeyCode::Semicolon => altitude -= 0.1 * altitude,
                    event::VirtualKeyCode::Left => {
                        lat += -angle.sin() * -(0.0000001 * altitude).min(0.01);
                        long += angle.cos() * -(0.0000001 * altitude).min(0.01);
                    }
                    event::VirtualKeyCode::Right => {
                        lat += -angle.sin() * (0.0000001 * altitude).min(0.01);
                        long += angle.cos() * (0.0000001 * altitude).min(0.01);
                    }
                    event::VirtualKeyCode::Up => {
                        lat += angle.cos() * (0.0000001 * altitude).min(0.01);
                        long += -angle.sin() * (0.0000001 * altitude).min(0.01);
                    }
                    event::VirtualKeyCode::Down => {
                        lat += angle.cos() * -(0.0000001 * altitude).min(0.01);
                        long += -angle.sin() * -(0.0000001 * altitude).min(0.01);
                    }
                    _ => {}
                },
                event::WindowEvent::Resized(new_size) => {
                    size = new_size;
                    swap_chain = make_swapchain(&device, &surface, size.width, size.height);
                    depth_buffer = make_depth_buffer(&device, size.width, size.height);
                    proj = compute_projection_matrix(size.width as f32, size.height as f32);
                }
                _ => {}
            },
            event::Event::MainEventsCleared => {
                let frame = swap_chain.get_current_frame();
                let frame = match frame {
                    Ok(ref f) => &f.output.view,
                    Err(_) => return,
                };

                while let Some(gilrs::Event { id, event: _event, time: _ }) = gilrs.next_event() {
                    current_gamepad = Some(id);
                }
                if let Some(gamepad) = current_gamepad.map(|id| gilrs.gamepad(id)) {
                    lat += angle.cos() * gamepad.value(Axis::LeftStickY) as f64 * 0.01
                        - angle.sin() * gamepad.value(Axis::LeftStickX) as f64 * 0.01;

                    long += -angle.sin() * gamepad.value(Axis::LeftStickY) as f64 * 0.01
                        + angle.cos() * gamepad.value(Axis::LeftStickX) as f64 * 0.01;

                    angle -= gamepad.value(Axis::RightZ) as f64 * 0.01;

                    if gamepad.is_pressed(Button::DPadUp) {
                        altitude += 0.01 * altitude;
                    }
                    if gamepad.is_pressed(Button::DPadDown) {
                        altitude -= 0.01 * altitude;
                    }
                }

                lat = lat.max(-PI).min(PI);
                if long < -PI {
                    long += PI * 2.0;
                }
                if long > PI {
                    long -= PI * 2.0;
                }

                let surface_height = terrain.get_height(lat, long) as f64;
                let r = altitude + planet_radius + surface_height + 2.0;
                let eye = cgmath::Point3::new(
                    r * lat.cos() * long.cos(),
                    r * lat.cos() * long.sin(),
                    r * lat.sin(),
                );

                let dt = (planet_radius / (planet_radius + altitude)).acos() * 0.3;
                let latc = lat + angle.cos() * dt;
                let longc = long - angle.sin() * dt;

                let center = cgmath::Point3::new(
                    planet_radius * latc.cos() * longc.cos() - eye.x,
                    planet_radius * latc.cos() * longc.sin() - eye.y,
                    planet_radius * latc.sin() - eye.z,
                );
                let up = cgmath::Vector3::new(eye.x as f32, eye.y as f32, eye.z as f32);

                let view = cgmath::Matrix4::look_at_rh(
                    cgmath::Point3::origin(),
                    cgmath::Point3::new(center.x as f32, center.y as f32, center.z as f32),
                    up,
                );

                let view_proj = proj * view;
                let view_proj = mint::ColumnMatrix4 {
                    x: view_proj.x.into(),
                    y: view_proj.y.into(),
                    z: view_proj.z.into(),
                    w: view_proj.w.into(),
                };

                terrain.render(
                    &device,
                    &mut queue,
                    &frame,
                    &depth_buffer,
                    (size.width, size.height),
                    view_proj,
                    eye.into(),
                );

                if !set_visible {
                    window.set_visible(true);
                    set_visible = true;
                }
            }
            _ => (),
        }
    });
}
<commit_msg>Delay swapchain and depth buffer creation until first frame<commit_after>use cgmath::EuclideanSpace;
use gilrs::{Axis, Button, Gilrs};
use std::{f64::consts::PI, path::PathBuf};
use structopt::StructOpt;
use winit::{
    event,
    event_loop::{ControlFlow, EventLoop},
};

#[derive(Debug, StructOpt)]
struct Opt {
    #[structopt(short, long, default_value = "8FH495PF+29")]
    plus: String,
    #[structopt(short, long, default_value = "0")]
    heading: f64,
    #[structopt(short, long, default_value = "200000")]
    elevation: f64,
    #[structopt(long)]
    generate: Option<PathBuf>,
}

fn compute_projection_matrix(width: f32, height: f32) -> cgmath::Matrix4<f32> {
    let aspect = width / height;
    let f = 1.0 / (45.0f32.to_radians() / aspect).tan();
    let near = 0.1;

    #[cfg_attr(rustfmt, rustfmt_skip)]
    cgmath::Matrix4::new(
        f/aspect,  0.0,  0.0,   0.0,
        0.0,       f,    0.0,   0.0,
        0.0,       0.0,  0.0,  -1.0,
        0.0,       0.0,  near,  0.0)
}

fn make_swapchain(
    device: &wgpu::Device,
    surface: &wgpu::Surface,
    width: u32,
    height: u32,
) -> wgpu::SwapChain {
    device.create_swap_chain(
        &surface,
        &wgpu::SwapChainDescriptor {
            usage: wgpu::TextureUsage::RENDER_ATTACHMENT,
            format: wgpu::TextureFormat::Bgra8UnormSrgb,
            width,
            height,
            present_mode: wgpu::PresentMode::Fifo, // disable vsync by switching to Mailbox,
        },
    )
}
fn make_depth_buffer(device: &wgpu::Device, width: u32, height: u32) -> wgpu::TextureView {
    device
        .create_texture(&wgpu::TextureDescriptor {
            size: wgpu::Extent3d { width, height, depth: 1 },
            mip_level_count: 1,
            sample_count: 1,
            dimension: wgpu::TextureDimension::D2,
            format: wgpu::TextureFormat::Depth32Float,
            usage: wgpu::TextureUsage::RENDER_ATTACHMENT,
            label: None,
        })
        .create_view(&Default::default())
}

fn main() {
    // env_logger::init();

    let runtime = tokio::runtime::Runtime::new().unwrap();

    let trace_path: Option<&std::path::Path> = if cfg!(feature = "trace") {
        std::fs::create_dir_all("trace").unwrap();
        Some(std::path::Path::new("trace"))
    } else {
        None
    };

    let event_loop = EventLoop::new();
    let monitor = event_loop
        .available_monitors()
        .find(|monitor| monitor.video_modes().any(|mode| mode.size().width == 1920));
    let window = winit::window::WindowBuilder::new()
        .with_visible(false)
        .with_fullscreen(Some(winit::window::Fullscreen::Borderless(monitor)))
        .build(&event_loop)
        .unwrap();

    let instance = wgpu::Instance::new(wgpu::BackendBit::PRIMARY);
    let surface = unsafe { instance.create_surface(&window) };
    let adapter = runtime
        .block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {
            power_preference: wgpu::PowerPreference::HighPerformance,
            compatible_surface: Some(&surface),
        }))
        .expect("Unable to create compatible wgpu adapter");

    // Terra requires support for BC texture compression.
    assert!(adapter.features().contains(wgpu::Features::TEXTURE_COMPRESSION_BC));

    let features = if !adapter.features().contains(wgpu::Features::SHADER_FLOAT64)
        || cfg!(feature = "soft-float64")
    {
        wgpu::Features::TEXTURE_COMPRESSION_BC
    } else {
        wgpu::Features::TEXTURE_COMPRESSION_BC | wgpu::Features::SHADER_FLOAT64
    };

    let (device, mut queue) = runtime
        .block_on(adapter.request_device(
            &wgpu::DeviceDescriptor { features, limits: wgpu::Limits::default(), label: None },
            trace_path,
        ))
        .expect("Unable to create compatible wgpu device");

    let mut size = window.inner_size();
    let mut swap_chain = None;
    let mut depth_buffer = None;

    let mut gilrs = Gilrs::new().unwrap();
    let mut current_gamepad = None;
    for (_id, gamepad) in gilrs.gamepads() {
        current_gamepad = Some(gamepad.id());
    }

    let opt = Opt::from_args();
    let plus_center =
        open_location_code::decode(&opt.plus).expect("Failed to parse plus code").center;

    let planet_radius = 6371000.0;
    let mut angle = opt.heading.to_radians();
    let mut lat = plus_center.y().to_radians();
    let mut long = plus_center.x().to_radians();
    let mut altitude = opt.elevation;

    let mut terrain = terra::Terrain::new(&device, &mut queue).unwrap();

    if let Some(dataset_directory) = opt.generate {
        let pb = indicatif::ProgressBar::new(100);
        pb.set_style(
            indicatif::ProgressStyle::default_bar()
                .template("{msg} {pos}/{len} [{wide_bar}] {percent}% {per_sec} {eta}")
                .progress_chars("=> "),
        );
        let mut last_message = None;
        let mut progress_callback = |l: &str, i: usize, total: usize| {
            if last_message.is_none() || l != last_message.as_ref().unwrap() {
                pb.set_message(l);
                pb.reset_eta();
                last_message = Some(l.to_string());
            }
            pb.set_length(total as u64);
            pb.set_position(i as u64);
        };

        runtime
            .block_on(terrain.generate_heightmaps(
                dataset_directory.join("ETOPO1_Ice_c_geotiff.zip"),
                dataset_directory.join("strm3"),
                &mut progress_callback,
            ))
            .unwrap();
        runtime
            .block_on(
                terrain
                    .generate_albedos(dataset_directory.join("bluemarble"), &mut progress_callback),
            )
            .unwrap();
        runtime.block_on(terrain.generate_roughness(&mut progress_callback)).unwrap();
    }

    {
        let r = altitude + planet_radius;
        let eye = cgmath::Point3::new(
            r * lat.cos() * long.cos(),
            r * lat.cos() * long.sin(),
            r * lat.sin(),
        );
        while terrain.poll_loading_status(&device, &mut queue, eye.into()) {
            std::thread::sleep(std::time::Duration::from_millis(10));
        }
    }

    let mut set_visible = false;
    event_loop.run(move |event, _, control_flow| {
        *control_flow = if cfg!(feature = "metal-auto-capture") {
            ControlFlow::Exit
        } else {
            ControlFlow::Poll
        };
        match event {
            event::Event::WindowEvent { event, .. } => match event {
                event::WindowEvent::CloseRequested => {
                    *control_flow = ControlFlow::Exit;
                }
                event::WindowEvent::KeyboardInput {
                    input:
                        event::KeyboardInput {
                            virtual_keycode: Some(keycode),
                            state: event::ElementState::Pressed,
                            ..
                        },
                    ..
                } => match keycode {
                    event::VirtualKeyCode::Escape => *control_flow = ControlFlow::Exit,
                    event::VirtualKeyCode::Space => altitude += 0.1 * altitude,
                    event::VirtualKeyCode::Semicolon => altitude -= 0.1 * altitude,
                    event::VirtualKeyCode::Left => {
                        lat += -angle.sin() * -(0.0000001 * altitude).min(0.01);
                        long += angle.cos() * -(0.0000001 * altitude).min(0.01);
                    }
                    event::VirtualKeyCode::Right => {
                        lat += -angle.sin() * (0.0000001 * altitude).min(0.01);
                        long += angle.cos() * (0.0000001 * altitude).min(0.01);
                    }
                    event::VirtualKeyCode::Up => {
                        lat += angle.cos() * (0.0000001 * altitude).min(0.01);
                        long += -angle.sin() * (0.0000001 * altitude).min(0.01);
                    }
                    event::VirtualKeyCode::Down => {
                        lat += angle.cos() * -(0.0000001 * altitude).min(0.01);
                        long += -angle.sin() * -(0.0000001 * altitude).min(0.01);
                    }
                    _ => {}
                },
                event::WindowEvent::Resized(new_size) => {
                    size = new_size;
                    swap_chain = None;
                    depth_buffer = None;
                }
                _ => {}
            },
            event::Event::MainEventsCleared => {
                if swap_chain.is_none() {
                    swap_chain = Some(make_swapchain(&device, &surface, size.width, size.height));
                }
                if depth_buffer.is_none() {
                    depth_buffer = Some(make_depth_buffer(&device, size.width, size.height));
                }

                let frame = swap_chain.as_ref().unwrap().get_current_frame();
                let frame = match frame {
                    Ok(ref f) => &f.output.view,
                    Err(_) => return,
                };

                while let Some(gilrs::Event { id, event: _event, time: _ }) = gilrs.next_event() {
                    current_gamepad = Some(id);
                }
                if let Some(gamepad) = current_gamepad.map(|id| gilrs.gamepad(id)) {
                    lat += angle.cos() * gamepad.value(Axis::LeftStickY) as f64 * 0.01
                        - angle.sin() * gamepad.value(Axis::LeftStickX) as f64 * 0.01;

                    long += -angle.sin() * gamepad.value(Axis::LeftStickY) as f64 * 0.01
                        + angle.cos() * gamepad.value(Axis::LeftStickX) as f64 * 0.01;

                    angle -= gamepad.value(Axis::RightZ) as f64 * 0.01;

                    if gamepad.is_pressed(Button::DPadUp) {
                        altitude += 0.01 * altitude;
                    }
                    if gamepad.is_pressed(Button::DPadDown) {
                        altitude -= 0.01 * altitude;
                    }
                }

                lat = lat.max(-PI).min(PI);
                if long < -PI {
                    long += PI * 2.0;
                }
                if long > PI {
                    long -= PI * 2.0;
                }

                let surface_height = terrain.get_height(lat, long) as f64;
                let r = altitude + planet_radius + surface_height + 2.0;
                let eye = cgmath::Point3::new(
                    r * lat.cos() * long.cos(),
                    r * lat.cos() * long.sin(),
                    r * lat.sin(),
                );

                let dt = (planet_radius / (planet_radius + altitude)).acos() * 0.3;
                let latc = lat + angle.cos() * dt;
                let longc = long - angle.sin() * dt;

                let center = cgmath::Point3::new(
                    planet_radius * latc.cos() * longc.cos() - eye.x,
                    planet_radius * latc.cos() * longc.sin() - eye.y,
                    planet_radius * latc.sin() - eye.z,
                );
                let up = cgmath::Vector3::new(eye.x as f32, eye.y as f32, eye.z as f32);

                let view = cgmath::Matrix4::look_at_rh(
                    cgmath::Point3::origin(),
                    cgmath::Point3::new(center.x as f32, center.y as f32, center.z as f32),
                    up,
                );

                let proj = compute_projection_matrix(size.width as f32, size.height as f32);
                let view_proj = proj * view;
                let view_proj = mint::ColumnMatrix4 {
                    x: view_proj.x.into(),
                    y: view_proj.y.into(),
                    z: view_proj.z.into(),
                    w: view_proj.w.into(),
                };

                terrain.render(
                    &device,
                    &mut queue,
                    &frame,
                    depth_buffer.as_ref().unwrap(),
                    (size.width, size.height),
                    view_proj,
                    eye.into(),
                );

                if !set_visible {
                    window.set_visible(true);
                    set_visible = true;
                }
            }
            _ => (),
        }
    });
}
<|endoftext|>


<commit_before>use itertools::Itertools;
use lightning::map::{Map, ObjectMap};
use rayon::prelude::*;
use std::{
    collections::{BTreeSet, HashMap, HashSet},
    sync::Arc,
};
use tokio_stream::StreamExt;

use dovahkiin::types::{OwnedValue, SharedValue};

use crate::ram::{
    cell::{header_from_chunk_raw, select_from_chunk_raw},
    chunk::Chunk,
};

mod histogram;
pub mod sm;

pub struct SchemaStatistics {
    histogram: HashMap<u64, [OwnedValue; 10]>,
    count: usize,
    segs: usize,
    bytes: usize,
    timestamp: u64,
}

pub struct ChunkStatistics {
    schemas: ObjectMap<Arc<SchemaStatistics>>,
}

const HISTOGRAM_PARTITATION_SIZE: usize = 1024;
const HISTOGRAM_PARTITATION_BUCKETS: usize = 256;
const HISTOGRAM_TARGET_BUCKETS: usize = 100;

type HistogramKey = [u8; 8];

impl ChunkStatistics {
    pub fn from_chunk(chunk: &Chunk) -> Self {
        let histogram_partitations = chunk
            .cell_index
            .entries()
            .chunks(HISTOGRAM_PARTITATION_SIZE)
            .map(|s| s.to_vec())
            .collect_vec();
        let partitations: Vec<_> = histogram_partitations
            .into_par_iter()
            .map(|partitation| {
                // Build exact histogram for each of the partitation and then approximate overall histogram
                let mut sizes = HashMap::new();
                let mut segs = HashMap::new();
                let mut exact_accumlators = HashMap::new();
                let partitation_size = partitation.len();
                for (hash, _) in partitation {
                    let loc = if let Ok(ptr) = chunk.location_for_read(hash as u64) {
                        ptr
                    } else {
                        trace!("Cannot obtain cell lock {} for statistics", hash);
                        continue;
                    };
                    match header_from_chunk_raw(*loc) {
                        Ok((header, _, entry_header)) => {
                            let cell_size = entry_header.content_length as usize;
                            let cell_seg = chunk.allocator.id_by_addr(*loc);
                            let schema_id = header.schema;
                            if let Some(schema) = chunk.meta.schemas.get(&schema_id) {
                                let fields = schema.index_fields.keys().cloned().collect_vec();
                                if let Ok(partial_cell) =
                                    select_from_chunk_raw(*loc, chunk, fields.as_slice())
                                {
                                    let field_array = if fields.len() == 1 {
                                        vec![partial_cell]
                                    } else if let SharedValue::Array(arr) = partial_cell {
                                        arr
                                    } else {
                                        error!(
                                            "Cannot decode partial cell for statistics {:?}",
                                            partial_cell
                                        );
                                        continue;
                                    };
                                    for (i, val) in field_array.into_iter().enumerate() {
                                        if val == SharedValue::Null || val == SharedValue::NA {
                                            continue;
                                        }
                                        let field_id = fields[i];
                                        exact_accumlators
                                            .entry(schema_id)
                                            .or_insert_with(|| HashMap::new())
                                            .entry(field_id)
                                            .or_insert_with(|| Vec::with_capacity(partitation_size))
                                            .push(val.feature());
                                    }
                                    *sizes.entry(schema_id).or_insert(0) += cell_size;
                                    segs.entry(schema_id)
                                        .or_insert_with(|| HashSet::new())
                                        .insert(cell_seg);
                                }
                            } else {
                                warn!("Cannot get schema {} for statistics", schema_id);
                            }
                        }
                        Err(e) => {
                            warn!("Failed to read {} for statistics, error: {:?}", hash, e);
                        }
                    }
                }
                let histograms: HashMap<_, _> = exact_accumlators
                    .into_iter()
                    .map(|(schema_id, schema_histograms)| {
                        let compiled_histograms = schema_histograms
                            .into_iter()
                            .map(|(field, mut items)| {
                                items.sort();
                                let depth = items.len() / HISTOGRAM_PARTITATION_BUCKETS;
                                let mut histogram = (0..HISTOGRAM_PARTITATION_BUCKETS)
                                    .map(|tile| items[tile * depth])
                                    .collect_vec();
                                let last_item = &items[items.len() - 1];
                                if histogram.last().unwrap() != last_item {
                                    histogram.push(*last_item);
                                }
                                (field, (histogram, items.len(), depth))
                            })
                            .collect::<HashMap<_, _>>();
                        (schema_id, compiled_histograms)
                    })
                    .collect::<HashMap<_, _>>();
                (sizes, segs, histograms)
            })
            .collect();
        let schema_ids: Vec<_> = partitations
            .iter()
            .map(|(sizes, _, _)| sizes.keys())
            .flatten()
            .dedup()
            .collect();
        let total_size = schema_ids
            .iter()
            .map(|sid| {
                (
                    *sid,
                    partitations
                        .iter()
                        .map(|(sizes, _, _)| sizes.get(sid).unwrap_or(&0))
                        .sum::<usize>(),
                )
            })
            .collect::<HashMap<_, _>>();
        let total_segs = schema_ids
            .iter()
            .map(|sid| {
                (
                    *sid,
                    partitations
                        .iter()
                        .map(|(_, segs, _)| segs.get(sid).map(|set| set.len()).unwrap_or(0))
                        .sum::<usize>(),
                )
            })
            .collect::<HashMap<_, _>>();
        let empty_histo = Default::default();
        let schema_histograms = schema_ids
            .iter()
            .map(|sid| {
                (*sid, {
                    let parted_histos = partitations
                        .iter()
                        .map(|(_, _, histo)| histo.get(sid).unwrap_or(&empty_histo))
                        .collect_vec();
                    let field_ids = parted_histos
                        .iter()
                        .map(|histo_map| histo_map.keys())
                        .flatten()
                        .dedup()
                        .collect::<Vec<_>>();
                    field_ids
                        .par_iter()
                        .map(|field_id| {
                            let schema_field_histograms = parted_histos
                                .iter()
                                .map(|histo_map| &histo_map[field_id])
                                .collect_vec();
                            (**field_id, build_histogram(schema_field_histograms))
                        })
                        .collect::<HashMap<u64, _>>()
                })
            })
            .collect::<HashMap<_, _>>();
        unimplemented!()
    }
}

fn build_histogram(
    partitations: Vec<&(Vec<HistogramKey>, usize, usize)>,
) -> [HistogramKey; HISTOGRAM_TARGET_BUCKETS + 1] {
    // Build the approximated histogram from partitation histograms
    // https://arxiv.org/abs/1606.05633
    let mut part_idxs = vec![0; partitations.len()];
    let part_histos = partitations
        .iter()
        .map(|(histo, _, _)| histo)
        .filter(|histo| !histo.is_empty())
        .collect_vec();
    let num_total = partitations.iter().map(|(_, num, _)| num).sum::<usize>();
    let part_depths = partitations
        .iter()
        .map(|(_, _, depth)| *depth)
        .collect_vec();
    let target_width = num_total / HISTOGRAM_TARGET_BUCKETS;
    let mut target_histogram = [[0u8; 8]; HISTOGRAM_TARGET_BUCKETS + 1];
    // Perform a merge sort for sorted pre-histogram
    let mut filled = target_width;
    let mut last_key = Default::default();
    'HISTO_CONST: for i in 0..HISTOGRAM_PARTITATION_BUCKETS {
        loop {
            let (key, ended) = if let Some((part_idx, histo)) = part_histos
                .iter()
                .enumerate()
                .filter(|(i, h)| {
                    let idx = part_idxs[*i];
                    idx < h.len()
                })
                .min_by(|(i1, h1), (i2, h2)| {
                    let h1_idx = part_idxs[*i1];
                    let h2_idx = part_idxs[*i2];
                    h1[h1_idx].cmp(&h2[h2_idx])
                })
            {
                let histo_idx = part_idxs[part_idx];
                part_idxs[part_idx] += 1;
                ((histo[histo_idx], part_idx), false)
            } else {
                (last_key, true)
            };
            last_key = key;
            let idx = last_key.1;
            if filled >= target_width || ended {
                target_histogram[i] = last_key.0;
                filled = 0;
                continue 'HISTO_CONST;
            }
            filled += part_depths[idx];
        }
    }
    target_histogram
}
<commit_msg>complete histogram from chunk<commit_after>use itertools::Itertools;
use lightning::map::{Map, ObjectMap};
use rayon::prelude::*;
use std::{
    collections::{BTreeSet, HashMap, HashSet},
    sync::Arc,
};
use tokio_stream::StreamExt;

use dovahkiin::types::{OwnedValue, SharedValue};

use crate::ram::{
    cell::{header_from_chunk_raw, select_from_chunk_raw},
    chunk::Chunk,
    clock::now,
};

mod histogram;
pub mod sm;

pub struct SchemaStatistics {
    pub histogram: HashMap<u64, [HistogramKey; HISTOGRAM_TARGET_BUCKETS + 1]>,
    pub count: usize,
    pub segs: usize,
    pub bytes: usize,
    pub timestamp: u32,
}

pub struct ChunkStatistics {
    pub schemas: ObjectMap<Arc<SchemaStatistics>>,
}

const HISTOGRAM_PARTITATION_SIZE: usize = 1024;
const HISTOGRAM_PARTITATION_BUCKETS: usize = 256;
const HISTOGRAM_TARGET_BUCKETS: usize = 100;

type HistogramKey = [u8; 8];

impl ChunkStatistics {
    pub fn from_chunk(chunk: &Chunk) -> Self {
        let histogram_partitations = chunk
            .cell_index
            .entries()
            .chunks(HISTOGRAM_PARTITATION_SIZE)
            .map(|s| s.to_vec())
            .collect_vec();
        let partitations: Vec<_> = histogram_partitations
            .into_par_iter()
            .map(|partitation| {
                // Build exact histogram for each of the partitation and then approximate overall histogram
                let mut sizes = HashMap::new();
                let mut segs = HashMap::new();
                let mut counts = HashMap::new();
                let mut exact_accumlators = HashMap::new();
                let partitation_size = partitation.len();
                for (hash, _) in partitation {
                    let loc = if let Ok(ptr) = chunk.location_for_read(hash as u64) {
                        ptr
                    } else {
                        trace!("Cannot obtain cell lock {} for statistics", hash);
                        continue;
                    };
                    match header_from_chunk_raw(*loc) {
                        Ok((header, _, entry_header)) => {
                            let cell_size = entry_header.content_length as usize;
                            let cell_seg = chunk.allocator.id_by_addr(*loc);
                            let schema_id = header.schema;
                            if let Some(schema) = chunk.meta.schemas.get(&schema_id) {
                                let fields = schema.index_fields.keys().cloned().collect_vec();
                                if let Ok(partial_cell) =
                                    select_from_chunk_raw(*loc, chunk, fields.as_slice())
                                {
                                    let field_array = if fields.len() == 1 {
                                        vec![partial_cell]
                                    } else if let SharedValue::Array(arr) = partial_cell {
                                        arr
                                    } else {
                                        error!(
                                            "Cannot decode partial cell for statistics {:?}",
                                            partial_cell
                                        );
                                        continue;
                                    };
                                    for (i, val) in field_array.into_iter().enumerate() {
                                        if val == SharedValue::Null || val == SharedValue::NA {
                                            continue;
                                        }
                                        let field_id = fields[i];
                                        exact_accumlators
                                            .entry(schema_id)
                                            .or_insert_with(|| HashMap::new())
                                            .entry(field_id)
                                            .or_insert_with(|| Vec::with_capacity(partitation_size))
                                            .push(val.feature());
                                    }
                                    *counts.entry(schema_id).or_insert(0) += 1;
                                    *sizes.entry(schema_id).or_insert(0) += cell_size;
                                    segs.entry(schema_id)
                                        .or_insert_with(|| HashSet::new())
                                        .insert(cell_seg);
                                }
                            } else {
                                warn!("Cannot get schema {} for statistics", schema_id);
                            }
                        }
                        Err(e) => {
                            warn!("Failed to read {} for statistics, error: {:?}", hash, e);
                        }
                    }
                }
                let histograms: HashMap<_, _> = exact_accumlators
                    .into_iter()
                    .map(|(schema_id, schema_histograms)| {
                        let compiled_histograms = schema_histograms
                            .into_iter()
                            .map(|(field, mut items)| {
                                items.sort();
                                let depth = items.len() / HISTOGRAM_PARTITATION_BUCKETS;
                                let mut histogram = (0..HISTOGRAM_PARTITATION_BUCKETS)
                                    .map(|tile| items[tile * depth])
                                    .collect_vec();
                                let last_item = &items[items.len() - 1];
                                if histogram.last().unwrap() != last_item {
                                    histogram.push(*last_item);
                                }
                                (field, (histogram, items.len(), depth))
                            })
                            .collect::<HashMap<_, _>>();
                        (schema_id, compiled_histograms)
                    })
                    .collect::<HashMap<_, _>>();
                (sizes, segs, counts, histograms)
            })
            .collect();
        let schema_ids: Vec<_> = partitations
            .iter()
            .map(|(sizes, _, _, _)| sizes.keys())
            .flatten()
            .dedup()
            .collect();
        let total_size = schema_ids
            .iter()
            .map(|sid| {
                (
                    *sid,
                    partitations
                        .iter()
                        .map(|(sizes, _, _, _)| sizes.get(sid).unwrap_or(&0))
                        .sum::<usize>(),
                )
            })
            .collect::<HashMap<_, _>>();
        let total_counts = schema_ids
            .iter()
            .map(|sid| {
                (
                    *sid,
                    partitations
                        .iter()
                        .map(|(_, _, counts, _)| counts.get(sid).unwrap_or(&0))
                        .sum::<usize>(),
                )
            })
            .collect::<HashMap<_, _>>();
        let total_segs = schema_ids
            .iter()
            .map(|sid| {
                (
                    *sid,
                    partitations
                        .iter()
                        .map(|(_, segs, _, _)| segs.get(sid).map(|set| set.len()).unwrap_or(0))
                        .sum::<usize>(),
                )
            })
            .collect::<HashMap<_, _>>();
        let empty_histo = Default::default();
        let mut schema_histograms = schema_ids
            .iter()
            .map(|sid| {
                (*sid, {
                    let parted_histos = partitations
                        .iter()
                        .map(|(_, _, _, histo)| histo.get(sid).unwrap_or(&empty_histo))
                        .collect_vec();
                    let field_ids = parted_histos
                        .iter()
                        .map(|histo_map| histo_map.keys())
                        .flatten()
                        .dedup()
                        .collect::<Vec<_>>();
                    field_ids
                        .par_iter()
                        .map(|field_id| {
                            let schema_field_histograms = parted_histos
                                .iter()
                                .map(|histo_map| &histo_map[field_id])
                                .collect_vec();
                            (**field_id, build_histogram(schema_field_histograms))
                        })
                        .collect::<HashMap<u64, _>>()
                })
            })
            .collect::<HashMap<_, _>>();
        let schema_statistics = ObjectMap::<Arc<SchemaStatistics>>::with_capacity(
            schema_ids.capacity().next_power_of_two(),
        );
        for schema_id in schema_ids {
            let statistics = SchemaStatistics {
                histogram: schema_histograms.remove(&schema_id).unwrap(),
                count: *total_counts.get(&schema_id).unwrap(),
                segs: *total_segs.get(&schema_id).unwrap(),
                bytes: *total_size.get(&schema_id).unwrap(),
                timestamp: now(),
            };
            schema_statistics.insert(&(*schema_id as usize), Arc::new(statistics));
        }
        Self {
            schemas: schema_statistics
        }
    }
}

fn build_histogram(
    partitations: Vec<&(Vec<HistogramKey>, usize, usize)>,
) -> [HistogramKey; HISTOGRAM_TARGET_BUCKETS + 1] {
    // Build the approximated histogram from partitation histograms
    // https://arxiv.org/abs/1606.05633
    let mut part_idxs = vec![0; partitations.len()];
    let part_histos = partitations
        .iter()
        .map(|(histo, _, _)| histo)
        .filter(|histo| !histo.is_empty())
        .collect_vec();
    let num_total = partitations.iter().map(|(_, num, _)| num).sum::<usize>();
    let part_depths = partitations
        .iter()
        .map(|(_, _, depth)| *depth)
        .collect_vec();
    let target_width = num_total / HISTOGRAM_TARGET_BUCKETS;
    let mut target_histogram = [[0u8; 8]; HISTOGRAM_TARGET_BUCKETS + 1];
    // Perform a merge sort for sorted pre-histogram
    let mut filled = target_width;
    let mut last_key = Default::default();
    'HISTO_CONST: for i in 0..HISTOGRAM_PARTITATION_BUCKETS {
        loop {
            let (key, ended) = if let Some((part_idx, histo)) = part_histos
                .iter()
                .enumerate()
                .filter(|(i, h)| {
                    let idx = part_idxs[*i];
                    idx < h.len()
                })
                .min_by(|(i1, h1), (i2, h2)| {
                    let h1_idx = part_idxs[*i1];
                    let h2_idx = part_idxs[*i2];
                    h1[h1_idx].cmp(&h2[h2_idx])
                }) {
                let histo_idx = part_idxs[part_idx];
                part_idxs[part_idx] += 1;
                ((histo[histo_idx], part_idx), false)
            } else {
                (last_key, true)
            };
            last_key = key;
            let idx = last_key.1;
            if filled >= target_width || ended {
                target_histogram[i] = last_key.0;
                filled = 0;
                continue 'HISTO_CONST;
            }
            filled += part_depths[idx];
        }
    }
    target_histogram
}
<|endoftext|>


<commit_before>use std::io::Read;
use error::{Result, Error, ErrorKind};
use reader::{FbxEvent, PropertyValue};
use super::CommonState;

#[derive(Debug, Clone)]
pub struct BinaryParser {
    version: u32,
    end_offset_stack: Vec<u32>,
}

impl BinaryParser {
    pub fn new(version: u32) -> Self {
        BinaryParser {
            version: version,
            end_offset_stack: vec![],
        }
    }

    pub fn next<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<FbxEvent> {
        // Read a node record header.
        let node_record_header = try!(NodeRecordHeader::load(reader, &mut common.pos));
        if node_record_header.is_null_record() {
            // End of a node.
            return if let Some(expected_pos) = self.end_offset_stack.pop() {
                if common.pos == expected_pos as u64 {
                    Ok(FbxEvent::EndNode)
                } else {
                    // Data is collapsed (the node doesn't end at expected position).
                    Err(Error::new(common.pos, ErrorKind::DataError("Node does not end at expected position".to_string())))
                }
            } else {
                // Data is collapsed (an extra node end marker found).
                Err(Error::new(common.pos, ErrorKind::DataError("An extra node end marker found".to_string())))
            };
        } else {
            // Start of a node.
            self.end_offset_stack.push(node_record_header.end_offset);
        }

        // Read a node name.
        let name = try_read_fixstr!(common.pos, reader, node_record_header.name_len);

        // Read properties.
        let mut properties = Vec::<PropertyValue>::with_capacity(node_record_header.num_properties as usize);
        for _ in 0..node_record_header.num_properties {
            let prop = try!(self.read_property(reader, common));
            properties.push(prop);
        }

        Ok(FbxEvent::StartNode {
            name: name,
            properties: properties,
        })
    }

    fn read_property<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<PropertyValue> {
        Err(Error::new(common.pos, ErrorKind::Unimplemented("Parser for FBX node property is not implemented yet".to_string())))
    }
}

#[derive(Debug, Copy, Clone)]
struct NodeRecordHeader {
    end_offset: u32,
    num_properties: u32,
    property_list_len: u32,
    name_len: u8,
}

impl NodeRecordHeader {
    pub fn load<R: Read>(reader: &mut R, pos: &mut u64) -> Result<Self> {
        let end_offset = try_read_le_u32!(*pos, reader);
        let num_properties = try_read_le_u32!(*pos, reader);
        let property_list_len = try_read_le_u32!(*pos, reader);
        let name_len = try_read_le_u8!(*pos, reader);
        Ok(NodeRecordHeader {
            end_offset: end_offset,
            num_properties: num_properties,
            property_list_len: property_list_len,
            name_len: name_len,
        })
    }

    /// Check whether the header indicates there are no more children.
    pub fn is_null_record(&self) -> bool {
        self.end_offset == 0
            && self.num_properties == 0
            && self.property_list_len == 0
            && self.name_len == 0
    }
}
<commit_msg>Fix critical bug which ignores some `EndNode` events<commit_after>use std::io::Read;
use error::{Result, Error, ErrorKind};
use reader::{FbxEvent, PropertyValue};
use super::CommonState;

#[derive(Debug, Clone)]
pub struct BinaryParser {
    version: u32,
    end_offset_stack: Vec<u32>,
}

impl BinaryParser {
    pub fn new(version: u32) -> Self {
        BinaryParser {
            version: version,
            end_offset_stack: vec![],
        }
    }

    pub fn next<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<FbxEvent> {
        // Check if the previously read node ends here.
        if let Some(&end_pos_top) = self.end_offset_stack.last() {
            if end_pos_top as u64 == common.pos {
                // Reached the end of previously read node.
                self.end_offset_stack.pop();
                return Ok(FbxEvent::EndNode);
            }
        }

        // Read a node record header.
        let node_record_header = try!(NodeRecordHeader::load(reader, &mut common.pos));
        if node_record_header.is_null_record() {
            // End of a node.
            return if let Some(expected_pos) = self.end_offset_stack.pop() {
                if common.pos == expected_pos as u64 {
                    Ok(FbxEvent::EndNode)
                } else {
                    // Data is collapsed (the node doesn't end at expected position).
                    Err(Error::new(common.pos, ErrorKind::DataError("Node does not end at expected position".to_string())))
                }
            } else {
                // Data is collapsed (an extra node end marker found).
                Err(Error::new(common.pos, ErrorKind::DataError("An extra node end marker found".to_string())))
            };
        } else {
            // Start of a node.
            self.end_offset_stack.push(node_record_header.end_offset);
        }

        // Read a node name.
        let name = try_read_fixstr!(common.pos, reader, node_record_header.name_len);

        // Read properties.
        let mut properties = Vec::<PropertyValue>::with_capacity(node_record_header.num_properties as usize);
        for _ in 0..node_record_header.num_properties {
            let prop = try!(self.read_property(reader, common));
            properties.push(prop);
        }

        Ok(FbxEvent::StartNode {
            name: name,
            properties: properties,
        })
    }

    fn read_property<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<PropertyValue> {
        Err(Error::new(common.pos, ErrorKind::Unimplemented("Parser for FBX node property is not implemented yet".to_string())))
    }
}

#[derive(Debug, Copy, Clone)]
struct NodeRecordHeader {
    end_offset: u32,
    num_properties: u32,
    property_list_len: u32,
    name_len: u8,
}

impl NodeRecordHeader {
    pub fn load<R: Read>(reader: &mut R, pos: &mut u64) -> Result<Self> {
        let end_offset = try_read_le_u32!(*pos, reader);
        let num_properties = try_read_le_u32!(*pos, reader);
        let property_list_len = try_read_le_u32!(*pos, reader);
        let name_len = try_read_le_u8!(*pos, reader);
        Ok(NodeRecordHeader {
            end_offset: end_offset,
            num_properties: num_properties,
            property_list_len: property_list_len,
            name_len: name_len,
        })
    }

    /// Check whether the header indicates there are no more children.
    pub fn is_null_record(&self) -> bool {
        self.end_offset == 0
            && self.num_properties == 0
            && self.property_list_len == 0
            && self.name_len == 0
    }
}
<|endoftext|>


<commit_before>use std::io::Read;
use error::{Result, Error, ErrorKind};
use reader::FbxEvent;
use super::CommonState;

#[derive(Debug, Clone)]
pub struct BinaryParser {
    version: u32,
}

impl BinaryParser {
    pub fn new(version: u32) -> Self {
        BinaryParser {
            version: version,
        }
    }

    pub fn next<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<FbxEvent> {
        let node_record_header = try!(NodeRecordHeader::load(reader, &mut common.pos));
        Err(Error::new(common.pos, ErrorKind::Unimplemented("Parser for Binary FBX format is not implemented yet".to_string())))
    }
}

#[derive(Debug, Copy, Clone)]
struct NodeRecordHeader {
    end_offset: u32,
    num_properties: u32,
    property_list_len: u32,
    name_len: u8,
}

impl NodeRecordHeader {
    pub fn load<R: Read>(_reader: &mut R, pos: &mut u64) -> Result<Self> {
        Err(Error::new(*pos, ErrorKind::Unimplemented("Parser for NodeRecordHeader is not implemented yet".to_string())))
    }
}
<commit_msg>Add `NodeRecordHeader::is_null_record()`<commit_after>use std::io::Read;
use error::{Result, Error, ErrorKind};
use reader::FbxEvent;
use super::CommonState;

#[derive(Debug, Clone)]
pub struct BinaryParser {
    version: u32,
}

impl BinaryParser {
    pub fn new(version: u32) -> Self {
        BinaryParser {
            version: version,
        }
    }

    pub fn next<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<FbxEvent> {
        let node_record_header = try!(NodeRecordHeader::load(reader, &mut common.pos));
        Err(Error::new(common.pos, ErrorKind::Unimplemented("Parser for Binary FBX format is not implemented yet".to_string())))
    }
}

#[derive(Debug, Copy, Clone)]
struct NodeRecordHeader {
    end_offset: u32,
    num_properties: u32,
    property_list_len: u32,
    name_len: u8,
}

impl NodeRecordHeader {
    pub fn load<R: Read>(_reader: &mut R, pos: &mut u64) -> Result<Self> {
        Err(Error::new(*pos, ErrorKind::Unimplemented("Parser for NodeRecordHeader is not implemented yet".to_string())))
    }

    /// Check whether the header indicates there are no more children.
    pub fn is_null_record(&self) -> bool {
        self.end_offset == 0
            && self.num_properties == 0
            && self.property_list_len == 0
            && self.name_len == 0
    }
}
<|endoftext|>


<commit_before>extern crate term ;
extern crate event ;
extern crate system ;

use term::Offset2 ;
use term::smt::* ;
use term::smt::sync::* ;

use event::Event ;

use system::{ Sys, Prop } ;
use system::smt::Unroller ;

macro_rules! try_error {
  ($e:expr, $event:expr) => (
    match $e {
      Ok(()) => (),
      Err(e) => $event.log( & format!("error: {:?}", e) ),
    }
  )
}

pub fn run(
  sys: Sys, props: Vec<Prop>, event: Event
) {
  use term::{ Operator, SymMaker, OpMaker, VarMaker, Type } ;
  event.log(
    & format!("checking {} propertie(s) on system {}", props.len(), sys.sym())
  ) ;

  event.log("creating solver") ;

  let conf = SolverConf::z3().print_success() ;
  let factory = event.factory().clone() ;

  let k = Offset2::init() ;

  match Solver::mk(z3_cmd(), conf, factory.clone()) {
    Err(e) => event.log( & format!("error:\n  {:?}", e) ),
    Ok(mut solver) => {

      event.log("declaring functions") ;
      try_error!(
        sys.defclare_funs(& mut solver), event
      ) ;

      event.log("declaring state@0") ;
      try_error!(
        sys.declare_svars(& mut solver, & factory, k.curr()), event
      ) ;

      event.log( "asserting init@0" ) ;
      try_error!(
        solver.assert( & (sys.init_term(), & k) ), event
      ) ;

      let cpt = 0 ;

      loop {

        let actlit = factory.sym(format!("fresh_{}", cpt)) ;
        let actlit = factory.var_consign().var( actlit ) ;
        let mut neg_props = Vec::with_capacity(props.len()) ;
        for prop in props.iter() {
          neg_props.push(
            factory.op(
              Operator::Not, vec![prop.body().clone()]
            )
          )
        }
        let prop_term = factory.op(
          Operator::Or, neg_props
        ) ;
        event.log(
          & format!("defining actlit {}\nto imply {}", actlit, prop_term)
        ) ;

        try_error!(
          solver.declare_fun(
            & (& actlit, k.curr()),
            & Vec::<Type>::new(),
            & Type::Bool
          ), event
        ) ;

        let e = factory.op(
          Operator::Impl, vec![
            factory.mk_var(actlit.clone()),
            prop_term
          ]
        ) ;

        try_error!(
          solver.assert(& (& e, & k)), event
        ) ;

        event.log(
          & format!("check-sat assuming {}", actlit)
        ) ;

        match solver.check_sat_assuming( & vec![(& actlit, k.curr())] ) {
          Ok(true) => event.log("sat"),
          Ok(false) => event.log("unsat"),
          Err(e) => event.log( & format!("{:?}", e) ),
        } ;

        event.log("unrolling is not implemented, exiting") ;

        break

      }
    },
  }
}<commit_msg>minor log fix<commit_after>extern crate term ;
extern crate event ;
extern crate system ;

use term::Offset2 ;
use term::smt::* ;
use term::smt::sync::* ;

use event::Event ;

use system::{ Sys, Prop } ;
use system::smt::Unroller ;

macro_rules! try_error {
  ($e:expr, $event:expr) => (
    match $e {
      Ok(()) => (),
      Err(e) => $event.log( & format!("error: {:?}", e) ),
    }
  )
}

pub fn run(
  sys: Sys, props: Vec<Prop>, event: Event
) {
  use term::{ Operator, SymMaker, OpMaker, VarMaker, Type } ;
  event.log(
    & format!("checking {} propertie(s) on system {}", props.len(), sys.sym())
  ) ;

  event.log("creating solver") ;

  let conf = SolverConf::z3().print_success() ;
  let factory = event.factory().clone() ;

  let k = Offset2::init() ;

  match Solver::mk(z3_cmd(), conf, factory.clone()) {
    Err(e) => event.log( & format!("error:\n  {:?}", e) ),
    Ok(mut solver) => {

      event.log("declaring functions, init and trans") ;
      try_error!(
        sys.defclare_funs(& mut solver), event
      ) ;

      event.log("declaring state@0") ;
      try_error!(
        sys.declare_svars(& mut solver, & factory, k.curr()), event
      ) ;

      event.log( "asserting init@0" ) ;
      try_error!(
        solver.assert( & (sys.init_term(), & k) ), event
      ) ;

      let cpt = 0 ;

      loop {

        let actlit = factory.sym(format!("fresh_{}", cpt)) ;
        let actlit = factory.var_consign().var( actlit ) ;
        let mut neg_props = Vec::with_capacity(props.len()) ;
        for prop in props.iter() {
          neg_props.push(
            factory.op(
              Operator::Not, vec![prop.body().clone()]
            )
          )
        }
        let prop_term = factory.op(
          Operator::Or, neg_props
        ) ;
        event.log(
          & format!("defining actlit {}\nto imply {}", actlit, prop_term)
        ) ;

        try_error!(
          solver.declare_fun(
            & (& actlit, k.curr()),
            & Vec::<Type>::new(),
            & Type::Bool
          ), event
        ) ;

        let e = factory.op(
          Operator::Impl, vec![
            factory.mk_var(actlit.clone()),
            prop_term
          ]
        ) ;

        try_error!(
          solver.assert(& (& e, & k)), event
        ) ;

        event.log(
          & format!("check-sat assuming {}", actlit)
        ) ;

        match solver.check_sat_assuming( & vec![(& actlit, k.curr())] ) {
          Ok(true) => event.log("sat"),
          Ok(false) => event.log("unsat"),
          Err(e) => event.log( & format!("{:?}", e) ),
        } ;

        event.log("unrolling is not implemented, exiting") ;

        break

      }
    },
  }
}<|endoftext|>


<commit_before>use std::env;
use std::fs;
use std::io::Read;
use std::path;

use toml;

use utils::ServerError;

#[derive(Deserialize, Clone)]
pub struct Config {
    pub main: MainConfig,
}

#[derive(Deserialize, Clone)]
pub struct MainConfig {
    pub listen: String,
    pub data_path: path::PathBuf,
    pub use_proxy_headers: bool,
}

impl Config {
    pub fn read_file(path: &path::Path) -> Result<Self, ServerError> {
        let path = &env::current_dir().unwrap().join(path);
        let mut s = String::new();
        let mut f = try!(fs::File::open(path));
        try!(f.read_to_string(&mut s));
        Ok(try!(toml::from_str(&s)))
    }
}
<commit_msg>Fix config again<commit_after>use std::env;
use std::fs;
use std::io::Read;
use std::path;

use toml;

use utils::ServerError;

#[derive(Deserialize, Clone)]
pub struct Config {
    pub main: MainConfig,
}

#[derive(Deserialize, Clone)]
pub struct MainConfig {
    pub listen: String,
    pub data_path: path::PathBuf,
    pub use_proxy_headers: bool,
}

impl Config {
    pub fn read_file(path: &path::Path) -> Result<Self, ServerError> {
        let path = &env::current_dir().unwrap().join(path);
        let mut s = String::new();
        let mut f = try!(fs::File::open(path));
        try!(f.read_to_string(&mut s));
        let mut rv: Self = try!(toml::from_str(&s));
        rv.main.data_path = try!(rv.main.data_path.canonicalize());
        Ok(rv)
    }
}
<|endoftext|>


<commit_before>// xfail-stage0
// -*- rust -*-
use std;
import std::io;
import std::str;

fn test_simple() {
    let str tmpfile = "test/run-pass/lib-io-test-simple.tmp";
    log tmpfile;
    let str frood = "A hoopy frood who really knows where his towel is.";
    log frood;
    {
        let io::writer out = io::file_writer(tmpfile, [io::create,
                                                       io::truncate]);
        out.write_str(frood);
    }
    let io::reader inp = io::file_reader(tmpfile);
    let str frood2 = inp.read_c_str();
    log frood2;
    assert (str::eq(frood, frood2));
}

fn main() {
    test_simple();
}<commit_msg>Disable run-pass/lib-io.rs on mac<commit_after>// xfail-stage0
// -*- rust -*-
use std;
import std::io;
import std::str;

#[cfg(target_os = "linux")]
#[cfg(target_os = "win32")]
fn test_simple() {
    let str tmpfile = "test/run-pass/lib-io-test-simple.tmp";
    log tmpfile;
    let str frood = "A hoopy frood who really knows where his towel is.";
    log frood;
    {
        let io::writer out = io::file_writer(tmpfile, [io::create,
                                                       io::truncate]);
        out.write_str(frood);
    }
    let io::reader inp = io::file_reader(tmpfile);
    let str frood2 = inp.read_c_str();
    log frood2;
    assert (str::eq(frood, frood2));
}

// FIXME (726)
#[cfg(target_os = "macos")]
fn test_simple() {}

fn main() {
    test_simple();
}<|endoftext|>


<commit_before>use crate::io::{self, Error, ErrorKind};
use libc::{self, c_int, c_char};
use libc::{RTP_ID};
use crate::sys;
use crate::sys::cvt;
use crate::sys::process::process_common::*;
use crate::sys_common::thread;

////////////////////////////////////////////////////////////////////////////////
// Command
////////////////////////////////////////////////////////////////////////////////

impl Command {
    pub fn spawn(&mut self, default: Stdio, needs_stdin: bool)
                 -> io::Result<(Process, StdioPipes)> {
        use crate::sys::{cvt_r};
        const CLOEXEC_MSG_FOOTER: &'static [u8] = b"NOEX";

        if self.saw_nul() {
            return Err(io::Error::new(ErrorKind::InvalidInput,
                                      "nul byte found in provided data"));
        }
        let (ours, theirs) = self.setup_io(default, needs_stdin)?;
        let mut p = Process { pid: 0, status: None };

        unsafe {
            macro_rules! t {
                ($e:expr) => (match $e {
                    Ok(e) => e,
                    Err(e) => return Err(e.into()),
                })
            }

            let mut orig_stdin = libc::STDIN_FILENO;
            let mut orig_stdout = libc::STDOUT_FILENO;
            let mut orig_stderr = libc::STDERR_FILENO;

            if let Some(fd) = theirs.stdin.fd() {
                orig_stdin = t!(cvt_r(|| libc::dup(libc::STDIN_FILENO)));
                t!(cvt_r(|| libc::dup2(fd, libc::STDIN_FILENO)));
            }
            if let Some(fd) = theirs.stdout.fd() {
                orig_stdout = t!(cvt_r(|| libc::dup(libc::STDOUT_FILENO)));
                t!(cvt_r(|| libc::dup2(fd, libc::STDOUT_FILENO)));
            }
            if let Some(fd) = theirs.stderr.fd() {
                orig_stderr = t!(cvt_r(|| libc::dup(libc::STDERR_FILENO)));
                t!(cvt_r(|| libc::dup2(fd, libc::STDERR_FILENO)));
            }

            if let Some(ref cwd) = *self.get_cwd() {
                t!(cvt(libc::chdir(cwd.as_ptr())));
            }

            let ret = libc::rtpSpawn(
                self.get_argv()[0],                   // executing program
                self.get_argv().as_ptr() as *const _, // argv
                *sys::os::environ() as *const *const c_char,
                100 as c_int,                         // initial priority
                thread::min_stack(),                  // initial stack size.
                0,                                    // options
                0                                     // task options
            );

            // Because FileDesc was not used, each duplicated file descriptor
            // needs to be closed manually
            if orig_stdin != libc::STDIN_FILENO {
                t!(cvt_r(|| libc::dup2(orig_stdin, libc::STDIN_FILENO)));
                libc::close(orig_stdin);
            }
            if orig_stdout != libc::STDOUT_FILENO {
                t!(cvt_r(|| libc::dup2(orig_stdout, libc::STDOUT_FILENO)));
                libc::close(orig_stdout);
            }
            if orig_stderr != libc::STDERR_FILENO {
                t!(cvt_r(|| libc::dup2(orig_stderr, libc::STDERR_FILENO)));
                libc::close(orig_stderr);
            }

            if ret != libc::RTP_ID_ERROR {
                p.pid = ret;
                Ok((p, ours))
            } else {
                Err(io::Error::last_os_error())
            }
        }
    }

    pub fn exec(&mut self, default: Stdio) -> io::Error {
        let ret = Command::spawn(self, default, false);
        match ret {
            Ok(t) => unsafe {
                let mut status = 0 as c_int;
                libc::waitpid(t.0.pid, &mut status, 0);
                libc::exit(0);
            },
            Err(e) => e,
        }
    }
}

////////////////////////////////////////////////////////////////////////////////
// Processes
////////////////////////////////////////////////////////////////////////////////

/// The unique id of the process (this should never be negative).
pub struct Process {
    pid: RTP_ID,
    status: Option<ExitStatus>,
}

impl Process {
    pub fn id(&self) -> u32 {
        self.pid as u32
    }

    pub fn kill(&mut self) -> io::Result<()> {
        // If we've already waited on this process then the pid can be recycled
        // and used for another process, and we probably shouldn't be killing
        // random processes, so just return an error.
        if self.status.is_some() {
            Err(Error::new(ErrorKind::InvalidInput,
                           "invalid argument: can't kill an exited process"))
        } else {
            cvt(unsafe { libc::kill(self.pid, libc::SIGKILL) }).map(|_| ())
        }
    }

    pub fn wait(&mut self) -> io::Result<ExitStatus> {
        use crate::sys::cvt_r;
        if let Some(status) = self.status {
            return Ok(status)
        }
        let mut status = 0 as c_int;
        cvt_r(|| unsafe { libc::waitpid(self.pid, &mut status, 0) })?;
        self.status = Some(ExitStatus::new(status));
        Ok(ExitStatus::new(status))
    }

    pub fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
        if let Some(status) = self.status {
            return Ok(Some(status))
        }
        let mut status = 0 as c_int;
        let pid = cvt(unsafe {
            libc::waitpid(self.pid, &mut status, libc::WNOHANG)
        })?;
        if pid == 0 {
            Ok(None)
        } else {
            self.status = Some(ExitStatus::new(status));
            Ok(Some(ExitStatus::new(status)))
        }
    }
}
<commit_msg>Rollup merge of #64818 - Wind-River:master, r=alexcrichton<commit_after>use crate::io::{self, Error, ErrorKind};
use libc::{self, c_int, c_char};
use libc::{RTP_ID};
use crate::sys;
use crate::sys::cvt;
use crate::sys::process::process_common::*;
use crate::sys_common::thread;

////////////////////////////////////////////////////////////////////////////////
// Command
////////////////////////////////////////////////////////////////////////////////

impl Command {
    pub fn spawn(&mut self, default: Stdio, needs_stdin: bool)
                 -> io::Result<(Process, StdioPipes)> {
        use crate::sys::{cvt_r};
        const CLOEXEC_MSG_FOOTER: &'static [u8] = b"NOEX";

        if self.saw_nul() {
            return Err(io::Error::new(ErrorKind::InvalidInput,
                                      "nul byte found in provided data"));
        }
        let (ours, theirs) = self.setup_io(default, needs_stdin)?;
        let mut p = Process { pid: 0, status: None };

        unsafe {
            macro_rules! t {
                ($e:expr) => (match $e {
                    Ok(e) => e,
                    Err(e) => return Err(e.into()),
                })
            }

            let mut orig_stdin = libc::STDIN_FILENO;
            let mut orig_stdout = libc::STDOUT_FILENO;
            let mut orig_stderr = libc::STDERR_FILENO;

            if let Some(fd) = theirs.stdin.fd() {
                orig_stdin = t!(cvt_r(|| libc::dup(libc::STDIN_FILENO)));
                t!(cvt_r(|| libc::dup2(fd, libc::STDIN_FILENO)));
            }
            if let Some(fd) = theirs.stdout.fd() {
                orig_stdout = t!(cvt_r(|| libc::dup(libc::STDOUT_FILENO)));
                t!(cvt_r(|| libc::dup2(fd, libc::STDOUT_FILENO)));
            }
            if let Some(fd) = theirs.stderr.fd() {
                orig_stderr = t!(cvt_r(|| libc::dup(libc::STDERR_FILENO)));
                t!(cvt_r(|| libc::dup2(fd, libc::STDERR_FILENO)));
            }

            if let Some(ref cwd) = *self.get_cwd() {
                t!(cvt(libc::chdir(cwd.as_ptr())));
            }

            let ret = libc::rtpSpawn(
                self.get_argv()[0],                   // executing program
                self.get_argv().as_ptr() as *mut *const c_char, // argv
                *sys::os::environ() as *mut *const c_char,
                100 as c_int,                         // initial priority
                thread::min_stack(),                  // initial stack size.
                0,                                    // options
                0                                     // task options
            );

            // Because FileDesc was not used, each duplicated file descriptor
            // needs to be closed manually
            if orig_stdin != libc::STDIN_FILENO {
                t!(cvt_r(|| libc::dup2(orig_stdin, libc::STDIN_FILENO)));
                libc::close(orig_stdin);
            }
            if orig_stdout != libc::STDOUT_FILENO {
                t!(cvt_r(|| libc::dup2(orig_stdout, libc::STDOUT_FILENO)));
                libc::close(orig_stdout);
            }
            if orig_stderr != libc::STDERR_FILENO {
                t!(cvt_r(|| libc::dup2(orig_stderr, libc::STDERR_FILENO)));
                libc::close(orig_stderr);
            }

            if ret != libc::RTP_ID_ERROR {
                p.pid = ret;
                Ok((p, ours))
            } else {
                Err(io::Error::last_os_error())
            }
        }
    }

    pub fn exec(&mut self, default: Stdio) -> io::Error {
        let ret = Command::spawn(self, default, false);
        match ret {
            Ok(t) => unsafe {
                let mut status = 0 as c_int;
                libc::waitpid(t.0.pid, &mut status, 0);
                libc::exit(0);
            },
            Err(e) => e,
        }
    }
}

////////////////////////////////////////////////////////////////////////////////
// Processes
////////////////////////////////////////////////////////////////////////////////

/// The unique id of the process (this should never be negative).
pub struct Process {
    pid: RTP_ID,
    status: Option<ExitStatus>,
}

impl Process {
    pub fn id(&self) -> u32 {
        self.pid as u32
    }

    pub fn kill(&mut self) -> io::Result<()> {
        // If we've already waited on this process then the pid can be recycled
        // and used for another process, and we probably shouldn't be killing
        // random processes, so just return an error.
        if self.status.is_some() {
            Err(Error::new(ErrorKind::InvalidInput,
                           "invalid argument: can't kill an exited process"))
        } else {
            cvt(unsafe { libc::kill(self.pid, libc::SIGKILL) }).map(|_| ())
        }
    }

    pub fn wait(&mut self) -> io::Result<ExitStatus> {
        use crate::sys::cvt_r;
        if let Some(status) = self.status {
            return Ok(status)
        }
        let mut status = 0 as c_int;
        cvt_r(|| unsafe { libc::waitpid(self.pid, &mut status, 0) })?;
        self.status = Some(ExitStatus::new(status));
        Ok(ExitStatus::new(status))
    }

    pub fn try_wait(&mut self) -> io::Result<Option<ExitStatus>> {
        if let Some(status) = self.status {
            return Ok(Some(status))
        }
        let mut status = 0 as c_int;
        let pid = cvt(unsafe {
            libc::waitpid(self.pid, &mut status, libc::WNOHANG)
        })?;
        if pid == 0 {
            Ok(None)
        } else {
            self.status = Some(ExitStatus::new(status));
            Ok(Some(ExitStatus::new(status)))
        }
    }
}
<|endoftext|>