<commit_before>use std::f32::consts::PI;
use std::sync::{Arc, Mutex,
                mpsc::{Sender, Receiver, TryRecvError},
                atomic::{AtomicUsize, Ordering},
};

use cgmath::{Point3, Vector3, Vector4, prelude::*};

use glium::Rect;

use rand::{self, prelude::*};

use crate::bvh::BVHNode;
use crate::camera::Camera;
use crate::color::Color;
use crate::material::Material;
use crate::pt_renderer::{Intersect, Ray, RenderCoordinator};
use crate::scene::Scene;
use crate::triangle::Hit;

const EPSILON: f32 = 1e-5;
const _EB: f32 = 5.0; // Desired expectation value of bounces
// The matching survival probability from negative binomial distribution
const RR_PROB: f32 = _EB / (_EB + 1.0);

pub struct RenderWorker {
    scene: Arc<Scene>,
    camera: Camera,
    coordinator: Arc<Mutex<RenderCoordinator>>,
    message_rx: Receiver<()>,
    result_tx: Sender<(Rect, Vec<f32>)>,
    ray_count: Arc<AtomicUsize>,
}

impl RenderWorker {
    pub fn new(scene: Arc<Scene>, camera: Camera, coordinator: Arc<Mutex<RenderCoordinator>>,
               message_rx: Receiver<()>, result_tx: Sender<(Rect, Vec<f32>)>,
               ray_count: Arc<AtomicUsize>) -> RenderWorker {
        RenderWorker {
            scene, camera, coordinator,
            message_rx, result_tx,
            ray_count,
        }
    }

    pub fn run(&self) {
        let (width, height) = {
            let coordinator = self.coordinator.lock().unwrap();
            (coordinator.width, coordinator.height)
        };
        let clip_to_world = self.camera.get_world_to_clip().invert().unwrap();
        let mut node_stack: Vec<(&BVHNode, f32)> = Vec::new();
        loop {
            match self.message_rx.try_recv() {
                Err(TryRecvError::Empty) => (),
                Ok(_) => return,
                Err(TryRecvError::Disconnected) => {
                    println!("Threads were not properly stopped before disconnecting channel!");
                    return;
                }
            }
            let block = {
                let mut coordinator = self.coordinator.lock().unwrap();
                coordinator.next_block()
            };
            if let Some(rect) = block {
                let mut block = vec![0.0f32; (3 * rect.width * rect.height) as usize];
                for h in 0..rect.height {
                    for w in 0..rect.width {
                        let samples_per_dir = 2u32;
                        let mut c = Color::black();
                        for j in 0..samples_per_dir {
                            for i in 0..samples_per_dir {
                                let dx = (i as f32 + rand::random::<f32>()) / samples_per_dir as f32;
                                let dy = (j as f32 + rand::random::<f32>()) / samples_per_dir as f32;
                                let clip_x = 2.0 * ((rect.left + w) as f32 + dx) / width as f32 - 1.0;
                                let clip_y = 2.0 * ((rect.bottom + h) as f32 + dy) / height as f32 - 1.0;
                                let clip_p = Vector4::new(clip_x, clip_y, 1.0, 1.0);
                                let world_p = clip_to_world * clip_p;
                                let dir = ((world_p / world_p.w).truncate() - self.camera.pos.to_vec())
                                    .normalize();
                                let ray = Ray::new(self.camera.pos, dir, 10.0 * self.camera.scale);
                                c += self.trace_ray(&ray, &mut node_stack, 0);
                            }
                        }
                        c /= samples_per_dir.pow(2) as f32;
                        let pixel_i = 3 * (h * rect.width + w) as usize;
                        block[pixel_i]     = c.r;
                        block[pixel_i + 1] = c.g;
                        block[pixel_i + 2] = c.b;
                    }
                }
                self.result_tx.send((rect, block)).expect("Receiver closed!");
            } else {
                return;
            }
        }
    }

    fn trace_ray(&'a self, ray: &Ray, node_stack: &mut Vec<(&'a BVHNode, f32)>, bounce: u32) -> Color {
        let mut c = Color::black();
        if let Some(hit) = self.find_hit(&ray, node_stack) {
            let material = &self.scene.materials[hit.tri.material_i];
            let mut normal = hit.normal();
            // Flip the normal if its pointing to the opposite side from the hit
            if normal.dot(ray.dir) > 0.0 {
                normal *= -1.0;
            }
            if bounce == 0 {
                if let Some(emissive) = material.emissive {
                    c += normal.dot(-ray.dir).max(0.0) * emissive;
                }
            }
            let (emissive, light_pos, light_normal, light_pdf) = self.sample_light();
            let bump_pos = hit.pos() + EPSILON * normal;
            let hit_to_light = light_pos - bump_pos;
            let light_dir = hit_to_light.normalize();
            let shadow_ray = Ray::new(bump_pos, light_dir, hit_to_light.magnitude() - EPSILON);
            if shadow_ray.length > 0.1 && self.find_hit(&shadow_ray, node_stack).is_none() {
                let cos_l = light_normal.dot(-light_dir).max(0.0);
                let cos_t = normal.dot(light_dir).max(0.0);
                c += emissive * self.brdf(&hit, &ray, &shadow_ray, material)
                    * cos_l * cos_t / (hit_to_light.magnitude2() * light_pdf);
            }
            let rr: f32 = rand::random();
            if rr < RR_PROB {
                let (new_dir, mut pdf) = self.sample_dir(normal);
                pdf *= RR_PROB;
                let new_ray = Ray::new(bump_pos, new_dir, 10.0 * self.camera.scale);
                c += normal.dot(new_dir) * self.brdf(&hit, &ray, &new_ray, material)
                    * self.trace_ray(&new_ray, node_stack, bounce + 1) / pdf;
            }
        }
        c
    }

    fn brdf(&self, hit: &Hit, _in_ray: &Ray, _out_ray: &Ray, material: &Material) -> Color {
        let tex_coords = hit.tex_coords();
        material.diffuse(tex_coords) / PI
    }

    fn sample_dir(&self, normal: Vector3<f32>) -> (Vector3<f32>, f32) {
        let dir = 2.0 * PI * rand::random::<f32>();
        let length: f32 = rand::random();
        let x = length * dir.cos();
        let y = length * dir.sin();
        let z = (1.0 - length.powi(2)).sqrt();
        let nx = if normal.x.abs() > normal.y.abs() {
            Vector3::new(normal.z, 0.0, -normal.x).normalize()
        } else {
            Vector3::new(0.0, -normal.z, normal.y).normalize()
        };
        let ny = normal.cross(nx);
        (x * nx + y * ny + z * normal, z / PI)
    }

    pub fn sample_light(&self) -> (Color, Point3<f32>, Vector3<f32>, f32) {
        if self.scene.lights.is_empty() {
            let light_intensity = 10.0 * self.camera.scale;
            (light_intensity * Color::white(), self.camera.pos, self.camera.dir, 1.0)
        } else {
            let i = rand::thread_rng().gen_range(0, self.scene.lights.len());
            let light = &self.scene.lights[i];
            let pdf = 1.0 / (self.scene.lights.len() as f32 * light.area());
            let (point, normal) = light.random_point();
            let light_material = &self.scene.materials[light.material_i];
            let emissive = light_material.emissive.expect("Light wasn't emissive");
            (emissive, point, normal, pdf)
        }
    }

    fn find_hit(&'a self, ray: &Ray, node_stack: &mut Vec<(&'a BVHNode, f32)>) -> Option<Hit> {
        self.ray_count.fetch_add(1, Ordering::Relaxed);
        let bvh = &self.scene.bvh;
        node_stack.push((bvh.root(), 0.0f32));
        let mut closest_hit: Option<Hit> = None;
        while let Some((node, t)) = node_stack.pop() {
            // We've already found closer hit
            if closest_hit.as_ref().map_or(false, |closest| closest.t <= t) { continue }
            if node.is_leaf() {
                for tri in &self.scene.triangles[node.start_i..node.end_i] {
                    if let Some(hit) = tri.intersect(&ray) {
                        if let Some(closest) = closest_hit.take() {
                            if hit.t < closest.t {
                                closest_hit = Some(hit);
                            } else {
                                closest_hit = Some(closest);
                            }
                        } else {
                            closest_hit = Some(hit);
                        }
                    }
                }
            } else {
                // TODO: put closest hit on top of the stack
                let (left, right) = bvh.get_children(node).expect("Non leaf node had no child nodes!");
                if let Some(t_left) = left.intersect(&ray) {
                    node_stack.push((left, t_left));
                }
                if let Some(t_right) = right.intersect(&ray) {
                    node_stack.push((right, t_right));
                }
            }
        }
        closest_hit
    }
}
<commit_msg>Add option to sort bvh nodes during traversal<commit_after>use std::f32::consts::PI;
use std::sync::{Arc, Mutex,
                mpsc::{Sender, Receiver, TryRecvError},
                atomic::{AtomicUsize, Ordering},
};

use cgmath::{Point3, Vector3, Vector4, prelude::*};

use glium::Rect;

use rand::{self, prelude::*};

use crate::bvh::BVHNode;
use crate::camera::Camera;
use crate::color::Color;
use crate::material::Material;
use crate::pt_renderer::{Intersect, Ray, RenderCoordinator};
use crate::scene::Scene;
use crate::triangle::Hit;

const EPSILON: f32 = 1e-5;
const _EB: f32 = 5.0; // Desired expectation value of bounces
// The matching survival probability from negative binomial distribution
const RR_PROB: f32 = _EB / (_EB + 1.0);

pub struct RenderWorker {
    scene: Arc<Scene>,
    camera: Camera,
    coordinator: Arc<Mutex<RenderCoordinator>>,
    message_rx: Receiver<()>,
    result_tx: Sender<(Rect, Vec<f32>)>,
    ray_count: Arc<AtomicUsize>,
}

impl RenderWorker {
    pub fn new(scene: Arc<Scene>, camera: Camera, coordinator: Arc<Mutex<RenderCoordinator>>,
               message_rx: Receiver<()>, result_tx: Sender<(Rect, Vec<f32>)>,
               ray_count: Arc<AtomicUsize>) -> RenderWorker {
        RenderWorker {
            scene, camera, coordinator,
            message_rx, result_tx,
            ray_count,
        }
    }

    pub fn run(&self) {
        let (width, height) = {
            let coordinator = self.coordinator.lock().unwrap();
            (coordinator.width, coordinator.height)
        };
        let clip_to_world = self.camera.get_world_to_clip().invert().unwrap();
        let mut node_stack: Vec<(&BVHNode, f32)> = Vec::new();
        loop {
            match self.message_rx.try_recv() {
                Err(TryRecvError::Empty) => (),
                Ok(_) => return,
                Err(TryRecvError::Disconnected) => {
                    println!("Threads were not properly stopped before disconnecting channel!");
                    return;
                }
            }
            let block = {
                let mut coordinator = self.coordinator.lock().unwrap();
                coordinator.next_block()
            };
            if let Some(rect) = block {
                let mut block = vec![0.0f32; (3 * rect.width * rect.height) as usize];
                for h in 0..rect.height {
                    for w in 0..rect.width {
                        let samples_per_dir = 2u32;
                        let mut c = Color::black();
                        for j in 0..samples_per_dir {
                            for i in 0..samples_per_dir {
                                let dx = (i as f32 + rand::random::<f32>()) / samples_per_dir as f32;
                                let dy = (j as f32 + rand::random::<f32>()) / samples_per_dir as f32;
                                let clip_x = 2.0 * ((rect.left + w) as f32 + dx) / width as f32 - 1.0;
                                let clip_y = 2.0 * ((rect.bottom + h) as f32 + dy) / height as f32 - 1.0;
                                let clip_p = Vector4::new(clip_x, clip_y, 1.0, 1.0);
                                let world_p = clip_to_world * clip_p;
                                let dir = ((world_p / world_p.w).truncate() - self.camera.pos.to_vec())
                                    .normalize();
                                let ray = Ray::new(self.camera.pos, dir, 10.0 * self.camera.scale);
                                c += self.trace_ray(&ray, &mut node_stack, 0);
                            }
                        }
                        c /= samples_per_dir.pow(2) as f32;
                        let pixel_i = 3 * (h * rect.width + w) as usize;
                        block[pixel_i]     = c.r;
                        block[pixel_i + 1] = c.g;
                        block[pixel_i + 2] = c.b;
                    }
                }
                self.result_tx.send((rect, block)).expect("Receiver closed!");
            } else {
                return;
            }
        }
    }

    fn trace_ray(&'a self, ray: &Ray, node_stack: &mut Vec<(&'a BVHNode, f32)>, bounce: u32) -> Color {
        let mut c = Color::black();
        if let Some(hit) = self.find_hit(&ray, node_stack) {
            let material = &self.scene.materials[hit.tri.material_i];
            let mut normal = hit.normal();
            // Flip the normal if its pointing to the opposite side from the hit
            if normal.dot(ray.dir) > 0.0 {
                normal *= -1.0;
            }
            if bounce == 0 {
                if let Some(emissive) = material.emissive {
                    c += normal.dot(-ray.dir).max(0.0) * emissive;
                }
            }
            let (emissive, light_pos, light_normal, light_pdf) = self.sample_light();
            let bump_pos = hit.pos() + EPSILON * normal;
            let hit_to_light = light_pos - bump_pos;
            let light_dir = hit_to_light.normalize();
            let shadow_ray = Ray::new(bump_pos, light_dir, hit_to_light.magnitude() - EPSILON);
            if shadow_ray.length > 0.1 && self.find_hit(&shadow_ray, node_stack).is_none() {
                let cos_l = light_normal.dot(-light_dir).max(0.0);
                let cos_t = normal.dot(light_dir).max(0.0);
                c += emissive * self.brdf(&hit, &ray, &shadow_ray, material)
                    * cos_l * cos_t / (hit_to_light.magnitude2() * light_pdf);
            }
            let rr: f32 = rand::random();
            if rr < RR_PROB {
                let (new_dir, mut pdf) = self.sample_dir(normal);
                pdf *= RR_PROB;
                let new_ray = Ray::new(bump_pos, new_dir, 10.0 * self.camera.scale);
                c += normal.dot(new_dir) * self.brdf(&hit, &ray, &new_ray, material)
                    * self.trace_ray(&new_ray, node_stack, bounce + 1) / pdf;
            }
        }
        c
    }

    fn brdf(&self, hit: &Hit, _in_ray: &Ray, _out_ray: &Ray, material: &Material) -> Color {
        let tex_coords = hit.tex_coords();
        material.diffuse(tex_coords) / PI
    }

    fn sample_dir(&self, normal: Vector3<f32>) -> (Vector3<f32>, f32) {
        let dir = 2.0 * PI * rand::random::<f32>();
        let length: f32 = rand::random();
        let x = length * dir.cos();
        let y = length * dir.sin();
        let z = (1.0 - length.powi(2)).sqrt();
        let nx = if normal.x.abs() > normal.y.abs() {
            Vector3::new(normal.z, 0.0, -normal.x).normalize()
        } else {
            Vector3::new(0.0, -normal.z, normal.y).normalize()
        };
        let ny = normal.cross(nx);
        (x * nx + y * ny + z * normal, z / PI)
    }

    pub fn sample_light(&self) -> (Color, Point3<f32>, Vector3<f32>, f32) {
        if self.scene.lights.is_empty() {
            let light_intensity = 10.0 * self.camera.scale;
            (light_intensity * Color::white(), self.camera.pos, self.camera.dir, 1.0)
        } else {
            let i = rand::thread_rng().gen_range(0, self.scene.lights.len());
            let light = &self.scene.lights[i];
            let pdf = 1.0 / (self.scene.lights.len() as f32 * light.area());
            let (point, normal) = light.random_point();
            let light_material = &self.scene.materials[light.material_i];
            let emissive = light_material.emissive.expect("Light wasn't emissive");
            (emissive, point, normal, pdf)
        }
    }

    fn find_hit(&'a self, ray: &Ray, node_stack: &mut Vec<(&'a BVHNode, f32)>) -> Option<Hit> {
        self.ray_count.fetch_add(1, Ordering::Relaxed);
        let bvh = &self.scene.bvh;
        node_stack.push((bvh.root(), 0.0f32));
        let mut closest_hit: Option<Hit> = None;
        while let Some((node, t)) = node_stack.pop() {
            // We've already found closer hit
            if closest_hit.as_ref().map_or(false, |closest| closest.t <= t) { continue }
            if node.is_leaf() {
                for tri in &self.scene.triangles[node.start_i..node.end_i] {
                    if let Some(hit) = tri.intersect(&ray) {
                        if let Some(closest) = closest_hit.take() {
                            if hit.t < closest.t {
                                closest_hit = Some(hit);
                            } else {
                                closest_hit = Some(closest);
                            }
                        } else {
                            closest_hit = Some(hit);
                        }
                    }
                }
            } else {
                let (left, right) = bvh.get_children(node)
                    .expect("Non leaf node had no child nodes!");
                if true { // not sorted
                    if let Some(t_left) = left.intersect(&ray) {
                        node_stack.push((left, t_left));
                    }
                    if let Some(t_right) = right.intersect(&ray) {
                        node_stack.push((right, t_right));
                    }
                } else { // sort nodes
                    let left_intersect = left.intersect(&ray);
                    let right_intersect = right.intersect(&ray);
                    if let Some(t_left) = left_intersect {
                        if let Some(t_right) = right_intersect {
                            // Put the closer hit on top
                            if t_left >= t_right {
                                node_stack.push((left, t_left));
                                node_stack.push((right, t_right));
                            } else {
                                node_stack.push((right, t_right));
                                node_stack.push((left, t_left));
                            }
                        } else {
                            node_stack.push((left, t_left));
                        }
                    } else if let Some(t_right) = right_intersect {
                        node_stack.push((right, t_right));
                    }
                }
            }
        }
        closest_hit
    }
}
<|endoftext|>


<commit_before>extern crate futures;
extern crate rand;
extern crate tempfile;
extern crate tokio_fs;
extern crate tokio_io;

use tokio_fs::*;
use tokio_io::io;

use futures::future::poll_fn;
use futures::Future;
use rand::{distributions, thread_rng, Rng};
use tempfile::Builder as TmpBuilder;

use std::fs;
use std::io::SeekFrom;

mod pool;

#[test]
fn read_write() {
    const NUM_CHARS: usize = 16 * 1_024;

    let dir = TmpBuilder::new()
        .prefix("tokio-fs-tests")
        .tempdir()
        .unwrap();
    let file_path = dir.path().join("read_write.txt");

    let contents: Vec<u8> = thread_rng()
        .sample_iter(&distributions::Alphanumeric)
        .take(NUM_CHARS)
        .collect::<String>()
        .into();

    pool::run({
        let file_path = file_path.clone();
        let contents = contents.clone();

        File::create(file_path)
            .and_then(|file| file.metadata())
            .inspect(|&(_, ref metadata)| assert!(metadata.is_file()))
            .and_then(move |(file, _)| io::write_all(file, contents))
            .and_then(|(mut file, _)| poll_fn(move || file.poll_sync_all()))
            .then(|res| {
                let _ = res.unwrap();
                Ok(())
            })
    });

    let dst = fs::read(&file_path).unwrap();
    assert_eq!(dst, contents);

    pool::run({
        File::open(file_path)
            .and_then(|file| io::read_to_end(file, vec![]))
            .then(move |res| {
                let (_, buf) = res.unwrap();
                assert_eq!(buf, contents);
                Ok(())
            })
    });
}

#[test]
fn read_write_helpers() {
    const NUM_CHARS: usize = 16 * 1_024;

    let dir = TmpBuilder::new()
        .prefix("tokio-fs-tests")
        .tempdir()
        .unwrap();
    let file_path = dir.path().join("read_write_all.txt");

    let contents: Vec<u8> = thread_rng()
        .sample_iter(&distributions::Alphanumeric)
        .take(NUM_CHARS)
        .collect::<String>()
        .into();

    pool::run(write(file_path.clone(), contents.clone()).then(|res| {
        let _ = res.unwrap();
        Ok(())
    }));

    let dst = fs::read(&file_path).unwrap();
    assert_eq!(dst, contents);

    pool::run({
        read(file_path).then(move |res| {
            let buf = res.unwrap();
            assert_eq!(buf, contents);
            Ok(())
        })
    });
}

#[test]
fn metadata() {
    let dir = TmpBuilder::new()
        .prefix("tokio-fs-tests")
        .tempdir()
        .unwrap();
    let file_path = dir.path().join("metadata.txt");

    pool::run({
        let file_path = file_path.clone();
        let file_path2 = file_path.clone();
        let file_path3 = file_path.clone();

        tokio_fs::metadata(file_path)
            .then(|r| {
                let _ = r.err().unwrap();
                Ok(())
            })
            .and_then(|_| File::create(file_path2))
            .and_then(|_| tokio_fs::metadata(file_path3))
            .then(|r| {
                assert!(r.unwrap().is_file());
                Ok(())
            })
    });
}

#[test]
fn seek() {
    let dir = TmpBuilder::new()
        .prefix("tokio-fs-tests")
        .tempdir()
        .unwrap();
    let file_path = dir.path().join("seek.txt");

    pool::run({
        OpenOptions::new()
            .create(true)
            .read(true)
            .write(true)
            .open(file_path)
            .and_then(|file| io::write_all(file, "Hello, world!"))
            .and_then(|(file, _)| file.seek(SeekFrom::End(-6)))
            .and_then(|(file, _)| io::read_exact(file, vec![0; 5]))
            .and_then(|(file, buf)| {
                assert_eq!(buf, b"world");
                file.seek(SeekFrom::Start(0))
            })
            .and_then(|(file, _)| io::read_exact(file, vec![0; 5]))
            .and_then(|(_, buf)| {
                assert_eq!(buf, b"Hello");
                Ok(())
            })
            .then(|r| {
                let _ = r.unwrap();
                Ok(())
            })
    });
}

#[test]
fn clone() {
    let dir = TmpBuilder::new()
        .prefix("tokio-fs-tests")
        .tempdir()
        .unwrap();
    let file_path = dir.path().join("clone.txt");

    pool::run(
        File::create(file_path.clone())
            .and_then(|file| {
                file.try_clone()
                    .map_err(|(_file, err)| err)
                    .and_then(|(file, clone)| {
                        io::write_all(file, "clone ")
                            .and_then(|_| io::write_all(clone, "successful"))
                    })
            })
            .then(|res| {
                let _ = res.unwrap();
                Ok(())
            }),
    );

    let mut file = StdFile::open(&file_path).unwrap();

    let mut dst = vec![];
    file.read_to_end(&mut dst).unwrap();

    assert_eq!(dst, b"clone successful")
}
<commit_msg>fs: fix tests (#916)<commit_after>extern crate futures;
extern crate rand;
extern crate tempfile;
extern crate tokio_fs;
extern crate tokio_io;

use tokio_fs::*;
use tokio_io::io;

use futures::future::poll_fn;
use futures::Future;
use rand::{distributions, thread_rng, Rng};
use tempfile::Builder as TmpBuilder;

use std::fs;
use std::io::SeekFrom;

mod pool;

#[test]
fn read_write() {
    const NUM_CHARS: usize = 16 * 1_024;

    let dir = TmpBuilder::new()
        .prefix("tokio-fs-tests")
        .tempdir()
        .unwrap();
    let file_path = dir.path().join("read_write.txt");

    let contents: Vec<u8> = thread_rng()
        .sample_iter(&distributions::Alphanumeric)
        .take(NUM_CHARS)
        .collect::<String>()
        .into();

    pool::run({
        let file_path = file_path.clone();
        let contents = contents.clone();

        File::create(file_path)
            .and_then(|file| file.metadata())
            .inspect(|&(_, ref metadata)| assert!(metadata.is_file()))
            .and_then(move |(file, _)| io::write_all(file, contents))
            .and_then(|(mut file, _)| poll_fn(move || file.poll_sync_all()))
            .then(|res| {
                let _ = res.unwrap();
                Ok(())
            })
    });

    let dst = fs::read(&file_path).unwrap();
    assert_eq!(dst, contents);

    pool::run({
        File::open(file_path)
            .and_then(|file| io::read_to_end(file, vec![]))
            .then(move |res| {
                let (_, buf) = res.unwrap();
                assert_eq!(buf, contents);
                Ok(())
            })
    });
}

#[test]
fn read_write_helpers() {
    const NUM_CHARS: usize = 16 * 1_024;

    let dir = TmpBuilder::new()
        .prefix("tokio-fs-tests")
        .tempdir()
        .unwrap();
    let file_path = dir.path().join("read_write_all.txt");

    let contents: Vec<u8> = thread_rng()
        .sample_iter(&distributions::Alphanumeric)
        .take(NUM_CHARS)
        .collect::<String>()
        .into();

    pool::run(write(file_path.clone(), contents.clone()).then(|res| {
        let _ = res.unwrap();
        Ok(())
    }));

    let dst = fs::read(&file_path).unwrap();
    assert_eq!(dst, contents);

    pool::run({
        read(file_path).then(move |res| {
            let buf = res.unwrap();
            assert_eq!(buf, contents);
            Ok(())
        })
    });
}

#[test]
fn metadata() {
    let dir = TmpBuilder::new()
        .prefix("tokio-fs-tests")
        .tempdir()
        .unwrap();
    let file_path = dir.path().join("metadata.txt");

    pool::run({
        let file_path = file_path.clone();
        let file_path2 = file_path.clone();
        let file_path3 = file_path.clone();

        tokio_fs::metadata(file_path)
            .then(|r| {
                let _ = r.err().unwrap();
                Ok(())
            })
            .and_then(|_| File::create(file_path2))
            .and_then(|_| tokio_fs::metadata(file_path3))
            .then(|r| {
                assert!(r.unwrap().is_file());
                Ok(())
            })
    });
}

#[test]
fn seek() {
    let dir = TmpBuilder::new()
        .prefix("tokio-fs-tests")
        .tempdir()
        .unwrap();
    let file_path = dir.path().join("seek.txt");

    pool::run({
        OpenOptions::new()
            .create(true)
            .read(true)
            .write(true)
            .open(file_path)
            .and_then(|file| io::write_all(file, "Hello, world!"))
            .and_then(|(file, _)| file.seek(SeekFrom::End(-6)))
            .and_then(|(file, _)| io::read_exact(file, vec![0; 5]))
            .and_then(|(file, buf)| {
                assert_eq!(buf, b"world");
                file.seek(SeekFrom::Start(0))
            })
            .and_then(|(file, _)| io::read_exact(file, vec![0; 5]))
            .and_then(|(_, buf)| {
                assert_eq!(buf, b"Hello");
                Ok(())
            })
            .then(|r| {
                let _ = r.unwrap();
                Ok(())
            })
    });
}

#[test]
fn clone() {
    use std::io::prelude::*;

    let dir = TmpBuilder::new()
        .prefix("tokio-fs-tests")
        .tempdir()
        .unwrap();
    let file_path = dir.path().join("clone.txt");

    pool::run(
        File::create(file_path.clone())
            .and_then(|file| {
                file.try_clone()
                    .map_err(|(_file, err)| err)
                    .and_then(|(file, clone)| {
                        io::write_all(file, "clone ")
                            .and_then(|_| io::write_all(clone, "successful"))
                    })
            })
            .then(|res| {
                let _ = res.unwrap();
                Ok(())
            }),
    );

    let mut file = std::fs::File::open(&file_path).unwrap();

    let mut dst = vec![];
    file.read_to_end(&mut dst).unwrap();

    assert_eq!(dst, b"clone successful")
}
<|endoftext|>


<commit_before>#![stable(feature = "rust1", since = "1.0.0")]
#![allow(missing_docs)]

pub mod ffi;
pub mod fs;
pub mod io;
pub mod process;
pub mod raw;

#[stable(feature = "rust1", since = "1.0.0")]
pub mod prelude {
    #[doc(no_inline)]
    #[stable(feature = "rust1", since = "1.0.0")]
    pub use super::ffi::{OsStrExt, OsStringExt};
    #[doc(no_inline)]
    #[stable(feature = "rust1", since = "1.0.0")]
    pub use super::fs::{FileTypeExt, MetadataExt, OpenOptionsExt, PermissionsExt};
    #[doc(no_inline)]
    #[stable(feature = "rust1", since = "1.0.0")]
    pub use super::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
}
<commit_msg>Rollup merge of #67041 - Wind-River:master_base, r=alexcrichton<commit_after>#![stable(feature = "rust1", since = "1.0.0")]
#![allow(missing_docs)]

pub mod ffi;
pub mod fs;
pub mod io;
pub mod process;
pub mod raw;

#[stable(feature = "rust1", since = "1.0.0")]
pub mod prelude {
    #[doc(no_inline)]
    #[stable(feature = "rust1", since = "1.0.0")]
    pub use super::ffi::{OsStrExt, OsStringExt};
    #[doc(no_inline)]
    #[stable(feature = "rust1", since = "1.0.0")]
    pub use super::fs::{FileTypeExt, MetadataExt, OpenOptionsExt, PermissionsExt};
    #[doc(no_inline)]
    #[stable(feature = "rust1", since = "1.0.0")]
    pub use super::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
    #[doc(no_inline)]
    #[stable(feature = "rust1", since = "1.0.0")]
    pub use super::process::ExitStatusExt;
}
<|endoftext|>


<commit_before>extern crate cargo;
extern crate reqwest;
#[macro_use]
extern crate lazy_static;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
extern crate toml;

use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::collections::btree_map::Entry;
use std::fs::File;
use std::io::{Read, Write};

use cargo::core::{Dependency, Package, Source, SourceId};
use cargo::core::registry::PackageRegistry;
use cargo::core::resolver::{self, Method, Resolve};
use cargo::sources::RegistrySource;
use cargo::util::Config;

/// The list of crates from crates.io
#[derive(Debug, Deserialize)]
struct TopCrates {
    crates: Vec<Crate>,
}

/// A single crate from crates.io
#[derive(Debug, Deserialize)]
struct OneCrate {
    #[serde(rename="crate")]
    krate: Crate,
}

/// The shared description of a crate
#[derive(Debug, Deserialize)]
struct Crate {
    #[serde(rename="id")]
    name: String,
}

/// A Cargo.toml file.
#[derive(Serialize)]
struct TomlManifest {
    package: TomlPackage,
    profile: Profiles,
    #[serde(serialize_with = "toml::ser::tables_last")]
    dependencies: BTreeMap<String, DependencySpec>,
}

/// Header of Cargo.toml file.
#[derive(Serialize)]
struct TomlPackage {
    name: String,
    version: String,
    authors: Vec<String>,
}

/// A mapping of a crates name to its identifier used in source code
#[derive(Debug, Serialize)]
struct CrateInformation {
    name: String,
    version: String,
    id: String,
}

/// Hand-curated changes to the crate list
#[derive(Debug, Deserialize)]
struct Modifications {
    #[serde(default)]
    blacklist: Vec<String>,
    #[serde(default)]
    additions: BTreeSet<String>,
}

/// A profile section in a Cargo.toml file
#[derive(Serialize)]
#[serde(rename_all="kebab-case")]
struct Profile {
    codegen_units: u32,
    incremental: bool,
}

/// Available profile types
#[derive(Serialize)]
struct Profiles {
    dev: Profile,
    release: Profile,
}

/// `"1.0.0"` or `{ version = "1.0.0", features = ["..."] }`
#[derive(Serialize, Clone)]
#[serde(untagged)]
enum DependencySpec {
    String(String),
    #[serde(rename_all = "kebab-case")]
    Explicit {
        version: String,
        #[serde(skip_serializing_if = "Vec::is_empty")]
        features: Vec<String>,
        #[serde(skip_serializing_if = "is_true")]
        default_features: bool,
    }
}

impl DependencySpec {
    fn version(&self) -> String {
        match *self {
            DependencySpec::String(ref version) |
            DependencySpec::Explicit { ref version, .. } => version.clone(),
        }
    }
}

fn is_true(b: &bool) -> bool {
    *b
}

impl Modifications {
    fn blacklisted(&self, name: &str) -> bool {
        self.blacklist.iter().any(|n| n == name)
    }
}

lazy_static! {
    static ref MODIFICATIONS: Modifications = {
        let mut f = File::open("crate-modifications.toml")
            .expect("unable to open crate modifications file");

        let mut d = Vec::new();
        f.read_to_end(&mut d)
            .expect("unable to read crate modifications file");

        toml::from_slice(&d)
            .expect("unable to parse crate modifications file")
    };
}

impl TopCrates {
    /// List top 100 crates by number of downloads on crates.io.
    fn download() -> TopCrates {
        let resp =
            reqwest::get("https://crates.io/api/v1/crates?page=1&per_page=100&sort=downloads")
            .expect("Could not fetch top crates");
        assert!(resp.status().is_success());

        serde_json::from_reader(resp).expect("Invalid JSON")
    }

    /// Add crates that have been hand-picked
    fn add_curated_crates(&mut self) {
        self.crates.extend({
            MODIFICATIONS
                .additions
                .iter()
                .cloned()
                .map(|name| Crate { name })
        });
    }
}

fn unique_latest_crates(resolve: Resolve) -> BTreeMap<String, DependencySpec> {
    let mut uniqs = BTreeMap::new();
    for pkg in resolve.iter() {
        if MODIFICATIONS.blacklisted(pkg.name().as_str()) {
            continue;
        }

        match uniqs.entry(pkg.name()) {
            Entry::Vacant(entry) => {
                // First time seeing this package.
                entry.insert(pkg.version());
            }
            Entry::Occupied(mut entry) => {
                // Seen before, keep the newest version.
                if &pkg.version() > entry.get() {
                    entry.insert(pkg.version());
                }
            }
        }
    }

    uniqs.into_iter()
         .map(|(name, version)| (
             name.to_string(),
             DependencySpec::String(version.to_string()),
         ))
         .collect()
}

/// Updates `dep` to include features specified by the custom metadata of `pkg`.
///
/// Our custom metadata format looks like:
///
///     [package.metadata.playground]
///     default-features = true
///     features = ["std", "extra-traits"]
///     all-features = false
///
/// All fields are optional.
fn fill_playground_metadata_features(dep: &mut DependencySpec, pkg: &Package) {
    let custom_metadata = match pkg.manifest().custom_metadata() {
        Some(custom_metadata) => custom_metadata,
        None => return,
    };

    let playground_metadata = match custom_metadata.get("playground") {
        Some(playground_metadata) => playground_metadata,
        None => return,
    };

    #[derive(Deserialize)]
    #[serde(default, rename_all = "kebab-case")]
    struct Metadata {
        features: Vec<String>,
        default_features: bool,
        all_features: bool,
    }

    impl Default for Metadata {
        fn default() -> Self {
            Metadata {
                features: Vec::new(),
                default_features: true,
                all_features: false,
            }
        }
    }

    let metadata = match playground_metadata.clone().try_into::<Metadata>() {
        Ok(metadata) => metadata,
        Err(err) => {
            eprintln!(
                "Failed to parse custom metadata for {} {}: {}",
                pkg.name(), pkg.version(), err);
            return;
        }
    };

    // If `all-features` is set then we ignore `features`.
    let summary = pkg.summary();
    let mut enabled_features: BTreeSet<String> = if metadata.all_features {
        summary.features().keys().cloned().collect()
    } else {
        metadata.features.into_iter().collect()
    };

    // If not opting out of default features, remove default features from the
    // explicit features list. This avoids ongoing spurious diffs in our
    // generated Cargo.toml as default features are added to a library.
    if metadata.default_features {
        if let Some(default_feature_names) = summary.features().get("default") {
            enabled_features.remove("default");
            for feature in default_feature_names {
                enabled_features.remove(&feature.to_string(summary));
            }
        }
    }

    if !enabled_features.is_empty() || !metadata.default_features {
        *dep = DependencySpec::Explicit {
            version: dep.version(),
            features: enabled_features.into_iter().collect(),
            default_features: metadata.default_features,
        };
    }
}

fn write_manifest(manifest: TomlManifest, path: &str) {
    let mut f = File::create(path).expect("Unable to create Cargo.toml");
    let content = toml::to_vec(&manifest).expect("Couldn't serialize TOML");
    f.write_all(&content).expect("Couldn't write Cargo.toml");
}

fn main() {
    // Setup to interact with cargo.
    let config = Config::default().expect("Unable to create default Cargo config");
    let crates_io = SourceId::crates_io(&config).expect("Unable to create crates.io source ID");
    let mut source = RegistrySource::remote(&crates_io, &config);
    source.update().expect("Unable to update registry");

    let mut top = TopCrates::download();
    top.add_curated_crates();

    // Find the newest (non-prerelease, non-yanked) versions of all
    // the interesting crates.
    let mut summaries = Vec::new();
    for Crate { ref name } in top.crates {
        if MODIFICATIONS.blacklisted(name) {
            continue;
        }

        // Query the registry for a summary of this crate.
        // Usefully, this doesn't seem to include yanked versions
        let dep = Dependency::parse_no_deprecated(name, None, &crates_io)
            .unwrap_or_else(|e| panic!("Unable to parse dependency for {}: {}", name, e));

        let matches = source.query_vec(&dep).unwrap_or_else(|e| {
            panic!("Unable to query registry for {}: {}", name, e);
        });

        // Find the newest non-prelease version
        let summary = matches.into_iter()
            .filter(|summary| !summary.version().is_prerelease())
            .max_by_key(|summary| summary.version().clone())
            .unwrap_or_else(|| panic!("Registry has no viable versions of {}", name));

        // Add a dependency on this crate.
        summaries.push((summary, Method::Required {
            dev_deps: false,
            features: &[],
            uses_default_features: true,
            all_features: false,
        }));
    }

    // Resolve transitive dependencies.
    let mut registry = PackageRegistry::new(&config)
        .expect("Unable to create package registry");
    registry.lock_patches();
    let try_to_use = HashSet::new();
    let res = resolver::resolve(&summaries, &[], &mut registry, &try_to_use, None, true)
        .expect("Unable to resolve dependencies");

    // Construct playground's Cargo.toml.
    let mut unique_latest_crates = unique_latest_crates(res);

    let mut infos = Vec::new();

    for (name, spec) in &mut unique_latest_crates {
        let version = spec.version();

        let pkgid = cargo::core::PackageId::new(&name, &version, &crates_io)
            .unwrap_or_else(|e| panic!("Unable to build PackageId for {} {}: {}", name, version, e));

        let pkg = source.download(&pkgid)
            .unwrap_or_else(|e| panic!("Unable to download {} {}: {}", name, version, e));

        for target in pkg.targets() {
            if let cargo::core::TargetKind::Lib(_) = *target.kind() {
                infos.push(CrateInformation {
                    name: name.clone(),
                    version: version.clone(),
                    id: target.crate_name()
                })
            }
        }

        fill_playground_metadata_features(spec, &pkg);
    }

    let manifest = TomlManifest {
        package: TomlPackage {
            name: "playground".to_owned(),
            version: "0.0.1".to_owned(),
            authors: vec!["The Rust Playground".to_owned()],
        },
        profile: Profiles {
            dev: Profile { codegen_units: 1, incremental: false },
            release: Profile { codegen_units: 1, incremental: false },
        },
        dependencies: unique_latest_crates,
    };

    // Write manifest file.
    let cargo_toml = "../compiler/base/Cargo.toml";
    write_manifest(manifest, cargo_toml);
    println!("wrote {}", cargo_toml);

    let path = "../compiler/base/crate-information.json";
    let mut f = File::create(path)
        .unwrap_or_else(|e| panic!("Unable to create {}: {}", path, e));
    serde_json::to_writer_pretty(&mut f, &infos)
        .unwrap_or_else(|e| panic!("Unable to write {}: {}", path, e));
    println!("Wrote {}", path);
}
<commit_msg>Use exact versions in the Cargo.toml<commit_after>extern crate cargo;
extern crate reqwest;
#[macro_use]
extern crate lazy_static;
extern crate serde;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
extern crate toml;

use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::collections::btree_map::Entry;
use std::fs::File;
use std::io::{Read, Write};

use cargo::core::{Dependency, Package, Source, SourceId};
use cargo::core::registry::PackageRegistry;
use cargo::core::resolver::{self, Method, Resolve};
use cargo::sources::RegistrySource;
use cargo::util::Config;

use serde::Serialize;

/// The list of crates from crates.io
#[derive(Debug, Deserialize)]
struct TopCrates {
    crates: Vec<Crate>,
}

/// A single crate from crates.io
#[derive(Debug, Deserialize)]
struct OneCrate {
    #[serde(rename="crate")]
    krate: Crate,
}

/// The shared description of a crate
#[derive(Debug, Deserialize)]
struct Crate {
    #[serde(rename="id")]
    name: String,
}

/// A Cargo.toml file.
#[derive(Serialize)]
struct TomlManifest {
    package: TomlPackage,
    profile: Profiles,
    #[serde(serialize_with = "toml::ser::tables_last")]
    dependencies: BTreeMap<String, DependencySpec>,
}

/// Header of Cargo.toml file.
#[derive(Serialize)]
struct TomlPackage {
    name: String,
    version: String,
    authors: Vec<String>,
}

/// A mapping of a crates name to its identifier used in source code
#[derive(Debug, Serialize)]
struct CrateInformation {
    name: String,
    version: String,
    id: String,
}

/// Hand-curated changes to the crate list
#[derive(Debug, Deserialize)]
struct Modifications {
    #[serde(default)]
    blacklist: Vec<String>,
    #[serde(default)]
    additions: BTreeSet<String>,
}

/// A profile section in a Cargo.toml file
#[derive(Serialize)]
#[serde(rename_all="kebab-case")]
struct Profile {
    codegen_units: u32,
    incremental: bool,
}

/// Available profile types
#[derive(Serialize)]
struct Profiles {
    dev: Profile,
    release: Profile,
}

/// `"1.0.0"` or `{ version = "1.0.0", features = ["..."] }`
#[derive(Serialize, Clone)]
#[serde(untagged)]
enum DependencySpec {
    #[serde(serialize_with = "exact_version")]
    String(String),
    #[serde(rename_all = "kebab-case")]
    Explicit {
        #[serde(serialize_with = "exact_version")]
        version: String,
        #[serde(skip_serializing_if = "Vec::is_empty")]
        features: Vec<String>,
        #[serde(skip_serializing_if = "is_true")]
        default_features: bool,
    }
}

impl DependencySpec {
    fn version(&self) -> String {
        match *self {
            DependencySpec::String(ref version) |
            DependencySpec::Explicit { ref version, .. } => version.clone(),
        }
    }
}

fn exact_version<S>(version: &String, serializer: S) -> Result<S::Ok, S::Error>
where
    S: serde::Serializer,
{
    format!("={}", version).serialize(serializer)
}

fn is_true(b: &bool) -> bool {
    *b
}

impl Modifications {
    fn blacklisted(&self, name: &str) -> bool {
        self.blacklist.iter().any(|n| n == name)
    }
}

lazy_static! {
    static ref MODIFICATIONS: Modifications = {
        let mut f = File::open("crate-modifications.toml")
            .expect("unable to open crate modifications file");

        let mut d = Vec::new();
        f.read_to_end(&mut d)
            .expect("unable to read crate modifications file");

        toml::from_slice(&d)
            .expect("unable to parse crate modifications file")
    };
}

impl TopCrates {
    /// List top 100 crates by number of downloads on crates.io.
    fn download() -> TopCrates {
        let resp =
            reqwest::get("https://crates.io/api/v1/crates?page=1&per_page=100&sort=downloads")
            .expect("Could not fetch top crates");
        assert!(resp.status().is_success());

        serde_json::from_reader(resp).expect("Invalid JSON")
    }

    /// Add crates that have been hand-picked
    fn add_curated_crates(&mut self) {
        self.crates.extend({
            MODIFICATIONS
                .additions
                .iter()
                .cloned()
                .map(|name| Crate { name })
        });
    }
}

fn unique_latest_crates(resolve: Resolve) -> BTreeMap<String, DependencySpec> {
    let mut uniqs = BTreeMap::new();
    for pkg in resolve.iter() {
        if MODIFICATIONS.blacklisted(pkg.name().as_str()) {
            continue;
        }

        match uniqs.entry(pkg.name()) {
            Entry::Vacant(entry) => {
                // First time seeing this package.
                entry.insert(pkg.version());
            }
            Entry::Occupied(mut entry) => {
                // Seen before, keep the newest version.
                if &pkg.version() > entry.get() {
                    entry.insert(pkg.version());
                }
            }
        }
    }

    uniqs.into_iter()
         .map(|(name, version)| (
             name.to_string(),
             DependencySpec::String(version.to_string()),
         ))
         .collect()
}

/// Updates `dep` to include features specified by the custom metadata of `pkg`.
///
/// Our custom metadata format looks like:
///
///     [package.metadata.playground]
///     default-features = true
///     features = ["std", "extra-traits"]
///     all-features = false
///
/// All fields are optional.
fn fill_playground_metadata_features(dep: &mut DependencySpec, pkg: &Package) {
    let custom_metadata = match pkg.manifest().custom_metadata() {
        Some(custom_metadata) => custom_metadata,
        None => return,
    };

    let playground_metadata = match custom_metadata.get("playground") {
        Some(playground_metadata) => playground_metadata,
        None => return,
    };

    #[derive(Deserialize)]
    #[serde(default, rename_all = "kebab-case")]
    struct Metadata {
        features: Vec<String>,
        default_features: bool,
        all_features: bool,
    }

    impl Default for Metadata {
        fn default() -> Self {
            Metadata {
                features: Vec::new(),
                default_features: true,
                all_features: false,
            }
        }
    }

    let metadata = match playground_metadata.clone().try_into::<Metadata>() {
        Ok(metadata) => metadata,
        Err(err) => {
            eprintln!(
                "Failed to parse custom metadata for {} {}: {}",
                pkg.name(), pkg.version(), err);
            return;
        }
    };

    // If `all-features` is set then we ignore `features`.
    let summary = pkg.summary();
    let mut enabled_features: BTreeSet<String> = if metadata.all_features {
        summary.features().keys().cloned().collect()
    } else {
        metadata.features.into_iter().collect()
    };

    // If not opting out of default features, remove default features from the
    // explicit features list. This avoids ongoing spurious diffs in our
    // generated Cargo.toml as default features are added to a library.
    if metadata.default_features {
        if let Some(default_feature_names) = summary.features().get("default") {
            enabled_features.remove("default");
            for feature in default_feature_names {
                enabled_features.remove(&feature.to_string(summary));
            }
        }
    }

    if !enabled_features.is_empty() || !metadata.default_features {
        *dep = DependencySpec::Explicit {
            version: dep.version(),
            features: enabled_features.into_iter().collect(),
            default_features: metadata.default_features,
        };
    }
}

fn write_manifest(manifest: TomlManifest, path: &str) {
    let mut f = File::create(path).expect("Unable to create Cargo.toml");
    let content = toml::to_vec(&manifest).expect("Couldn't serialize TOML");
    f.write_all(&content).expect("Couldn't write Cargo.toml");
}

fn main() {
    // Setup to interact with cargo.
    let config = Config::default().expect("Unable to create default Cargo config");
    let crates_io = SourceId::crates_io(&config).expect("Unable to create crates.io source ID");
    let mut source = RegistrySource::remote(&crates_io, &config);
    source.update().expect("Unable to update registry");

    let mut top = TopCrates::download();
    top.add_curated_crates();

    // Find the newest (non-prerelease, non-yanked) versions of all
    // the interesting crates.
    let mut summaries = Vec::new();
    for Crate { ref name } in top.crates {
        if MODIFICATIONS.blacklisted(name) {
            continue;
        }

        // Query the registry for a summary of this crate.
        // Usefully, this doesn't seem to include yanked versions
        let dep = Dependency::parse_no_deprecated(name, None, &crates_io)
            .unwrap_or_else(|e| panic!("Unable to parse dependency for {}: {}", name, e));

        let matches = source.query_vec(&dep).unwrap_or_else(|e| {
            panic!("Unable to query registry for {}: {}", name, e);
        });

        // Find the newest non-prelease version
        let summary = matches.into_iter()
            .filter(|summary| !summary.version().is_prerelease())
            .max_by_key(|summary| summary.version().clone())
            .unwrap_or_else(|| panic!("Registry has no viable versions of {}", name));

        // Add a dependency on this crate.
        summaries.push((summary, Method::Required {
            dev_deps: false,
            features: &[],
            uses_default_features: true,
            all_features: false,
        }));
    }

    // Resolve transitive dependencies.
    let mut registry = PackageRegistry::new(&config)
        .expect("Unable to create package registry");
    registry.lock_patches();
    let try_to_use = HashSet::new();
    let res = resolver::resolve(&summaries, &[], &mut registry, &try_to_use, None, true)
        .expect("Unable to resolve dependencies");

    // Construct playground's Cargo.toml.
    let mut unique_latest_crates = unique_latest_crates(res);

    let mut infos = Vec::new();

    for (name, spec) in &mut unique_latest_crates {
        let version = spec.version();

        let pkgid = cargo::core::PackageId::new(&name, &version, &crates_io)
            .unwrap_or_else(|e| panic!("Unable to build PackageId for {} {}: {}", name, version, e));

        let pkg = source.download(&pkgid)
            .unwrap_or_else(|e| panic!("Unable to download {} {}: {}", name, version, e));

        for target in pkg.targets() {
            if let cargo::core::TargetKind::Lib(_) = *target.kind() {
                infos.push(CrateInformation {
                    name: name.clone(),
                    version: version.clone(),
                    id: target.crate_name()
                })
            }
        }

        fill_playground_metadata_features(spec, &pkg);
    }

    let manifest = TomlManifest {
        package: TomlPackage {
            name: "playground".to_owned(),
            version: "0.0.1".to_owned(),
            authors: vec!["The Rust Playground".to_owned()],
        },
        profile: Profiles {
            dev: Profile { codegen_units: 1, incremental: false },
            release: Profile { codegen_units: 1, incremental: false },
        },
        dependencies: unique_latest_crates,
    };

    // Write manifest file.
    let cargo_toml = "../compiler/base/Cargo.toml";
    write_manifest(manifest, cargo_toml);
    println!("wrote {}", cargo_toml);

    let path = "../compiler/base/crate-information.json";
    let mut f = File::create(path)
        .unwrap_or_else(|e| panic!("Unable to create {}: {}", path, e));
    serde_json::to_writer_pretty(&mut f, &infos)
        .unwrap_or_else(|e| panic!("Unable to write {}: {}", path, e));
    println!("Wrote {}", path);
}
<|endoftext|>


<commit_before>extern crate flv_toolbox_rs;

use flv_toolbox_rs::lib::{ FLVTagRead, FLVTag, FLVTagType, FLVTagWrite, format_seconds_ms };

use std::fs::File;
use std::io::Read;
use std::slice;
use std::collections::BTreeMap;

#[derive(Debug, Clone)]
pub struct FlvTimeInfo {
    pub ref_video_frame_duration: u64,
    pub ref_audio_frame_duration: u64,
    pub real_video_frame_duration: i64,
    pub real_audio_frame_duration: i64,
    pub timestamps: Vec<TagTimestamp>,
}

#[derive(Debug, Clone)]
pub enum TagTimestamp {
    Video(i64, i64),
    Audio(i64, i64),
}

fn get_info(path: &str) -> FlvTimeInfo {

    let mut file = File::open(path).unwrap();
    let file_info = file.metadata().unwrap();
    let file_len = file_info.len();
    let mut parser = FLVTagRead::new(&mut file);

    let mut ref_video_frame_duration: u64 = 0;
    let mut ref_audio_frame_duration: u64 = 0;
    let mut real_video_frame_duration: i64 = 0;
    let mut real_audio_frame_duration: i64 = 0;

    let mut last_video_tag: Option<FLVTag> = None;
    let mut video_duration_map: BTreeMap<i64, u64> = BTreeMap::new();

    let mut last_audio_tag: Option<FLVTag> = None;
    let mut audio_duration_map: BTreeMap<i64, u64> = BTreeMap::new();

    let mut timestamps: Vec<TagTimestamp> = vec![];

    loop {
        let position = parser.get_position();
        print!("progress: {: >3.0}%\r", position as f64 / file_len as f64 * 100.);
        let nxt: Option<FLVTag> = parser.next();
        if nxt.is_none() {
            break;
        }
        let tag: FLVTag = nxt.unwrap();

        match tag.get_tag_type() {
            FLVTagType::TAG_TYPE_VIDEO => {

                if last_video_tag.is_some() {
                    let time_delta = tag.get_timestamp() as i64 - last_video_tag.as_ref().unwrap().get_timestamp() as i64;
                    timestamps.push(TagTimestamp::Video(last_video_tag.as_ref().unwrap().get_timestamp() as i64, time_delta));
                    *(video_duration_map.entry(time_delta).or_insert(0)) += 1;
                }

                match tag.get_frame_type() {
                    1 => {
                        match tag.get_avc_packet_type() {
                            0 => { // AVC_PACKET_TYPE_SEQUENCE_HEADER

                            },
                            _ => { // normal keyframes
                                last_video_tag = Some(tag);
                            }
                        };
                    }
                    _ => { // normal frames
                        last_video_tag = Some(tag);
                    }
                };
            },
            FLVTagType::TAG_TYPE_AUDIO => {

                if last_audio_tag.is_some() {
                    let time_delta = tag.get_timestamp() as i64 - last_audio_tag.as_ref().unwrap().get_timestamp() as i64;
                    timestamps.push(TagTimestamp::Audio(last_audio_tag.as_ref().unwrap().get_timestamp() as i64, time_delta));
                    *(audio_duration_map.entry(time_delta).or_insert(0)) += 1;
                }

                if tag.is_acc_sequence_header() { // acc sequence header

                } else { // normal frames
                    last_audio_tag = Some(tag);
                }
            },
            FLVTagType::TAG_TYPE_SCRIPTDATAOBJECT => {

            }
        };
    }
    println!("complete!\r");

    real_video_frame_duration = top_duration(&video_duration_map);
    real_audio_frame_duration = top_duration(&audio_duration_map);

    return FlvTimeInfo {
        ref_video_frame_duration  ,
        ref_audio_frame_duration  ,
        real_video_frame_duration ,
        real_audio_frame_duration ,
        timestamps                ,
    };
}

fn top_duration(pairs: &BTreeMap<i64, u64>) -> i64 {
    let mut list: Vec<(&i64, &u64)> = pairs.iter().collect();
    match list.len() {
        0 => 0,
        1 => *(list[0].0),
        _ => {
            list.as_mut_slice().sort_by_key(|&(_, &count)| count );
            *(list.last().unwrap().0)
        },
    }
}

fn check_offset(info: &FlvTimeInfo) {
    let &FlvTimeInfo {
        ref ref_video_frame_duration  ,
        ref ref_audio_frame_duration  ,
        ref real_video_frame_duration ,
        ref real_audio_frame_duration ,
        ref timestamps                ,
    } = info;

    let mut video_time: i64 = 0;
    let mut audio_time: i64 = 0;

    for tm in timestamps.iter() {
        let mut changed = false;
        let mut current_time = 0;
        match tm {
            &TagTimestamp::Video(ref time, ref duration) => {
                current_time = *time;
                if (*duration - *real_video_frame_duration).abs() > 1 {
                    // println!("v offset {}", video_time - *time);
                    changed = true;
                    video_time += *real_video_frame_duration;
                } else {
                    video_time += *duration;
                }
            }
            &TagTimestamp::Audio(ref time, ref duration) => {
                current_time = *time;
                if (*duration - *real_audio_frame_duration).abs() > 1 {
                    // println!("a offset {}", audio_time - *time);
                    changed = true;
                    audio_time += *real_audio_frame_duration;
                } else {
                    audio_time += *duration;
                }
            }
        };
        if changed {
            println!("{}: av offset: {}", format_seconds_ms(current_time as u64), audio_time - video_time);
        }
    }
}

fn main() {
    let path = std::env::args().nth(1).unwrap();
    println!("checking flv: {}", path);
    let info = get_info(&path);
    // println!("{:?}", info);
    check_offset(&info);
}<commit_msg>update<commit_after>extern crate ffmpeg;
extern crate flv_toolbox_rs;

use flv_toolbox_rs::lib::{ FLVTagRead, FLVTag, FLVTagType, FLVTagWrite, format_seconds_ms, AudioSpecificConfig };

use std::fs::File;
use std::io::{ Read, Write };
use std::slice;
use std::collections::BTreeMap;

#[derive(Debug, Clone)]
pub struct FlvTimeInfo {
    pub ref_video_frame_duration: u64,
    pub ref_audio_frame_duration: u64,
    pub real_video_frame_duration: i64,
    pub real_audio_frame_duration: i64,
    pub timestamps: Vec<TagTimestamp>,
}

#[derive(Debug, Clone)]
pub enum TagTimestamp {
    Video(i64, i64),
    Audio(i64, i64, i64),
}

fn get_info(path: &str) -> FlvTimeInfo {

    let mut file = File::open(path).unwrap();
    let file_info = file.metadata().unwrap();
    let file_len = file_info.len();
    let mut parser = FLVTagRead::new(&mut file);

    let mut ref_video_frame_duration: u64 = 0;
    let mut ref_audio_frame_duration: u64 = 0;
    let mut real_video_frame_duration: i64 = 0;
    let mut real_audio_frame_duration: i64 = 0;

    let mut last_video_tag: Option<FLVTag> = None;
    let mut video_duration_map: BTreeMap<i64, u64> = BTreeMap::new();

    let mut last_audio_tag: Option<FLVTag> = None;
    let mut audio_duration_map: BTreeMap<i64, u64> = BTreeMap::new();
    let mut last_audio_decode_duration: i64 = 0;

    let mut timestamps: Vec<TagTimestamp> = vec![];

    // ffmpeg
    ffmpeg::init().unwrap();
    let codec = ffmpeg::decoder::find(ffmpeg::codec::id::Id::AAC).unwrap();
    let context = ffmpeg::codec::Context::new();
    let opened = context.decoder().open_as(codec).unwrap();
    let mut decoder = opened.audio().unwrap();
    // asc
    let mut asc: Option<AudioSpecificConfig> = None;

    loop {
        let position = parser.get_position();
        print!("progress: {: >3.0}%\r", position as f64 / file_len as f64 * 100.);
        let nxt: Option<FLVTag> = parser.next();
        if nxt.is_none() {
            break;
        }
        let tag: FLVTag = nxt.unwrap();

        match tag.get_tag_type() {
            FLVTagType::TAG_TYPE_VIDEO => {

                if last_video_tag.is_some() {
                    let time_delta = tag.get_timestamp() as i64 - last_video_tag.as_ref().unwrap().get_timestamp() as i64;
                    timestamps.push(TagTimestamp::Video(last_video_tag.as_ref().unwrap().get_timestamp() as i64, time_delta));
                    *(video_duration_map.entry(time_delta).or_insert(0)) += 1;
                }

                match tag.get_frame_type() {
                    1 => {
                        match tag.get_avc_packet_type() {
                            0 => { // AVC_PACKET_TYPE_SEQUENCE_HEADER

                            },
                            _ => { // normal keyframes
                                last_video_tag = Some(tag);
                            }
                        };
                    }
                    _ => { // normal frames
                        last_video_tag = Some(tag);
                    }
                };
            },
            FLVTagType::TAG_TYPE_AUDIO => {

                if last_audio_tag.is_some() {
                    let time_delta = tag.get_timestamp() as i64 - last_audio_tag.as_ref().unwrap().get_timestamp() as i64;
                    timestamps.push(TagTimestamp::Audio(last_audio_tag.as_ref().unwrap().get_timestamp() as i64, time_delta, last_audio_decode_duration));
                    *(audio_duration_map.entry(time_delta).or_insert(0)) += 1;
                }

                if tag.is_acc_sequence_header() { // acc sequence header
                    asc = Some(tag.get_sound_audio_specific_config());
                } else { // normal frames
                    // decode audio frame samples by ffmpeg
                    let mut audio_buffer: Vec<u8> = Vec::with_capacity(tag.get_sound_data_size() as usize + 7);
                    audio_buffer.write(&FLVTag::get_sound_adts_header_data(asc.as_ref().unwrap(), tag.get_sound_data_size() + 7));
                    audio_buffer.write(&tag.get_sound_data());
                    // println!("{:?}", audio_buffer);
                    // break;

                    let packet = ffmpeg::codec::packet::Packet::borrow(&audio_buffer);
                    let mut frame = ffmpeg::util::frame::Audio::empty();
                    let result = decoder.decode(&packet, &mut frame).unwrap();
                    let duration = 1000. * frame.samples() as f64 / frame.rate() as f64;
                    last_audio_decode_duration = duration as i64;
                    // println!("{:?}", (frame.channels(), frame.rate(), frame.samples(), duration));

                    last_audio_tag = Some(tag);
                }
            },
            FLVTagType::TAG_TYPE_SCRIPTDATAOBJECT => {

            }
        };
    }
    println!("complete!\r");

    real_video_frame_duration = top_duration(&video_duration_map);
    real_audio_frame_duration = top_duration(&audio_duration_map);

    return FlvTimeInfo {
        ref_video_frame_duration  ,
        ref_audio_frame_duration  ,
        real_video_frame_duration ,
        real_audio_frame_duration ,
        timestamps                ,
    };
}

fn top_duration(pairs: &BTreeMap<i64, u64>) -> i64 {
    let mut list: Vec<(&i64, &u64)> = pairs.iter().collect();
    match list.len() {
        0 => 0,
        1 => *(list[0].0),
        _ => {
            list.as_mut_slice().sort_by_key(|&(_, &count)| count );
            *(list.last().unwrap().0)
        },
    }
}

fn check_offset(info: &FlvTimeInfo) {
    let &FlvTimeInfo {
        ref ref_video_frame_duration  ,
        ref ref_audio_frame_duration  ,
        ref real_video_frame_duration ,
        ref real_audio_frame_duration ,
        ref timestamps                ,
    } = info;

    let mut video_time: i64 = 0;
    let mut audio_time: i64 = 0;
    let mut current_time = 0;

    for tm in timestamps.iter() {
        let mut changed = false;
        match tm {
            &TagTimestamp::Video(ref time, ref duration) => {
                // current_time = *time;
                if (*duration - *real_video_frame_duration).abs() > 1 {
                    // println!("v offset {}", video_time - *time);
                    // changed = true;
                    video_time += *real_video_frame_duration;
                } else {
                    video_time += *duration;
                }
            }
            &TagTimestamp::Audio(ref time, ref duration, ref decode_duration) => {
                current_time = *time;
                if (*duration - *decode_duration).abs() > 1 {
                    // println!("a offset {}", audio_time - *time);
                    changed = true;
                    audio_time += *decode_duration;
                } else {
                    audio_time += *decode_duration;
                }
            }
        };
        if changed {
            println!("{}: av offset: {} {} {}", format_seconds_ms(current_time as u64), audio_time - video_time, audio_time - current_time, video_time - current_time);
        }
    }
}

fn main() {
    let path = std::env::args().nth(1).unwrap();
    println!("checking flv: {}", path);
    let info = get_info(&path);
    // println!("{:?}", info);
    check_offset(&info);
}<|endoftext|>


<commit_before>#![crate_name="rustspec_assertions"]
#![crate_type="dylib"]
#![feature(macro_rules, plugin_registrar)]

extern crate syntax;
extern crate rustc;

pub use matchers::equals::Equals;
pub use matchers::equals::eq;
pub use matchers::gt::Gt;
pub use matchers::gt::be_gt;
pub use matchers::ge::Ge;
pub use matchers::ge::be_ge;
pub use matchers::lt::Lt;
pub use matchers::lt::be_lt;
pub use matchers::le::Le;
pub use matchers::le::be_le;
pub use expect::expect;

use syntax::ext::base;
use syntax::ext::base::{ExtCtxt, MacResult};
use syntax::codemap::{Span};
use syntax::ast::TokenTree;
use rustc::plugin::Registry;
use syntax::ext::build::AstBuilder;

#[plugin_registrar]
pub fn plugin_registrar(registry: &mut Registry) {
    registry.register_macro("expand_line", expand_line);
}

// report the appropiate line when assertion is summoned inside a macro.
pub fn expand_line(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult> {
    base::check_zero_tts(cx, sp, tts, "expand_line!");

    let topmost = cx.backtrace().unwrap()
        .call_site.expn_info.unwrap();

    let loc = cx.codemap().lookup_char_pos(topmost.call_site.lo);

    base::MacExpr::new(cx.expr_uint(topmost.call_site, loc.line))
}

pub mod expect;
mod matchers {
    pub mod equals;
    pub mod gt;
    pub mod ge;
    pub mod lt;
    pub mod le;
    pub mod matcher;
}
<commit_msg>go back to using line! until parser issue is resolved<commit_after>#![crate_name="rustspec_assertions"]
#![crate_type="dylib"]
#![feature(macro_rules, plugin_registrar)]

extern crate syntax;
extern crate rustc;

pub use matchers::equals::Equals;
pub use matchers::equals::eq;
pub use matchers::gt::Gt;
pub use matchers::gt::be_gt;
pub use matchers::ge::Ge;
pub use matchers::ge::be_ge;
pub use matchers::lt::Lt;
pub use matchers::lt::be_lt;
pub use matchers::le::Le;
pub use matchers::le::be_le;
pub use expect::expect;

// use syntax::ext::base;
// use syntax::ext::base::{ExtCtxt, MacResult};
// use syntax::codemap::{Span};
// use syntax::ast::TokenTree;
// use rustc::plugin::Registry;
// use syntax::ext::build::AstBuilder;

// #[plugin_registrar]
// pub fn plugin_registrar(registry: &mut Registry) {
//     registry.register_macro("expand_line", expand_line);
// }

// // report the appropiate line when assertion is summoned inside a macro.
// pub fn expand_line(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult> {
//     base::check_zero_tts(cx, sp, tts, "expand_line!");

//     let topmost = cx.backtrace().unwrap()
//         .call_site.expn_info.unwrap();

//     let loc = cx.codemap().lookup_char_pos(topmost.call_site.lo);

//     base::MacExpr::new(cx.expr_uint(topmost.call_site, loc.line))
// }

#[macro_export]
pub macro_rules! expand_line(
    () => (
        line!()
    );
)

pub mod expect;
mod matchers {
    pub mod equals;
    pub mod gt;
    pub mod ge;
    pub mod lt;
    pub mod le;
    pub mod matcher;
}
<|endoftext|>