<commit_before>use std::cmp;

use ctxt::SemContext;

use gc::root::IndirectObj;
use gc::swiper::card::{CardEntry, CardTable};
use gc::swiper::crossing::{CrossingEntry, CrossingMap};
use gc::swiper::in_kilo;
use gc::swiper::large::LargeSpace;
use gc::swiper::old::OldGen;
use gc::swiper::young::YoungGen;
use gc::swiper::{CardIdx, Region, CARD_SIZE};
use gc::Address;

use mem;
use object::Obj;
use timer::{in_ms, Timer};

pub struct MinorCollector<'a, 'ast: 'a> {
    ctxt: &'a SemContext<'ast>,
    young: &'a YoungGen,
    old: &'a OldGen,
    large: &'a LargeSpace,
    rootset: &'a [IndirectObj],
    card_table: &'a CardTable,
    crossing_map: &'a CrossingMap,

    free: Address,
    promotion_failed: bool,
    promoted_size: usize,
    young_from: Region,
}

impl<'a, 'ast> MinorCollector<'a, 'ast> {
    pub fn new(
        ctxt: &'a SemContext<'ast>,
        young: &'a YoungGen,
        old: &'a OldGen,
        large: &'a LargeSpace,
        card_table: &'a CardTable,
        crossing_map: &'a CrossingMap,
        rootset: &'a [IndirectObj],
    ) -> MinorCollector<'a, 'ast> {
        MinorCollector {
            ctxt: ctxt,
            young: young,
            old: old,
            large: large,
            rootset: rootset,
            card_table: card_table,
            crossing_map: crossing_map,
            free: Address::null(),
            promotion_failed: false,
            promoted_size: 0,
            young_from: young.from_space(),
        }
    }

    pub fn collect(&mut self) {
        let mut timer = Timer::new(self.ctxt.args.flag_gc_verbose);
        let init_size = self.heap_size();
        let young_init_size = self.young.used_region().size();
        self.free = self.young.to_space().start;

        self.young.unprotect_to_space();

        self.visit_roots();
        self.copy_dirty_cards();
        // self.visit_large_objects();
        self.visit_copied_objects();
        self.young.swap_spaces(self.free);

        self.young.protect_to_space();

        timer.stop_with(|dur| {
            let new_size = self.heap_size();
            let young_new_size = self.young.used_region().size();
            let garbage = young_init_size - young_new_size - self.promoted_size;
            let garbage_ratio = (garbage as f64 / young_init_size as f64) * 100f64;

            println!(
                "GC: Minor GC ({:.2} ms, {:.1}K->{:.1}K, young {:.1}K->{:.1}K, \
                 {:.1}K promoted, {:.1}K/{:.0}% garbage)",
                in_ms(dur),
                in_kilo(init_size),
                in_kilo(new_size),
                in_kilo(young_init_size),
                in_kilo(young_new_size),
                in_kilo(self.promoted_size),
                in_kilo(garbage),
                garbage_ratio
            );
        });
    }

    fn heap_size(&self) -> usize {
        self.young.used_region().size() + self.old.used_region().size()
    }

    fn visit_roots(&mut self) {
        // detect all references from roots into young generation
        for &root in self.rootset {
            let root_ptr = root.get();

            if self.young.contains(Address::from_ptr(root_ptr)) {
                root.set(self.copy(root_ptr));
            }
        }
    }

    fn visit_large_objects(&mut self) {
        self.large.visit_objects(|addr| {
            let object = unsafe { &mut *addr.to_mut_ptr::<Obj>() };

            if object.is_array_ref() {
                self.visit_large_object_array(object, addr);

            } else {
                self.visit_large_object(object, addr);

            }
        })
    }

    fn visit_large_object_array(&mut self, object: &mut Obj, object_start: Address) {
        let object_end = object_start.offset(object.size() as usize);
        let (start_card_idx, end_card_idx) = self.card_table.card_indices(object_start, object_end);

        for card_idx in start_card_idx..end_card_idx {
            let card_idx = card_idx.into();

            if self.card_table.get(card_idx).is_clean() {
                continue;
            }

            let card_start = self.card_table.to_address(card_idx);
            let card_end = card_start.offset(CARD_SIZE);
            let end = cmp::min(card_end, object_end);

            self.copy_card(card_idx, card_start, end, false);
        }
    }

    fn visit_large_object(&mut self, object: &mut Obj, object_start: Address) {
        if !self.card_table.is_dirty(object_start) { return; }

        object.visit_reference_fields(|field| {
            let field_ptr = field.get();

            if self.young.contains(Address::from_ptr(field_ptr)) {
                field.set(self.copy(field_ptr));
            }
        });
    }

    fn visit_copied_objects(&mut self) {
        let mut scan = self.young.to_space().start;

        // visit all fields in copied objects
        while scan < self.free {
            let object = unsafe { &mut *scan.to_mut_ptr::<Obj>() };

            object.visit_reference_fields(|field| {
                let field_ptr = field.get();

                if self.young.contains(Address::from_ptr(field_ptr)) {
                    field.set(self.copy(field_ptr));
                }
            });

            scan = scan.offset(object.size());
        }
    }

    // copy all references from old- into young-generation.
    fn copy_dirty_cards(&mut self) {
        self.card_table.visit_dirty_in_old(self.old.free(), |card| {
            let crossing_entry = self.crossing_map.get(card);
            let card_start = self.card_table.to_address(card);
            let card_end = card_start.offset(CARD_SIZE);

            match crossing_entry {
                CrossingEntry::NoRefs => panic!("card dirty without any refs"),
                CrossingEntry::LeadingRefs(refs) => {
                    let mut ptr = card_start;
                    let mut ref_to_young_gen = false;

                    for _ in 0..refs {
                        let ind_ptr = IndirectObj::from_address(ptr);
                        let dir_ptr = ind_ptr.get();

                        if self.young_from.contains(Address::from_ptr(dir_ptr)) {
                            let copied_obj = self.copy(dir_ptr);
                            ind_ptr.set(copied_obj);

                            if self.young.contains(Address::from_ptr(copied_obj)) {
                                ref_to_young_gen = true;
                            }
                        }

                        ptr = ptr.offset(mem::ptr_width() as usize);
                    }

                    // copy all objects from this card
                    self.copy_card(card, ptr, card_end, ref_to_young_gen);
                }

                CrossingEntry::FirstObject(offset) => {
                    let ptr = card_start.offset(offset as usize * mem::ptr_width_usize());

                    // copy all objects from this card
                    self.copy_card(card, ptr, card_end, false);
                }

                CrossingEntry::ArrayStart(offset) => {
                    let ptr = card_start.to_usize() - (offset as usize * mem::ptr_width_usize());

                    // copy all objects from this card
                    self.copy_card(card, ptr.into(), card_end, false);
                }
            }
        });
    }

    fn copy_card(
        &mut self,
        card: CardIdx,
        mut ptr: Address,
        card_end: Address,
        mut ref_to_young_gen: bool,
    ) {
        let old_end: Address = self.old.free();
        let mut end = cmp::min(card_end, old_end);

        loop {
            while ptr < end {
                let object = unsafe { &mut *ptr.to_mut_ptr::<Obj>() };

                object.visit_reference_fields_within(end, |field| {
                    let field_ptr = field.get();

                    if self.young_from.contains(Address::from_ptr(field_ptr)) {
                        let copied_obj = self.copy(field_ptr);
                        field.set(copied_obj);

                        // determine if copied object is still in young generation
                        if self.young.contains(Address::from_ptr(copied_obj)) {
                            ref_to_young_gen = true;
                        }
                    }
                });

                ptr = ptr.offset(object.size());
            }

            // if we are in the last card of the old generation, promoted objects
            // will increase `end` towards `card_end`. Those newly promoted objects
            // need also be handled.
            if end == card_end {
                break;
            }

            let old_end = self.old.free();
            let next_end = cmp::min(card_end, old_end);

            if end != next_end {
                end = next_end;
            } else {
                break;
            }
        }

        // if there are no references to the young generation in this card,
        // set the card to clean.
        if !ref_to_young_gen {
            self.card_table.set(card, CardEntry::Clean);
        }
    }

    fn copy(&mut self, obj: *mut Obj) -> *mut Obj {
        let obj_addr = Address::from_ptr(obj);
        let obj = unsafe { &mut *obj };

        if let Some(fwd) = obj.header().forwarded() {
            return fwd.to_mut_ptr();
        }

        let obj_size = obj.size();
        debug_assert!(
            self.young_from.contains(obj_addr),
            "copy objects only from from-space."
        );

        // if object is old enough we copy it into the old generation
        if self.young.should_be_promoted(obj_addr) {
            let copy_addr = self.old.alloc(obj_size, obj.is_array_ref());

            // if there isn't enough space in old gen keep it in the
            // young generation for now
            if copy_addr.is_null() {
                self.promotion_failed = true;
            } else {
                self.promote_object(obj, copy_addr, obj_size);
                return copy_addr.to_mut_ptr();
            }
        }

        let copy_addr = self.free;
        self.free = copy_addr.offset(obj_size);

        obj.copy_to(copy_addr, obj_size);
        obj.header_mut().forward_to(copy_addr);

        copy_addr.to_mut_ptr()
    }

    fn promote_object(&mut self, obj: &mut Obj, copy_addr: Address, obj_size: usize) {
        obj.copy_to(copy_addr, obj_size);
        self.promoted_size += obj_size;

        // Promoted object can have references to the young generation.
        // Set the card table entry to dirty if this is the case.
        self.handle_promoted_object(copy_addr);

        obj.header_mut().forward_to(copy_addr);
    }

    fn handle_promoted_object(&mut self, addr: Address) {
        let card_idx = self.card_table.card_idx(addr);

        // card is already dirty, nothing left to do.
        if self.card_table.get(card_idx).is_dirty() {
            return;
        }

        let object: &mut Obj = unsafe { &mut *addr.to_mut_ptr() };
        let mut old_to_young_ref = false;

        object.visit_reference_fields(|field| {
            let field_ptr = field.get();

            if self.young.contains(Address::from_ptr(field_ptr)) {
                old_to_young_ref = true;
            }
        });

        if old_to_young_ref {
            self.card_table.set(card_idx, CardEntry::Dirty);
        }
    }

    pub fn promotion_failed(&self) -> bool {
        self.promotion_failed
    }
}
<commit_msg>swiper: move code into copy_refs<commit_after>use std::cmp;

use ctxt::SemContext;

use gc::root::IndirectObj;
use gc::swiper::card::{CardEntry, CardTable};
use gc::swiper::crossing::{CrossingEntry, CrossingMap};
use gc::swiper::in_kilo;
use gc::swiper::large::LargeSpace;
use gc::swiper::old::OldGen;
use gc::swiper::young::YoungGen;
use gc::swiper::{CardIdx, Region, CARD_SIZE};
use gc::Address;

use mem;
use object::Obj;
use timer::{in_ms, Timer};

pub struct MinorCollector<'a, 'ast: 'a> {
    ctxt: &'a SemContext<'ast>,
    young: &'a YoungGen,
    old: &'a OldGen,
    large: &'a LargeSpace,
    rootset: &'a [IndirectObj],
    card_table: &'a CardTable,
    crossing_map: &'a CrossingMap,

    free: Address,
    promotion_failed: bool,
    promoted_size: usize,
    young_from: Region,
}

impl<'a, 'ast> MinorCollector<'a, 'ast> {
    pub fn new(
        ctxt: &'a SemContext<'ast>,
        young: &'a YoungGen,
        old: &'a OldGen,
        large: &'a LargeSpace,
        card_table: &'a CardTable,
        crossing_map: &'a CrossingMap,
        rootset: &'a [IndirectObj],
    ) -> MinorCollector<'a, 'ast> {
        MinorCollector {
            ctxt: ctxt,
            young: young,
            old: old,
            large: large,
            rootset: rootset,
            card_table: card_table,
            crossing_map: crossing_map,
            free: Address::null(),
            promotion_failed: false,
            promoted_size: 0,
            young_from: young.from_space(),
        }
    }

    pub fn collect(&mut self) {
        let mut timer = Timer::new(self.ctxt.args.flag_gc_verbose);
        let init_size = self.heap_size();
        let young_init_size = self.young.used_region().size();
        self.free = self.young.to_space().start;

        self.young.unprotect_to_space();

        self.visit_roots();
        self.copy_dirty_cards();
        // self.visit_large_objects();
        self.visit_copied_objects();
        self.young.swap_spaces(self.free);

        self.young.protect_to_space();

        timer.stop_with(|dur| {
            let new_size = self.heap_size();
            let young_new_size = self.young.used_region().size();
            let garbage = young_init_size - young_new_size - self.promoted_size;
            let garbage_ratio = (garbage as f64 / young_init_size as f64) * 100f64;

            println!(
                "GC: Minor GC ({:.2} ms, {:.1}K->{:.1}K, young {:.1}K->{:.1}K, \
                 {:.1}K promoted, {:.1}K/{:.0}% garbage)",
                in_ms(dur),
                in_kilo(init_size),
                in_kilo(new_size),
                in_kilo(young_init_size),
                in_kilo(young_new_size),
                in_kilo(self.promoted_size),
                in_kilo(garbage),
                garbage_ratio
            );
        });
    }

    fn heap_size(&self) -> usize {
        self.young.used_region().size() + self.old.used_region().size()
    }

    fn visit_roots(&mut self) {
        // detect all references from roots into young generation
        for &root in self.rootset {
            let root_ptr = root.get();

            if self.young.contains(Address::from_ptr(root_ptr)) {
                root.set(self.copy(root_ptr));
            }
        }
    }

    fn visit_large_objects(&mut self) {
        self.large.visit_objects(|addr| {
            let object = unsafe { &mut *addr.to_mut_ptr::<Obj>() };

            if object.is_array_ref() {
                self.visit_large_object_array(object, addr);

            } else {
                self.visit_large_object(object, addr);

            }
        })
    }

    fn visit_large_object_array(&mut self, object: &mut Obj, object_start: Address) {
        let object_end = object_start.offset(object.size() as usize);
        let (start_card_idx, end_card_idx) = self.card_table.card_indices(object_start, object_end);

        for card_idx in start_card_idx..end_card_idx {
            let card_idx = card_idx.into();

            if self.card_table.get(card_idx).is_clean() {
                continue;
            }

            let card_start = self.card_table.to_address(card_idx);
            let card_end = card_start.offset(CARD_SIZE);
            let end = cmp::min(card_end, object_end);

            if card_idx.to_usize() == start_card_idx {
                self.copy_card(card_idx, object_start, end, false);

            } else {
                // all but the first card are full with references
                let refs = end.offset_from(card_start) / mem::ptr_width_usize();
                let mut ref_to_young_gen = false;

                self.copy_refs(card_start, refs, &mut ref_to_young_gen);
                self.update_card(card_idx, ref_to_young_gen);
            }
        }
    }

    fn visit_large_object(&mut self, object: &mut Obj, object_start: Address) {
        if !self.card_table.is_dirty(object_start) { return; }

        object.visit_reference_fields(|field| {
            let field_ptr = field.get();

            if self.young.contains(Address::from_ptr(field_ptr)) {
                field.set(self.copy(field_ptr));
            }
        });
    }

    fn visit_copied_objects(&mut self) {
        let mut scan = self.young.to_space().start;

        // visit all fields in copied objects
        while scan < self.free {
            let object = unsafe { &mut *scan.to_mut_ptr::<Obj>() };

            object.visit_reference_fields(|field| {
                let field_ptr = field.get();

                if self.young.contains(Address::from_ptr(field_ptr)) {
                    field.set(self.copy(field_ptr));
                }
            });

            scan = scan.offset(object.size());
        }
    }

    // copy all references from old- into young-generation.
    fn copy_dirty_cards(&mut self) {
        self.card_table.visit_dirty_in_old(self.old.free(), |card| {
            let crossing_entry = self.crossing_map.get(card);
            let card_start = self.card_table.to_address(card);
            let card_end = card_start.offset(CARD_SIZE);

            match crossing_entry {
                CrossingEntry::NoRefs => panic!("card dirty without any refs"),
                CrossingEntry::LeadingRefs(refs) => {
                    let mut ref_to_young_gen = false;

                    // copy references at start of card
                    let first_obect = self.copy_refs(card_start, refs as usize, &mut ref_to_young_gen);

                    // copy all objects from this card
                    self.copy_card(card, first_obect, card_end, ref_to_young_gen);
                }

                CrossingEntry::FirstObject(offset) => {
                    let ptr = card_start.offset(offset as usize * mem::ptr_width_usize());

                    // copy all objects from this card
                    self.copy_card(card, ptr, card_end, false);
                }

                CrossingEntry::ArrayStart(offset) => {
                    let ptr = card_start.to_usize() - (offset as usize * mem::ptr_width_usize());

                    // copy all objects from this card
                    self.copy_card(card, ptr.into(), card_end, false);
                }
            }
        });
    }

    fn copy_refs(&mut self, mut ptr: Address, refs: usize, ref_to_young_gen: &mut bool) -> Address {
        for _ in 0..refs {
            let ind_ptr = IndirectObj::from_address(ptr);
            let dir_ptr = ind_ptr.get();

            if self.young_from.contains(Address::from_ptr(dir_ptr)) {
                let copied_obj = self.copy(dir_ptr);
                ind_ptr.set(copied_obj);

                if self.young.contains(Address::from_ptr(copied_obj)) {
                    *ref_to_young_gen = true;
                }
            }

            ptr = ptr.offset(mem::ptr_width_usize());
        }

        ptr
    }

    fn copy_card(
        &mut self,
        card: CardIdx,
        mut ptr: Address,
        card_end: Address,
        mut ref_to_young_gen: bool,
    ) {
        let old_end: Address = self.old.free();
        let mut end = cmp::min(card_end, old_end);

        loop {
            while ptr < end {
                let object = unsafe { &mut *ptr.to_mut_ptr::<Obj>() };

                object.visit_reference_fields_within(end, |field| {
                    let field_ptr = field.get();

                    if self.young_from.contains(Address::from_ptr(field_ptr)) {
                        let copied_obj = self.copy(field_ptr);
                        field.set(copied_obj);

                        // determine if copied object is still in young generation
                        if self.young.contains(Address::from_ptr(copied_obj)) {
                            ref_to_young_gen = true;
                        }
                    }
                });

                ptr = ptr.offset(object.size());
            }

            // if we are in the last card of the old generation, promoted objects
            // will increase `end` towards `card_end`. Those newly promoted objects
            // need also be handled.
            if end == card_end {
                break;
            }

            let old_end = self.old.free();
            let next_end = cmp::min(card_end, old_end);

            if end != next_end {
                end = next_end;
            } else {
                break;
            }
        }

        self.update_card(card, ref_to_young_gen);
    }

    fn update_card(&mut self, card_idx: CardIdx, ref_to_young_gen: bool) {
        // if there are no references to the young generation in this card,
        // set the card to clean.
        if !ref_to_young_gen {
            self.card_table.set(card_idx, CardEntry::Clean);
        }
    }

    fn copy(&mut self, obj: *mut Obj) -> *mut Obj {
        let obj_addr = Address::from_ptr(obj);
        let obj = unsafe { &mut *obj };

        if let Some(fwd) = obj.header().forwarded() {
            return fwd.to_mut_ptr();
        }

        let obj_size = obj.size();
        debug_assert!(
            self.young_from.contains(obj_addr),
            "copy objects only from from-space."
        );

        // if object is old enough we copy it into the old generation
        if self.young.should_be_promoted(obj_addr) {
            let copy_addr = self.old.alloc(obj_size, obj.is_array_ref());

            // if there isn't enough space in old gen keep it in the
            // young generation for now
            if copy_addr.is_null() {
                self.promotion_failed = true;
            } else {
                self.promote_object(obj, copy_addr, obj_size);
                return copy_addr.to_mut_ptr();
            }
        }

        let copy_addr = self.free;
        self.free = copy_addr.offset(obj_size);

        obj.copy_to(copy_addr, obj_size);
        obj.header_mut().forward_to(copy_addr);

        copy_addr.to_mut_ptr()
    }

    fn promote_object(&mut self, obj: &mut Obj, copy_addr: Address, obj_size: usize) {
        obj.copy_to(copy_addr, obj_size);
        self.promoted_size += obj_size;

        // Promoted object can have references to the young generation.
        // Set the card table entry to dirty if this is the case.
        self.handle_promoted_object(copy_addr);

        obj.header_mut().forward_to(copy_addr);
    }

    fn handle_promoted_object(&mut self, addr: Address) {
        let card_idx = self.card_table.card_idx(addr);

        // card is already dirty, nothing left to do.
        if self.card_table.get(card_idx).is_dirty() {
            return;
        }

        let object: &mut Obj = unsafe { &mut *addr.to_mut_ptr() };
        let mut old_to_young_ref = false;

        object.visit_reference_fields(|field| {
            let field_ptr = field.get();

            if self.young.contains(Address::from_ptr(field_ptr)) {
                old_to_young_ref = true;
            }
        });

        if old_to_young_ref {
            self.card_table.set(card_idx, CardEntry::Dirty);
        }
    }

    pub fn promotion_failed(&self) -> bool {
        self.promotion_failed
    }
}
<|endoftext|>


<commit_before>use crate::builders::CrateBuilder;
use crate::util::{ChaosProxy, FreshSchema};
use anyhow::Error;
use cargo_registry::models::{NewUser, User};
use diesel::prelude::*;
use reqwest::blocking::{Client, Response};
use std::collections::HashMap;
use std::io::{BufRead, BufReader, Read};
use std::process::{Child, Command, Stdio};
use std::sync::{mpsc::Sender, Arc};
use std::time::Duration;
use url::Url;

const SERVER_BOOT_TIMEOUT_SECONDS: u64 = 30;

#[test]
fn normal_startup() -> Result<(), Error> {
    let server_bin = ServerBin::prepare()?;
    initialize_dummy_crate(&server_bin.db()?);

    let running_server = server_bin.start()?;

    // Ensure the application correctly responds to download requests
    let resp = running_server.get("api/v1/crates/FOO/1.0.0/download")?;
    assert!(resp.status().is_redirection());
    assert!(resp
        .headers()
        .get("location")
        .unwrap()
        .to_str()?
        .ends_with("/crates/foo/foo-1.0.0.crate"));

    Ok(())
}

#[test]
fn startup_without_database() -> Result<(), Error> {
    let server_bin = ServerBin::prepare()?;
    initialize_dummy_crate(&server_bin.db()?);

    // Break the networking *before* starting the binary, to ensure the binary can fully startup
    // without a database connection. Most of crates.io should not work when started without a
    // database, but unconditional redirects will work.
    server_bin.chaosproxy.break_networking();

    let running_server = server_bin.start()?;

    // Ensure unconditional redirects work.
    let resp = running_server.get("api/v1/crates/FOO/1.0.0/download")?;
    assert!(resp.status().is_redirection());
    assert!(resp
        .headers()
        .get("location")
        .unwrap()
        .to_str()?
        .ends_with("/crates/FOO/FOO-1.0.0.crate"));

    Ok(())
}

fn initialize_dummy_crate(conn: &PgConnection) {
    use cargo_registry::schema::users;

    let user: User = diesel::insert_into(users::table)
        .values(NewUser {
            gh_id: 0,
            gh_login: "user",
            ..NewUser::default()
        })
        .get_result(conn)
        .expect("failed to create dummy user");

    CrateBuilder::new("foo", user.id)
        .version("1.0.0")
        .build(conn)
        .expect("failed to create dummy crate");
}

struct ServerBin {
    chaosproxy: Arc<ChaosProxy>,
    db_url: String,
    env: HashMap<String, String>,
    fresh_schema: FreshSchema,
}

impl ServerBin {
    fn prepare() -> Result<Self, Error> {
        let mut env = dotenv::vars().collect::<HashMap<_, _>>();
        // Bind a random port every time the server is started.
        env.insert("PORT".into(), "0".into());
        // Avoid creating too many database connections.
        env.insert("DB_PRIMARY_POOL_SIZE".into(), "2".into());
        env.insert("DB_REPLICA_POOL_SIZE".into(), "1".into());
        env.remove("DB_MIN_SIZE");
        // Other configuration variables needed for the application to boot.
        env.insert("WEB_ALLOWED_ORIGINS".into(), "http://localhost:8888".into());
        env.insert("SESSION_KEY".into(), "a".repeat(32));
        env.insert("GH_CLIENT_ID".into(), String::new());
        env.insert("GH_CLIENT_SECRET".into(), String::new());

        // Use a proxied fresh schema as the database url.
        let fresh_schema = FreshSchema::new(env.get("TEST_DATABASE_URL").unwrap());
        let (chaosproxy, db_url) = ChaosProxy::proxy_database_url(fresh_schema.database_url())?;
        env.remove("TEST_DATABASE_URL");
        env.insert("DATABASE_URL".into(), db_url.clone());
        env.insert("READ_ONLY_REPLICA_URL".into(), db_url.clone());

        Ok(ServerBin {
            chaosproxy,
            db_url,
            env,
            fresh_schema,
        })
    }

    fn db(&self) -> Result<PgConnection, Error> {
        Ok(PgConnection::establish(&self.db_url)?)
    }

    fn start(self) -> Result<RunningServer, Error> {
        let mut process = Command::new(env!("CARGO_BIN_EXE_server"))
            .env_clear()
            .envs(self.env.into_iter())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()?;

        let (port_send, port_recv) = std::sync::mpsc::channel();
        stream_processor(process.stdout.take().unwrap(), "stdout", Some(port_send));
        stream_processor(process.stderr.take().unwrap(), "stderr", None);

        // Possible causes for this to fail:
        // - the server binary failed to start
        // - the server binary requires a database connection now
        // - the server binary doesn't print "listening on port {port}" anymore
        let port: u16 = port_recv
            .recv_timeout(Duration::from_secs(SERVER_BOOT_TIMEOUT_SECONDS))
            .map_err(|_| anyhow::anyhow!("the server took too much time to initialize"))?
            .parse()?;

        let http = Client::builder()
            .redirect(reqwest::redirect::Policy::none())
            .build()?;

        Ok(RunningServer {
            process,
            port,
            http,
            _chaosproxy: self.chaosproxy,
            _fresh_schema: self.fresh_schema,
        })
    }
}

struct RunningServer {
    process: Child,
    port: u16,
    http: Client,

    // Keep these two items at the bottom in this order to drop everything in the correct order.
    _chaosproxy: Arc<ChaosProxy>,
    _fresh_schema: FreshSchema,
}

impl RunningServer {
    fn get(&self, url: &str) -> Result<Response, Error> {
        Ok(self
            .http
            .get(format!("http://127.0.0.1:{}/{}", self.port, url))
            .header("User-Agent", "crates.io test suite")
            .send()?)
    }
}

impl Drop for RunningServer {
    fn drop(&mut self) {
        self.process
            .kill()
            .expect("failed to kill the server binary");
    }
}

fn stream_processor<R>(stream: R, kind: &'static str, port_send: Option<Sender<String>>)
where
    R: Read + Send + 'static,
{
    std::thread::spawn(move || {
        let stream = BufReader::new(stream);
        for line in stream.lines() {
            let line = match line {
                Ok(line) => line,
                // We receive an EOF when the process terminates
                Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break,
                Err(err) => panic!("unexpected error while reading process {}: {}", kind, err),
            };

            // If we expect the port number to be logged into this stream, look for it and send it
            // over the channel as soon as it's found.
            if let Some(port_send) = &port_send {
                let pattern = "Listening at ";
                if let Some(idx) = line.find(pattern) {
                    let start = idx + pattern.len();
                    let url = &line[start..];
                    let url = Url::parse(url).unwrap();
                    let port = url.port().unwrap();
                    port_send
                        .send(port.to_string())
                        .expect("failed to send the port to the test thread")
                }
            }

            println!("[server {kind}] {line}");
        }
    });
}
<commit_msg>tests/server_binary: Set `RUST_LOG=info` env var<commit_after>use crate::builders::CrateBuilder;
use crate::util::{ChaosProxy, FreshSchema};
use anyhow::Error;
use cargo_registry::models::{NewUser, User};
use diesel::prelude::*;
use reqwest::blocking::{Client, Response};
use std::collections::HashMap;
use std::io::{BufRead, BufReader, Read};
use std::process::{Child, Command, Stdio};
use std::sync::{mpsc::Sender, Arc};
use std::time::Duration;
use url::Url;

const SERVER_BOOT_TIMEOUT_SECONDS: u64 = 30;

#[test]
fn normal_startup() -> Result<(), Error> {
    let server_bin = ServerBin::prepare()?;
    initialize_dummy_crate(&server_bin.db()?);

    let running_server = server_bin.start()?;

    // Ensure the application correctly responds to download requests
    let resp = running_server.get("api/v1/crates/FOO/1.0.0/download")?;
    assert!(resp.status().is_redirection());
    assert!(resp
        .headers()
        .get("location")
        .unwrap()
        .to_str()?
        .ends_with("/crates/foo/foo-1.0.0.crate"));

    Ok(())
}

#[test]
fn startup_without_database() -> Result<(), Error> {
    let server_bin = ServerBin::prepare()?;
    initialize_dummy_crate(&server_bin.db()?);

    // Break the networking *before* starting the binary, to ensure the binary can fully startup
    // without a database connection. Most of crates.io should not work when started without a
    // database, but unconditional redirects will work.
    server_bin.chaosproxy.break_networking();

    let running_server = server_bin.start()?;

    // Ensure unconditional redirects work.
    let resp = running_server.get("api/v1/crates/FOO/1.0.0/download")?;
    assert!(resp.status().is_redirection());
    assert!(resp
        .headers()
        .get("location")
        .unwrap()
        .to_str()?
        .ends_with("/crates/FOO/FOO-1.0.0.crate"));

    Ok(())
}

fn initialize_dummy_crate(conn: &PgConnection) {
    use cargo_registry::schema::users;

    let user: User = diesel::insert_into(users::table)
        .values(NewUser {
            gh_id: 0,
            gh_login: "user",
            ..NewUser::default()
        })
        .get_result(conn)
        .expect("failed to create dummy user");

    CrateBuilder::new("foo", user.id)
        .version("1.0.0")
        .build(conn)
        .expect("failed to create dummy crate");
}

struct ServerBin {
    chaosproxy: Arc<ChaosProxy>,
    db_url: String,
    env: HashMap<String, String>,
    fresh_schema: FreshSchema,
}

impl ServerBin {
    fn prepare() -> Result<Self, Error> {
        let mut env = dotenv::vars().collect::<HashMap<_, _>>();
        // Bind a random port every time the server is started.
        env.insert("PORT".into(), "0".into());
        // Avoid creating too many database connections.
        env.insert("DB_PRIMARY_POOL_SIZE".into(), "2".into());
        env.insert("DB_REPLICA_POOL_SIZE".into(), "1".into());
        env.remove("DB_MIN_SIZE");
        // Other configuration variables needed for the application to boot.
        env.insert("WEB_ALLOWED_ORIGINS".into(), "http://localhost:8888".into());
        env.insert("SESSION_KEY".into(), "a".repeat(32));
        env.insert("GH_CLIENT_ID".into(), String::new());
        env.insert("GH_CLIENT_SECRET".into(), String::new());

        // Use a proxied fresh schema as the database url.
        let fresh_schema = FreshSchema::new(env.get("TEST_DATABASE_URL").unwrap());
        let (chaosproxy, db_url) = ChaosProxy::proxy_database_url(fresh_schema.database_url())?;
        env.remove("TEST_DATABASE_URL");
        env.insert("DATABASE_URL".into(), db_url.clone());
        env.insert("READ_ONLY_REPLICA_URL".into(), db_url.clone());

        Ok(ServerBin {
            chaosproxy,
            db_url,
            env,
            fresh_schema,
        })
    }

    fn db(&self) -> Result<PgConnection, Error> {
        Ok(PgConnection::establish(&self.db_url)?)
    }

    fn start(self) -> Result<RunningServer, Error> {
        let mut process = Command::new(env!("CARGO_BIN_EXE_server"))
            .env_clear()
            .envs(self.env.into_iter())
            .env("RUST_LOG", "info")
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()?;

        let (port_send, port_recv) = std::sync::mpsc::channel();
        stream_processor(process.stdout.take().unwrap(), "stdout", Some(port_send));
        stream_processor(process.stderr.take().unwrap(), "stderr", None);

        // Possible causes for this to fail:
        // - the server binary failed to start
        // - the server binary requires a database connection now
        // - the server binary doesn't print "listening on port {port}" anymore
        let port: u16 = port_recv
            .recv_timeout(Duration::from_secs(SERVER_BOOT_TIMEOUT_SECONDS))
            .map_err(|_| anyhow::anyhow!("the server took too much time to initialize"))?
            .parse()?;

        let http = Client::builder()
            .redirect(reqwest::redirect::Policy::none())
            .build()?;

        Ok(RunningServer {
            process,
            port,
            http,
            _chaosproxy: self.chaosproxy,
            _fresh_schema: self.fresh_schema,
        })
    }
}

struct RunningServer {
    process: Child,
    port: u16,
    http: Client,

    // Keep these two items at the bottom in this order to drop everything in the correct order.
    _chaosproxy: Arc<ChaosProxy>,
    _fresh_schema: FreshSchema,
}

impl RunningServer {
    fn get(&self, url: &str) -> Result<Response, Error> {
        Ok(self
            .http
            .get(format!("http://127.0.0.1:{}/{}", self.port, url))
            .header("User-Agent", "crates.io test suite")
            .send()?)
    }
}

impl Drop for RunningServer {
    fn drop(&mut self) {
        self.process
            .kill()
            .expect("failed to kill the server binary");
    }
}

fn stream_processor<R>(stream: R, kind: &'static str, port_send: Option<Sender<String>>)
where
    R: Read + Send + 'static,
{
    std::thread::spawn(move || {
        let stream = BufReader::new(stream);
        for line in stream.lines() {
            let line = match line {
                Ok(line) => line,
                // We receive an EOF when the process terminates
                Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break,
                Err(err) => panic!("unexpected error while reading process {}: {}", kind, err),
            };

            // If we expect the port number to be logged into this stream, look for it and send it
            // over the channel as soon as it's found.
            if let Some(port_send) = &port_send {
                let pattern = "Listening at ";
                if let Some(idx) = line.find(pattern) {
                    let start = idx + pattern.len();
                    let url = &line[start..];
                    let url = Url::parse(url).unwrap();
                    let port = url.port().unwrap();
                    port_send
                        .send(port.to_string())
                        .expect("failed to send the port to the test thread")
                }
            }

            println!("[server {kind}] {line}");
        }
    });
}
<|endoftext|>


<commit_before>use tokenizer::TomlFragment;
use super::parser_whitespace::parse_whitespace;
use super::parser_comment::parse_comment;
use super::parser_boolean::parse_boolean;
use super::parser_integer::parse_integer;
use super::parser_table::parse_table;
use self::TokenizeResult::{Success, Error};

#[derive(PartialEq, Show)]
pub enum TokenizeResult<'a> {
    Success(Vec<TomlFragment<'a>>),
    Error(&'a str),
}

pub fn tokenize(s : &str) -> TokenizeResult {
    let mut tokens = Vec::with_capacity(20);

    let mut rest = s;

    while !rest.is_empty() {
        match parse_whitespace(rest) {
            None => {},
            Some(result) => { tokens.push(result.fragment); rest = result.remainder; continue; }
        }

        match parse_comment(rest) {
            None => {},
            Some(result) => { tokens.push(result.fragment); rest = result.remainder; continue; }
        }

        match parse_boolean(rest) {
            None => {},
            Some(result) => { tokens.push(result.fragment); rest = result.remainder; continue; }
        }

        match parse_integer(rest) {
            None => {},
            Some(result) => { tokens.push(result.fragment); rest = result.remainder; continue; }
        }

        match parse_table(rest) {
            None => {},
            Some(result) => { tokens.push(result.fragment); rest = result.remainder; continue; }
        }

        return Error("Tokenize failed");
    }

    Success(tokens)
}

#[test]
fn tokenize_error() {
    assert_eq!(Error("Tokenize failed"), tokenize("!23121312awefa"));
}

#[test]
fn tokenize_empty_str() {
    assert_eq!(Success(vec![]), tokenize(""));
}

#[test]
fn tokenize_whitespace() {
    let tokens = vec![TomlFragment::Whitespace(" ")];
    assert_eq!(Success(tokens), tokenize(" "));
}

#[test]
fn tokenize_comment() {
    let tokens = vec![TomlFragment::Comment("this is a comment")];
    assert_eq!(Success(tokens), tokenize("#this is a comment"));
}

#[test]
fn tokenize_booleans() {
    let tokens = vec![
        TomlFragment::Boolean("true"),
        TomlFragment::Whitespace(" "),
        TomlFragment::Boolean("false"),
    ];
    assert_eq!(Success(tokens), tokenize("true false"));
}

#[test]
fn tokenize_whitespace_comment() {
    let tokens = vec![
        TomlFragment::Whitespace("    \t"),
        TomlFragment::Comment("just some whitespace"),
    ];
    assert_eq!(Success(tokens), tokenize("    \t#just some whitespace"));
}

#[test]
fn tokenize_positive_int_whitespace_negative_int() {
    let tokens = vec![
        TomlFragment::Integer("123"),
        TomlFragment::Whitespace(" "),
        TomlFragment::Integer("-456"),
    ];
    assert_eq!(Success(tokens), tokenize("123 -456"));
}

#[test]
fn tokenize_table() {
    let tokens = vec![TomlFragment::Table("table")];
    assert_eq!(Success(tokens), tokenize("[table]"));
}
<commit_msg>[Tokenizer] Clean up some of the test code<commit_after>use tokenizer::TomlFragment;
use tokenizer::TomlFragment::*;
use super::parser_whitespace::parse_whitespace;
use super::parser_comment::parse_comment;
use super::parser_boolean::parse_boolean;
use super::parser_integer::parse_integer;
use super::parser_table::parse_table;
use self::TokenizeResult::{Success, Error};

#[derive(PartialEq, Show)]
pub enum TokenizeResult<'a> {
    Success(Vec<TomlFragment<'a>>),
    Error(&'a str),
}

pub fn tokenize(s : &str) -> TokenizeResult {
    let mut tokens = Vec::with_capacity(20);

    let mut rest = s;

    while !rest.is_empty() {
        match parse_whitespace(rest) {
            None => {},
            Some(result) => { tokens.push(result.fragment); rest = result.remainder; continue; }
        }

        match parse_comment(rest) {
            None => {},
            Some(result) => { tokens.push(result.fragment); rest = result.remainder; continue; }
        }

        match parse_boolean(rest) {
            None => {},
            Some(result) => { tokens.push(result.fragment); rest = result.remainder; continue; }
        }

        match parse_integer(rest) {
            None => {},
            Some(result) => { tokens.push(result.fragment); rest = result.remainder; continue; }
        }

        match parse_table(rest) {
            None => {},
            Some(result) => { tokens.push(result.fragment); rest = result.remainder; continue; }
        }

        return Error("Tokenize failed");
    }

    Success(tokens)
}

#[test]
fn tokenize_error() {
    assert_eq!(Error("Tokenize failed"), tokenize("!23121312awefa"));
}

#[test]
fn tokenize_empty_str() {
    assert_eq!(Success(vec![]), tokenize(""));
}

#[test]
fn tokenize_whitespace() {
    let tokens = vec![Whitespace(" ")];
    assert_eq!(Success(tokens), tokenize(" "));
}

#[test]
fn tokenize_comment() {
    let tokens = vec![Comment("this is a comment")];
    assert_eq!(Success(tokens), tokenize("#this is a comment"));
}

#[test]
fn tokenize_booleans() {
    let tokens = vec![
        Boolean("true"),
        Whitespace(" "),
        Boolean("false"),
    ];
    assert_eq!(Success(tokens), tokenize("true false"));
}

#[test]
fn tokenize_whitespace_comment() {
    let tokens = vec![
        Whitespace("    \t"),
        Comment("just some whitespace"),
    ];
    assert_eq!(Success(tokens), tokenize("    \t#just some whitespace"));
}

#[test]
fn tokenize_positive_int_whitespace_negative_int() {
    let tokens = vec![
        Integer("123"),
        Whitespace(" "),
        Integer("-456"),
    ];
    assert_eq!(Success(tokens), tokenize("123 -456"));
}

#[test]
fn tokenize_table() {
    let tokens = vec![Table("table")];
    assert_eq!(Success(tokens), tokenize("[table]"));
}
<|endoftext|>


<commit_before>//! Checks the licenses of third-party dependencies by inspecting vendors.

use std::collections::{BTreeSet, HashSet, HashMap};
use std::fs;
use std::path::Path;
use std::process::Command;

use serde_json;

const LICENSES: &[&str] = &[
    "MIT/Apache-2.0",
    "MIT / Apache-2.0",
    "Apache-2.0/MIT",
    "Apache-2.0 / MIT",
    "MIT OR Apache-2.0",
    "MIT",
    "Unlicense/MIT",
    "Unlicense OR MIT",
];

/// These are exceptions to Rust's permissive licensing policy, and
/// should be considered bugs. Exceptions are only allowed in Rust
/// tooling. It is _crucial_ that no exception crates be dependencies
/// of the Rust runtime (std/test).
const EXCEPTIONS: &[&str] = &[
    "mdbook",             // MPL2, mdbook
    "openssl",            // BSD+advertising clause, cargo, mdbook
    "pest",               // MPL2, mdbook via handlebars
    "arrayref",           // BSD-2-Clause, mdbook via handlebars via pest
    "thread-id",          // Apache-2.0, mdbook
    "toml-query",         // MPL-2.0, mdbook
    "is-match",           // MPL-2.0, mdbook
    "cssparser",          // MPL-2.0, rustdoc
    "smallvec",           // MPL-2.0, rustdoc
    "fuchsia-zircon-sys", // BSD-3-Clause, rustdoc, rustc, cargo
    "fuchsia-zircon",     // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir)
    "cssparser-macros",   // MPL-2.0, rustdoc
    "selectors",          // MPL-2.0, rustdoc
    "clippy_lints",       // MPL-2.0, rls
    "colored",            // MPL-2.0, rustfmt
    "ordslice",           // Apache-2.0, rls
    "cloudabi",           // BSD-2-Clause, (rls -> crossbeam-channel 0.2 -> rand 0.5)
    "ryu",                // Apache-2.0, rls/cargo/... (because of serde)
    "bytesize",           // Apache-2.0, cargo
    "im-rc",              // MPL-2.0+, cargo
    "adler32",            // BSD-3-Clause AND Zlib, cargo dep that isn't used
    "fortanix-sgx-abi",   // MPL-2.0+, libstd but only for `sgx` target
];

/// Which crates to check against the whitelist?
const WHITELIST_CRATES: &[CrateVersion] = &[
    CrateVersion("rustc", "0.0.0"),
    CrateVersion("rustc_codegen_llvm", "0.0.0"),
];

/// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible.
const WHITELIST: &[Crate] = &[
    Crate("adler32"),
    Crate("aho-corasick"),
    Crate("arrayvec"),
    Crate("atty"),
    Crate("backtrace"),
    Crate("backtrace-sys"),
    Crate("bitflags"),
    Crate("build_const"),
    Crate("byteorder"),
    Crate("cc"),
    Crate("cfg-if"),
    Crate("chalk-engine"),
    Crate("chalk-macros"),
    Crate("cloudabi"),
    Crate("cmake"),
    Crate("compiler_builtins"),
    Crate("crc"),
    Crate("crc32fast"),
    Crate("crossbeam-deque"),
    Crate("crossbeam-epoch"),
    Crate("crossbeam-utils"),
    Crate("datafrog"),
    Crate("either"),
    Crate("ena"),
    Crate("env_logger"),
    Crate("filetime"),
    Crate("flate2"),
    Crate("fuchsia-zircon"),
    Crate("fuchsia-zircon-sys"),
    Crate("getopts"),
    Crate("humantime"),
    Crate("jobserver"),
    Crate("kernel32-sys"),
    Crate("lazy_static"),
    Crate("libc"),
    Crate("libz-sys"),
    Crate("lock_api"),
    Crate("log"),
    Crate("log_settings"),
    Crate("memchr"),
    Crate("memmap"),
    Crate("memoffset"),
    Crate("miniz-sys"),
    Crate("miniz_oxide"),
    Crate("miniz_oxide_c_api"),
    Crate("nodrop"),
    Crate("num_cpus"),
    Crate("owning_ref"),
    Crate("parking_lot"),
    Crate("parking_lot_core"),
    Crate("pkg-config"),
    Crate("polonius-engine"),
    Crate("proc-macro2"),
    Crate("quick-error"),
    Crate("quote"),
    Crate("rand"),
    Crate("rand_chacha"),
    Crate("rand_core"),
    Crate("rand_hc"),
    Crate("rand_isaac"),
    Crate("rand_pcg"),
    Crate("rand_xorshift"),
    Crate("redox_syscall"),
    Crate("redox_termios"),
    Crate("regex"),
    Crate("regex-syntax"),
    Crate("remove_dir_all"),
    Crate("rustc-demangle"),
    Crate("rustc-hash"),
    Crate("rustc-rayon"),
    Crate("rustc-rayon-core"),
    Crate("rustc_version"),
    Crate("scoped-tls"),
    Crate("scopeguard"),
    Crate("semver"),
    Crate("semver-parser"),
    Crate("serde"),
    Crate("serde_derive"),
    Crate("smallvec"),
    Crate("stable_deref_trait"),
    Crate("syn"),
    Crate("tempfile"),
    Crate("termcolor"),
    Crate("terminon"),
    Crate("termion"),
    Crate("thread_local"),
    Crate("ucd-util"),
    Crate("unicode-width"),
    Crate("unicode-xid"),
    Crate("unreachable"),
    Crate("utf8-ranges"),
    Crate("vcpkg"),
    Crate("version_check"),
    Crate("void"),
    Crate("winapi"),
    Crate("winapi-build"),
    Crate("winapi-i686-pc-windows-gnu"),
    Crate("winapi-util"),
    Crate("winapi-x86_64-pc-windows-gnu"),
    Crate("wincolor"),
];

// Some types for Serde to deserialize the output of `cargo metadata` to.

#[derive(Deserialize)]
struct Output {
    resolve: Resolve,
}

#[derive(Deserialize)]
struct Resolve {
    nodes: Vec<ResolveNode>,
}

#[derive(Deserialize)]
struct ResolveNode {
    id: String,
    dependencies: Vec<String>,
}

/// A unique identifier for a crate.
#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
struct Crate<'a>(&'a str); // (name)

#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
struct CrateVersion<'a>(&'a str, &'a str); // (name, version)

impl<'a> Crate<'a> {
    pub fn id_str(&self) -> String {
        format!("{} ", self.0)
    }
}

impl<'a> CrateVersion<'a> {
    /// Returns the struct and whether or not the dependency is in-tree.
    pub fn from_str(s: &'a str) -> (Self, bool) {
        let mut parts = s.split(' ');
        let name = parts.next().unwrap();
        let version = parts.next().unwrap();
        let path = parts.next().unwrap();

        let is_path_dep = path.starts_with("(path+");

        (CrateVersion(name, version), is_path_dep)
    }

    pub fn id_str(&self) -> String {
        format!("{} {}", self.0, self.1)
    }
}

impl<'a> From<CrateVersion<'a>> for Crate<'a> {
    fn from(cv: CrateVersion<'a>) -> Crate<'a> {
        Crate(cv.0)
    }
}

/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed.
///
/// Specifically, this checks that the license is correct.
pub fn check(path: &Path, bad: &mut bool) {
    // Check licences.
    let path = path.join("../vendor");
    assert!(path.exists(), "vendor directory missing");
    let mut saw_dir = false;
    for dir in t!(path.read_dir()) {
        saw_dir = true;
        let dir = t!(dir);

        // Skip our exceptions.
        let is_exception = EXCEPTIONS.iter().any(|exception| {
            dir.path()
                .to_str()
                .unwrap()
                .contains(&format!("vendor/{}", exception))
        });
        if is_exception {
            continue;
        }

        let toml = dir.path().join("Cargo.toml");
        *bad = *bad || !check_license(&toml);
    }
    assert!(saw_dir, "no vendored source");
}

/// Checks the dependency of `WHITELIST_CRATES` at the given path. Changes `bad` to `true` if a
/// check failed.
///
/// Specifically, this checks that the dependencies are on the `WHITELIST`.
pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) {
    // Get dependencies from Cargo metadata.
    let resolve = get_deps(path, cargo);

    // Get the whitelist in a convenient form.
    let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect();

    // Check dependencies.
    let mut visited = BTreeSet::new();
    let mut unapproved = BTreeSet::new();
    for &krate in WHITELIST_CRATES.iter() {
        let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate, false);
        unapproved.append(&mut bad);
    }

    if !unapproved.is_empty() {
        println!("Dependencies not on the whitelist:");
        for dep in unapproved {
            println!("* {}", dep.id_str());
        }
        *bad = true;
    }

    check_crate_duplicate(&resolve, bad);
}

fn check_license(path: &Path) -> bool {
    if !path.exists() {
        panic!("{} does not exist", path.display());
    }
    let contents = t!(fs::read_to_string(&path));

    let mut found_license = false;
    for line in contents.lines() {
        if !line.starts_with("license") {
            continue;
        }
        let license = extract_license(line);
        if !LICENSES.contains(&&*license) {
            println!("invalid license {} in {}", license, path.display());
            return false;
        }
        found_license = true;
        break;
    }
    if !found_license {
        println!("no license in {}", path.display());
        return false;
    }

    true
}

fn extract_license(line: &str) -> String {
    let first_quote = line.find('"');
    let last_quote = line.rfind('"');
    if let (Some(f), Some(l)) = (first_quote, last_quote) {
        let license = &line[f + 1..l];
        license.into()
    } else {
        "bad-license-parse".into()
    }
}

/// Gets the dependencies of the crate at the given path using `cargo metadata`.
fn get_deps(path: &Path, cargo: &Path) -> Resolve {
    // Run `cargo metadata` to get the set of dependencies.
    let output = Command::new(cargo)
        .arg("metadata")
        .arg("--format-version")
        .arg("1")
        .arg("--manifest-path")
        .arg(path.join("../Cargo.toml"))
        .output()
        .expect("Unable to run `cargo metadata`")
        .stdout;
    let output = String::from_utf8_lossy(&output);
    let output: Output = serde_json::from_str(&output).unwrap();

    output.resolve
}

/// Checks the dependencies of the given crate from the given cargo metadata to see if they are on
/// the whitelist. Returns a list of illegal dependencies.
fn check_crate_whitelist<'a, 'b>(
    whitelist: &'a HashSet<Crate>,
    resolve: &'a Resolve,
    visited: &'b mut BTreeSet<CrateVersion<'a>>,
    krate: CrateVersion<'a>,
    must_be_on_whitelist: bool,
) -> BTreeSet<Crate<'a>> {
    // This will contain bad deps.
    let mut unapproved = BTreeSet::new();

    // Check if we have already visited this crate.
    if visited.contains(&krate) {
        return unapproved;
    }

    visited.insert(krate);

    // If this path is in-tree, we don't require it to be on the whitelist.
    if must_be_on_whitelist {
        // If this dependency is not on `WHITELIST`, add to bad set.
        if !whitelist.contains(&krate.into()) {
            unapproved.insert(krate.into());
        }
    }

    // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!).
    let to_check = resolve
        .nodes
        .iter()
        .find(|n| n.id.starts_with(&krate.id_str()))
        .expect("crate does not exist");

    for dep in to_check.dependencies.iter() {
        let (krate, is_path_dep) = CrateVersion::from_str(dep);

        let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate, !is_path_dep);
        unapproved.append(&mut bad);
    }

    unapproved
}

fn check_crate_duplicate(resolve: &Resolve, bad: &mut bool) {
    const FORBIDDEN_TO_HAVE_DUPLICATES: &[&str] = &[
        // These two crates take quite a long time to build, so don't allow two versions of them
        // to accidentally sneak into our dependency graph, in order to ensure we keep our CI times
        // under control.

        // "cargo", // FIXME(#53005)
        "rustc-ap-syntax",
    ];
    let mut name_to_id: HashMap<_, Vec<_>> = HashMap::new();
    for node in resolve.nodes.iter() {
        name_to_id.entry(node.id.split_whitespace().next().unwrap())
            .or_default()
            .push(&node.id);
    }

    for name in FORBIDDEN_TO_HAVE_DUPLICATES {
        if name_to_id[name].len() <= 1 {
            continue
        }
        println!("crate `{}` is duplicated in `Cargo.lock`", name);
        for id in name_to_id[name].iter() {
            println!("  * {}", id);
        }
        *bad = true;
    }
}
<commit_msg>Add constant_time_eq to license exception list of tidy tool<commit_after>//! Checks the licenses of third-party dependencies by inspecting vendors.

use std::collections::{BTreeSet, HashSet, HashMap};
use std::fs;
use std::path::Path;
use std::process::Command;

use serde_json;

const LICENSES: &[&str] = &[
    "MIT/Apache-2.0",
    "MIT / Apache-2.0",
    "Apache-2.0/MIT",
    "Apache-2.0 / MIT",
    "MIT OR Apache-2.0",
    "MIT",
    "Unlicense/MIT",
    "Unlicense OR MIT",
];

/// These are exceptions to Rust's permissive licensing policy, and
/// should be considered bugs. Exceptions are only allowed in Rust
/// tooling. It is _crucial_ that no exception crates be dependencies
/// of the Rust runtime (std/test).
const EXCEPTIONS: &[&str] = &[
    "mdbook",             // MPL2, mdbook
    "openssl",            // BSD+advertising clause, cargo, mdbook
    "pest",               // MPL2, mdbook via handlebars
    "arrayref",           // BSD-2-Clause, mdbook via handlebars via pest
    "thread-id",          // Apache-2.0, mdbook
    "toml-query",         // MPL-2.0, mdbook
    "is-match",           // MPL-2.0, mdbook
    "cssparser",          // MPL-2.0, rustdoc
    "smallvec",           // MPL-2.0, rustdoc
    "fuchsia-zircon-sys", // BSD-3-Clause, rustdoc, rustc, cargo
    "fuchsia-zircon",     // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir)
    "cssparser-macros",   // MPL-2.0, rustdoc
    "selectors",          // MPL-2.0, rustdoc
    "clippy_lints",       // MPL-2.0, rls
    "colored",            // MPL-2.0, rustfmt
    "ordslice",           // Apache-2.0, rls
    "cloudabi",           // BSD-2-Clause, (rls -> crossbeam-channel 0.2 -> rand 0.5)
    "ryu",                // Apache-2.0, rls/cargo/... (because of serde)
    "bytesize",           // Apache-2.0, cargo
    "im-rc",              // MPL-2.0+, cargo
    "adler32",            // BSD-3-Clause AND Zlib, cargo dep that isn't used
    "fortanix-sgx-abi",   // MPL-2.0+, libstd but only for `sgx` target
    "constant_time_eq",   // CC0-1.0, rustfmt
];

/// Which crates to check against the whitelist?
const WHITELIST_CRATES: &[CrateVersion] = &[
    CrateVersion("rustc", "0.0.0"),
    CrateVersion("rustc_codegen_llvm", "0.0.0"),
];

/// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible.
const WHITELIST: &[Crate] = &[
    Crate("adler32"),
    Crate("aho-corasick"),
    Crate("arrayvec"),
    Crate("atty"),
    Crate("backtrace"),
    Crate("backtrace-sys"),
    Crate("bitflags"),
    Crate("build_const"),
    Crate("byteorder"),
    Crate("cc"),
    Crate("cfg-if"),
    Crate("chalk-engine"),
    Crate("chalk-macros"),
    Crate("cloudabi"),
    Crate("cmake"),
    Crate("compiler_builtins"),
    Crate("crc"),
    Crate("crc32fast"),
    Crate("crossbeam-deque"),
    Crate("crossbeam-epoch"),
    Crate("crossbeam-utils"),
    Crate("datafrog"),
    Crate("either"),
    Crate("ena"),
    Crate("env_logger"),
    Crate("filetime"),
    Crate("flate2"),
    Crate("fuchsia-zircon"),
    Crate("fuchsia-zircon-sys"),
    Crate("getopts"),
    Crate("humantime"),
    Crate("jobserver"),
    Crate("kernel32-sys"),
    Crate("lazy_static"),
    Crate("libc"),
    Crate("libz-sys"),
    Crate("lock_api"),
    Crate("log"),
    Crate("log_settings"),
    Crate("memchr"),
    Crate("memmap"),
    Crate("memoffset"),
    Crate("miniz-sys"),
    Crate("miniz_oxide"),
    Crate("miniz_oxide_c_api"),
    Crate("nodrop"),
    Crate("num_cpus"),
    Crate("owning_ref"),
    Crate("parking_lot"),
    Crate("parking_lot_core"),
    Crate("pkg-config"),
    Crate("polonius-engine"),
    Crate("proc-macro2"),
    Crate("quick-error"),
    Crate("quote"),
    Crate("rand"),
    Crate("rand_chacha"),
    Crate("rand_core"),
    Crate("rand_hc"),
    Crate("rand_isaac"),
    Crate("rand_pcg"),
    Crate("rand_xorshift"),
    Crate("redox_syscall"),
    Crate("redox_termios"),
    Crate("regex"),
    Crate("regex-syntax"),
    Crate("remove_dir_all"),
    Crate("rustc-demangle"),
    Crate("rustc-hash"),
    Crate("rustc-rayon"),
    Crate("rustc-rayon-core"),
    Crate("rustc_version"),
    Crate("scoped-tls"),
    Crate("scopeguard"),
    Crate("semver"),
    Crate("semver-parser"),
    Crate("serde"),
    Crate("serde_derive"),
    Crate("smallvec"),
    Crate("stable_deref_trait"),
    Crate("syn"),
    Crate("tempfile"),
    Crate("termcolor"),
    Crate("terminon"),
    Crate("termion"),
    Crate("thread_local"),
    Crate("ucd-util"),
    Crate("unicode-width"),
    Crate("unicode-xid"),
    Crate("unreachable"),
    Crate("utf8-ranges"),
    Crate("vcpkg"),
    Crate("version_check"),
    Crate("void"),
    Crate("winapi"),
    Crate("winapi-build"),
    Crate("winapi-i686-pc-windows-gnu"),
    Crate("winapi-util"),
    Crate("winapi-x86_64-pc-windows-gnu"),
    Crate("wincolor"),
];

// Some types for Serde to deserialize the output of `cargo metadata` to.

#[derive(Deserialize)]
struct Output {
    resolve: Resolve,
}

#[derive(Deserialize)]
struct Resolve {
    nodes: Vec<ResolveNode>,
}

#[derive(Deserialize)]
struct ResolveNode {
    id: String,
    dependencies: Vec<String>,
}

/// A unique identifier for a crate.
#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
struct Crate<'a>(&'a str); // (name)

#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
struct CrateVersion<'a>(&'a str, &'a str); // (name, version)

impl<'a> Crate<'a> {
    pub fn id_str(&self) -> String {
        format!("{} ", self.0)
    }
}

impl<'a> CrateVersion<'a> {
    /// Returns the struct and whether or not the dependency is in-tree.
    pub fn from_str(s: &'a str) -> (Self, bool) {
        let mut parts = s.split(' ');
        let name = parts.next().unwrap();
        let version = parts.next().unwrap();
        let path = parts.next().unwrap();

        let is_path_dep = path.starts_with("(path+");

        (CrateVersion(name, version), is_path_dep)
    }

    pub fn id_str(&self) -> String {
        format!("{} {}", self.0, self.1)
    }
}

impl<'a> From<CrateVersion<'a>> for Crate<'a> {
    fn from(cv: CrateVersion<'a>) -> Crate<'a> {
        Crate(cv.0)
    }
}

/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed.
///
/// Specifically, this checks that the license is correct.
pub fn check(path: &Path, bad: &mut bool) {
    // Check licences.
    let path = path.join("../vendor");
    assert!(path.exists(), "vendor directory missing");
    let mut saw_dir = false;
    for dir in t!(path.read_dir()) {
        saw_dir = true;
        let dir = t!(dir);

        // Skip our exceptions.
        let is_exception = EXCEPTIONS.iter().any(|exception| {
            dir.path()
                .to_str()
                .unwrap()
                .contains(&format!("vendor/{}", exception))
        });
        if is_exception {
            continue;
        }

        let toml = dir.path().join("Cargo.toml");
        *bad = *bad || !check_license(&toml);
    }
    assert!(saw_dir, "no vendored source");
}

/// Checks the dependency of `WHITELIST_CRATES` at the given path. Changes `bad` to `true` if a
/// check failed.
///
/// Specifically, this checks that the dependencies are on the `WHITELIST`.
pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) {
    // Get dependencies from Cargo metadata.
    let resolve = get_deps(path, cargo);

    // Get the whitelist in a convenient form.
    let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect();

    // Check dependencies.
    let mut visited = BTreeSet::new();
    let mut unapproved = BTreeSet::new();
    for &krate in WHITELIST_CRATES.iter() {
        let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate, false);
        unapproved.append(&mut bad);
    }

    if !unapproved.is_empty() {
        println!("Dependencies not on the whitelist:");
        for dep in unapproved {
            println!("* {}", dep.id_str());
        }
        *bad = true;
    }

    check_crate_duplicate(&resolve, bad);
}

fn check_license(path: &Path) -> bool {
    if !path.exists() {
        panic!("{} does not exist", path.display());
    }
    let contents = t!(fs::read_to_string(&path));

    let mut found_license = false;
    for line in contents.lines() {
        if !line.starts_with("license") {
            continue;
        }
        let license = extract_license(line);
        if !LICENSES.contains(&&*license) {
            println!("invalid license {} in {}", license, path.display());
            return false;
        }
        found_license = true;
        break;
    }
    if !found_license {
        println!("no license in {}", path.display());
        return false;
    }

    true
}

fn extract_license(line: &str) -> String {
    let first_quote = line.find('"');
    let last_quote = line.rfind('"');
    if let (Some(f), Some(l)) = (first_quote, last_quote) {
        let license = &line[f + 1..l];
        license.into()
    } else {
        "bad-license-parse".into()
    }
}

/// Gets the dependencies of the crate at the given path using `cargo metadata`.
fn get_deps(path: &Path, cargo: &Path) -> Resolve {
    // Run `cargo metadata` to get the set of dependencies.
    let output = Command::new(cargo)
        .arg("metadata")
        .arg("--format-version")
        .arg("1")
        .arg("--manifest-path")
        .arg(path.join("../Cargo.toml"))
        .output()
        .expect("Unable to run `cargo metadata`")
        .stdout;
    let output = String::from_utf8_lossy(&output);
    let output: Output = serde_json::from_str(&output).unwrap();

    output.resolve
}

/// Checks the dependencies of the given crate from the given cargo metadata to see if they are on
/// the whitelist. Returns a list of illegal dependencies.
fn check_crate_whitelist<'a, 'b>(
    whitelist: &'a HashSet<Crate>,
    resolve: &'a Resolve,
    visited: &'b mut BTreeSet<CrateVersion<'a>>,
    krate: CrateVersion<'a>,
    must_be_on_whitelist: bool,
) -> BTreeSet<Crate<'a>> {
    // This will contain bad deps.
    let mut unapproved = BTreeSet::new();

    // Check if we have already visited this crate.
    if visited.contains(&krate) {
        return unapproved;
    }

    visited.insert(krate);

    // If this path is in-tree, we don't require it to be on the whitelist.
    if must_be_on_whitelist {
        // If this dependency is not on `WHITELIST`, add to bad set.
        if !whitelist.contains(&krate.into()) {
            unapproved.insert(krate.into());
        }
    }

    // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!).
    let to_check = resolve
        .nodes
        .iter()
        .find(|n| n.id.starts_with(&krate.id_str()))
        .expect("crate does not exist");

    for dep in to_check.dependencies.iter() {
        let (krate, is_path_dep) = CrateVersion::from_str(dep);

        let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate, !is_path_dep);
        unapproved.append(&mut bad);
    }

    unapproved
}

fn check_crate_duplicate(resolve: &Resolve, bad: &mut bool) {
    const FORBIDDEN_TO_HAVE_DUPLICATES: &[&str] = &[
        // These two crates take quite a long time to build, so don't allow two versions of them
        // to accidentally sneak into our dependency graph, in order to ensure we keep our CI times
        // under control.

        // "cargo", // FIXME(#53005)
        "rustc-ap-syntax",
    ];
    let mut name_to_id: HashMap<_, Vec<_>> = HashMap::new();
    for node in resolve.nodes.iter() {
        name_to_id.entry(node.id.split_whitespace().next().unwrap())
            .or_default()
            .push(&node.id);
    }

    for name in FORBIDDEN_TO_HAVE_DUPLICATES {
        if name_to_id[name].len() <= 1 {
            continue
        }
        println!("crate `{}` is duplicated in `Cargo.lock`", name);
        for id in name_to_id[name].iter() {
            println!("  * {}", id);
        }
        *bad = true;
    }
}
<|endoftext|>


<commit_before>extern crate cairo;

extern crate gtk;
use gtk::{Inhibit, ToolButtonExt, WidgetExt};

#[cfg(feature = "profiling-audio-draw")]
use chrono::Utc;

use std::boxed::Box;

use std::rc::Rc;
use std::cell::RefCell;

use std::sync::{Arc, Mutex};

use media::{PlaybackContext, DoubleAudioBuffer, SampleExtractor};

use toc::Timestamp;

use super::{ControllerState, DoubleWaveformBuffer, MainController, WaveformBuffer,
            WaveformConditions, BACKGROUND_COLOR};

const BUFFER_DURATION: u64 = 60_000_000_000; // 60 s
const MIN_REQ_DURATION: f64 = 1_953_125f64; // 2 ms / 1000 px
const MAX_REQ_DURATION: f64 = 32_000_000_000f64; // 32 s / 1000 px
const INIT_REQ_DURATION: f64 = 4_000_000_000f64; // 4 s / 1000 px
const STEP_REQ_DURATION: f64 = 2f64;

pub struct AudioController {
    container: gtk::Box,
    drawingarea: gtk::DrawingArea,
    zoom_in_btn: gtk::ToolButton,
    zoom_out_btn: gtk::ToolButton,

    is_active: bool,
    playback_needs_refresh: bool,

    requested_duration: f64,
    cursor_pos: u64,
    last_visible_pos: u64,

    waveform_mtx: Arc<Mutex<Box<SampleExtractor>>>,
    dbl_buffer_mtx: Arc<Mutex<DoubleAudioBuffer>>,
}

impl AudioController {
    pub fn new(builder: &gtk::Builder) -> Rc<RefCell<Self>> {
        let dbl_buffer_mtx = DoubleWaveformBuffer::new_mutex(BUFFER_DURATION);
        let waveform_mtx = dbl_buffer_mtx
            .lock()
            .expect("AudioController::new: couldn't lock dbl_buffer_mtx")
            .get_exposed_buffer_mtx();

        Rc::new(RefCell::new(AudioController {
            container: builder.get_object("audio-container").unwrap(),
            drawingarea: builder.get_object("audio-drawingarea").unwrap(),
            zoom_in_btn: builder.get_object("audio_zoom_in-toolbutton").unwrap(),
            zoom_out_btn: builder.get_object("audio_zoom_out-toolbutton").unwrap(),

            is_active: false,
            playback_needs_refresh: false,

            requested_duration: INIT_REQ_DURATION,
            cursor_pos: 0,
            last_visible_pos: 0,

            waveform_mtx: waveform_mtx,
            dbl_buffer_mtx: dbl_buffer_mtx,
        }))
    }

    pub fn register_callbacks(
        this_rc: &Rc<RefCell<Self>>,
        main_ctrl: &Rc<RefCell<MainController>>,
    ) {
        let this = this_rc.borrow();

        // draw
        let this_clone = Rc::clone(this_rc);
        this.drawingarea.connect_draw(
            move |drawing_area, cairo_ctx| {
                let mut this = this_clone.borrow_mut();
                this.draw(drawing_area, cairo_ctx)
            },
        );

        // widget size changed
        let main_ctrl_clone = Rc::clone(main_ctrl);
        let this_clone = Rc::clone(this_rc);
        this.drawingarea.connect_size_allocate(move |_, _| {
            this_clone.borrow_mut().refresh(&main_ctrl_clone);
        });

        // click in drawing_area
        let main_ctrl_clone = Rc::clone(main_ctrl);
        let this_clone = Rc::clone(this_rc);
        this.drawingarea.connect_button_press_event(
            move |_, event_button| {
                match event_button.get_button() {
                    1 => { // left click => seek
                        let position_opt = this_clone.borrow()
                            .get_position_at(event_button.get_position().0);
                        if let Some(position) = position_opt {
                            main_ctrl_clone.borrow_mut().seek(position, true); // accurate (slow)
                        }
                    }
                    3 => { // right click => segment playback
                        let (position_opt, cursor_pos, last_pos) = {
                            let this = this_clone.borrow();
                            (
                                this.get_position_at(event_button.get_position().0),
                                this.cursor_pos,
                                this.last_visible_pos,
                            )
                        };
                        if let Some(position) = position_opt {
                            main_ctrl_clone.borrow_mut()
                                .play_range(position, last_pos, cursor_pos);
                        }
                    }
                    _ => (),
                }
                Inhibit(true)
            },
        );

        // click zoom in
        let main_ctrl_clone = Rc::clone(main_ctrl);
        let this_clone = Rc::clone(this_rc);
        this.zoom_in_btn.connect_clicked(move |_| {
            let mut this = this_clone.borrow_mut();
            this.requested_duration /= STEP_REQ_DURATION;
            if this.requested_duration >= MIN_REQ_DURATION {
                this.refresh(&main_ctrl_clone);
            } else {
                this.requested_duration = MIN_REQ_DURATION;
            }
        });

        // click zoom out
        let main_ctrl_clone = Rc::clone(main_ctrl);
        let this_clone = Rc::clone(this_rc);
        this.zoom_out_btn.connect_clicked(move |_| {
            let mut this = this_clone.borrow_mut();
            this.requested_duration *= STEP_REQ_DURATION;
            if this.requested_duration <= MAX_REQ_DURATION {
                this.refresh(&main_ctrl_clone);
            } else {
                this.requested_duration = MAX_REQ_DURATION;
            }
        });
    }

    pub fn cleanup(&mut self) {
        self.is_active = false;
        self.playback_needs_refresh = false;
        {
            self.dbl_buffer_mtx
                .lock()
                .expect("AudioController::cleanup: Couldn't lock dbl_buffer_mtx")
                .cleanup();
        }
        self.requested_duration = INIT_REQ_DURATION;
        self.cursor_pos = 0;
        self.last_visible_pos = 0;
        self.drawingarea.queue_draw();
    }

    pub fn get_dbl_buffer_mtx(&self) -> Arc<Mutex<DoubleAudioBuffer>> {
        Arc::clone(&self.dbl_buffer_mtx)
    }

    pub fn new_media(&mut self, context: &PlaybackContext) {
        let has_audio = context
            .info
            .lock()
            .expect(
                "Failed to lock media info while initializing audio controller",
            )
            .audio_best
            .is_some();

        if has_audio {
            let allocation = self.drawingarea.get_allocation();
            {
                // init the buffers in order to render the waveform in current conditions
                let requested_duration = self.requested_duration;
                self.dbl_buffer_mtx
                    .lock()
                    .expect(
                        "AudioController::size-allocate: couldn't lock dbl_buffer_mtx",
                    )
                    .set_conditions(Box::new(WaveformConditions::new(
                        requested_duration,
                        allocation.width,
                        allocation.height,
                    )));
            }

            self.is_active = true;
            self.container.show();
        } else {
            self.container.hide();
        }
    }

    pub fn seek(&mut self, position: u64, state: &ControllerState) {
        {
            self.waveform_mtx
                .lock()
                .expect("AudioController::seek: Couldn't lock waveform_mtx")
                .as_mut_any()
                .downcast_mut::<WaveformBuffer>()
                .expect(
                    "AudioController::seek: SamplesExtratctor is not a WaveformBuffer",
                )
                .seek(position, *state == ControllerState::Playing);
        }

        if *state == ControllerState::Paused {
            // refresh the buffer in order to render the waveform
            // with samples that might not be rendered in current WaveformImage yet
            self.dbl_buffer_mtx
                .lock()
                .expect("AudioController::seek: couldn't lock dbl_buffer_mtx")
                .refresh(false); // don't keep continuity
        }
        self.drawingarea.queue_draw();
    }

    pub fn start_play_range(&mut self) {
        self.waveform_mtx
            .lock()
            .expect("AudioController::seek: Couldn't lock waveform_mtx")
            .as_mut_any()
            .downcast_mut::<WaveformBuffer>()
            .expect(
                "AudioController::seek: SamplesExtratctor is not a WaveformBuffer",
            )
            .start_play_range();
    }

    pub fn tick(&mut self) {
        if self.is_active {
            if self.playback_needs_refresh {
                #[cfg(feature = "trace-audio-controller")]
                println!("AudioController::tick forcing refresh");

                self.dbl_buffer_mtx
                    .lock()
                    .expect("AudioController::tick: couldn't lock dbl_buffer_mtx")
                    .refresh(true); // keep continuity
            }

            self.drawingarea.queue_draw();
        }
    }

    fn get_position_at(&self, x: f64) -> Option<u64> {
        let waveform_buffer_grd = &mut *self.waveform_mtx.lock().expect(
            "Couldn't lock waveform buffer in audio controller draw",
        );
        waveform_buffer_grd
            .as_any()
            .downcast_ref::<WaveformBuffer>()
            .expect(concat!(
                "AudioController::get_current_and_sample_at ",
                "SamplesExtratctor is not a waveform buffer",
            ))
            .get_position_at(x)
    }

    fn clean_cairo_context(&self, cr: &cairo::Context) {
        cr.set_source_rgb(BACKGROUND_COLOR.0, BACKGROUND_COLOR.1, BACKGROUND_COLOR.2);
        cr.paint();
    }

    fn draw(&mut self, drawingarea: &gtk::DrawingArea, cr: &cairo::Context) -> Inhibit {
        #[cfg(feature = "profiling-audio-draw")]
        let before_init = Utc::now();

        let allocation = drawingarea.get_allocation();
        if allocation.width.is_negative() {
            #[cfg(feature = "trace-audio-controller")]
            println!(
                "AudioController::draw negative allocation.width: {}",
                allocation.width
            );

            self.clean_cairo_context(cr);
            return Inhibit(true);
        }

        #[cfg(feature = "profiling-audio-draw")]
        let before_lock = Utc::now();
        #[cfg(feature = "profiling-audio-draw")]
        let mut _before_cndt = Utc::now();
        #[cfg(feature = "profiling-audio-draw")]
        let mut _before_image = Utc::now();

        let image_positions = {
            let waveform_grd = &mut *self.waveform_mtx.lock().expect(
                "AudioController::draw: couldn't lock waveform_mtx",
            );
            let waveform_buffer = waveform_grd
                .as_mut_any()
                .downcast_mut::<WaveformBuffer>()
                .expect(
                    "AudioController::draw: SamplesExtratctor is not a WaveformBuffer",
                );

            #[cfg(feature = "profiling-audio-draw")]
            let _before_cndt = Utc::now();

            waveform_buffer.update_conditions(
                self.requested_duration,
                allocation.width,
                allocation.height,
            );

            self.playback_needs_refresh = waveform_buffer.playback_needs_refresh;

            match waveform_buffer.get_image() {
                Some((image, image_positions)) => {
                    #[cfg(feature = "profiling-audio-draw")]
                    let _before_image = Utc::now();

                    cr.set_source_surface(image, -image_positions.first.x, 0f64);
                    cr.paint();

                    image_positions
                }
                None => {
                    self.clean_cairo_context(cr);

                    #[cfg(feature = "trace-audio-controller")]
                    println!("AudioController::draw no image");

                    return Inhibit(true);
                }
            }
        };

        #[cfg(feature = "profiling-audio-draw")]
        let before_pos = Utc::now();

        let height = f64::from(allocation.height);
        let width = f64::from(allocation.width);
        cr.scale(1f64, 1f64);
        cr.set_source_rgb(1f64, 1f64, 0f64);
        cr.set_font_size(14f64);

        // first position
        let first_text = Timestamp::format(image_positions.first.timestamp, false);
        let first_text_width = 2f64 + cr.text_extents(&first_text).width;
        cr.move_to(2f64, 30f64);
        cr.show_text(&first_text);

        // last position
        if let Some(last_pos) = image_positions.last {
            let last_text = Timestamp::format(last_pos.timestamp, false);
            // align actual width to a 15px multiple box in order to avoid
            // variations due to actual size of individual digits
            let last_text_width = (cr.text_extents(&last_text).width / 15f64).ceil() * 15f64;
            if last_pos.x - last_text_width > first_text_width + 5f64 {
                // last text won't overlap with first text
                cr.move_to(last_pos.x - last_text_width, 30f64);
                cr.show_text(&last_text);
            }

            self.last_visible_pos = last_pos.timestamp;

            // make sure the rest of the image is filled with background's color
            cr.set_source_rgb(BACKGROUND_COLOR.0, BACKGROUND_COLOR.1, BACKGROUND_COLOR.2);
            cr.rectangle(last_pos.x, 0f64, width - last_pos.x, height);
            cr.fill();
        }

        if let Some(current_pos) = image_positions.current {
            // draw current pos
            cr.set_source_rgb(1f64, 1f64, 0f64);

            let cursor_text = Timestamp::format(current_pos.timestamp, true);
            let cursor_text_width = 5f64 + cr.text_extents(&cursor_text).width;
            let cursor_text_x = if current_pos.x + cursor_text_width < width {
                current_pos.x + 5f64
            } else {
                current_pos.x - cursor_text_width
            };
            cr.move_to(cursor_text_x, 15f64);
            cr.show_text(&cursor_text);

            cr.set_line_width(1f64);
            cr.move_to(current_pos.x, 0f64);
            cr.line_to(current_pos.x, height);
            cr.stroke();

            self.cursor_pos = current_pos.timestamp;
        }

        #[cfg(feature = "profiling-audio-draw")]
        let end = Utc::now();

        #[cfg(feature = "profiling-audio-draw")]
        println!(
            "audio-draw,{},{},{},{},{},{}",
            before_init.time().format("%H:%M:%S%.6f"),
            before_lock.time().format("%H:%M:%S%.6f"),
            _before_cndt.time().format("%H:%M:%S%.6f"),
            _before_image.time().format("%H:%M:%S%.6f"),
            before_pos.time().format("%H:%M:%S%.6f"),
            end.time().format("%H:%M:%S%.6f"),
        );

        Inhibit(true)
    }

    fn refresh(&mut self, main_ctrl: &Rc<RefCell<MainController>>) {
        let allocation = self.drawingarea.get_allocation();
        {
            // refresh the buffer in order to render the waveform
            // in latest conditions
            let requested_duration = self.requested_duration;
            self.dbl_buffer_mtx
                .lock()
                .expect(
                    "AudioController::size-allocate: couldn't lock dbl_buffer_mtx",
                )
                .refresh_with_conditions(
                    Box::new(WaveformConditions::new(
                        requested_duration,
                        allocation.width,
                        allocation.height,
                    )),
                    *main_ctrl.borrow().get_state() == ControllerState::Playing,
                );
        }

        self.drawingarea.queue_draw();
    }
}
<commit_msg>Range playback: play a minimum hearable range (#84)<commit_after>extern crate cairo;

extern crate gtk;
use gtk::{Inhibit, ToolButtonExt, WidgetExt};

#[cfg(feature = "profiling-audio-draw")]
use chrono::Utc;

use std::boxed::Box;

use std::rc::Rc;
use std::cell::RefCell;

use std::sync::{Arc, Mutex};

use media::{PlaybackContext, DoubleAudioBuffer, SampleExtractor};

use toc::Timestamp;

use super::{ControllerState, DoubleWaveformBuffer, MainController, WaveformBuffer,
            WaveformConditions, BACKGROUND_COLOR};

const BUFFER_DURATION: u64 = 60_000_000_000; // 60 s
const MIN_REQ_DURATION: f64 = 1_953_125f64; // 2 ms / 1000 px
const MAX_REQ_DURATION: f64 = 32_000_000_000f64; // 32 s / 1000 px
const INIT_REQ_DURATION: f64 = 4_000_000_000f64; // 4 s / 1000 px
const STEP_REQ_DURATION: f64 = 2f64;

const MIN_RANGE_DURATION: u64 = 100_000_000; // 100 ms

pub struct AudioController {
    container: gtk::Box,
    drawingarea: gtk::DrawingArea,
    zoom_in_btn: gtk::ToolButton,
    zoom_out_btn: gtk::ToolButton,

    is_active: bool,
    playback_needs_refresh: bool,

    requested_duration: f64,
    cursor_pos: u64,
    last_visible_pos: u64,

    waveform_mtx: Arc<Mutex<Box<SampleExtractor>>>,
    dbl_buffer_mtx: Arc<Mutex<DoubleAudioBuffer>>,
}

impl AudioController {
    pub fn new(builder: &gtk::Builder) -> Rc<RefCell<Self>> {
        let dbl_buffer_mtx = DoubleWaveformBuffer::new_mutex(BUFFER_DURATION);
        let waveform_mtx = dbl_buffer_mtx
            .lock()
            .expect("AudioController::new: couldn't lock dbl_buffer_mtx")
            .get_exposed_buffer_mtx();

        Rc::new(RefCell::new(AudioController {
            container: builder.get_object("audio-container").unwrap(),
            drawingarea: builder.get_object("audio-drawingarea").unwrap(),
            zoom_in_btn: builder.get_object("audio_zoom_in-toolbutton").unwrap(),
            zoom_out_btn: builder.get_object("audio_zoom_out-toolbutton").unwrap(),

            is_active: false,
            playback_needs_refresh: false,

            requested_duration: INIT_REQ_DURATION,
            cursor_pos: 0,
            last_visible_pos: 0,

            waveform_mtx: waveform_mtx,
            dbl_buffer_mtx: dbl_buffer_mtx,
        }))
    }

    pub fn register_callbacks(
        this_rc: &Rc<RefCell<Self>>,
        main_ctrl: &Rc<RefCell<MainController>>,
    ) {
        let this = this_rc.borrow();

        // draw
        let this_clone = Rc::clone(this_rc);
        this.drawingarea.connect_draw(
            move |drawing_area, cairo_ctx| {
                let mut this = this_clone.borrow_mut();
                this.draw(drawing_area, cairo_ctx)
            },
        );

        // widget size changed
        let main_ctrl_clone = Rc::clone(main_ctrl);
        let this_clone = Rc::clone(this_rc);
        this.drawingarea.connect_size_allocate(move |_, _| {
            this_clone.borrow_mut().refresh(&main_ctrl_clone);
        });

        // click in drawing_area
        let main_ctrl_clone = Rc::clone(main_ctrl);
        let this_clone = Rc::clone(this_rc);
        this.drawingarea.connect_button_press_event(
            move |_, event_button| {
                match event_button.get_button() {
                    1 => { // left click => seek
                        let position_opt = this_clone.borrow()
                            .get_position_at(event_button.get_position().0);
                        if let Some(position) = position_opt {
                            main_ctrl_clone.borrow_mut().seek(position, true); // accurate (slow)
                        }
                    }
                    3 => { // right click => segment playback
                        let (position_opt, cursor_pos, last_pos) = {
                            let this = this_clone.borrow();
                            (
                                this.get_position_at(event_button.get_position().0),
                                this.cursor_pos,
                                this.last_visible_pos,
                            )
                        };
                        if let Some(start_pos) = position_opt {
                            // get a reasonable range so that we can still hear
                            // something even when there are few samples in current window
                            let end_pos = start_pos + MIN_RANGE_DURATION.max(last_pos - start_pos);
                            main_ctrl_clone.borrow_mut()
                                .play_range(start_pos, end_pos, cursor_pos);
                        }
                    }
                    _ => (),
                }
                Inhibit(true)
            },
        );

        // click zoom in
        let main_ctrl_clone = Rc::clone(main_ctrl);
        let this_clone = Rc::clone(this_rc);
        this.zoom_in_btn.connect_clicked(move |_| {
            let mut this = this_clone.borrow_mut();
            this.requested_duration /= STEP_REQ_DURATION;
            if this.requested_duration >= MIN_REQ_DURATION {
                this.refresh(&main_ctrl_clone);
            } else {
                this.requested_duration = MIN_REQ_DURATION;
            }
        });

        // click zoom out
        let main_ctrl_clone = Rc::clone(main_ctrl);
        let this_clone = Rc::clone(this_rc);
        this.zoom_out_btn.connect_clicked(move |_| {
            let mut this = this_clone.borrow_mut();
            this.requested_duration *= STEP_REQ_DURATION;
            if this.requested_duration <= MAX_REQ_DURATION {
                this.refresh(&main_ctrl_clone);
            } else {
                this.requested_duration = MAX_REQ_DURATION;
            }
        });
    }

    pub fn cleanup(&mut self) {
        self.is_active = false;
        self.playback_needs_refresh = false;
        {
            self.dbl_buffer_mtx
                .lock()
                .expect("AudioController::cleanup: Couldn't lock dbl_buffer_mtx")
                .cleanup();
        }
        self.requested_duration = INIT_REQ_DURATION;
        self.cursor_pos = 0;
        self.last_visible_pos = 0;
        self.drawingarea.queue_draw();
    }

    pub fn get_dbl_buffer_mtx(&self) -> Arc<Mutex<DoubleAudioBuffer>> {
        Arc::clone(&self.dbl_buffer_mtx)
    }

    pub fn new_media(&mut self, context: &PlaybackContext) {
        let has_audio = context
            .info
            .lock()
            .expect(
                "Failed to lock media info while initializing audio controller",
            )
            .audio_best
            .is_some();

        if has_audio {
            let allocation = self.drawingarea.get_allocation();
            {
                // init the buffers in order to render the waveform in current conditions
                let requested_duration = self.requested_duration;
                self.dbl_buffer_mtx
                    .lock()
                    .expect(
                        "AudioController::size-allocate: couldn't lock dbl_buffer_mtx",
                    )
                    .set_conditions(Box::new(WaveformConditions::new(
                        requested_duration,
                        allocation.width,
                        allocation.height,
                    )));
            }

            self.is_active = true;
            self.container.show();
        } else {
            self.container.hide();
        }
    }

    pub fn seek(&mut self, position: u64, state: &ControllerState) {
        {
            self.waveform_mtx
                .lock()
                .expect("AudioController::seek: Couldn't lock waveform_mtx")
                .as_mut_any()
                .downcast_mut::<WaveformBuffer>()
                .expect(
                    "AudioController::seek: SamplesExtratctor is not a WaveformBuffer",
                )
                .seek(position, *state == ControllerState::Playing);
        }

        if *state == ControllerState::Paused {
            // refresh the buffer in order to render the waveform
            // with samples that might not be rendered in current WaveformImage yet
            self.dbl_buffer_mtx
                .lock()
                .expect("AudioController::seek: couldn't lock dbl_buffer_mtx")
                .refresh(false); // don't keep continuity
        }
        self.drawingarea.queue_draw();
    }

    pub fn start_play_range(&mut self) {
        self.waveform_mtx
            .lock()
            .expect("AudioController::seek: Couldn't lock waveform_mtx")
            .as_mut_any()
            .downcast_mut::<WaveformBuffer>()
            .expect(
                "AudioController::seek: SamplesExtratctor is not a WaveformBuffer",
            )
            .start_play_range();
    }

    pub fn tick(&mut self) {
        if self.is_active {
            if self.playback_needs_refresh {
                #[cfg(feature = "trace-audio-controller")]
                println!("AudioController::tick forcing refresh");

                self.dbl_buffer_mtx
                    .lock()
                    .expect("AudioController::tick: couldn't lock dbl_buffer_mtx")
                    .refresh(true); // keep continuity
            }

            self.drawingarea.queue_draw();
        }
    }

    fn get_position_at(&self, x: f64) -> Option<u64> {
        let waveform_buffer_grd = &mut *self.waveform_mtx.lock().expect(
            "Couldn't lock waveform buffer in audio controller draw",
        );
        waveform_buffer_grd
            .as_any()
            .downcast_ref::<WaveformBuffer>()
            .expect(concat!(
                "AudioController::get_current_and_sample_at ",
                "SamplesExtratctor is not a waveform buffer",
            ))
            .get_position_at(x)
    }

    fn clean_cairo_context(&self, cr: &cairo::Context) {
        cr.set_source_rgb(BACKGROUND_COLOR.0, BACKGROUND_COLOR.1, BACKGROUND_COLOR.2);
        cr.paint();
    }

    fn draw(&mut self, drawingarea: &gtk::DrawingArea, cr: &cairo::Context) -> Inhibit {
        #[cfg(feature = "profiling-audio-draw")]
        let before_init = Utc::now();

        let allocation = drawingarea.get_allocation();
        if allocation.width.is_negative() {
            #[cfg(feature = "trace-audio-controller")]
            println!(
                "AudioController::draw negative allocation.width: {}",
                allocation.width
            );

            self.clean_cairo_context(cr);
            return Inhibit(true);
        }

        #[cfg(feature = "profiling-audio-draw")]
        let before_lock = Utc::now();
        #[cfg(feature = "profiling-audio-draw")]
        let mut _before_cndt = Utc::now();
        #[cfg(feature = "profiling-audio-draw")]
        let mut _before_image = Utc::now();

        let image_positions = {
            let waveform_grd = &mut *self.waveform_mtx.lock().expect(
                "AudioController::draw: couldn't lock waveform_mtx",
            );
            let waveform_buffer = waveform_grd
                .as_mut_any()
                .downcast_mut::<WaveformBuffer>()
                .expect(
                    "AudioController::draw: SamplesExtratctor is not a WaveformBuffer",
                );

            #[cfg(feature = "profiling-audio-draw")]
            let _before_cndt = Utc::now();

            waveform_buffer.update_conditions(
                self.requested_duration,
                allocation.width,
                allocation.height,
            );

            self.playback_needs_refresh = waveform_buffer.playback_needs_refresh;

            match waveform_buffer.get_image() {
                Some((image, image_positions)) => {
                    #[cfg(feature = "profiling-audio-draw")]
                    let _before_image = Utc::now();

                    cr.set_source_surface(image, -image_positions.first.x, 0f64);
                    cr.paint();

                    image_positions
                }
                None => {
                    self.clean_cairo_context(cr);

                    #[cfg(feature = "trace-audio-controller")]
                    println!("AudioController::draw no image");

                    return Inhibit(true);
                }
            }
        };

        #[cfg(feature = "profiling-audio-draw")]
        let before_pos = Utc::now();

        let height = f64::from(allocation.height);
        let width = f64::from(allocation.width);
        cr.scale(1f64, 1f64);
        cr.set_source_rgb(1f64, 1f64, 0f64);
        cr.set_font_size(14f64);

        // first position
        let first_text = Timestamp::format(image_positions.first.timestamp, false);
        let first_text_width = 2f64 + cr.text_extents(&first_text).width;
        cr.move_to(2f64, 30f64);
        cr.show_text(&first_text);

        // last position
        if let Some(last_pos) = image_positions.last {
            let last_text = Timestamp::format(last_pos.timestamp, false);
            // align actual width to a 15px multiple box in order to avoid
            // variations due to actual size of individual digits
            let last_text_width = (cr.text_extents(&last_text).width / 15f64).ceil() * 15f64;
            if last_pos.x - last_text_width > first_text_width + 5f64 {
                // last text won't overlap with first text
                cr.move_to(last_pos.x - last_text_width, 30f64);
                cr.show_text(&last_text);
            }

            self.last_visible_pos = last_pos.timestamp;

            // make sure the rest of the image is filled with background's color
            cr.set_source_rgb(BACKGROUND_COLOR.0, BACKGROUND_COLOR.1, BACKGROUND_COLOR.2);
            cr.rectangle(last_pos.x, 0f64, width - last_pos.x, height);
            cr.fill();
        }

        if let Some(current_pos) = image_positions.current {
            // draw current pos
            cr.set_source_rgb(1f64, 1f64, 0f64);

            let cursor_text = Timestamp::format(current_pos.timestamp, true);
            let cursor_text_width = 5f64 + cr.text_extents(&cursor_text).width;
            let cursor_text_x = if current_pos.x + cursor_text_width < width {
                current_pos.x + 5f64
            } else {
                current_pos.x - cursor_text_width
            };
            cr.move_to(cursor_text_x, 15f64);
            cr.show_text(&cursor_text);

            cr.set_line_width(1f64);
            cr.move_to(current_pos.x, 0f64);
            cr.line_to(current_pos.x, height);
            cr.stroke();

            self.cursor_pos = current_pos.timestamp;
        }

        #[cfg(feature = "profiling-audio-draw")]
        let end = Utc::now();

        #[cfg(feature = "profiling-audio-draw")]
        println!(
            "audio-draw,{},{},{},{},{},{}",
            before_init.time().format("%H:%M:%S%.6f"),
            before_lock.time().format("%H:%M:%S%.6f"),
            _before_cndt.time().format("%H:%M:%S%.6f"),
            _before_image.time().format("%H:%M:%S%.6f"),
            before_pos.time().format("%H:%M:%S%.6f"),
            end.time().format("%H:%M:%S%.6f"),
        );

        Inhibit(true)
    }

    fn refresh(&mut self, main_ctrl: &Rc<RefCell<MainController>>) {
        let allocation = self.drawingarea.get_allocation();
        {
            // refresh the buffer in order to render the waveform
            // in latest conditions
            let requested_duration = self.requested_duration;
            self.dbl_buffer_mtx
                .lock()
                .expect(
                    "AudioController::size-allocate: couldn't lock dbl_buffer_mtx",
                )
                .refresh_with_conditions(
                    Box::new(WaveformConditions::new(
                        requested_duration,
                        allocation.width,
                        allocation.height,
                    )),
                    *main_ctrl.borrow().get_state() == ControllerState::Playing,
                );
        }

        self.drawingarea.queue_draw();
    }
}
<|endoftext|>