<commit_before>//! Background layer rendering

use super::{Ppu, Rgb};

/// Collected background settings
struct BgSettings {
    /// Mosaic pixel size (1-16). 1 = Normal pixels.
    /// FIXME: I think there's a difference between disabled and enabled with 1x1 mosaic size in
    /// some modes (highres presumably)
    #[allow(dead_code)] // FIXME NYI
    mosaic: u8,
    /// Tilemap word address in VRAM
    /// "Starting at the tilemap address, the first $800 bytes are for tilemap A. Then come the
    /// $800 bytes for B, then C then D."
    tilemap_word_addr: u16,
    /// When `true`, this BGs tilemaps are mirrored sideways
    tilemap_mirror_h: bool,
    /// When `true`, this BGs tilemaps are mirrored downwards
    tilemap_mirror_v: bool,
    /// If `true`, BG tiles are 16x16 pixels. If `false`, they are 8x8 pixels.
    tile_size_16: bool,
    /// Character Data start address in VRAM
    chr_addr: u16,
    hofs: u16,
    vofs: u16,
}

/// Unpacked tilemap entry for internal (rendering) use
struct TilemapEntry {
    #[allow(dead_code)] // FIXME
    vflip: bool,
    #[allow(dead_code)]
    hflip: bool,
    /// Priority bit (0-1)
    priority: u8,
    /// Tile palette (0-7)
    palette: u8,
    /// Index into the character/tile data, where the actual tile is stored
    tile_number: u16,
}

impl Ppu {
    /// Determines whether the given BG layer (1-4) is enabled
    fn bg_enabled(&self, bg: u8) -> bool { self.tm & (1 << (bg - 1)) != 0 }

    /// Reads the tilemap entry at the given VRAM word address.
    ///     vhopppcc cccccccc (high, low)
    ///     v/h        = Vertical/Horizontal flip this tile.
    ///     o          = Tile priority.
    ///     ppp        = Tile palette base.
    ///     cccccccccc = Tile number.
    fn tilemap_entry(&self, word_address: u16) -> TilemapEntry {
        let byte_address = word_address << 1;
        let lo = self.vram[byte_address];
        let hi = self.vram[byte_address + 1];

        TilemapEntry {
            vflip: hi & 0x80 != 0,
            hflip: hi & 0x40 != 0,
            priority: (hi & 0x20) >> 5,
            palette: (hi & 0x1c) >> 2,
            tile_number: ((hi as u16 & 0x03) << 8) | lo as u16,
        }
    }

    /// Collects properties of a background layer
    fn bg_settings(&self, bg: u8) -> BgSettings {
        // The BGxSC register for our background layer
        let bgsc = match bg {
            1 => self.bg1sc,
            2 => self.bg2sc,
            3 => self.bg3sc,
            4 => self.bg4sc,
            _ => unreachable!(),
        };
        // Chr (Tileset, not Tilemap) start (word?) address >> 12
        let chr = match bg {
            1 => self.bg12nba & 0x0f,
            2 => (self.bg12nba & 0xf0) >> 4,
            3 => self.bg34nba & 0x0f,
            4 => (self.bg34nba & 0xf0) >> 4,
            _ => unreachable!(),
        };
        let (hofs, vofs) = match bg {
            1 => (self.bg1hofs, self.bg1vofs),
            2 => (self.bg2hofs, self.bg2vofs),
            3 => (self.bg3hofs, self.bg3vofs),
            4 => (self.bg4hofs, self.bg4vofs),
            _ => unreachable!(),
        };

        BgSettings {
            mosaic: if self.mosaic & (1 << (bg-1)) == 0 {
                1
            } else {
                ((self.mosaic & 0xf0) >> 4) + 1
            },
            // FIXME: This looks more like the byte address to me!
            tilemap_word_addr: ((bgsc as u16 & 0xfc) >> 2) << 10,
            tilemap_mirror_h: bgsc & 0b01 == 0, // inverted bit value
            tilemap_mirror_v: bgsc & 0b10 == 0, // inverted bit value
            tile_size_16: match self.bg_mode() {
                // "If the BG character size for BG1/BG2/BG3/BG4 bit is set, then the BG is made of
                // 16x16 tiles. Otherwise, 8x8 tiles are used. However, note that Modes 5 and 6
                // always use 16-pixel wide tiles, and Mode 7 always uses 8x8 tiles."
                5 | 6 => true,
                7 => false,
                _ => {
                    // BGMODE: `4321----` (`-` = not relevant here) - Use 16x16 tiles?
                    self.bgmode & (1 << (bg + 3)) != 0
                }
            },
            chr_addr: (chr as u16) << 12,
            hofs: hofs,
            vofs: vofs,
        }
    }

    /// Returns the number of colors in the given BG layer in the current BG mode (4, 16, 128 or
    /// 256). `X` denotes a BG for offset-per-tile data.
    ///
    ///     Mode    # Colors for BG
    ///              1   2   3   4
    ///     ======---=---=---=---=
    ///     0        4   4   4   4
    ///     1       16  16   4   -
    ///     2       16  16   X   -
    ///     3      256  16   -   -
    ///     4      256   4   X   -
    ///     5       16   4   -   -
    ///     6       16   -   X   -
    ///     7      256   -   -   -
    ///     7EXTBG 256 128   -   -
    fn color_count_for_bg(&self, bg: u8) -> u16 {
        match self.bg_mode() {
            0 => 4,
            1 => match bg {
                1 | 2 => 16,
                3 => 4,
                _ => unreachable!(),
            },
            2 => 16,
            3 => match bg {
                1 => 256,
                2 => 16,
                _ => unreachable!(),
            },
            4 => match bg {
                1 => 256,
                2 => 4,
                _ => unreachable!(),
            },
            5 => match bg {
                1 => 16,
                2 => 4,
                _ => unreachable!(),
            },
            6 => 16,
            7 => panic!("NYI: color_count_for_bg for mode 7"),   // (make sure to handle EXTBG)
            _ => unreachable!(),
        }
    }

    /// Calculates the palette base index for a tile in the given background layer. `tile_palette`
    /// is the palette number stored in the tilemap entry (the 3 `p` bits).
    fn palette_base_for_bg_tile(&self, bg: u8, palette_num: u8) -> u8 {
        debug_assert!(bg >= 1 && bg <= 4);
        match self.bg_mode() {
            0 => palette_num * 4 + (bg - 1) * 32,
            1 | 5 => palette_num * self.color_count_for_bg(bg) as u8,   // doesn't have 256 colors
            2 => palette_num * 16,
            3 => match bg {
                1 => 0,
                2 => palette_num * 16,
                _ => unreachable!(),    // no BG3/4
            },
            4 => match bg {
                1 => 0,
                2 => palette_num * 4,
                _ => unreachable!(),    // no BG3/4
            },
            6 => palette_num * 16,      // BG1 has 16 colors
            7 => panic!("NYI: palette_base_for_bg_tile for mode 7"),
            _ => unreachable!(),
        }
    }

    /// Lookup the color of the given background layer (1-4) at the current pixel, using the given
    /// priority (0-1) only. This will also scroll backgrounds accordingly and apply color math.
    ///
    /// Returns `None` if the pixel is transparent, `Some(Rgb)` otherwise.
    pub fn lookup_bg_color(&self, bg_num: u8, prio: u8) -> Option<Rgb> {
        debug_assert!(bg_num >= 1 && bg_num <= 4);
        debug_assert!(prio == 0 || prio == 1);
        if !self.bg_enabled(bg_num) { return None }

        // Apply BG scrolling and get the tile coordinates
        // FIXME Apply mosaic filter
        // FIXME Fix this: "Note that many games will set their vertical scroll values to -1 rather
        // than 0. This is because the SNES loads OBJ data for each scanline during the previous
        // scanline. The very first line, though, wouldn’t have any OBJ data loaded! So the SNES
        // doesn’t actually output scanline 0, although it does everything to render it. These
        // games want the first line of their tilemap to be the first line output, so they set
        // their VOFS registers in this manner. Note that an interlace screen needs -2 rather than
        // -1 to properly correct for the missing line 0 (and an emulator would need to add 2
        // instead of 1 to account for this)."
        let x = self.x;
        let y = self.scanline;
        let bg = self.bg_settings(bg_num);
        let tile_size = if bg.tile_size_16 { 16 } else { 8 };
        let (hofs, vofs) = (bg.hofs, bg.vofs);
        let tile_x = x.wrapping_add(hofs) / tile_size as u16;
        let tile_y = y.wrapping_add(vofs) / tile_size as u16;
        let off_x = (x.wrapping_add(hofs) % tile_size as u16) as u8;
        let off_y = (y.wrapping_add(vofs) % tile_size as u16) as u8;
        let (sx, sy) = (!bg.tilemap_mirror_h, !bg.tilemap_mirror_v);

        // Calculate the VRAM word address, where the tilemap entry for our tile is stored
        // FIXME Check if this really is correct
        let tilemap_entry_word_address =
            bg.tilemap_word_addr |
            ((tile_y & 0x1f) << 5) |
            (tile_x & 0x1f) |
            if sy {(tile_y & 0x20) << if sx {6} else {5}} else {0} |
            if sx {(tile_x & 0x20) << 5} else {0};
        let tilemap_entry = self.tilemap_entry(tilemap_entry_word_address);
        if tilemap_entry.priority != prio { return None }

        let color_count = self.color_count_for_bg(bg_num);
        debug_assert!(color_count.is_power_of_two());  // should be power of two
        if color_count == 256 {
            debug_assert!(self.cgwsel & 0x01 == 0, "NYI: direct color mode");
        }

        // Calculate the number of bitplanes needed to store a color in this BG
        let bitplane_count = (color_count - 1).count_ones() as u16;

        // FIXME: Formula taken from the wiki, is this correct? In particular: `chr_addr<<1`?
        let bitplane_start_addr =
            (bg.chr_addr << 1) +
            (tilemap_entry.tile_number * 8 * bitplane_count);   // 8 bytes per bitplane

        let palette_base = self.palette_base_for_bg_tile(bg_num, tilemap_entry.palette);
        let palette_index = self.read_chr_entry(bitplane_count as u8,
                                                bitplane_start_addr,
                                                tile_size,
                                                (off_x, off_y));

        match palette_index {
            0 => None,
            _ => Some(self.lookup_color(palette_base + palette_index)),
        }
    }
}
<commit_msg>ignore accidental test<commit_after>//! Background layer rendering

use super::{Ppu, Rgb};

/// Collected background settings
struct BgSettings {
    /// Mosaic pixel size (1-16). 1 = Normal pixels.
    /// FIXME: I think there's a difference between disabled and enabled with 1x1 mosaic size in
    /// some modes (highres presumably)
    #[allow(dead_code)] // FIXME NYI
    mosaic: u8,
    /// Tilemap word address in VRAM
    /// "Starting at the tilemap address, the first $800 bytes are for tilemap A. Then come the
    /// $800 bytes for B, then C then D."
    tilemap_word_addr: u16,
    /// When `true`, this BGs tilemaps are mirrored sideways
    tilemap_mirror_h: bool,
    /// When `true`, this BGs tilemaps are mirrored downwards
    tilemap_mirror_v: bool,
    /// If `true`, BG tiles are 16x16 pixels. If `false`, they are 8x8 pixels.
    tile_size_16: bool,
    /// Character Data start address in VRAM
    chr_addr: u16,
    hofs: u16,
    vofs: u16,
}

/// Unpacked tilemap entry for internal (rendering) use
struct TilemapEntry {
    #[allow(dead_code)] // FIXME
    vflip: bool,
    #[allow(dead_code)]
    hflip: bool,
    /// Priority bit (0-1)
    priority: u8,
    /// Tile palette (0-7)
    palette: u8,
    /// Index into the character/tile data, where the actual tile is stored
    tile_number: u16,
}

impl Ppu {
    /// Determines whether the given BG layer (1-4) is enabled
    fn bg_enabled(&self, bg: u8) -> bool { self.tm & (1 << (bg - 1)) != 0 }

    /// Reads the tilemap entry at the given VRAM word address.
    ///     vhopppcc cccccccc (high, low)
    ///     v/h        = Vertical/Horizontal flip this tile.
    ///     o          = Tile priority.
    ///     ppp        = Tile palette base.
    ///     cccccccccc = Tile number.
    fn tilemap_entry(&self, word_address: u16) -> TilemapEntry {
        let byte_address = word_address << 1;
        let lo = self.vram[byte_address];
        let hi = self.vram[byte_address + 1];

        TilemapEntry {
            vflip: hi & 0x80 != 0,
            hflip: hi & 0x40 != 0,
            priority: (hi & 0x20) >> 5,
            palette: (hi & 0x1c) >> 2,
            tile_number: ((hi as u16 & 0x03) << 8) | lo as u16,
        }
    }

    /// Collects properties of a background layer
    fn bg_settings(&self, bg: u8) -> BgSettings {
        // The BGxSC register for our background layer
        let bgsc = match bg {
            1 => self.bg1sc,
            2 => self.bg2sc,
            3 => self.bg3sc,
            4 => self.bg4sc,
            _ => unreachable!(),
        };
        // Chr (Tileset, not Tilemap) start (word?) address >> 12
        let chr = match bg {
            1 => self.bg12nba & 0x0f,
            2 => (self.bg12nba & 0xf0) >> 4,
            3 => self.bg34nba & 0x0f,
            4 => (self.bg34nba & 0xf0) >> 4,
            _ => unreachable!(),
        };
        let (hofs, vofs) = match bg {
            1 => (self.bg1hofs, self.bg1vofs),
            2 => (self.bg2hofs, self.bg2vofs),
            3 => (self.bg3hofs, self.bg3vofs),
            4 => (self.bg4hofs, self.bg4vofs),
            _ => unreachable!(),
        };

        BgSettings {
            mosaic: if self.mosaic & (1 << (bg-1)) == 0 {
                1
            } else {
                ((self.mosaic & 0xf0) >> 4) + 1
            },
            // FIXME: This looks more like the byte address to me!
            tilemap_word_addr: ((bgsc as u16 & 0xfc) >> 2) << 10,
            tilemap_mirror_h: bgsc & 0b01 == 0, // inverted bit value
            tilemap_mirror_v: bgsc & 0b10 == 0, // inverted bit value
            tile_size_16: match self.bg_mode() {
                // "If the BG character size for BG1/BG2/BG3/BG4 bit is set, then the BG is made of
                // 16x16 tiles. Otherwise, 8x8 tiles are used. However, note that Modes 5 and 6
                // always use 16-pixel wide tiles, and Mode 7 always uses 8x8 tiles."
                5 | 6 => true,
                7 => false,
                _ => {
                    // BGMODE: `4321----` (`-` = not relevant here) - Use 16x16 tiles?
                    self.bgmode & (1 << (bg + 3)) != 0
                }
            },
            chr_addr: (chr as u16) << 12,
            hofs: hofs,
            vofs: vofs,
        }
    }

    /// Returns the number of colors in the given BG layer in the current BG mode (4, 16, 128 or
    /// 256). `X` denotes a BG for offset-per-tile data.
    ///
    /// ```ignore
    /// Mode    # Colors for BG
    ///          1   2   3   4
    /// ======---=---=---=---=
    /// 0        4   4   4   4
    /// 1       16  16   4   -
    /// 2       16  16   X   -
    /// 3      256  16   -   -
    /// 4      256   4   X   -
    /// 5       16   4   -   -
    /// 6       16   -   X   -
    /// 7      256   -   -   -
    /// 7EXTBG 256 128   -   -
    /// ```
    fn color_count_for_bg(&self, bg: u8) -> u16 {
        match self.bg_mode() {
            0 => 4,
            1 => match bg {
                1 | 2 => 16,
                3 => 4,
                _ => unreachable!(),
            },
            2 => 16,
            3 => match bg {
                1 => 256,
                2 => 16,
                _ => unreachable!(),
            },
            4 => match bg {
                1 => 256,
                2 => 4,
                _ => unreachable!(),
            },
            5 => match bg {
                1 => 16,
                2 => 4,
                _ => unreachable!(),
            },
            6 => 16,
            7 => panic!("NYI: color_count_for_bg for mode 7"),   // (make sure to handle EXTBG)
            _ => unreachable!(),
        }
    }

    /// Calculates the palette base index for a tile in the given background layer. `tile_palette`
    /// is the palette number stored in the tilemap entry (the 3 `p` bits).
    fn palette_base_for_bg_tile(&self, bg: u8, palette_num: u8) -> u8 {
        debug_assert!(bg >= 1 && bg <= 4);
        match self.bg_mode() {
            0 => palette_num * 4 + (bg - 1) * 32,
            1 | 5 => palette_num * self.color_count_for_bg(bg) as u8,   // doesn't have 256 colors
            2 => palette_num * 16,
            3 => match bg {
                1 => 0,
                2 => palette_num * 16,
                _ => unreachable!(),    // no BG3/4
            },
            4 => match bg {
                1 => 0,
                2 => palette_num * 4,
                _ => unreachable!(),    // no BG3/4
            },
            6 => palette_num * 16,      // BG1 has 16 colors
            7 => panic!("NYI: palette_base_for_bg_tile for mode 7"),
            _ => unreachable!(),
        }
    }

    /// Lookup the color of the given background layer (1-4) at the current pixel, using the given
    /// priority (0-1) only. This will also scroll backgrounds accordingly and apply color math.
    ///
    /// Returns `None` if the pixel is transparent, `Some(Rgb)` otherwise.
    pub fn lookup_bg_color(&self, bg_num: u8, prio: u8) -> Option<Rgb> {
        debug_assert!(bg_num >= 1 && bg_num <= 4);
        debug_assert!(prio == 0 || prio == 1);
        if !self.bg_enabled(bg_num) { return None }

        // Apply BG scrolling and get the tile coordinates
        // FIXME Apply mosaic filter
        // FIXME Fix this: "Note that many games will set their vertical scroll values to -1 rather
        // than 0. This is because the SNES loads OBJ data for each scanline during the previous
        // scanline. The very first line, though, wouldn’t have any OBJ data loaded! So the SNES
        // doesn’t actually output scanline 0, although it does everything to render it. These
        // games want the first line of their tilemap to be the first line output, so they set
        // their VOFS registers in this manner. Note that an interlace screen needs -2 rather than
        // -1 to properly correct for the missing line 0 (and an emulator would need to add 2
        // instead of 1 to account for this)."
        let x = self.x;
        let y = self.scanline;
        let bg = self.bg_settings(bg_num);
        let tile_size = if bg.tile_size_16 { 16 } else { 8 };
        let (hofs, vofs) = (bg.hofs, bg.vofs);
        let tile_x = x.wrapping_add(hofs) / tile_size as u16;
        let tile_y = y.wrapping_add(vofs) / tile_size as u16;
        let off_x = (x.wrapping_add(hofs) % tile_size as u16) as u8;
        let off_y = (y.wrapping_add(vofs) % tile_size as u16) as u8;
        let (sx, sy) = (!bg.tilemap_mirror_h, !bg.tilemap_mirror_v);

        // Calculate the VRAM word address, where the tilemap entry for our tile is stored
        // FIXME Check if this really is correct
        let tilemap_entry_word_address =
            bg.tilemap_word_addr |
            ((tile_y & 0x1f) << 5) |
            (tile_x & 0x1f) |
            if sy {(tile_y & 0x20) << if sx {6} else {5}} else {0} |
            if sx {(tile_x & 0x20) << 5} else {0};
        let tilemap_entry = self.tilemap_entry(tilemap_entry_word_address);
        if tilemap_entry.priority != prio { return None }

        let color_count = self.color_count_for_bg(bg_num);
        debug_assert!(color_count.is_power_of_two());  // should be power of two
        if color_count == 256 {
            debug_assert!(self.cgwsel & 0x01 == 0, "NYI: direct color mode");
        }

        // Calculate the number of bitplanes needed to store a color in this BG
        let bitplane_count = (color_count - 1).count_ones() as u16;

        // FIXME: Formula taken from the wiki, is this correct? In particular: `chr_addr<<1`?
        let bitplane_start_addr =
            (bg.chr_addr << 1) +
            (tilemap_entry.tile_number * 8 * bitplane_count);   // 8 bytes per bitplane

        let palette_base = self.palette_base_for_bg_tile(bg_num, tilemap_entry.palette);
        let palette_index = self.read_chr_entry(bitplane_count as u8,
                                                bitplane_start_addr,
                                                tile_size,
                                                (off_x, off_y));

        match palette_index {
            0 => None,
            _ => Some(self.lookup_color(palette_base + palette_index)),
        }
    }
}
<|endoftext|>


<commit_before>//! Services and MakeServices
//!
//! - A [`Service`](service::Service) is a trait representing an asynchronous
//!   function of a request to a response. It's similar to
//!   `async fn(Request) -> Result<Response, Error>`.
//! - A [`MakeService`](service::MakeService) is a trait creating specific
//!   instances of a `Service`.
//!
//! These types are conceptually similar to those in
//! [tower](https://crates.io/crates/tower), while being specific to hyper.
//!
//! # Service
//!
//! In hyper, especially in the server setting, a `Service` is usually bound
//! to a single connection. It defines how to respond to **all** requests that
//! connection will receive.
//!
//! While it's possible to implement `Service` for a type manually, the helpers
//! [`service_fn`](service::service_fn) and
//! [`service_fn_ok`](service::service_fn_ok) should be sufficient for most
//! cases.
//!
//! # MakeService
//!
//! Since a `Service` is bound to a single connection, a [`Server`](::Server)
//! needs a way to make them as it accepts connections. This is what a
//! `MakeService` does.
//!
//! Resources that need to be shared by all `Service`s can be put into a
//! `MakeService`, and then passed to individual `Service`s when `make_service`
//! is called.

mod make_service;
mod service;

pub use self::make_service::{make_service_fn, MakeService, MakeServiceRef};
pub use self::service::{service_fn, Service};
<commit_msg>docs(service): remove reference to service_fn_ok (#1978)<commit_after>//! Services and MakeServices
//!
//! - A [`Service`](service::Service) is a trait representing an asynchronous
//!   function of a request to a response. It's similar to
//!   `async fn(Request) -> Result<Response, Error>`.
//! - A [`MakeService`](service::MakeService) is a trait creating specific
//!   instances of a `Service`.
//!
//! These types are conceptually similar to those in
//! [tower](https://crates.io/crates/tower), while being specific to hyper.
//!
//! # Service
//!
//! In hyper, especially in the server setting, a `Service` is usually bound
//! to a single connection. It defines how to respond to **all** requests that
//! connection will receive.
//!
//! While it's possible to implement `Service` for a type manually, the helper
//! [`service_fn`](service::service_fn) should be sufficient for most cases.
//!
//! # MakeService
//!
//! Since a `Service` is bound to a single connection, a [`Server`](::Server)
//! needs a way to make them as it accepts connections. This is what a
//! `MakeService` does.
//!
//! Resources that need to be shared by all `Service`s can be put into a
//! `MakeService`, and then passed to individual `Service`s when `make_service`
//! is called.

mod make_service;
mod service;

pub use self::make_service::{make_service_fn, MakeService, MakeServiceRef};
pub use self::service::{service_fn, Service};
<|endoftext|>


<commit_before>use std::hash::Hash;
use std::fmt::Debug;
use std::borrow::Borrow;
use std::marker::PhantomData;

use perlin_core::index::posting::{Posting, PostingIterator};
use perlin_core::utils::seeking_iterator::{PeekableSeekable, SeekingIterator};
use perlin_core::utils::progress::Progress;
use perlin_core::index::Index;

use language::CanApply;
use query::{ToOperands, Operand, Operator, ChainingOperator};
use field::Fields;

/// Mimics the functionality of the `try!` macro for `Option`s.
/// Evaluates `Some(x)` to x. Else it returns `None`.
macro_rules! try_option{
    ($operand:expr) => {
        if let Some(x) = $operand {
            x
        } else {
            return None;
        }
    }
}

#[derive(Debug)]
pub enum Combinator {
    All,
    Any,
}

pub struct SplitFunnel<'a, T: 'a + Hash + Eq, CB> {
    index: &'a Index<T>,
    combinator: Combinator,
    chaining_operator: ChainingOperator,
    result: Vec<PeekableSeekable<Operand<'a>>>,
    callback: CB,
}

impl<'a, T: 'a + Hash + Eq, CB> SplitFunnel<'a, T, CB> {
    pub fn create(chaining_operator: ChainingOperator,
                  combinator: Combinator,
                  index: &'a Index<T>,
                  cb: CB)
                  -> Self {
        SplitFunnel {
            index: index,
            combinator: combinator,
            chaining_operator: chaining_operator,
            callback: cb,
            result: Vec::new(),
        }
    }
}

impl<'a, T: 'a + Hash + Eq + Ord, CB> CanApply<T> for SplitFunnel<'a, T, CB>
    where CB: CanApply<T>
{
    type Output = CB::Output;

    fn apply(&mut self, term: T) {
        // Query index
        match self.index.query_atom(&term) {
            PostingIterator::Empty => self.result.push(PeekableSeekable::new(Operand::Empty)),
            PostingIterator::Decoder(decoder) => {
                self.result.push(PeekableSeekable::new(Operand::Term(decoder)))
            }
        }
        // And keep going
        self.callback.apply(term);
    }
}

impl<'a, T: 'a + Hash + Eq, CB> ToOperands<'a> for SplitFunnel<'a, T, CB>
    where CB: ToOperands<'a>
{
    fn to_operands(self) -> Vec<(ChainingOperator, PeekableSeekable<Operand<'a>>)> {
        let mut other = self.callback.to_operands();
        match self.combinator {
            Combinator::All => {
                other.push((self.chaining_operator,
                            PeekableSeekable::new(Operand::Operated(Box::new(And {}),
                                                                    self.result))))
            }
            Combinator::Any => {
                other.push((self.chaining_operator,
                            PeekableSeekable::new(Operand::Operated(Box::new(Or {}), self.result))))
            }
        }
        other
    }
}

/// This funnel is used at an end of a query pipeline
/// It calls `index.query_atom` and stores the result, which is lazy
/// When `to_operand` is then called, it packs everything into an operator!
// TODO: In this struct lies an opportunity to optimize
// If the operator is AND and one term returns an empty posting iterator
// We could skip the rest
// If the operator is Or and one term returns an empty posting iterator we
// could discard it
pub struct Funnel<'a, T: 'a, TIndex: 'a> {
    index: &'a TIndex,
    combinator: Combinator,
    chaining_operator: ChainingOperator,
    result: Vec<PeekableSeekable<Operand<'a>>>,
    _term: PhantomData<T>,
}

impl<'a, T: 'a, TIndex: 'a> Funnel<'a, T, TIndex> {
    pub fn create(chaining_operator: ChainingOperator,
                  combinator: Combinator,
                  index: &'a TIndex)
                  -> Self {
        Funnel {
            index: index,
            combinator: combinator,
            chaining_operator: chaining_operator,
            result: Vec::new(),
            _term: PhantomData,
        }
    }
}

impl<'a: 'b, 'b, T: 'a + Hash + Eq + Ord + Debug> CanApply<&'b T>
    for Funnel<'a, T, Fields<T>> {
    type Output = T;

    fn apply(&mut self, term: &'b T) {
        for index in self.index.fields.values() {
            match index.query_atom(&term) {
                PostingIterator::Empty => self.result.push(PeekableSeekable::new(Operand::Empty)),
                PostingIterator::Decoder(decoder) => {
                    self.result.push(PeekableSeekable::new(Operand::Term(decoder)))
                }
            }
        }
    }
}

impl<'a, T: 'a + Hash + Eq + Ord + Debug> CanApply<T>
    for Funnel<'a, T, Fields<T>> {
    type Output = T;

    fn apply(&mut self, term: T) {
        for index in self.index.fields.values() {
            match index.query_atom(&term) {
                PostingIterator::Empty => self.result.push(PeekableSeekable::new(Operand::Empty)),
                PostingIterator::Decoder(decoder) => {
                    self.result.push(PeekableSeekable::new(Operand::Term(decoder)))
                }
            }
        }
    }
}

impl<'a: 'b, 'b, T: 'a + Hash + Eq + Ord + Debug, TIndex> CanApply<&'b T> for Funnel<'a, T, TIndex>
    where TIndex: Borrow<Index<T>>
{
    type Output = T;

    fn apply(&mut self, term: &'b T) {
        match (self.index.borrow()).query_atom(&term) {
            PostingIterator::Empty => self.result.push(PeekableSeekable::new(Operand::Empty)),
            PostingIterator::Decoder(decoder) => {
                self.result.push(PeekableSeekable::new(Operand::Term(decoder)))
            }
        }
    }
}

impl<'a, T: 'a + Hash + Eq + Ord + Debug, TIndex> CanApply<T> for Funnel<'a, T, TIndex>
    where TIndex: Borrow<Index<T>>
{
    type Output = T;

    fn apply(&mut self, term: T) {
        match (self.index.borrow()).query_atom(&term) {
            PostingIterator::Empty => self.result.push(PeekableSeekable::new(Operand::Empty)),
            PostingIterator::Decoder(decoder) => {
                self.result.push(PeekableSeekable::new(Operand::Term(decoder)))
            }
        }
    }
}

impl<'a, T: 'a, TIndex> ToOperands<'a> for Funnel<'a, T, TIndex> {
    fn to_operands(self) -> Vec<(ChainingOperator, PeekableSeekable<Operand<'a>>)> {
        if self.result.is_empty() {
            return vec![];
        }
        match self.combinator {
            Combinator::All => {
                vec![(self.chaining_operator,
                      PeekableSeekable::new(Operand::Operated(Box::new(And {}), self.result)))]
            }
            Combinator::Any => {
                vec![(self.chaining_operator,
                      PeekableSeekable::new(Operand::Operated(Box::new(Or {}), self.result)))]
            }
        }

    }
}
/// END FUNNEL
#[derive(Debug, Copy, Clone)]
pub struct And;

impl Operator for And {
    fn next(&mut self, operands: &mut [PeekableSeekable<Operand>]) -> Option<Posting> {
        let mut focus = try_option!(operands[0].next()); // Acts as temporary to be compared against
        let mut last_iter = 0; // The iterator that last set 'focus'
        'possible_documents: loop {
            // For every term
            for (i, input) in operands.iter_mut().enumerate() {
                // If the focus was set by the current iterator, we have a match
                // We cycled through all the iterators once
                if last_iter == i {
                    continue;
                }

                let v = try_option!(input.next_seek(&focus));
                if v.0 > focus.0 {
                    // If it is larger, we are now looking at a different focus.
                    // Reset focus and last_iter. Then start from the beginning
                    focus = v;
                    last_iter = i;
                    continue 'possible_documents;
                }
            }
            return Some(focus);
        }
    }

    fn next_seek(&mut self,
                 operands: &mut [PeekableSeekable<Operand>],
                 target: &Posting)
                 -> Option<Posting> {
        // Advance operands to `target`
        for op in operands.iter_mut() {
            op.peek_seek(target);
        }
        self.next(operands)
    }

    fn progress(&self, operands: &[PeekableSeekable<Operand>]) -> Progress {
        operands.iter()
            .map(|op| op.inner().progress())
            .max()
            .unwrap_or(Progress::done())
    }
}

#[derive(Debug, Copy, Clone)]
pub struct Or;

impl Operator for Or {
    fn next(&mut self, operands: &mut [PeekableSeekable<Operand>]) -> Option<Posting> {
        // TODO: Probably improveable
        // Find the smallest current value of all operands
        let min_doc_id = operands.iter_mut()
            .map(|op| op.peek())
            .filter(|val| val.is_some())
            .map(|val| val.unwrap().doc_id())
            .min();

        // Walk over all operands. Advance those who emit the min value
        if min_doc_id.is_some() {
            let mut i = 0;
            let mut tmp = None;
            while i < operands.len() {
                let v = operands[i].peek().map(|p| p.doc_id());
                if !v.is_none() && v == min_doc_id {
                    tmp = operands[i].next();
                }
                i += 1;
            }
            tmp
        } else {
            None
        }
    }

    fn next_seek(&mut self,
                 operands: &mut [PeekableSeekable<Operand>],
                 target: &Posting)
                 -> Option<Posting> {
        // Advance operands to `target`
        for op in operands.iter_mut() {
            op.peek_seek(target);
        }
        self.next(operands)
    }

    fn progress(&self, operands: &[PeekableSeekable<Operand>]) -> Progress {
        operands.iter()
            .map(|op| op.inner().progress())
            .min()
            .unwrap_or(Progress::done())
    }
}
<commit_msg>format<commit_after>use std::hash::Hash;
use std::fmt::Debug;
use std::borrow::Borrow;
use std::marker::PhantomData;

use perlin_core::index::posting::{Posting, PostingIterator};
use perlin_core::utils::seeking_iterator::{PeekableSeekable, SeekingIterator};
use perlin_core::utils::progress::Progress;
use perlin_core::index::Index;

use language::CanApply;
use query::{ToOperands, Operand, Operator, ChainingOperator};
use field::Fields;

/// Mimics the functionality of the `try!` macro for `Option`s.
/// Evaluates `Some(x)` to x. Else it returns `None`.
macro_rules! try_option{
    ($operand:expr) => {
        if let Some(x) = $operand {
            x
        } else {
            return None;
        }
    }
}

#[derive(Debug)]
pub enum Combinator {
    All,
    Any,
}

pub struct SplitFunnel<'a, T: 'a + Hash + Eq, CB> {
    index: &'a Index<T>,
    combinator: Combinator,
    chaining_operator: ChainingOperator,
    result: Vec<PeekableSeekable<Operand<'a>>>,
    callback: CB,
}

impl<'a, T: 'a + Hash + Eq, CB> SplitFunnel<'a, T, CB> {
    pub fn create(chaining_operator: ChainingOperator,
                  combinator: Combinator,
                  index: &'a Index<T>,
                  cb: CB)
                  -> Self {
        SplitFunnel {
            index: index,
            combinator: combinator,
            chaining_operator: chaining_operator,
            callback: cb,
            result: Vec::new(),
        }
    }
}

impl<'a, T: 'a + Hash + Eq + Ord, CB> CanApply<T> for SplitFunnel<'a, T, CB>
    where CB: CanApply<T>
{
    type Output = CB::Output;

    fn apply(&mut self, term: T) {
        // Query index
        match self.index.query_atom(&term) {
            PostingIterator::Empty => self.result.push(PeekableSeekable::new(Operand::Empty)),
            PostingIterator::Decoder(decoder) => {
                self.result.push(PeekableSeekable::new(Operand::Term(decoder)))
            }
        }
        // And keep going
        self.callback.apply(term);
    }
}

impl<'a, T: 'a + Hash + Eq, CB> ToOperands<'a> for SplitFunnel<'a, T, CB>
    where CB: ToOperands<'a>
{
    fn to_operands(self) -> Vec<(ChainingOperator, PeekableSeekable<Operand<'a>>)> {
        let mut other = self.callback.to_operands();
        match self.combinator {
            Combinator::All => {
                other.push((self.chaining_operator,
                            PeekableSeekable::new(Operand::Operated(Box::new(And {}),
                                                                    self.result))))
            }
            Combinator::Any => {
                other.push((self.chaining_operator,
                            PeekableSeekable::new(Operand::Operated(Box::new(Or {}), self.result))))
            }
        }
        other
    }
}

/// This funnel is used at an end of a query pipeline
/// It calls `index.query_atom` and stores the result, which is lazy
/// When `to_operand` is then called, it packs everything into an operator!
// TODO: In this struct lies an opportunity to optimize
// If the operator is AND and one term returns an empty posting iterator
// We could skip the rest
// If the operator is Or and one term returns an empty posting iterator we
// could discard it
pub struct Funnel<'a, T: 'a, TIndex: 'a> {
    index: &'a TIndex,
    combinator: Combinator,
    chaining_operator: ChainingOperator,
    result: Vec<PeekableSeekable<Operand<'a>>>,
    _term: PhantomData<T>,
}

impl<'a, T: 'a, TIndex: 'a> Funnel<'a, T, TIndex> {
    pub fn create(chaining_operator: ChainingOperator,
                  combinator: Combinator,
                  index: &'a TIndex)
                  -> Self {
        Funnel {
            index: index,
            combinator: combinator,
            chaining_operator: chaining_operator,
            result: Vec::new(),
            _term: PhantomData,
        }
    }
}

impl<'a: 'b, 'b, T: 'a + Hash + Eq + Ord + Debug> CanApply<&'b T> for Funnel<'a, T, Fields<T>> {
    type Output = T;

    fn apply(&mut self, term: &'b T) {
        for index in self.index.fields.values() {
            match index.query_atom(&term) {
                PostingIterator::Empty => self.result.push(PeekableSeekable::new(Operand::Empty)),
                PostingIterator::Decoder(decoder) => {
                    self.result.push(PeekableSeekable::new(Operand::Term(decoder)))
                }
            }
        }
    }
}

impl<'a, T: 'a + Hash + Eq + Ord + Debug> CanApply<T> for Funnel<'a, T, Fields<T>> {
    type Output = T;

    fn apply(&mut self, term: T) {
        for index in self.index.fields.values() {
            match index.query_atom(&term) {
                PostingIterator::Empty => self.result.push(PeekableSeekable::new(Operand::Empty)),
                PostingIterator::Decoder(decoder) => {
                    self.result.push(PeekableSeekable::new(Operand::Term(decoder)))
                }
            }
        }
    }
}

impl<'a: 'b, 'b, T: 'a + Hash + Eq + Ord + Debug, TIndex> CanApply<&'b T> for Funnel<'a, T, TIndex>
    where TIndex: Borrow<Index<T>>
{
    type Output = T;

    fn apply(&mut self, term: &'b T) {
        match (self.index.borrow()).query_atom(&term) {
            PostingIterator::Empty => self.result.push(PeekableSeekable::new(Operand::Empty)),
            PostingIterator::Decoder(decoder) => {
                self.result.push(PeekableSeekable::new(Operand::Term(decoder)))
            }
        }
    }
}

impl<'a, T: 'a + Hash + Eq + Ord + Debug, TIndex> CanApply<T> for Funnel<'a, T, TIndex>
    where TIndex: Borrow<Index<T>>
{
    type Output = T;

    fn apply(&mut self, term: T) {
        match (self.index.borrow()).query_atom(&term) {
            PostingIterator::Empty => self.result.push(PeekableSeekable::new(Operand::Empty)),
            PostingIterator::Decoder(decoder) => {
                self.result.push(PeekableSeekable::new(Operand::Term(decoder)))
            }
        }
    }
}

impl<'a, T: 'a, TIndex> ToOperands<'a> for Funnel<'a, T, TIndex> {
    fn to_operands(self) -> Vec<(ChainingOperator, PeekableSeekable<Operand<'a>>)> {
        if self.result.is_empty() {
            return vec![];
        }
        match self.combinator {
            Combinator::All => {
                vec![(self.chaining_operator,
                      PeekableSeekable::new(Operand::Operated(Box::new(And {}), self.result)))]
            }
            Combinator::Any => {
                vec![(self.chaining_operator,
                      PeekableSeekable::new(Operand::Operated(Box::new(Or {}), self.result)))]
            }
        }

    }
}
/// END FUNNEL
#[derive(Debug, Copy, Clone)]
pub struct And;

impl Operator for And {
    fn next(&mut self, operands: &mut [PeekableSeekable<Operand>]) -> Option<Posting> {
        let mut focus = try_option!(operands[0].next()); // Acts as temporary to be compared against
        let mut last_iter = 0; // The iterator that last set 'focus'
        'possible_documents: loop {
            // For every term
            for (i, input) in operands.iter_mut().enumerate() {
                // If the focus was set by the current iterator, we have a match
                // We cycled through all the iterators once
                if last_iter == i {
                    continue;
                }

                let v = try_option!(input.next_seek(&focus));
                if v.0 > focus.0 {
                    // If it is larger, we are now looking at a different focus.
                    // Reset focus and last_iter. Then start from the beginning
                    focus = v;
                    last_iter = i;
                    continue 'possible_documents;
                }
            }
            return Some(focus);
        }
    }

    fn next_seek(&mut self,
                 operands: &mut [PeekableSeekable<Operand>],
                 target: &Posting)
                 -> Option<Posting> {
        // Advance operands to `target`
        for op in operands.iter_mut() {
            op.peek_seek(target);
        }
        self.next(operands)
    }

    fn progress(&self, operands: &[PeekableSeekable<Operand>]) -> Progress {
        operands.iter()
            .map(|op| op.inner().progress())
            .max()
            .unwrap_or(Progress::done())
    }
}

#[derive(Debug, Copy, Clone)]
pub struct Or;

impl Operator for Or {
    fn next(&mut self, operands: &mut [PeekableSeekable<Operand>]) -> Option<Posting> {
        // TODO: Probably improveable
        // Find the smallest current value of all operands
        let min_doc_id = operands.iter_mut()
            .map(|op| op.peek())
            .filter(|val| val.is_some())
            .map(|val| val.unwrap().doc_id())
            .min();

        // Walk over all operands. Advance those who emit the min value
        if min_doc_id.is_some() {
            let mut i = 0;
            let mut tmp = None;
            while i < operands.len() {
                let v = operands[i].peek().map(|p| p.doc_id());
                if !v.is_none() && v == min_doc_id {
                    tmp = operands[i].next();
                }
                i += 1;
            }
            tmp
        } else {
            None
        }
    }

    fn next_seek(&mut self,
                 operands: &mut [PeekableSeekable<Operand>],
                 target: &Posting)
                 -> Option<Posting> {
        // Advance operands to `target`
        for op in operands.iter_mut() {
            op.peek_seek(target);
        }
        self.next(operands)
    }

    fn progress(&self, operands: &[PeekableSeekable<Operand>]) -> Progress {
        operands.iter()
            .map(|op| op.inner().progress())
            .min()
            .unwrap_or(Progress::done())
    }
}
<|endoftext|>


<commit_before>use core::marker::PhantomData;
use core::{cmp, iter, slice, str};

use crate::endian::LittleEndian as LE;
use crate::pe;
use crate::read::{
    self, CompressedData, CompressedFileRange, ObjectSection, ObjectSegment, ReadError, ReadRef,
    Relocation, Result, SectionFlags, SectionIndex, SectionKind, SegmentFlags,
};

use super::{ImageNtHeaders, PeFile, SectionTable};

/// An iterator over the loadable sections of a `PeFile32`.
pub type PeSegmentIterator32<'data, 'file, R = &'data [u8]> =
    PeSegmentIterator<'data, 'file, pe::ImageNtHeaders32, R>;
/// An iterator over the loadable sections of a `PeFile64`.
pub type PeSegmentIterator64<'data, 'file, R = &'data [u8]> =
    PeSegmentIterator<'data, 'file, pe::ImageNtHeaders64, R>;

/// An iterator over the loadable sections of a `PeFile`.
#[derive(Debug)]
pub struct PeSegmentIterator<'data, 'file, Pe, R = &'data [u8]>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    pub(super) file: &'file PeFile<'data, Pe, R>,
    pub(super) iter: slice::Iter<'data, pe::ImageSectionHeader>,
}

impl<'data, 'file, Pe, R> Iterator for PeSegmentIterator<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    type Item = PeSegment<'data, 'file, Pe, R>;

    fn next(&mut self) -> Option<Self::Item> {
        self.iter.next().map(|section| PeSegment {
            file: self.file,
            section,
        })
    }
}

/// A loadable section of a `PeFile32`.
pub type PeSegment32<'data, 'file, R = &'data [u8]> =
    PeSegment<'data, 'file, pe::ImageNtHeaders32, R>;
/// A loadable section of a `PeFile64`.
pub type PeSegment64<'data, 'file, R = &'data [u8]> =
    PeSegment<'data, 'file, pe::ImageNtHeaders64, R>;

/// A loadable section of a `PeFile`.
#[derive(Debug)]
pub struct PeSegment<'data, 'file, Pe, R = &'data [u8]>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    file: &'file PeFile<'data, Pe, R>,
    section: &'data pe::ImageSectionHeader,
}

impl<'data, 'file, Pe, R> read::private::Sealed for PeSegment<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
}

impl<'data, 'file, Pe, R> ObjectSegment<'data> for PeSegment<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    #[inline]
    fn address(&self) -> u64 {
        u64::from(self.section.virtual_address.get(LE)).wrapping_add(self.file.common.image_base)
    }

    #[inline]
    fn size(&self) -> u64 {
        u64::from(self.section.virtual_size.get(LE))
    }

    #[inline]
    fn align(&self) -> u64 {
        self.file.section_alignment()
    }

    #[inline]
    fn file_range(&self) -> (u64, u64) {
        let (offset, size) = self.section.pe_file_range();
        (u64::from(offset), u64::from(size))
    }

    fn data(&self) -> Result<&'data [u8]> {
        self.section.pe_data(self.file.data)
    }

    fn data_range(&self, address: u64, size: u64) -> Result<Option<&'data [u8]>> {
        Ok(read::util::data_range(
            self.data()?,
            self.address(),
            address,
            size,
        ))
    }

    #[inline]
    fn name_bytes(&self) -> Result<Option<&[u8]>> {
        self.section
            .name(self.file.common.symbols.strings())
            .map(Some)
    }

    #[inline]
    fn name(&self) -> Result<Option<&str>> {
        let name = self.section.name(self.file.common.symbols.strings())?;
        Ok(Some(
            str::from_utf8(name)
                .ok()
                .read_error("Non UTF-8 PE section name")?,
        ))
    }

    #[inline]
    fn flags(&self) -> SegmentFlags {
        let characteristics = self.section.characteristics.get(LE);
        SegmentFlags::Coff { characteristics }
    }
}

/// An iterator over the sections of a `PeFile32`.
pub type PeSectionIterator32<'data, 'file, R = &'data [u8]> =
    PeSectionIterator<'data, 'file, pe::ImageNtHeaders32, R>;
/// An iterator over the sections of a `PeFile64`.
pub type PeSectionIterator64<'data, 'file, R = &'data [u8]> =
    PeSectionIterator<'data, 'file, pe::ImageNtHeaders64, R>;

/// An iterator over the sections of a `PeFile`.
#[derive(Debug)]
pub struct PeSectionIterator<'data, 'file, Pe, R = &'data [u8]>
where
    'data: 'file,
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    pub(super) file: &'file PeFile<'data, Pe, R>,
    pub(super) iter: iter::Enumerate<slice::Iter<'data, pe::ImageSectionHeader>>,
}

impl<'data, 'file, Pe, R> Iterator for PeSectionIterator<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    type Item = PeSection<'data, 'file, Pe, R>;

    fn next(&mut self) -> Option<Self::Item> {
        self.iter.next().map(|(index, section)| PeSection {
            file: self.file,
            index: SectionIndex(index + 1),
            section,
        })
    }
}

/// A section of a `PeFile32`.
pub type PeSection32<'data, 'file, R = &'data [u8]> =
    PeSection<'data, 'file, pe::ImageNtHeaders32, R>;
/// A section of a `PeFile64`.
pub type PeSection64<'data, 'file, R = &'data [u8]> =
    PeSection<'data, 'file, pe::ImageNtHeaders64, R>;

/// A section of a `PeFile`.
#[derive(Debug)]
pub struct PeSection<'data, 'file, Pe, R = &'data [u8]>
where
    'data: 'file,
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    pub(super) file: &'file PeFile<'data, Pe, R>,
    pub(super) index: SectionIndex,
    pub(super) section: &'data pe::ImageSectionHeader,
}

impl<'data, 'file, Pe, R> read::private::Sealed for PeSection<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
}

impl<'data, 'file, Pe, R> ObjectSection<'data> for PeSection<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    type RelocationIterator = PeRelocationIterator<'data, 'file, R>;

    #[inline]
    fn index(&self) -> SectionIndex {
        self.index
    }

    #[inline]
    fn address(&self) -> u64 {
        u64::from(self.section.virtual_address.get(LE)).wrapping_add(self.file.common.image_base)
    }

    #[inline]
    fn size(&self) -> u64 {
        u64::from(self.section.virtual_size.get(LE))
    }

    #[inline]
    fn align(&self) -> u64 {
        self.file.section_alignment()
    }

    #[inline]
    fn file_range(&self) -> Option<(u64, u64)> {
        let (offset, size) = self.section.pe_file_range();
        if size == 0 {
            None
        } else {
            Some((u64::from(offset), u64::from(size)))
        }
    }

    fn data(&self) -> Result<&'data [u8]> {
        self.section.pe_data(self.file.data)
    }

    fn data_range(&self, address: u64, size: u64) -> Result<Option<&'data [u8]>> {
        Ok(read::util::data_range(
            self.data()?,
            self.address(),
            address,
            size,
        ))
    }

    #[inline]
    fn compressed_file_range(&self) -> Result<CompressedFileRange> {
        Ok(CompressedFileRange::none(self.file_range()))
    }

    #[inline]
    fn compressed_data(&self) -> Result<CompressedData<'data>> {
        self.data().map(CompressedData::none)
    }

    #[inline]
    fn name_bytes(&self) -> Result<&[u8]> {
        self.section.name(self.file.common.symbols.strings())
    }

    #[inline]
    fn name(&self) -> Result<&str> {
        let name = self.name_bytes()?;
        str::from_utf8(name)
            .ok()
            .read_error("Non UTF-8 PE section name")
    }

    #[inline]
    fn segment_name_bytes(&self) -> Result<Option<&[u8]>> {
        Ok(None)
    }

    #[inline]
    fn segment_name(&self) -> Result<Option<&str>> {
        Ok(None)
    }

    #[inline]
    fn kind(&self) -> SectionKind {
        self.section.kind()
    }

    fn relocations(&self) -> PeRelocationIterator<'data, 'file, R> {
        PeRelocationIterator(PhantomData)
    }

    fn flags(&self) -> SectionFlags {
        SectionFlags::Coff {
            characteristics: self.section.characteristics.get(LE),
        }
    }
}

impl<'data> SectionTable<'data> {
    /// Return the file offset of the given virtual address, and the size up
    /// to the end of the section containing it.
    ///
    /// Returns `None` if no section contains the address.
    pub fn pe_file_range_at(&self, va: u32) -> Option<(u32, u32)> {
        self.iter().find_map(|section| section.pe_file_range_at(va))
    }

    /// Return the data starting at the given virtual address, up to the end of the
    /// section containing it.
    ///
    /// Ignores sections with invalid data.
    ///
    /// Returns `None` if no section contains the address.
    pub fn pe_data_at<R: ReadRef<'data>>(&self, data: R, va: u32) -> Option<&'data [u8]> {
        self.iter().find_map(|section| section.pe_data_at(data, va))
    }

    /// Return the data of the section that contains the given virtual address in a PE file.
    ///
    /// Also returns the virtual address of that section.
    ///
    /// Ignores sections with invalid data.
    pub fn pe_data_containing<R: ReadRef<'data>>(
        &self,
        data: R,
        va: u32,
    ) -> Option<(&'data [u8], u32)> {
        self.iter()
            .find_map(|section| section.pe_data_containing(data, va))
    }
}

impl pe::ImageSectionHeader {
    /// Return the offset and size of the section in a PE file.
    ///
    /// The size of the range will be the minimum of the file size and virtual size.
    pub fn pe_file_range(&self) -> (u32, u32) {
        // Pointer and size will be zero for uninitialized data; we don't need to validate this.
        let offset = self.pointer_to_raw_data.get(LE);
        let size = cmp::min(self.virtual_size.get(LE), self.size_of_raw_data.get(LE));
        (offset, size)
    }

    /// Return the file offset of the given virtual address, and the remaining size up
    /// to the end of the section.
    ///
    /// Returns `None` if the section does not contain the address.
    pub fn pe_file_range_at(&self, va: u32) -> Option<(u32, u32)> {
        let section_va = self.virtual_address.get(LE);
        let offset = va.checked_sub(section_va)?;
        let (section_offset, section_size) = self.pe_file_range();
        // Address must be within section (and not at its end).
        if offset < section_size {
            Some((section_offset.checked_add(offset)?, section_size - offset))
        } else {
            None
        }
    }

    /// Return the virtual address and size of the section.
    pub fn pe_address_range(&self) -> (u32, u32) {
        (self.virtual_address.get(LE), self.virtual_size.get(LE))
    }

    /// Return the section data in a PE file.
    ///
    /// The length of the data will be the minimum of the file size and virtual size.
    pub fn pe_data<'data, R: ReadRef<'data>>(&self, data: R) -> Result<&'data [u8]> {
        let (offset, size) = self.pe_file_range();
        data.read_bytes_at(offset.into(), size.into())
            .read_error("Invalid PE section offset or size")
    }

    /// Return the data starting at the given virtual address, up to the end of the
    /// section.
    ///
    /// Ignores sections with invalid data.
    ///
    /// Returns `None` if the section does not contain the address.
    pub fn pe_data_at<'data, R: ReadRef<'data>>(&self, data: R, va: u32) -> Option<&'data [u8]> {
        let (offset, size) = self.pe_file_range_at(va)?;
        data.read_bytes_at(offset.into(), size.into()).ok()
    }

    /// Return the section data if it contains the given virtual address.
    ///
    /// Also returns the virtual address of that section.
    ///
    /// Ignores sections with invalid data.
    pub fn pe_data_containing<'data, R: ReadRef<'data>>(
        &self,
        data: R,
        va: u32,
    ) -> Option<(&'data [u8], u32)> {
        let section_va = self.virtual_address.get(LE);
        let offset = va.checked_sub(section_va)?;
        let (section_offset, section_size) = self.pe_file_range();
        // Address must be within section (and not at its end).
        if offset < section_size {
            let section_data = data
                .read_bytes_at(section_offset.into(), section_size.into())
                .ok()?;
            Some((section_data, section_va))
        } else {
            None
        }
    }
}

/// An iterator over the relocations in an `PeSection`.
#[derive(Debug)]
pub struct PeRelocationIterator<'data, 'file, R = &'data [u8]>(
    PhantomData<(&'data (), &'file (), R)>,
);

impl<'data, 'file, R> Iterator for PeRelocationIterator<'data, 'file, R> {
    type Item = (u64, Relocation);

    fn next(&mut self) -> Option<Self::Item> {
        None
    }
}
<commit_msg>read/pe: add SectionTable::section_containing (#420)<commit_after>use core::marker::PhantomData;
use core::{cmp, iter, slice, str};

use crate::endian::LittleEndian as LE;
use crate::pe;
use crate::pe::ImageSectionHeader;
use crate::read::{
    self, CompressedData, CompressedFileRange, ObjectSection, ObjectSegment, ReadError, ReadRef,
    Relocation, Result, SectionFlags, SectionIndex, SectionKind, SegmentFlags,
};

use super::{ImageNtHeaders, PeFile, SectionTable};

/// An iterator over the loadable sections of a `PeFile32`.
pub type PeSegmentIterator32<'data, 'file, R = &'data [u8]> =
    PeSegmentIterator<'data, 'file, pe::ImageNtHeaders32, R>;
/// An iterator over the loadable sections of a `PeFile64`.
pub type PeSegmentIterator64<'data, 'file, R = &'data [u8]> =
    PeSegmentIterator<'data, 'file, pe::ImageNtHeaders64, R>;

/// An iterator over the loadable sections of a `PeFile`.
#[derive(Debug)]
pub struct PeSegmentIterator<'data, 'file, Pe, R = &'data [u8]>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    pub(super) file: &'file PeFile<'data, Pe, R>,
    pub(super) iter: slice::Iter<'data, pe::ImageSectionHeader>,
}

impl<'data, 'file, Pe, R> Iterator for PeSegmentIterator<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    type Item = PeSegment<'data, 'file, Pe, R>;

    fn next(&mut self) -> Option<Self::Item> {
        self.iter.next().map(|section| PeSegment {
            file: self.file,
            section,
        })
    }
}

/// A loadable section of a `PeFile32`.
pub type PeSegment32<'data, 'file, R = &'data [u8]> =
    PeSegment<'data, 'file, pe::ImageNtHeaders32, R>;
/// A loadable section of a `PeFile64`.
pub type PeSegment64<'data, 'file, R = &'data [u8]> =
    PeSegment<'data, 'file, pe::ImageNtHeaders64, R>;

/// A loadable section of a `PeFile`.
#[derive(Debug)]
pub struct PeSegment<'data, 'file, Pe, R = &'data [u8]>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    file: &'file PeFile<'data, Pe, R>,
    section: &'data pe::ImageSectionHeader,
}

impl<'data, 'file, Pe, R> read::private::Sealed for PeSegment<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
}

impl<'data, 'file, Pe, R> ObjectSegment<'data> for PeSegment<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    #[inline]
    fn address(&self) -> u64 {
        u64::from(self.section.virtual_address.get(LE)).wrapping_add(self.file.common.image_base)
    }

    #[inline]
    fn size(&self) -> u64 {
        u64::from(self.section.virtual_size.get(LE))
    }

    #[inline]
    fn align(&self) -> u64 {
        self.file.section_alignment()
    }

    #[inline]
    fn file_range(&self) -> (u64, u64) {
        let (offset, size) = self.section.pe_file_range();
        (u64::from(offset), u64::from(size))
    }

    fn data(&self) -> Result<&'data [u8]> {
        self.section.pe_data(self.file.data)
    }

    fn data_range(&self, address: u64, size: u64) -> Result<Option<&'data [u8]>> {
        Ok(read::util::data_range(
            self.data()?,
            self.address(),
            address,
            size,
        ))
    }

    #[inline]
    fn name_bytes(&self) -> Result<Option<&[u8]>> {
        self.section
            .name(self.file.common.symbols.strings())
            .map(Some)
    }

    #[inline]
    fn name(&self) -> Result<Option<&str>> {
        let name = self.section.name(self.file.common.symbols.strings())?;
        Ok(Some(
            str::from_utf8(name)
                .ok()
                .read_error("Non UTF-8 PE section name")?,
        ))
    }

    #[inline]
    fn flags(&self) -> SegmentFlags {
        let characteristics = self.section.characteristics.get(LE);
        SegmentFlags::Coff { characteristics }
    }
}

/// An iterator over the sections of a `PeFile32`.
pub type PeSectionIterator32<'data, 'file, R = &'data [u8]> =
    PeSectionIterator<'data, 'file, pe::ImageNtHeaders32, R>;
/// An iterator over the sections of a `PeFile64`.
pub type PeSectionIterator64<'data, 'file, R = &'data [u8]> =
    PeSectionIterator<'data, 'file, pe::ImageNtHeaders64, R>;

/// An iterator over the sections of a `PeFile`.
#[derive(Debug)]
pub struct PeSectionIterator<'data, 'file, Pe, R = &'data [u8]>
where
    'data: 'file,
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    pub(super) file: &'file PeFile<'data, Pe, R>,
    pub(super) iter: iter::Enumerate<slice::Iter<'data, pe::ImageSectionHeader>>,
}

impl<'data, 'file, Pe, R> Iterator for PeSectionIterator<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    type Item = PeSection<'data, 'file, Pe, R>;

    fn next(&mut self) -> Option<Self::Item> {
        self.iter.next().map(|(index, section)| PeSection {
            file: self.file,
            index: SectionIndex(index + 1),
            section,
        })
    }
}

/// A section of a `PeFile32`.
pub type PeSection32<'data, 'file, R = &'data [u8]> =
    PeSection<'data, 'file, pe::ImageNtHeaders32, R>;
/// A section of a `PeFile64`.
pub type PeSection64<'data, 'file, R = &'data [u8]> =
    PeSection<'data, 'file, pe::ImageNtHeaders64, R>;

/// A section of a `PeFile`.
#[derive(Debug)]
pub struct PeSection<'data, 'file, Pe, R = &'data [u8]>
where
    'data: 'file,
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    pub(super) file: &'file PeFile<'data, Pe, R>,
    pub(super) index: SectionIndex,
    pub(super) section: &'data pe::ImageSectionHeader,
}

impl<'data, 'file, Pe, R> read::private::Sealed for PeSection<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
}

impl<'data, 'file, Pe, R> ObjectSection<'data> for PeSection<'data, 'file, Pe, R>
where
    Pe: ImageNtHeaders,
    R: ReadRef<'data>,
{
    type RelocationIterator = PeRelocationIterator<'data, 'file, R>;

    #[inline]
    fn index(&self) -> SectionIndex {
        self.index
    }

    #[inline]
    fn address(&self) -> u64 {
        u64::from(self.section.virtual_address.get(LE)).wrapping_add(self.file.common.image_base)
    }

    #[inline]
    fn size(&self) -> u64 {
        u64::from(self.section.virtual_size.get(LE))
    }

    #[inline]
    fn align(&self) -> u64 {
        self.file.section_alignment()
    }

    #[inline]
    fn file_range(&self) -> Option<(u64, u64)> {
        let (offset, size) = self.section.pe_file_range();
        if size == 0 {
            None
        } else {
            Some((u64::from(offset), u64::from(size)))
        }
    }

    fn data(&self) -> Result<&'data [u8]> {
        self.section.pe_data(self.file.data)
    }

    fn data_range(&self, address: u64, size: u64) -> Result<Option<&'data [u8]>> {
        Ok(read::util::data_range(
            self.data()?,
            self.address(),
            address,
            size,
        ))
    }

    #[inline]
    fn compressed_file_range(&self) -> Result<CompressedFileRange> {
        Ok(CompressedFileRange::none(self.file_range()))
    }

    #[inline]
    fn compressed_data(&self) -> Result<CompressedData<'data>> {
        self.data().map(CompressedData::none)
    }

    #[inline]
    fn name_bytes(&self) -> Result<&[u8]> {
        self.section.name(self.file.common.symbols.strings())
    }

    #[inline]
    fn name(&self) -> Result<&str> {
        let name = self.name_bytes()?;
        str::from_utf8(name)
            .ok()
            .read_error("Non UTF-8 PE section name")
    }

    #[inline]
    fn segment_name_bytes(&self) -> Result<Option<&[u8]>> {
        Ok(None)
    }

    #[inline]
    fn segment_name(&self) -> Result<Option<&str>> {
        Ok(None)
    }

    #[inline]
    fn kind(&self) -> SectionKind {
        self.section.kind()
    }

    fn relocations(&self) -> PeRelocationIterator<'data, 'file, R> {
        PeRelocationIterator(PhantomData)
    }

    fn flags(&self) -> SectionFlags {
        SectionFlags::Coff {
            characteristics: self.section.characteristics.get(LE),
        }
    }
}

impl<'data> SectionTable<'data> {
    /// Return the file offset of the given virtual address, and the size up
    /// to the end of the section containing it.
    ///
    /// Returns `None` if no section contains the address.
    pub fn pe_file_range_at(&self, va: u32) -> Option<(u32, u32)> {
        self.iter().find_map(|section| section.pe_file_range_at(va))
    }

    /// Return the data starting at the given virtual address, up to the end of the
    /// section containing it.
    ///
    /// Ignores sections with invalid data.
    ///
    /// Returns `None` if no section contains the address.
    pub fn pe_data_at<R: ReadRef<'data>>(&self, data: R, va: u32) -> Option<&'data [u8]> {
        self.iter().find_map(|section| section.pe_data_at(data, va))
    }

    /// Return the data of the section that contains the given virtual address in a PE file.
    ///
    /// Also returns the virtual address of that section.
    ///
    /// Ignores sections with invalid data.
    pub fn pe_data_containing<R: ReadRef<'data>>(
        &self,
        data: R,
        va: u32,
    ) -> Option<(&'data [u8], u32)> {
        self.iter()
            .find_map(|section| section.pe_data_containing(data, va))
    }

    /// Return the section that contains a given virtual address.
    pub fn section_containing(&self, va: u32) -> Option<&'data ImageSectionHeader> {
        self.iter().find(|section| section.contains_rva(va))
    }
}

impl pe::ImageSectionHeader {
    /// Return the offset and size of the section in a PE file.
    ///
    /// The size of the range will be the minimum of the file size and virtual size.
    pub fn pe_file_range(&self) -> (u32, u32) {
        // Pointer and size will be zero for uninitialized data; we don't need to validate this.
        let offset = self.pointer_to_raw_data.get(LE);
        let size = cmp::min(self.virtual_size.get(LE), self.size_of_raw_data.get(LE));
        (offset, size)
    }

    /// Return the file offset of the given virtual address, and the remaining size up
    /// to the end of the section.
    ///
    /// Returns `None` if the section does not contain the address.
    pub fn pe_file_range_at(&self, va: u32) -> Option<(u32, u32)> {
        let section_va = self.virtual_address.get(LE);
        let offset = va.checked_sub(section_va)?;
        let (section_offset, section_size) = self.pe_file_range();
        // Address must be within section (and not at its end).
        if offset < section_size {
            Some((section_offset.checked_add(offset)?, section_size - offset))
        } else {
            None
        }
    }

    /// Return the virtual address and size of the section.
    pub fn pe_address_range(&self) -> (u32, u32) {
        (self.virtual_address.get(LE), self.virtual_size.get(LE))
    }

    /// Return the section data in a PE file.
    ///
    /// The length of the data will be the minimum of the file size and virtual size.
    pub fn pe_data<'data, R: ReadRef<'data>>(&self, data: R) -> Result<&'data [u8]> {
        let (offset, size) = self.pe_file_range();
        data.read_bytes_at(offset.into(), size.into())
            .read_error("Invalid PE section offset or size")
    }

    /// Return the data starting at the given virtual address, up to the end of the
    /// section.
    ///
    /// Ignores sections with invalid data.
    ///
    /// Returns `None` if the section does not contain the address.
    pub fn pe_data_at<'data, R: ReadRef<'data>>(&self, data: R, va: u32) -> Option<&'data [u8]> {
        let (offset, size) = self.pe_file_range_at(va)?;
        data.read_bytes_at(offset.into(), size.into()).ok()
    }

    /// Tests whether a given RVA is part of this section
    pub fn contains_rva(&self, va: u32) -> bool {
        let section_va = self.virtual_address.get(LE);
        match va.checked_sub(section_va) {
            None => false,
            Some(offset) => {
                // Address must be within section (and not at its end).
                offset < self.virtual_size.get(LE)
            }
        }
    }

    /// Return the section data if it contains the given virtual address.
    ///
    /// Also returns the virtual address of that section.
    ///
    /// Ignores sections with invalid data.
    pub fn pe_data_containing<'data, R: ReadRef<'data>>(
        &self,
        data: R,
        va: u32,
    ) -> Option<(&'data [u8], u32)> {
        let section_va = self.virtual_address.get(LE);
        let offset = va.checked_sub(section_va)?;
        let (section_offset, section_size) = self.pe_file_range();
        // Address must be within section (and not at its end).
        if offset < section_size {
            let section_data = data
                .read_bytes_at(section_offset.into(), section_size.into())
                .ok()?;
            Some((section_data, section_va))
        } else {
            None
        }
    }
}

/// An iterator over the relocations in an `PeSection`.
#[derive(Debug)]
pub struct PeRelocationIterator<'data, 'file, R = &'data [u8]>(
    PhantomData<(&'data (), &'file (), R)>,
);

impl<'data, 'file, R> Iterator for PeRelocationIterator<'data, 'file, R> {
    type Item = (u64, Relocation);

    fn next(&mut self) -> Option<Self::Item> {
        None
    }
}
<|endoftext|>


<commit_before>use std::collections::HashSet;
use std::collections::hash_map::HashMap;
use std::fmt;
use std::io::Write;
use std::sync::mpsc::{channel, Sender, Receiver};

use crossbeam::{self, Scope};
use term::color::YELLOW;

use core::{PackageId, Target, Profile};
use util::{Config, DependencyQueue, Fresh, Dirty, Freshness};
use util::{CargoResult, ProcessBuilder, profile, internal};

use super::{Context, Kind, Unit};
use super::job::Job;

/// A management structure of the entire dependency graph to compile.
///
/// This structure is backed by the `DependencyQueue` type and manages the
/// actual compilation step of each package. Packages enqueue units of work and
/// then later on the entire graph is processed and compiled.
pub struct JobQueue<'a> {
    jobs: usize,
    queue: DependencyQueue<Key<'a>, Vec<(Job, Freshness)>>,
    tx: Sender<(Key<'a>, Message)>,
    rx: Receiver<(Key<'a>, Message)>,
    active: usize,
    pending: HashMap<Key<'a>, PendingBuild>,
    compiled: HashSet<&'a PackageId>,
    documented: HashSet<&'a PackageId>,
    counts: HashMap<&'a PackageId, usize>,
    is_release: bool,
}

/// A helper structure for metadata about the state of a building package.
struct PendingBuild {
    /// Number of jobs currently active
    amt: usize,
    /// Current freshness state of this package. Any dirty target within a
    /// package will cause the entire package to become dirty.
    fresh: Freshness,
}

#[derive(Clone, Copy, Eq, PartialEq, Hash)]
struct Key<'a> {
    pkg: &'a PackageId,
    target: &'a Target,
    profile: &'a Profile,
    kind: Kind,
}

pub struct JobState<'a> {
    tx: Sender<(Key<'a>, Message)>,
    key: Key<'a>,
}

enum Message {
    Run(String),
    Stdout(String),
    Stderr(String),
    Finish(CargoResult<()>),
}

impl<'a> JobState<'a> {
    pub fn running(&self, cmd: &ProcessBuilder) {
        let _ = self.tx.send((self.key, Message::Run(cmd.to_string())));
    }

    pub fn stdout(&self, out: &str) {
        let _ = self.tx.send((self.key, Message::Stdout(out.to_string())));
    }

    pub fn stderr(&self, err: &str) {
        let _ = self.tx.send((self.key, Message::Stderr(err.to_string())));
    }
}

impl<'a> JobQueue<'a> {
    pub fn new<'cfg>(cx: &Context<'a, 'cfg>) -> JobQueue<'a> {
        let (tx, rx) = channel();
        JobQueue {
            jobs: cx.jobs() as usize,
            queue: DependencyQueue::new(),
            tx: tx,
            rx: rx,
            active: 0,
            pending: HashMap::new(),
            compiled: HashSet::new(),
            documented: HashSet::new(),
            counts: HashMap::new(),
            is_release: cx.build_config.release,
        }
    }

    pub fn enqueue<'cfg>(&mut self,
                         cx: &Context<'a, 'cfg>,
                         unit: &Unit<'a>,
                         job: Job,
                         fresh: Freshness) -> CargoResult<()> {
        let key = Key::new(unit);
        let deps = key.dependencies(cx)?;
        self.queue.queue(Fresh, key, Vec::new(), &deps).push((job, fresh));
        *self.counts.entry(key.pkg).or_insert(0) += 1;
        Ok(())
    }

    /// Execute all jobs necessary to build the dependency graph.
    ///
    /// This function will spawn off `config.jobs()` workers to build all of the
    /// necessary dependencies, in order. Freshness is propagated as far as
    /// possible along each dependency chain.
    pub fn execute(&mut self, cx: &mut Context) -> CargoResult<()> {
        let _p = profile::start("executing the job graph");

        crossbeam::scope(|scope| {
            self.drain_the_queue(cx, scope)
        })
    }

    fn drain_the_queue(&mut self, cx: &mut Context, scope: &Scope<'a>)
                       -> CargoResult<()> {
        use std::time::Instant;

        let mut queue = Vec::new();
        trace!("queue: {:#?}", self.queue);

        // Iteratively execute the entire dependency graph. Each turn of the
        // loop starts out by scheduling as much work as possible (up to the
        // maximum number of parallel jobs). A local queue is maintained
        // separately from the main dependency queue as one dequeue may actually
        // dequeue quite a bit of work (e.g. 10 binaries in one project).
        //
        // After a job has finished we update our internal state if it was
        // successful and otherwise wait for pending work to finish if it failed
        // and then immediately return.
        let mut error = None;
        let start_time = Instant::now();
        loop {
            while error.is_none() && self.active < self.jobs {
                if !queue.is_empty() {
                    let (key, job, fresh) = queue.remove(0);
                    self.run(key, fresh, job, cx.config, scope)?;
                } else if let Some((fresh, key, jobs)) = self.queue.dequeue() {
                    let total_fresh = jobs.iter().fold(fresh, |fresh, &(_, f)| {
                        f.combine(fresh)
                    });
                    self.pending.insert(key, PendingBuild {
                        amt: jobs.len(),
                        fresh: total_fresh,
                    });
                    queue.extend(jobs.into_iter().map(|(job, f)| {
                        (key, job, f.combine(fresh))
                    }));
                } else {
                    break
                }
            }
            if self.active == 0 {
                break
            }

            let (key, msg) = self.rx.recv().unwrap();

            match msg {
                Message::Run(cmd) => {
                    cx.config.shell().verbose(|c| c.status("Running", &cmd))?;
                }
                Message::Stdout(out) => {
                    if cx.config.extra_verbose() {
                        writeln!(cx.config.shell().out(), "{}", out)?;
                    }
                }
                Message::Stderr(err) => {
                    if cx.config.extra_verbose() {
                        writeln!(cx.config.shell().err(), "{}", err)?;
                    }
                }
                Message::Finish(result) => {
                    info!("end: {:?}", key);
                    self.active -= 1;
                    match result {
                        Ok(()) => self.finish(key, cx)?,
                        Err(e) => {
                            if self.active > 0 {
                                cx.config.shell().say(
                                            "Build failed, waiting for other \
                                             jobs to finish...", YELLOW)?;
                            }
                            if error.is_none() {
                                error = Some(e);
                            }
                        }
                    }
                }
            }
        }

        let build_type = if self.is_release { "release" } else { "dev" };
        let profile = cx.lib_profile();
        let mut opt_type = String::from(if profile.opt_level == "0" { "unoptimized" }
                                        else { "optimized" });
        if profile.debuginfo.is_some() {
            opt_type = opt_type + " + debuginfo";
        }
        let duration = start_time.elapsed();
        let time_elapsed = format!("{}.{1:.2} secs",
                                   duration.as_secs(),
                                   duration.subsec_nanos() / 10000000);
        if self.queue.is_empty() {
            let message = format!("{} [{}] target(s) in {}",
                                  build_type,
                                  opt_type,
                                  time_elapsed);
            cx.config.shell().status("Finished", message)?;
            Ok(())
        } else if let Some(e) = error {
            Err(e)
        } else {
            debug!("queue: {:#?}", self.queue);
            Err(internal("finished with jobs still left in the queue"))
        }
    }

    /// Executes a job in the `scope` given, pushing the spawned thread's
    /// handled onto `threads`.
    fn run(&mut self,
           key: Key<'a>,
           fresh: Freshness,
           job: Job,
           config: &Config,
           scope: &Scope<'a>) -> CargoResult<()> {
        info!("start: {:?}", key);

        self.active += 1;
        *self.counts.get_mut(key.pkg).unwrap() -= 1;

        let my_tx = self.tx.clone();
        scope.spawn(move || {
            let res = job.run(fresh, &JobState {
                tx: my_tx.clone(),
                key: key,
            });
            my_tx.send((key, Message::Finish(res))).unwrap();
        });

        // Print out some nice progress information
        self.note_working_on(config, &key, fresh)?;

        Ok(())
    }

    fn finish(&mut self, key: Key<'a>, cx: &mut Context) -> CargoResult<()> {
        if key.profile.run_custom_build && cx.show_warnings(key.pkg) {
            let output = cx.build_state.outputs.lock().unwrap();
            if let Some(output) = output.get(&(key.pkg.clone(), key.kind)) {
                for warning in output.warnings.iter() {
                    cx.config.shell().warn(warning)?;
                }
            }
        }
        let state = self.pending.get_mut(&key).unwrap();
        state.amt -= 1;
        if state.amt == 0 {
            self.queue.finish(&key, state.fresh);
        }
        Ok(())
    }

    // This isn't super trivial because we don't want to print loads and
    // loads of information to the console, but we also want to produce a
    // faithful representation of what's happening. This is somewhat nuanced
    // as a package can start compiling *very* early on because of custom
    // build commands and such.
    //
    // In general, we try to print "Compiling" for the first nontrivial task
    // run for a package, regardless of when that is. We then don't print
    // out any more information for a package after we've printed it once.
    fn note_working_on(&mut self,
                       config: &Config,
                       key: &Key<'a>,
                       fresh: Freshness) -> CargoResult<()> {
        if (self.compiled.contains(key.pkg) && !key.profile.doc) ||
            (self.documented.contains(key.pkg) && key.profile.doc) {
            return Ok(())
        }

        match fresh {
            // Any dirty stage which runs at least one command gets printed as
            // being a compiled package
            Dirty => {
                if key.profile.doc {
                    if !key.profile.test {
                        self.documented.insert(key.pkg);
                        config.shell().status("Documenting", key.pkg)?;
                    }
                } else {
                    self.compiled.insert(key.pkg);
                    config.shell().status("Compiling", key.pkg)?;
                }
            }
            Fresh if self.counts[key.pkg] == 0 => {
                self.compiled.insert(key.pkg);
                config.shell().verbose(|c| c.status("Fresh", key.pkg))?;
            }
            Fresh => {}
        }
        Ok(())
    }
}

impl<'a> Key<'a> {
    fn new(unit: &Unit<'a>) -> Key<'a> {
        Key {
            pkg: unit.pkg.package_id(),
            target: unit.target,
            profile: unit.profile,
            kind: unit.kind,
        }
    }

    fn dependencies<'cfg>(&self, cx: &Context<'a, 'cfg>)
                          -> CargoResult<Vec<Key<'a>>> {
        let unit = Unit {
            pkg: cx.get_package(self.pkg)?,
            target: self.target,
            profile: self.profile,
            kind: self.kind,
        };
        let targets = cx.dep_targets(&unit)?;
        Ok(targets.iter().filter_map(|unit| {
            // Binaries aren't actually needed to *compile* tests, just to run
            // them, so we don't include this dependency edge in the job graph.
            if self.target.is_test() && unit.target.is_bin() {
                None
            } else {
                Some(Key::new(unit))
            }
        }).collect())
    }
}

impl<'a> fmt::Debug for Key<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{} => {}/{} => {:?}", self.pkg, self.target, self.profile,
               self.kind)
    }
}
<commit_msg>When an error occured in a JobQueue print it directly<commit_after>use std::collections::HashSet;
use std::collections::hash_map::HashMap;
use std::fmt;
use std::io::Write;
use std::sync::mpsc::{channel, Sender, Receiver};

use crossbeam::{self, Scope};
use term::color::YELLOW;

use core::{PackageId, Target, Profile};
use util::{Config, DependencyQueue, Fresh, Dirty, Freshness};
use util::{CargoResult, ProcessBuilder, profile, internal, human};

use super::{Context, Kind, Unit};
use super::job::Job;

/// A management structure of the entire dependency graph to compile.
///
/// This structure is backed by the `DependencyQueue` type and manages the
/// actual compilation step of each package. Packages enqueue units of work and
/// then later on the entire graph is processed and compiled.
pub struct JobQueue<'a> {
    jobs: usize,
    queue: DependencyQueue<Key<'a>, Vec<(Job, Freshness)>>,
    tx: Sender<(Key<'a>, Message)>,
    rx: Receiver<(Key<'a>, Message)>,
    active: usize,
    pending: HashMap<Key<'a>, PendingBuild>,
    compiled: HashSet<&'a PackageId>,
    documented: HashSet<&'a PackageId>,
    counts: HashMap<&'a PackageId, usize>,
    is_release: bool,
}

/// A helper structure for metadata about the state of a building package.
struct PendingBuild {
    /// Number of jobs currently active
    amt: usize,
    /// Current freshness state of this package. Any dirty target within a
    /// package will cause the entire package to become dirty.
    fresh: Freshness,
}

#[derive(Clone, Copy, Eq, PartialEq, Hash)]
struct Key<'a> {
    pkg: &'a PackageId,
    target: &'a Target,
    profile: &'a Profile,
    kind: Kind,
}

pub struct JobState<'a> {
    tx: Sender<(Key<'a>, Message)>,
    key: Key<'a>,
}

enum Message {
    Run(String),
    Stdout(String),
    Stderr(String),
    Finish(CargoResult<()>),
}

impl<'a> JobState<'a> {
    pub fn running(&self, cmd: &ProcessBuilder) {
        let _ = self.tx.send((self.key, Message::Run(cmd.to_string())));
    }

    pub fn stdout(&self, out: &str) {
        let _ = self.tx.send((self.key, Message::Stdout(out.to_string())));
    }

    pub fn stderr(&self, err: &str) {
        let _ = self.tx.send((self.key, Message::Stderr(err.to_string())));
    }
}

impl<'a> JobQueue<'a> {
    pub fn new<'cfg>(cx: &Context<'a, 'cfg>) -> JobQueue<'a> {
        let (tx, rx) = channel();
        JobQueue {
            jobs: cx.jobs() as usize,
            queue: DependencyQueue::new(),
            tx: tx,
            rx: rx,
            active: 0,
            pending: HashMap::new(),
            compiled: HashSet::new(),
            documented: HashSet::new(),
            counts: HashMap::new(),
            is_release: cx.build_config.release,
        }
    }

    pub fn enqueue<'cfg>(&mut self,
                         cx: &Context<'a, 'cfg>,
                         unit: &Unit<'a>,
                         job: Job,
                         fresh: Freshness) -> CargoResult<()> {
        let key = Key::new(unit);
        let deps = key.dependencies(cx)?;
        self.queue.queue(Fresh, key, Vec::new(), &deps).push((job, fresh));
        *self.counts.entry(key.pkg).or_insert(0) += 1;
        Ok(())
    }

    /// Execute all jobs necessary to build the dependency graph.
    ///
    /// This function will spawn off `config.jobs()` workers to build all of the
    /// necessary dependencies, in order. Freshness is propagated as far as
    /// possible along each dependency chain.
    pub fn execute(&mut self, cx: &mut Context) -> CargoResult<()> {
        let _p = profile::start("executing the job graph");

        crossbeam::scope(|scope| {
            self.drain_the_queue(cx, scope)
        })
    }

    fn drain_the_queue(&mut self, cx: &mut Context, scope: &Scope<'a>)
                       -> CargoResult<()> {
        use std::time::Instant;

        let mut queue = Vec::new();
        trace!("queue: {:#?}", self.queue);

        // Iteratively execute the entire dependency graph. Each turn of the
        // loop starts out by scheduling as much work as possible (up to the
        // maximum number of parallel jobs). A local queue is maintained
        // separately from the main dependency queue as one dequeue may actually
        // dequeue quite a bit of work (e.g. 10 binaries in one project).
        //
        // After a job has finished we update our internal state if it was
        // successful and otherwise wait for pending work to finish if it failed
        // and then immediately return.
        let mut error = None;
        let start_time = Instant::now();
        loop {
            while error.is_none() && self.active < self.jobs {
                if !queue.is_empty() {
                    let (key, job, fresh) = queue.remove(0);
                    self.run(key, fresh, job, cx.config, scope)?;
                } else if let Some((fresh, key, jobs)) = self.queue.dequeue() {
                    let total_fresh = jobs.iter().fold(fresh, |fresh, &(_, f)| {
                        f.combine(fresh)
                    });
                    self.pending.insert(key, PendingBuild {
                        amt: jobs.len(),
                        fresh: total_fresh,
                    });
                    queue.extend(jobs.into_iter().map(|(job, f)| {
                        (key, job, f.combine(fresh))
                    }));
                } else {
                    break
                }
            }
            if self.active == 0 {
                break
            }

            let (key, msg) = self.rx.recv().unwrap();

            match msg {
                Message::Run(cmd) => {
                    cx.config.shell().verbose(|c| c.status("Running", &cmd))?;
                }
                Message::Stdout(out) => {
                    if cx.config.extra_verbose() {
                        writeln!(cx.config.shell().out(), "{}", out)?;
                    }
                }
                Message::Stderr(err) => {
                    if cx.config.extra_verbose() {
                        writeln!(cx.config.shell().err(), "{}", err)?;
                    }
                }
                Message::Finish(result) => {
                    info!("end: {:?}", key);
                    self.active -= 1;
                    match result {
                        Ok(()) => self.finish(key, cx)?,
                        Err(e) => {
                            cx.config.shell().error(format!("{}", e))?;
                            if self.active > 0 {
                                cx.config.shell().say(
                                            "Build failed, waiting for other \
                                             jobs to finish...", YELLOW)?;
                            }
                            if error.is_none() {
                                error = Some(human("build failed"));
                            }
                        }
                    }
                }
            }
        }

        let build_type = if self.is_release { "release" } else { "dev" };
        let profile = cx.lib_profile();
        let mut opt_type = String::from(if profile.opt_level == "0" { "unoptimized" }
                                        else { "optimized" });
        if profile.debuginfo.is_some() {
            opt_type = opt_type + " + debuginfo";
        }
        let duration = start_time.elapsed();
        let time_elapsed = format!("{}.{1:.2} secs",
                                   duration.as_secs(),
                                   duration.subsec_nanos() / 10000000);
        if self.queue.is_empty() {
            let message = format!("{} [{}] target(s) in {}",
                                  build_type,
                                  opt_type,
                                  time_elapsed);
            cx.config.shell().status("Finished", message)?;
            Ok(())
        } else if let Some(e) = error {
            Err(e)
        } else {
            debug!("queue: {:#?}", self.queue);
            Err(internal("finished with jobs still left in the queue"))
        }
    }

    /// Executes a job in the `scope` given, pushing the spawned thread's
    /// handled onto `threads`.
    fn run(&mut self,
           key: Key<'a>,
           fresh: Freshness,
           job: Job,
           config: &Config,
           scope: &Scope<'a>) -> CargoResult<()> {
        info!("start: {:?}", key);

        self.active += 1;
        *self.counts.get_mut(key.pkg).unwrap() -= 1;

        let my_tx = self.tx.clone();
        scope.spawn(move || {
            let res = job.run(fresh, &JobState {
                tx: my_tx.clone(),
                key: key,
            });
            my_tx.send((key, Message::Finish(res))).unwrap();
        });

        // Print out some nice progress information
        self.note_working_on(config, &key, fresh)?;

        Ok(())
    }

    fn finish(&mut self, key: Key<'a>, cx: &mut Context) -> CargoResult<()> {
        if key.profile.run_custom_build && cx.show_warnings(key.pkg) {
            let output = cx.build_state.outputs.lock().unwrap();
            if let Some(output) = output.get(&(key.pkg.clone(), key.kind)) {
                for warning in output.warnings.iter() {
                    cx.config.shell().warn(warning)?;
                }
            }
        }
        let state = self.pending.get_mut(&key).unwrap();
        state.amt -= 1;
        if state.amt == 0 {
            self.queue.finish(&key, state.fresh);
        }
        Ok(())
    }

    // This isn't super trivial because we don't want to print loads and
    // loads of information to the console, but we also want to produce a
    // faithful representation of what's happening. This is somewhat nuanced
    // as a package can start compiling *very* early on because of custom
    // build commands and such.
    //
    // In general, we try to print "Compiling" for the first nontrivial task
    // run for a package, regardless of when that is. We then don't print
    // out any more information for a package after we've printed it once.
    fn note_working_on(&mut self,
                       config: &Config,
                       key: &Key<'a>,
                       fresh: Freshness) -> CargoResult<()> {
        if (self.compiled.contains(key.pkg) && !key.profile.doc) ||
            (self.documented.contains(key.pkg) && key.profile.doc) {
            return Ok(())
        }

        match fresh {
            // Any dirty stage which runs at least one command gets printed as
            // being a compiled package
            Dirty => {
                if key.profile.doc {
                    if !key.profile.test {
                        self.documented.insert(key.pkg);
                        config.shell().status("Documenting", key.pkg)?;
                    }
                } else {
                    self.compiled.insert(key.pkg);
                    config.shell().status("Compiling", key.pkg)?;
                }
            }
            Fresh if self.counts[key.pkg] == 0 => {
                self.compiled.insert(key.pkg);
                config.shell().verbose(|c| c.status("Fresh", key.pkg))?;
            }
            Fresh => {}
        }
        Ok(())
    }
}

impl<'a> Key<'a> {
    fn new(unit: &Unit<'a>) -> Key<'a> {
        Key {
            pkg: unit.pkg.package_id(),
            target: unit.target,
            profile: unit.profile,
            kind: unit.kind,
        }
    }

    fn dependencies<'cfg>(&self, cx: &Context<'a, 'cfg>)
                          -> CargoResult<Vec<Key<'a>>> {
        let unit = Unit {
            pkg: cx.get_package(self.pkg)?,
            target: self.target,
            profile: self.profile,
            kind: self.kind,
        };
        let targets = cx.dep_targets(&unit)?;
        Ok(targets.iter().filter_map(|unit| {
            // Binaries aren't actually needed to *compile* tests, just to run
            // them, so we don't include this dependency edge in the job graph.
            if self.target.is_test() && unit.target.is_bin() {
                None
            } else {
                Some(Key::new(unit))
            }
        }).collect())
    }
}

impl<'a> fmt::Debug for Key<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{} => {}/{} => {:?}", self.pkg, self.target, self.profile,
               self.kind)
    }
}
<|endoftext|>