<commit_before>//! Legalize instructions.
//!
//! A legal instruction is one that can be mapped directly to a machine code instruction for the
//! target ISA. The `legalize_function()` function takes as input any function and transforms it
//! into an equivalent function using only legal instructions.
//!
//! The characteristics of legal instructions depend on the target ISA, so any given instruction
//! can be legal for one ISA and illegal for another.
//!
//! Besides transforming instructions, the legalizer also fills out the `function.encodings` map
//! which provides a legal encoding recipe for every instruction.
//!
//! The legalizer does not deal with register allocation constraints. These constraints are derived
//! from the encoding recipes, and solved later by the register allocator.

use flowgraph::ControlFlowGraph;
use ir::{Function, Cursor, DataFlowGraph, InstructionData, Opcode, InstBuilder};
use ir::condcodes::IntCC;
use isa::{TargetIsa, Legalize};

mod boundary;
mod split;

/// Legalize `func` for `isa`.
///
/// - Transform any instructions that don't have a legal representation in `isa`.
/// - Fill out `func.encodings`.
///
pub fn legalize_function(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &TargetIsa) {
    boundary::legalize_signatures(func, isa);

    // TODO: This is very simplified and incomplete.
    func.encodings.resize(func.dfg.num_insts());
    let mut pos = Cursor::new(&mut func.layout);
    while let Some(_ebb) = pos.next_ebb() {
        // Keep track of the cursor position before the instruction being processed, so we can
        // double back when replacing instructions.
        let mut prev_pos = pos.position();

        while let Some(inst) = pos.next_inst() {
            let opcode = func.dfg[inst].opcode();

            // Check for ABI boundaries that need to be converted to the legalized signature.
            if opcode.is_call() && boundary::handle_call_abi(&mut func.dfg, cfg, &mut pos) {
                // Go back and legalize the inserted argument conversion instructions.
                pos.set_position(prev_pos);
                continue;
            }

            if opcode.is_return() &&
               boundary::handle_return_abi(&mut func.dfg, cfg, &mut pos, &func.signature) {
                // Go back and legalize the inserted return value conversion instructions.
                pos.set_position(prev_pos);
                continue;
            }

            if opcode.is_branch() {
                split::simplify_branch_arguments(&mut func.dfg, inst);
            }

            match isa.encode(&func.dfg, &func.dfg[inst]) {
                Ok(encoding) => *func.encodings.ensure(inst) = encoding,
                Err(action) => {
                    // We should transform the instruction into legal equivalents.
                    // Possible strategies are:
                    // 1. Legalize::Expand: Expand instruction into sequence of legal instructions.
                    //    Possibly iteratively. ()
                    // 2. Legalize::Narrow: Split the controlling type variable into high and low
                    //    parts. This applies both to SIMD vector types which can be halved and to
                    //    integer types such as `i64` used on a 32-bit ISA. ().
                    // 3. TODO: Promote the controlling type variable to a larger type. This
                    //    typically means expressing `i8` and `i16` arithmetic in terms if `i32`
                    //    operations on RISC targets. (It may or may not be beneficial to promote
                    //    small vector types versus splitting them.)
                    // 4. TODO: Convert to library calls. For example, floating point operations on
                    //    an ISA with no IEEE 754 support.
                    let changed = match action {
                        Legalize::Expand => expand(&mut func.dfg, cfg, &mut pos),
                        Legalize::Narrow => narrow(&mut func.dfg, cfg, &mut pos),
                    };
                    // If the current instruction was replaced, we need to double back and revisit
                    // the expanded sequence. This is both to assign encodings and possible to
                    // expand further.
                    // There's a risk of infinite looping here if the legalization patterns are
                    // unsound. Should we attempt to detect that?
                    if changed {
                        pos.set_position(prev_pos);
                    }
                }
            }

            // Remember this position in case we need to double back.
            prev_pos = pos.position();
        }
    }
}

// Include legalization patterns that were generated by `gen_legalizer.py` from the `XForms` in
// `meta/cretonne/legalize.py`.
//
// Concretely, this defines private functions `narrow()`, and `expand()`.
include!(concat!(env!("OUT_DIR"), "/legalizer.rs"));
<commit_msg>Legalize EBBs in a reverse post-order.<commit_after>//! Legalize instructions.
//!
//! A legal instruction is one that can be mapped directly to a machine code instruction for the
//! target ISA. The `legalize_function()` function takes as input any function and transforms it
//! into an equivalent function using only legal instructions.
//!
//! The characteristics of legal instructions depend on the target ISA, so any given instruction
//! can be legal for one ISA and illegal for another.
//!
//! Besides transforming instructions, the legalizer also fills out the `function.encodings` map
//! which provides a legal encoding recipe for every instruction.
//!
//! The legalizer does not deal with register allocation constraints. These constraints are derived
//! from the encoding recipes, and solved later by the register allocator.

use flowgraph::ControlFlowGraph;
use ir::{Function, Cursor, DataFlowGraph, InstructionData, Opcode, InstBuilder};
use ir::condcodes::IntCC;
use isa::{TargetIsa, Legalize};

mod boundary;
mod split;

/// Legalize `func` for `isa`.
///
/// - Transform any instructions that don't have a legal representation in `isa`.
/// - Fill out `func.encodings`.
///
pub fn legalize_function(func: &mut Function, cfg: &mut ControlFlowGraph, isa: &TargetIsa) {
    boundary::legalize_signatures(func, isa);

    func.encodings.resize(func.dfg.num_insts());

    // Process EBBs in a reverse post-order. This minimizes the number of split instructions we
    // need.
    let mut postorder = cfg.postorder_ebbs();
    let mut pos = Cursor::new(&mut func.layout);

    while let Some(ebb) = postorder.pop() {
        pos.goto_top(ebb);

        // Keep track of the cursor position before the instruction being processed, so we can
        // double back when replacing instructions.
        let mut prev_pos = pos.position();

        while let Some(inst) = pos.next_inst() {
            let opcode = func.dfg[inst].opcode();

            // Check for ABI boundaries that need to be converted to the legalized signature.
            if opcode.is_call() && boundary::handle_call_abi(&mut func.dfg, cfg, &mut pos) {
                // Go back and legalize the inserted argument conversion instructions.
                pos.set_position(prev_pos);
                continue;
            }

            if opcode.is_return() &&
               boundary::handle_return_abi(&mut func.dfg, cfg, &mut pos, &func.signature) {
                // Go back and legalize the inserted return value conversion instructions.
                pos.set_position(prev_pos);
                continue;
            }

            if opcode.is_branch() {
                split::simplify_branch_arguments(&mut func.dfg, inst);
            }

            match isa.encode(&func.dfg, &func.dfg[inst]) {
                Ok(encoding) => *func.encodings.ensure(inst) = encoding,
                Err(action) => {
                    // We should transform the instruction into legal equivalents.
                    // Possible strategies are:
                    // 1. Legalize::Expand: Expand instruction into sequence of legal instructions.
                    //    Possibly iteratively. ()
                    // 2. Legalize::Narrow: Split the controlling type variable into high and low
                    //    parts. This applies both to SIMD vector types which can be halved and to
                    //    integer types such as `i64` used on a 32-bit ISA. ().
                    // 3. TODO: Promote the controlling type variable to a larger type. This
                    //    typically means expressing `i8` and `i16` arithmetic in terms if `i32`
                    //    operations on RISC targets. (It may or may not be beneficial to promote
                    //    small vector types versus splitting them.)
                    // 4. TODO: Convert to library calls. For example, floating point operations on
                    //    an ISA with no IEEE 754 support.
                    let changed = match action {
                        Legalize::Expand => expand(&mut func.dfg, cfg, &mut pos),
                        Legalize::Narrow => narrow(&mut func.dfg, cfg, &mut pos),
                    };
                    // If the current instruction was replaced, we need to double back and revisit
                    // the expanded sequence. This is both to assign encodings and possible to
                    // expand further.
                    // There's a risk of infinite looping here if the legalization patterns are
                    // unsound. Should we attempt to detect that?
                    if changed {
                        pos.set_position(prev_pos);
                    }
                }
            }

            // Remember this position in case we need to double back.
            prev_pos = pos.position();
        }
    }
}

// Include legalization patterns that were generated by `gen_legalizer.py` from the `XForms` in
// `meta/cretonne/legalize.py`.
//
// Concretely, this defines private functions `narrow()`, and `expand()`.
include!(concat!(env!("OUT_DIR"), "/legalizer.rs"));
<|endoftext|>


<commit_before>#![allow(missing_copy_implementations)]

#[cfg(test)]
mod tests;

use crate::fmt;
use crate::io::{
    self, BufRead, Initializer, IoSlice, IoSliceMut, Read, Seek, SeekFrom, SizeHint, Write,
};

/// A reader which is always at EOF.
///
/// This struct is generally created by calling [`empty()`]. Please see
/// the documentation of [`empty()`] for more details.
#[stable(feature = "rust1", since = "1.0.0")]
#[non_exhaustive]
pub struct Empty;

/// Constructs a new handle to an empty reader.
///
/// All reads from the returned reader will return [`Ok`]`(0)`.
///
/// # Examples
///
/// A slightly sad example of not reading anything into a buffer:
///
/// ```
/// use std::io::{self, Read};
///
/// let mut buffer = String::new();
/// io::empty().read_to_string(&mut buffer).unwrap();
/// assert!(buffer.is_empty());
/// ```
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_const_unstable(feature = "const_io_structs", issue = "78812")]
pub const fn empty() -> Empty {
    Empty
}

#[stable(feature = "rust1", since = "1.0.0")]
impl Read for Empty {
    #[inline]
    fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
        Ok(0)
    }

    #[inline]
    unsafe fn initializer(&self) -> Initializer {
        Initializer::nop()
    }
}
#[stable(feature = "rust1", since = "1.0.0")]
impl BufRead for Empty {
    #[inline]
    fn fill_buf(&mut self) -> io::Result<&[u8]> {
        Ok(&[])
    }
    #[inline]
    fn consume(&mut self, _n: usize) {}
}

#[stable(feature = "empty_seek", since = "1.51.0")]
impl Seek for Empty {
    fn seek(&mut self, _pos: SeekFrom) -> io::Result<u64> {
        Ok(0)
    }

    fn stream_len(&mut self) -> io::Result<u64> {
        Ok(0)
    }

    fn stream_position(&mut self) -> io::Result<u64> {
        Ok(0)
    }
}

#[stable(feature = "std_debug", since = "1.16.0")]
impl fmt::Debug for Empty {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("Empty").finish_non_exhaustive()
    }
}

impl SizeHint for Empty {
    #[inline]
    fn upper_bound(&self) -> Option<usize> {
        Some(0)
    }
}

/// A reader which yields one byte over and over and over and over and over and...
///
/// This struct is generally created by calling [`repeat()`]. Please
/// see the documentation of [`repeat()`] for more details.
#[stable(feature = "rust1", since = "1.0.0")]
pub struct Repeat {
    byte: u8,
}

/// Creates an instance of a reader that infinitely repeats one byte.
///
/// All reads from this reader will succeed by filling the specified buffer with
/// the given byte.
///
/// # Examples
///
/// ```
/// use std::io::{self, Read};
///
/// let mut buffer = [0; 3];
/// io::repeat(0b101).read_exact(&mut buffer).unwrap();
/// assert_eq!(buffer, [0b101, 0b101, 0b101]);
/// ```
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_const_unstable(feature = "const_io_structs", issue = "78812")]
pub const fn repeat(byte: u8) -> Repeat {
    Repeat { byte }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl Read for Repeat {
    #[inline]
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        for slot in &mut *buf {
            *slot = self.byte;
        }
        Ok(buf.len())
    }

    #[inline]
    fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result<usize> {
        let mut nwritten = 0;
        for buf in bufs {
            nwritten += self.read(buf)?;
        }
        Ok(nwritten)
    }

    #[inline]
    fn is_read_vectored(&self) -> bool {
        true
    }

    #[inline]
    unsafe fn initializer(&self) -> Initializer {
        Initializer::nop()
    }
}

impl SizeHint for Repeat {
    #[inline]
    fn lower_bound(&self) -> usize {
        usize::MAX
    }

    #[inline]
    fn upper_bound(&self) -> Option<usize> {
        None
    }
}

#[stable(feature = "std_debug", since = "1.16.0")]
impl fmt::Debug for Repeat {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("Repeat").finish_non_exhaustive()
    }
}

/// A writer which will move data into the void.
///
/// This struct is generally created by calling [`sink`]. Please
/// see the documentation of [`sink()`] for more details.
#[stable(feature = "rust1", since = "1.0.0")]
#[non_exhaustive]
pub struct Sink;

/// Creates an instance of a writer which will successfully consume all data.
///
/// All calls to [`write`] on the returned instance will return `Ok(buf.len())`
/// and the contents of the buffer will not be inspected.
///
/// [`write`]: Write::write
///
/// # Examples
///
/// ```rust
/// use std::io::{self, Write};
///
/// let buffer = vec![1, 2, 3, 5, 8];
/// let num_bytes = io::sink().write(&buffer).unwrap();
/// assert_eq!(num_bytes, 5);
/// ```
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_const_unstable(feature = "const_io_structs", issue = "78812")]
pub const fn sink() -> Sink {
    Sink
}

#[stable(feature = "rust1", since = "1.0.0")]
impl Write for Sink {
    #[inline]
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        Ok(buf.len())
    }

    #[inline]
    fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result<usize> {
        let total_len = bufs.iter().map(|b| b.len()).sum();
        Ok(total_len)
    }

    #[inline]
    fn is_write_vectored(&self) -> bool {
        true
    }

    #[inline]
    fn flush(&mut self) -> io::Result<()> {
        Ok(())
    }
}

#[stable(feature = "write_mt", since = "1.48.0")]
impl Write for &Sink {
    #[inline]
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        Ok(buf.len())
    }

    #[inline]
    fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result<usize> {
        let total_len = bufs.iter().map(|b| b.len()).sum();
        Ok(total_len)
    }

    #[inline]
    fn is_write_vectored(&self) -> bool {
        true
    }

    #[inline]
    fn flush(&mut self) -> io::Result<()> {
        Ok(())
    }
}

#[stable(feature = "std_debug", since = "1.16.0")]
impl fmt::Debug for Sink {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("Sink").finish_non_exhaustive()
    }
}
<commit_msg>impl Default, Copy, Clone for std::io::Sink and Empty<commit_after>#![allow(missing_copy_implementations)]

#[cfg(test)]
mod tests;

use crate::fmt;
use crate::io::{
    self, BufRead, Initializer, IoSlice, IoSliceMut, Read, Seek, SeekFrom, SizeHint, Write,
};

/// A reader which is always at EOF.
///
/// This struct is generally created by calling [`empty()`]. Please see
/// the documentation of [`empty()`] for more details.
#[stable(feature = "rust1", since = "1.0.0")]
#[non_exhaustive]
#[derive(Copy, Clone, Default)]
pub struct Empty;

/// Constructs a new handle to an empty reader.
///
/// All reads from the returned reader will return [`Ok`]`(0)`.
///
/// # Examples
///
/// A slightly sad example of not reading anything into a buffer:
///
/// ```
/// use std::io::{self, Read};
///
/// let mut buffer = String::new();
/// io::empty().read_to_string(&mut buffer).unwrap();
/// assert!(buffer.is_empty());
/// ```
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_const_unstable(feature = "const_io_structs", issue = "78812")]
pub const fn empty() -> Empty {
    Empty
}

#[stable(feature = "rust1", since = "1.0.0")]
impl Read for Empty {
    #[inline]
    fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
        Ok(0)
    }

    #[inline]
    unsafe fn initializer(&self) -> Initializer {
        Initializer::nop()
    }
}
#[stable(feature = "rust1", since = "1.0.0")]
impl BufRead for Empty {
    #[inline]
    fn fill_buf(&mut self) -> io::Result<&[u8]> {
        Ok(&[])
    }
    #[inline]
    fn consume(&mut self, _n: usize) {}
}

#[stable(feature = "empty_seek", since = "1.51.0")]
impl Seek for Empty {
    fn seek(&mut self, _pos: SeekFrom) -> io::Result<u64> {
        Ok(0)
    }

    fn stream_len(&mut self) -> io::Result<u64> {
        Ok(0)
    }

    fn stream_position(&mut self) -> io::Result<u64> {
        Ok(0)
    }
}

#[stable(feature = "std_debug", since = "1.16.0")]
impl fmt::Debug for Empty {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("Empty").finish_non_exhaustive()
    }
}

impl SizeHint for Empty {
    #[inline]
    fn upper_bound(&self) -> Option<usize> {
        Some(0)
    }
}

/// A reader which yields one byte over and over and over and over and over and...
///
/// This struct is generally created by calling [`repeat()`]. Please
/// see the documentation of [`repeat()`] for more details.
#[stable(feature = "rust1", since = "1.0.0")]
pub struct Repeat {
    byte: u8,
}

/// Creates an instance of a reader that infinitely repeats one byte.
///
/// All reads from this reader will succeed by filling the specified buffer with
/// the given byte.
///
/// # Examples
///
/// ```
/// use std::io::{self, Read};
///
/// let mut buffer = [0; 3];
/// io::repeat(0b101).read_exact(&mut buffer).unwrap();
/// assert_eq!(buffer, [0b101, 0b101, 0b101]);
/// ```
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_const_unstable(feature = "const_io_structs", issue = "78812")]
pub const fn repeat(byte: u8) -> Repeat {
    Repeat { byte }
}

#[stable(feature = "rust1", since = "1.0.0")]
impl Read for Repeat {
    #[inline]
    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
        for slot in &mut *buf {
            *slot = self.byte;
        }
        Ok(buf.len())
    }

    #[inline]
    fn read_vectored(&mut self, bufs: &mut [IoSliceMut<'_>]) -> io::Result<usize> {
        let mut nwritten = 0;
        for buf in bufs {
            nwritten += self.read(buf)?;
        }
        Ok(nwritten)
    }

    #[inline]
    fn is_read_vectored(&self) -> bool {
        true
    }

    #[inline]
    unsafe fn initializer(&self) -> Initializer {
        Initializer::nop()
    }
}

impl SizeHint for Repeat {
    #[inline]
    fn lower_bound(&self) -> usize {
        usize::MAX
    }

    #[inline]
    fn upper_bound(&self) -> Option<usize> {
        None
    }
}

#[stable(feature = "std_debug", since = "1.16.0")]
impl fmt::Debug for Repeat {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("Repeat").finish_non_exhaustive()
    }
}

/// A writer which will move data into the void.
///
/// This struct is generally created by calling [`sink`]. Please
/// see the documentation of [`sink()`] for more details.
#[stable(feature = "rust1", since = "1.0.0")]
#[non_exhaustive]
#[derive(Copy, Clone, Default)]
pub struct Sink;

/// Creates an instance of a writer which will successfully consume all data.
///
/// All calls to [`write`] on the returned instance will return `Ok(buf.len())`
/// and the contents of the buffer will not be inspected.
///
/// [`write`]: Write::write
///
/// # Examples
///
/// ```rust
/// use std::io::{self, Write};
///
/// let buffer = vec![1, 2, 3, 5, 8];
/// let num_bytes = io::sink().write(&buffer).unwrap();
/// assert_eq!(num_bytes, 5);
/// ```
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_const_unstable(feature = "const_io_structs", issue = "78812")]
pub const fn sink() -> Sink {
    Sink
}

#[stable(feature = "rust1", since = "1.0.0")]
impl Write for Sink {
    #[inline]
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        Ok(buf.len())
    }

    #[inline]
    fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result<usize> {
        let total_len = bufs.iter().map(|b| b.len()).sum();
        Ok(total_len)
    }

    #[inline]
    fn is_write_vectored(&self) -> bool {
        true
    }

    #[inline]
    fn flush(&mut self) -> io::Result<()> {
        Ok(())
    }
}

#[stable(feature = "write_mt", since = "1.48.0")]
impl Write for &Sink {
    #[inline]
    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
        Ok(buf.len())
    }

    #[inline]
    fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> io::Result<usize> {
        let total_len = bufs.iter().map(|b| b.len()).sum();
        Ok(total_len)
    }

    #[inline]
    fn is_write_vectored(&self) -> bool {
        true
    }

    #[inline]
    fn flush(&mut self) -> io::Result<()> {
        Ok(())
    }
}

#[stable(feature = "std_debug", since = "1.16.0")]
impl fmt::Debug for Sink {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("Sink").finish_non_exhaustive()
    }
}
<|endoftext|>


<commit_before>use criterion::*;

use rand::distributions::Standard;
use rand::rngs::SmallRng;
use rand::{Rng, SeedableRng};

use phf_generator::generate_hash;

fn gen_vec(len: usize) -> Vec<u64> {
    SmallRng::seed_from_u64(0xAAAAAAAAAAAAAAAA).sample_iter(Standard).take(len).collect()
}

fn bench_hash(b: &mut Bencher, len: &usize) {
    let vec = gen_vec(*len);
    b.iter(|| generate_hash(&vec))
}

fn gen_hash_small(c: &mut Criterion) {
    let sizes = vec![0, 1, 2, 5, 10, 25, 50, 75];
    c.bench_function_over_inputs("gen_hash_small", bench_hash, sizes);
}

fn gen_hash_med(c: &mut Criterion) {
    let sizes = vec![100, 250, 500, 1000, 2500, 5000, 7500];
    c.bench_function_over_inputs("gen_hash_medium", bench_hash, sizes);
}

fn gen_hash_large(c: &mut Criterion) {
    let sizes = vec![10_000, 25_000, 50_000, 75_000];
    c.bench_function_over_inputs("gen_hash_large", bench_hash, sizes);
}

fn gen_hash_xlarge(c: &mut Criterion) {
    let sizes = vec![100_000, 250_000, 500_000, 750_000, 1_000_000];
    c.bench_function_over_inputs("gen_hash_xlarge", bench_hash, sizes);
}

criterion_group!(benches, gen_hash_small, gen_hash_med, gen_hash_large, gen_hash_xlarge);

#[cfg(not(feature = "rayon"))]
criterion_main!(benches);

#[cfg(feature = "rayon")]
criterion_main!(benches, rayon::benches);

#[cfg(feature = "rayon")]
mod rayon {
    use criterion::*;

    use phf_generator::generate_hash_rayon;

    use super::gen_vec;

    fn bench_hash(b: &mut Bencher, len: &usize) {
        let vec = gen_vec(*len);
        b.iter(|| generate_hash_rayon(&vec))
    }

    fn gen_hash_small(c: &mut Criterion) {
        let sizes = vec![0, 1, 2, 5, 10, 25, 50, 75];
        c.bench_function_over_inputs("gen_hash_small_rayon", bench_hash, sizes);
    }

    fn gen_hash_med(c: &mut Criterion) {
        let sizes = vec![100, 250, 500, 1000, 2500, 5000, 7500];
        c.bench_function_over_inputs("gen_hash_medium_rayon", bench_hash, sizes);
    }

    fn gen_hash_large(c: &mut Criterion) {
        let sizes = vec![10_000, 25_000, 50_000, 75_000];
        c.bench_function_over_inputs("gen_hash_large_rayon", bench_hash, sizes);
    }

    fn gen_hash_xlarge(c: &mut Criterion) {
        let sizes = vec![100_000, 250_000, 500_000, 750_000, 1_000_000];
        c.bench_function_over_inputs("gen_hash_xlarge_rayon", bench_hash, sizes);
    }

    criterion_group!(benches, gen_hash_small, gen_hash_med, gen_hash_large, gen_hash_xlarge);
}<commit_msg>Tweak benchmarks<commit_after>use criterion::measurement::Measurement;
use criterion::{criterion_group, criterion_main, Bencher, BenchmarkId, Criterion};

use rand::distributions::Standard;
use rand::rngs::SmallRng;
use rand::{Rng, SeedableRng};

use phf_generator::generate_hash;

fn gen_vec(len: usize) -> Vec<u64> {
    SmallRng::seed_from_u64(0xAAAAAAAAAAAAAAAA)
        .sample_iter(Standard)
        .take(len)
        .collect()
}

fn bench_hash<M: Measurement>(b: &mut Bencher<M>, len: &usize) {
    let vec = gen_vec(*len);
    b.iter(|| generate_hash(&vec))
}

fn gen_hash_small(c: &mut Criterion) {
    let sizes = vec![0, 1, 2, 5, 10, 25, 50, 75];
    for size in &sizes {
        c.bench_with_input(BenchmarkId::new("gen_hash_small", *size), size, bench_hash);
    }
}

fn gen_hash_med(c: &mut Criterion) {
    let sizes = vec![100, 250, 500, 1000, 2500, 5000, 7500];
    for size in &sizes {
        c.bench_with_input(BenchmarkId::new("gen_hash_medium", *size), size, bench_hash);
    }
}

fn gen_hash_large(c: &mut Criterion) {
    let sizes = vec![10_000, 25_000, 50_000, 75_000];
    for size in &sizes {
        c.bench_with_input(BenchmarkId::new("gen_hash_large", *size), size, bench_hash);
    }
}

fn gen_hash_xlarge(c: &mut Criterion) {
    let sizes = vec![100_000, 250_000, 500_000, 750_000, 1_000_000];
    for size in &sizes {
        c.bench_with_input(BenchmarkId::new("gen_hash_xlarge", *size), size, bench_hash);
    }
}

criterion_group!(
    benches,
    gen_hash_small,
    gen_hash_med,
    gen_hash_large,
    gen_hash_xlarge
);

criterion_main!(benches);
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate docopt;
extern crate csv;
extern crate rustc_serialize;
extern crate mimir;
#[macro_use]
extern crate log;
extern crate geo;

use std::path::Path;
use mimir::rubber::Rubber;
use mimir::objects::Admin;
use std::fs;
use std::rc::Rc;
use std::collections::BTreeMap;

type AdminFromInsee = BTreeMap<String, Rc<Admin>>;

#[derive(RustcDecodable, RustcEncodable)]
pub struct Bano {
    pub id: String,
    pub nb: String,
    pub street: String,
    pub zip: String,
    pub city: String,
    pub src: String,
    pub lat: f64,
    pub lon: f64,
}

impl Bano {
    pub fn insee(&self) -> &str {
        assert!(self.id.len() >= 5);
        self.id[..5].trim_left_matches('0')
    }
    pub fn fantoir(&self) -> &str {
        assert!(self.id.len() >= 10);
        &self.id[..10]
    }
    pub fn into_addr(self, admins: &AdminFromInsee) -> mimir::Addr {
        let street_name = format!("{} ({})", self.street, self.city);
        let addr_name = format!("{} {}", self.nb, self.street);
        let addr_label = format!("{} ({})", addr_name, self.city);
        let street_id = format!("street:{}", self.fantoir().to_string());
        let admin = admins.get(&format!("admin:fr:{}", self.insee()));

        let street = mimir::Street {
            id: street_id,
            street_name: self.street,
            label: street_name.to_string(),
            administrative_regions: admin.map_or(vec![], |a| vec![a.clone()]),
            weight: 1,
            zip_codes: vec![self.zip.clone()],
            coord: mimir::Coord::new(self.lat, self.lon),
        };
        mimir::Addr {
            id: format!("addr:{};{}", self.lon, self.lat),
            house_number: self.nb,
            street: street,
            label: addr_label,
            coord: mimir::Coord::new(self.lat, self.lon),
            weight: 1,
            zip_codes: vec![self.zip.clone()],
        }
    }
}

fn index_bano<I>(cnx_string: &str, dataset: &str, files: I)
    where I: Iterator<Item = std::path::PathBuf>
{
    let doc_type = "addr";
    let mut rubber = Rubber::new(cnx_string);

    let admins_by_insee = rubber.get_admins_from_dataset(dataset)
        .unwrap_or_else(|err| {
            info!("Administratives regions not found in es db for dataset {}. (error: {})",
                  dataset,
                  err);
            vec![]
        })
        .into_iter()
        .map(|a| (a.id.to_string(), Rc::new(a)))
        .collect();

    let addr_index = rubber.make_index(doc_type, dataset).unwrap();
    info!("Add data in elasticsearch db.");
    for f in files {
        info!("importing {:?}...", &f);
        let mut rdr = csv::Reader::from_file(&f).unwrap().has_headers(false);

        let iter = rdr.decode().map(|r| {
            let b: Bano = r.unwrap();
            b.into_addr(&admins_by_insee)
        });
        match rubber.bulk_index(&addr_index, iter) {
            Err(e) => panic!("failed to bulk insert file {:?} because: {}", &f, e),
            Ok(nb) => info!("importing {:?}: {} addresses added.", &f, nb),
        }
    }
    rubber.publish_index(doc_type, dataset, addr_index).unwrap();
}

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_connection_string: String,
    flag_dataset: String,
}

static USAGE: &'static str = "
Usage:
    bano2mimir --input=<input> [--connection-string=<connection-string>] [--dataset=<dataset>]

    -i, --input=<input>           Bano files. Can be either a directory or a file.
    -c, --connection-string=<connection-string>
                                  Elasticsearch parameters, [default: http://localhost:9200/munin]
    -d, --dataset=<dataset>       Name of the dataset, [default: fr]
";

fn main() {
    mimir::logger_init().unwrap();
    info!("importing bano into Mimir");

    let args: Args = docopt::Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());

    let file_path = Path::new(&args.flag_input);
    if file_path.is_dir() {
        let paths: std::fs::ReadDir = fs::read_dir(&args.flag_input).unwrap();
        index_bano(&args.flag_connection_string,
                   &args.flag_dataset,
                   paths.map(|p| p.unwrap().path()));
    } else {
        index_bano(&args.flag_connection_string,
                   &args.flag_dataset,
                   std::iter::once(std::path::PathBuf::from(&args.flag_input)));
    }
}
<commit_msg>forget the boundary for performance purpose<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

extern crate docopt;
extern crate csv;
extern crate rustc_serialize;
extern crate mimir;
#[macro_use]
extern crate log;
extern crate geo;

use std::path::Path;
use mimir::rubber::Rubber;
use mimir::objects::Admin;
use std::fs;
use std::rc::Rc;
use std::collections::BTreeMap;

type AdminFromInsee = BTreeMap<String, Rc<Admin>>;

#[derive(RustcDecodable, RustcEncodable)]
pub struct Bano {
    pub id: String,
    pub nb: String,
    pub street: String,
    pub zip: String,
    pub city: String,
    pub src: String,
    pub lat: f64,
    pub lon: f64,
}

impl Bano {
    pub fn insee(&self) -> &str {
        assert!(self.id.len() >= 5);
        self.id[..5].trim_left_matches('0')
    }
    pub fn fantoir(&self) -> &str {
        assert!(self.id.len() >= 10);
        &self.id[..10]
    }
    pub fn into_addr(self, admins: &AdminFromInsee) -> mimir::Addr {
        let street_name = format!("{} ({})", self.street, self.city);
        let addr_name = format!("{} {}", self.nb, self.street);
        let addr_label = format!("{} ({})", addr_name, self.city);
        let street_id = format!("street:{}", self.fantoir().to_string());
        let admin = admins.get(&format!("admin:fr:{}", self.insee()));

        let street = mimir::Street {
            id: street_id,
            street_name: self.street,
            label: street_name.to_string(),
            administrative_regions: admin.map_or(vec![], |a| vec![a.clone()]),
            weight: 1,
            zip_codes: vec![self.zip.clone()],
            coord: mimir::Coord::new(self.lat, self.lon),
        };
        mimir::Addr {
            id: format!("addr:{};{}", self.lon, self.lat),
            house_number: self.nb,
            street: street,
            label: addr_label,
            coord: mimir::Coord::new(self.lat, self.lon),
            weight: 1,
            zip_codes: vec![self.zip.clone()],
        }
    }
}

fn index_bano<I>(cnx_string: &str, dataset: &str, files: I)
    where I: Iterator<Item = std::path::PathBuf>
{
    let doc_type = "addr";
    let mut rubber = Rubber::new(cnx_string);

    let admins_by_insee = rubber.get_admins_from_dataset(dataset)
        .unwrap_or_else(|err| {
            info!("Administratives regions not found in es db for dataset {}. (error: {})",
                  dataset,
                  err);
            vec![]
        })
        .into_iter()
        .map(|mut a| {
            a.boundary = None; // to save some space we remove the admin boundary
            (a.id.to_string(), Rc::new(a))
        })
        .collect();

    let addr_index = rubber.make_index(doc_type, dataset).unwrap();
    info!("Add data in elasticsearch db.");
    for f in files {
        info!("importing {:?}...", &f);
        let mut rdr = csv::Reader::from_file(&f).unwrap().has_headers(false);

        let iter = rdr.decode().map(|r| {
            let b: Bano = r.unwrap();
            b.into_addr(&admins_by_insee)
        });
        match rubber.bulk_index(&addr_index, iter) {
            Err(e) => panic!("failed to bulk insert file {:?} because: {}", &f, e),
            Ok(nb) => info!("importing {:?}: {} addresses added.", &f, nb),
        }
    }
    rubber.publish_index(doc_type, dataset, addr_index).unwrap();
}

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_connection_string: String,
    flag_dataset: String,
}

static USAGE: &'static str = "
Usage:
    bano2mimir --input=<input> [--connection-string=<connection-string>] [--dataset=<dataset>]

    -i, --input=<input>           Bano files. Can be either a directory or a file.
    -c, --connection-string=<connection-string>
                                  Elasticsearch parameters, [default: http://localhost:9200/munin]
    -d, --dataset=<dataset>       Name of the dataset, [default: fr]
";

fn main() {
    mimir::logger_init().unwrap();
    info!("importing bano into Mimir");

    let args: Args = docopt::Docopt::new(USAGE)
        .and_then(|d| d.decode())
        .unwrap_or_else(|e| e.exit());

    let file_path = Path::new(&args.flag_input);
    if file_path.is_dir() {
        let paths: std::fs::ReadDir = fs::read_dir(&args.flag_input).unwrap();
        index_bano(&args.flag_connection_string,
                   &args.flag_dataset,
                   paths.map(|p| p.unwrap().path()));
    } else {
        index_bano(&args.flag_connection_string,
                   &args.flag_dataset,
                   std::iter::once(std::path::PathBuf::from(&args.flag_input)));
    }
}
<|endoftext|>


<commit_before><commit_msg>Use absolute path for bundle path<commit_after><|endoftext|>


<commit_before>// Copyright 2015 Brendan Zabarauskas and the gl-rs developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use registry::Registry;
use std::io;

#[allow(missing_copy_implementations)]
pub struct GlobalGenerator;

impl super::Generator for GlobalGenerator {
    fn write<W>(&self, registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
        try!(write_header(dest));
        try!(write_metaloadfn(dest));
        try!(write_type_aliases(registry, dest));
        try!(write_enums(registry, dest));
        try!(write_fns(registry, dest));
        try!(write_fnptr_struct_def(dest));
        try!(write_ptrs(registry, dest));
        try!(write_fn_mods(registry, dest));
        try!(write_panicking_fns(registry, dest));
        try!(write_load_fn(registry, dest));
        Ok(())
    }
}

/// Creates a `__gl_imports` module which contains all the external symbols that we need for the
///  bindings.
fn write_header<W>(dest: &mut W) -> io::Result<()> where W: io::Write {
    writeln!(dest, r#"
        mod __gl_imports {{
            pub use std::mem;
            pub use std::os::raw;
        }}
    "#)
}

/// Creates the metaloadfn function for fallbacks
fn write_metaloadfn<W>(dest: &mut W) -> io::Result<()> where W: io::Write {
    writeln!(dest, r#"
        #[inline(never)]
        fn metaloadfn(mut loadfn: &mut FnMut(&str) -> *const __gl_imports::raw::c_void,
                      symbol: &str,
                      fallbacks: &[&str]) -> *const __gl_imports::raw::c_void {{
            let mut ptr = loadfn(symbol);
            if ptr.is_null() {{
                for &sym in fallbacks {{
                    ptr = loadfn(sym);
                    if !ptr.is_null() {{ break; }}
                }}
            }}
            ptr
        }}
    "#)
}

/// Creates a `types` module which contains all the type aliases.
///
/// See also `generators::gen_types`.
fn write_type_aliases<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    try!(writeln!(dest, r#"
        pub mod types {{
            #![allow(non_camel_case_types, non_snake_case, dead_code, missing_copy_implementations)]
    "#));

    try!(super::gen_types(registry.api, dest));

    writeln!(dest, "
        }}
    ")
}

/// Creates all the `<enum>` elements at the root of the bindings.
fn write_enums<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    for enm in &registry.enums {
        try!(super::gen_enum_item(enm, "types::", dest));
    }

    Ok(())
}

/// Creates the functions corresponding to the GL commands.
///
/// The function calls the corresponding function pointer stored in the `storage` module created
///  by `write_ptrs`.
fn write_fns<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    for cmd in &registry.cmds {
        if let Some(v) = registry.aliases.get(&cmd.proto.ident) {
            try!(writeln!(dest, "/// Fallbacks: {}", v.join(", ")));
        }

        try!(writeln!(dest,
            "#[allow(non_snake_case, unused_variables, dead_code)] #[inline]
            pub unsafe fn {name}({params}) -> {return_suffix} {{ \
                __gl_imports::mem::transmute::<_, extern \"system\" fn({typed_params}) -> {return_suffix}>\
                    (storage::{name}.f)({idents}) \
            }}",
            name = cmd.proto.ident,
            params = super::gen_parameters(cmd, true, true).join(", "),
            typed_params = super::gen_parameters(cmd, false, true).join(", "),
            return_suffix = cmd.proto.ty,
            idents = super::gen_parameters(cmd, true, false).join(", "),
        ));
    }

    Ok(())
}

/// Creates a `FnPtr` structure which contains the store for a single binding.
fn write_fnptr_struct_def<W>(dest: &mut W) -> io::Result<()> where W: io::Write {
    writeln!(dest, "
        #[allow(missing_copy_implementations)]
        pub struct FnPtr {{
            /// The function pointer that will be used when calling the function.
            f: *const __gl_imports::raw::c_void,
            /// True if the pointer points to a real function, false if points to a `panic!` fn.
            is_loaded: bool,
        }}

        impl FnPtr {{
            /// Creates a `FnPtr` from a load attempt.
            pub fn new(ptr: *const __gl_imports::raw::c_void) -> FnPtr {{
                if ptr.is_null() {{
                    FnPtr {{ f: missing_fn_panic as *const __gl_imports::raw::c_void, is_loaded: false }}
                }} else {{
                    FnPtr {{ f: ptr, is_loaded: true }}
                }}
            }}
        }}
    ")
}

/// Creates a `storage` module which contains a static `FnPtr` per GL command in the registry.
fn write_ptrs<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {

    try!(writeln!(dest,
        "mod storage {{
            #![allow(non_snake_case)]
            use super::__gl_imports::raw;
            use super::FnPtr;"));

    for c in &registry.cmds {
        try!(writeln!(dest,
            "pub static mut {name}: FnPtr = FnPtr {{
                f: super::missing_fn_panic as *const raw::c_void,
                is_loaded: false
            }};",
            name = c.proto.ident
        ));
    }

    writeln!(dest, "}}")
}

/// Creates one module for each GL command.
///
/// Each module contains `is_loaded` and `load_with` which interact with the `storage` module
///  created by `write_ptrs`.
fn write_fn_mods<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    for c in &registry.cmds {
        let fallbacks = match registry.aliases.get(&c.proto.ident) {
            Some(v) => {
                let names = v.iter().map(|name| format!("\"{}\"", super::gen_symbol_name(registry.api, &name[..]))).collect::<Vec<_>>();
                format!("&[{}]", names.join(", "))
            }, None => "&[]".to_string(),
        };
        let fnname = &c.proto.ident[..];
        let symbol = super::gen_symbol_name(registry.api, &c.proto.ident[..]);
        let symbol = &symbol[..];

        try!(writeln!(dest, r##"
            #[allow(non_snake_case)]
            pub mod {fnname} {{
                use super::{{storage, metaloadfn}};
                use super::__gl_imports::raw;
                use super::FnPtr;

                #[inline]
                #[allow(dead_code)]
                pub fn is_loaded() -> bool {{
                    unsafe {{ storage::{fnname}.is_loaded }}
                }}

                #[allow(dead_code)]
                pub fn load_with<F>(mut loadfn: F) where F: FnMut(&str) -> *const raw::c_void {{
                    unsafe {{
                        storage::{fnname} = FnPtr::new(metaloadfn(&mut loadfn, "{symbol}", {fallbacks}))
                    }}
                }}
            }}
        "##, fnname = fnname, fallbacks = fallbacks, symbol = symbol));
    }

    Ok(())
}

/// Creates a `missing_fn_panic` function.
///
/// This function is the mock that is called if the real function could not be called.
fn write_panicking_fns<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    writeln!(dest,
        "#[inline(never)]
        fn missing_fn_panic() -> ! {{
            panic!(\"{api} function was not loaded\")
        }}
        ", api = registry.api)
}

/// Creates the `load_with` function.
///
/// The function calls `load_with` in each module created by `write_fn_mods`.
fn write_load_fn<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    try!(writeln!(dest, "
        /// Load each OpenGL symbol using a custom load function. This allows for the
        /// use of functions like `glfwGetProcAddress` or `SDL_GL_GetProcAddress`.
        /// ~~~ignore
        /// gl::load_with(|s| glfw.get_proc_address(s));
        /// ~~~
        #[allow(dead_code)]
        pub fn load_with<F>(mut loadfn: F) where F: FnMut(&str) -> *const __gl_imports::raw::c_void {{
    "));

    for c in &registry.cmds {
        try!(writeln!(dest, "{cmd_name}::load_with(|s| loadfn(s));",
                      cmd_name = &c.proto.ident[..]));
    }

    writeln!(dest, "
        }}
    ")
}
<commit_msg>silence non-uppercase globals warning<commit_after>// Copyright 2015 Brendan Zabarauskas and the gl-rs developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use registry::Registry;
use std::io;

#[allow(missing_copy_implementations)]
pub struct GlobalGenerator;

impl super::Generator for GlobalGenerator {
    fn write<W>(&self, registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
        try!(write_header(dest));
        try!(write_metaloadfn(dest));
        try!(write_type_aliases(registry, dest));
        try!(write_enums(registry, dest));
        try!(write_fns(registry, dest));
        try!(write_fnptr_struct_def(dest));
        try!(write_ptrs(registry, dest));
        try!(write_fn_mods(registry, dest));
        try!(write_panicking_fns(registry, dest));
        try!(write_load_fn(registry, dest));
        Ok(())
    }
}

/// Creates a `__gl_imports` module which contains all the external symbols that we need for the
///  bindings.
fn write_header<W>(dest: &mut W) -> io::Result<()> where W: io::Write {
    writeln!(dest, r#"
        mod __gl_imports {{
            pub use std::mem;
            pub use std::os::raw;
        }}
    "#)
}

/// Creates the metaloadfn function for fallbacks
fn write_metaloadfn<W>(dest: &mut W) -> io::Result<()> where W: io::Write {
    writeln!(dest, r#"
        #[inline(never)]
        fn metaloadfn(mut loadfn: &mut FnMut(&str) -> *const __gl_imports::raw::c_void,
                      symbol: &str,
                      fallbacks: &[&str]) -> *const __gl_imports::raw::c_void {{
            let mut ptr = loadfn(symbol);
            if ptr.is_null() {{
                for &sym in fallbacks {{
                    ptr = loadfn(sym);
                    if !ptr.is_null() {{ break; }}
                }}
            }}
            ptr
        }}
    "#)
}

/// Creates a `types` module which contains all the type aliases.
///
/// See also `generators::gen_types`.
fn write_type_aliases<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    try!(writeln!(dest, r#"
        pub mod types {{
            #![allow(non_camel_case_types, non_snake_case, dead_code, missing_copy_implementations)]
    "#));

    try!(super::gen_types(registry.api, dest));

    writeln!(dest, "
        }}
    ")
}

/// Creates all the `<enum>` elements at the root of the bindings.
fn write_enums<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    for enm in &registry.enums {
        try!(super::gen_enum_item(enm, "types::", dest));
    }

    Ok(())
}

/// Creates the functions corresponding to the GL commands.
///
/// The function calls the corresponding function pointer stored in the `storage` module created
///  by `write_ptrs`.
fn write_fns<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    for cmd in &registry.cmds {
        if let Some(v) = registry.aliases.get(&cmd.proto.ident) {
            try!(writeln!(dest, "/// Fallbacks: {}", v.join(", ")));
        }

        try!(writeln!(dest,
            "#[allow(non_snake_case, unused_variables, dead_code)] #[inline]
            pub unsafe fn {name}({params}) -> {return_suffix} {{ \
                __gl_imports::mem::transmute::<_, extern \"system\" fn({typed_params}) -> {return_suffix}>\
                    (storage::{name}.f)({idents}) \
            }}",
            name = cmd.proto.ident,
            params = super::gen_parameters(cmd, true, true).join(", "),
            typed_params = super::gen_parameters(cmd, false, true).join(", "),
            return_suffix = cmd.proto.ty,
            idents = super::gen_parameters(cmd, true, false).join(", "),
        ));
    }

    Ok(())
}

/// Creates a `FnPtr` structure which contains the store for a single binding.
fn write_fnptr_struct_def<W>(dest: &mut W) -> io::Result<()> where W: io::Write {
    writeln!(dest, "
        #[allow(missing_copy_implementations)]
        pub struct FnPtr {{
            /// The function pointer that will be used when calling the function.
            f: *const __gl_imports::raw::c_void,
            /// True if the pointer points to a real function, false if points to a `panic!` fn.
            is_loaded: bool,
        }}

        impl FnPtr {{
            /// Creates a `FnPtr` from a load attempt.
            pub fn new(ptr: *const __gl_imports::raw::c_void) -> FnPtr {{
                if ptr.is_null() {{
                    FnPtr {{ f: missing_fn_panic as *const __gl_imports::raw::c_void, is_loaded: false }}
                }} else {{
                    FnPtr {{ f: ptr, is_loaded: true }}
                }}
            }}
        }}
    ")
}

/// Creates a `storage` module which contains a static `FnPtr` per GL command in the registry.
fn write_ptrs<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {

    try!(writeln!(dest,
        "mod storage {{
            #![allow(non_snake_case)]
            #![allow(non_upper_case_globals)]
            use super::__gl_imports::raw;
            use super::FnPtr;"));

    for c in &registry.cmds {
        try!(writeln!(dest,
            "pub static mut {name}: FnPtr = FnPtr {{
                f: super::missing_fn_panic as *const raw::c_void,
                is_loaded: false
            }};",
            name = c.proto.ident
        ));
    }

    writeln!(dest, "}}")
}

/// Creates one module for each GL command.
///
/// Each module contains `is_loaded` and `load_with` which interact with the `storage` module
///  created by `write_ptrs`.
fn write_fn_mods<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    for c in &registry.cmds {
        let fallbacks = match registry.aliases.get(&c.proto.ident) {
            Some(v) => {
                let names = v.iter().map(|name| format!("\"{}\"", super::gen_symbol_name(registry.api, &name[..]))).collect::<Vec<_>>();
                format!("&[{}]", names.join(", "))
            }, None => "&[]".to_string(),
        };
        let fnname = &c.proto.ident[..];
        let symbol = super::gen_symbol_name(registry.api, &c.proto.ident[..]);
        let symbol = &symbol[..];

        try!(writeln!(dest, r##"
            #[allow(non_snake_case)]
            pub mod {fnname} {{
                use super::{{storage, metaloadfn}};
                use super::__gl_imports::raw;
                use super::FnPtr;

                #[inline]
                #[allow(dead_code)]
                pub fn is_loaded() -> bool {{
                    unsafe {{ storage::{fnname}.is_loaded }}
                }}

                #[allow(dead_code)]
                pub fn load_with<F>(mut loadfn: F) where F: FnMut(&str) -> *const raw::c_void {{
                    unsafe {{
                        storage::{fnname} = FnPtr::new(metaloadfn(&mut loadfn, "{symbol}", {fallbacks}))
                    }}
                }}
            }}
        "##, fnname = fnname, fallbacks = fallbacks, symbol = symbol));
    }

    Ok(())
}

/// Creates a `missing_fn_panic` function.
///
/// This function is the mock that is called if the real function could not be called.
fn write_panicking_fns<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    writeln!(dest,
        "#[inline(never)]
        fn missing_fn_panic() -> ! {{
            panic!(\"{api} function was not loaded\")
        }}
        ", api = registry.api)
}

/// Creates the `load_with` function.
///
/// The function calls `load_with` in each module created by `write_fn_mods`.
fn write_load_fn<W>(registry: &Registry, dest: &mut W) -> io::Result<()> where W: io::Write {
    try!(writeln!(dest, "
        /// Load each OpenGL symbol using a custom load function. This allows for the
        /// use of functions like `glfwGetProcAddress` or `SDL_GL_GetProcAddress`.
        /// ~~~ignore
        /// gl::load_with(|s| glfw.get_proc_address(s));
        /// ~~~
        #[allow(dead_code)]
        pub fn load_with<F>(mut loadfn: F) where F: FnMut(&str) -> *const __gl_imports::raw::c_void {{
    "));

    for c in &registry.cmds {
        try!(writeln!(dest, "{cmd_name}::load_with(|s| loadfn(s));",
                      cmd_name = &c.proto.ident[..]));
    }

    writeln!(dest, "
        }}
    ")
}
<|endoftext|>


<commit_before>mod fuzzer;
mod generator;
mod utils;

pub use self::fuzzer::{Fuzzer, Status};
use crate::{
    database::Database,
    module::Module,
    vm::{use_provider::DbUseProvider, Closure, Vm},
};
use itertools::Itertools;
use tracing::{error, info};

pub async fn fuzz(db: &Database, module: Module) {
    let (fuzzables_heap, fuzzables) = {
        let result = Vm::new_for_running_module_closure(
            &DbUseProvider { db },
            Closure::of_module(db, module.clone()).unwrap(),
        )
        .run_synchronously_until_completion(db);
        (result.heap, result.fuzzable_closures)
    };

    info!(
        "Now, the fuzzing begins. So far, we have {} closures to fuzz.",
        fuzzables.len()
    );

    for (id, closure) in fuzzables {
        let mut fuzzer = Fuzzer::new(db, &fuzzables_heap, closure, id.clone());
        fuzzer.run(db, 1000);
        match fuzzer.status() {
            Status::StillFuzzing { .. } => {}
            Status::PanickedForArguments {
                heap,
                arguments,
                reason,
                tracer,
            } => {
                info!("The fuzzer discovered an input that crashes {id}:");
                info!(
                    "Calling `{id} {}` doesn't work because {reason}.",
                    arguments
                        .iter()
                        .map(|argument| argument.format(heap))
                        .join(" "),
                );
                info!("This was the stack trace:");
                tracer.dump_stack_trace(db, heap);

                module.dump_associated_debug_file("trace", &tracer.format_call_tree(heap));
            }
        }
    }
}
<commit_msg>Remove unused import<commit_after>mod fuzzer;
mod generator;
mod utils;

pub use self::fuzzer::{Fuzzer, Status};
use crate::{
    database::Database,
    module::Module,
    vm::{use_provider::DbUseProvider, Closure, Vm},
};
use itertools::Itertools;
use tracing::info;

pub async fn fuzz(db: &Database, module: Module) {
    let (fuzzables_heap, fuzzables) = {
        let result = Vm::new_for_running_module_closure(
            &DbUseProvider { db },
            Closure::of_module(db, module.clone()).unwrap(),
        )
        .run_synchronously_until_completion(db);
        (result.heap, result.fuzzable_closures)
    };

    info!(
        "Now, the fuzzing begins. So far, we have {} closures to fuzz.",
        fuzzables.len()
    );

    for (id, closure) in fuzzables {
        let mut fuzzer = Fuzzer::new(db, &fuzzables_heap, closure, id.clone());
        fuzzer.run(db, 1000);
        match fuzzer.status() {
            Status::StillFuzzing { .. } => {}
            Status::PanickedForArguments {
                heap,
                arguments,
                reason,
                tracer,
            } => {
                info!("The fuzzer discovered an input that crashes {id}:");
                info!(
                    "Calling `{id} {}` doesn't work because {reason}.",
                    arguments
                        .iter()
                        .map(|argument| argument.format(heap))
                        .join(" "),
                );
                info!("This was the stack trace:");
                tracer.dump_stack_trace(db, heap);

                module.dump_associated_debug_file("trace", &tracer.format_call_tree(heap));
            }
        }
    }
}
<|endoftext|>


<commit_before><commit_msg>Update struct modified<commit_after><|endoftext|>


<commit_before>#[doc(
    brief = "Provides all access to AST-related, non-sendable info",
    desc =
    "Rustdoc is intended to be parallel, and the rustc AST is filled \
     with shared boxes. The AST service attempts to provide a single \
     place to query AST-related information, shielding the rest of \
     Rustdoc from its non-sendableness."
)];

import rustc::driver::session;
import rustc::driver::driver;
import rustc::driver::diagnostic;
import rustc::syntax::ast;
import rustc::middle::ast_map;
import rustc::back::link;
import rustc::util::filesearch;
import rustc::front;

export ctxt;
export ctxt_handler;
export srv;
export mk_srv_from_str;
export mk_srv_from_file;
export exec;

type ctxt = {
    ast: @ast::crate,
    ast_map: ast_map::map
};

type ctxt_handler<T> = fn~(ctxt: ctxt) -> T;

type srv = {
    ctxt: ctxt
};

fn mk_srv_from_str(source: str) -> srv {
    let sess = build_session();
    {
        ctxt: build_ctxt(sess, parse::from_str_sess(sess, source))
    }
}

fn mk_srv_from_file(file: str) -> srv {
    let sess = build_session();
    {
        ctxt: build_ctxt(sess, parse::from_file_sess(sess, file))
    }
}

fn build_ctxt(sess: session::session, ast: @ast::crate) -> ctxt {

    import rustc::front::config;

    let ast = config::strip_unconfigured_items(ast);
    let ast = front::test::modify_for_testing(sess, ast);
    let ast_map = ast_map::map_crate(*ast);

    {
        ast: ast,
        ast_map: ast_map,
    }
}

fn build_session() -> session::session {
    let sopts: @session::options = @{
        crate_type: session::lib_crate,
        static: false,
        optimize: 0u,
        debuginfo: false,
        extra_debuginfo: false,
        verify: false,
        lint_opts: [],
        save_temps: false,
        stats: false,
        time_passes: false,
        time_llvm_passes: false,
        output_type: link::output_type_exe,
        addl_lib_search_paths: [],
        maybe_sysroot: none,
        target_triple: driver::host_triple(),
        cfg: [],
        test: false,
        parse_only: false,
        no_trans: false,
        do_gc: false,
        no_asm_comments: false,
        warn_unused_imports: false
    };
    driver::build_session(sopts, ".", diagnostic::emit)
}

#[test]
fn should_prune_unconfigured_items() {
    let source = "#[cfg(shut_up_and_leave_me_alone)]fn a() { }";
    let srv = mk_srv_from_str(source);
    exec(srv) {|ctxt|
        assert vec::is_empty(ctxt.ast.node.module.items);
    }
}

#[test]
#[ignore]
fn srv_should_build_ast_map() {
    // FIXME
    /*let source = "fn a() { }";
    let srv = mk_srv_from_str(source);
    exec(srv) {|ctxt|
        assert ctxt.ast_map.size() != 0u
    };*/
}

#[test]
#[ignore]
fn srv_should_build_reexport_map() {
    // FIXME
    /*let source = "import a::b; export b; mod a { mod b { } }";
    let srv = mk_srv_from_str(source);
    exec(srv) {|ctxt|
        assert ctxt.exp_map.size() != 0u
    };*/
}

#[test]
fn srv_should_resolve_external_crates() {
    let source = "use std;\
                  fn f() -> std::sha1::sha1 {\
                  std::sha1::mk_sha1() }";
    // Just testing that resolve doesn't crash
    mk_srv_from_str(source);
}

#[test]
fn srv_should_resolve_core_crate() {
    let source = "fn a() -> option { fail }";
    // Just testing that resolve doesn't crash
    mk_srv_from_str(source);
}

fn exec<T>(
    srv: srv,
    f: fn~(ctxt: ctxt) -> T
) -> T {
    f(srv.ctxt)
}

#[test]
fn srv_should_return_request_result() {
    let source = "fn a() { }";
    let srv = mk_srv_from_str(source);
    let result = exec(srv) {|_ctxt| 1000};
    assert result == 1000;
}
<commit_msg>rustdoc: Reenable a working test<commit_after>#[doc(
    brief = "Provides all access to AST-related, non-sendable info",
    desc =
    "Rustdoc is intended to be parallel, and the rustc AST is filled \
     with shared boxes. The AST service attempts to provide a single \
     place to query AST-related information, shielding the rest of \
     Rustdoc from its non-sendableness."
)];

import rustc::driver::session;
import rustc::driver::driver;
import rustc::driver::diagnostic;
import rustc::syntax::ast;
import rustc::middle::ast_map;
import rustc::back::link;
import rustc::util::filesearch;
import rustc::front;

export ctxt;
export ctxt_handler;
export srv;
export mk_srv_from_str;
export mk_srv_from_file;
export exec;

type ctxt = {
    ast: @ast::crate,
    ast_map: ast_map::map
};

type ctxt_handler<T> = fn~(ctxt: ctxt) -> T;

type srv = {
    ctxt: ctxt
};

fn mk_srv_from_str(source: str) -> srv {
    let sess = build_session();
    {
        ctxt: build_ctxt(sess, parse::from_str_sess(sess, source))
    }
}

fn mk_srv_from_file(file: str) -> srv {
    let sess = build_session();
    {
        ctxt: build_ctxt(sess, parse::from_file_sess(sess, file))
    }
}

fn build_ctxt(sess: session::session, ast: @ast::crate) -> ctxt {

    import rustc::front::config;

    let ast = config::strip_unconfigured_items(ast);
    let ast = front::test::modify_for_testing(sess, ast);
    let ast_map = ast_map::map_crate(*ast);

    {
        ast: ast,
        ast_map: ast_map,
    }
}

fn build_session() -> session::session {
    let sopts: @session::options = @{
        crate_type: session::lib_crate,
        static: false,
        optimize: 0u,
        debuginfo: false,
        extra_debuginfo: false,
        verify: false,
        lint_opts: [],
        save_temps: false,
        stats: false,
        time_passes: false,
        time_llvm_passes: false,
        output_type: link::output_type_exe,
        addl_lib_search_paths: [],
        maybe_sysroot: none,
        target_triple: driver::host_triple(),
        cfg: [],
        test: false,
        parse_only: false,
        no_trans: false,
        do_gc: false,
        no_asm_comments: false,
        warn_unused_imports: false
    };
    driver::build_session(sopts, ".", diagnostic::emit)
}

#[test]
fn should_prune_unconfigured_items() {
    let source = "#[cfg(shut_up_and_leave_me_alone)]fn a() { }";
    let srv = mk_srv_from_str(source);
    exec(srv) {|ctxt|
        assert vec::is_empty(ctxt.ast.node.module.items);
    }
}

#[test]
fn srv_should_build_ast_map() {
    let source = "fn a() { }";
    let srv = mk_srv_from_str(source);
    exec(srv) {|ctxt|
        assert ctxt.ast_map.size() != 0u
    };
}

#[test]
#[ignore]
fn srv_should_build_reexport_map() {
    // FIXME
    /*let source = "import a::b; export b; mod a { mod b { } }";
    let srv = mk_srv_from_str(source);
    exec(srv) {|ctxt|
        assert ctxt.exp_map.size() != 0u
    };*/
}

#[test]
fn srv_should_resolve_external_crates() {
    let source = "use std;\
                  fn f() -> std::sha1::sha1 {\
                  std::sha1::mk_sha1() }";
    // Just testing that resolve doesn't crash
    mk_srv_from_str(source);
}

#[test]
fn srv_should_resolve_core_crate() {
    let source = "fn a() -> option { fail }";
    // Just testing that resolve doesn't crash
    mk_srv_from_str(source);
}

fn exec<T>(
    srv: srv,
    f: fn~(ctxt: ctxt) -> T
) -> T {
    f(srv.ctxt)
}

#[test]
fn srv_should_return_request_result() {
    let source = "fn a() { }";
    let srv = mk_srv_from_str(source);
    let result = exec(srv) {|_ctxt| 1000};
    assert result == 1000;
}
<|endoftext|>


<commit_before><commit_msg>fix node::insert() method, missing next element<commit_after><|endoftext|>


<commit_before><commit_msg>Tapa: enhance generator<commit_after><|endoftext|>


<commit_before>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use channel::diem_channel::{self, Receiver};
use diem_config::{config::RoleType, network_id::NetworkContext};
use diem_crypto::x25519;
use diem_logger::prelude::*;
use diem_metrics::{
    register_histogram, register_int_counter_vec, DurationHistogram, IntCounterVec,
};
use diem_network_address::NetworkAddress;
use diem_network_address_encryption::{Encryptor, Error as EncryptorError};
use diem_types::on_chain_config::{OnChainConfigPayload, ValidatorSet, ON_CHAIN_CONFIG_REGISTRY};
use futures::{sink::SinkExt, StreamExt};
use network::{
    connectivity_manager::{ConnectivityRequest, DiscoverySource},
    counters::inc_by_with_context,
    logging::NetworkSchema,
};
use once_cell::sync::Lazy;
use std::{
    collections::{HashMap, HashSet},
    sync::Arc,
    time::Instant,
};
use subscription_service::ReconfigSubscription;

pub mod builder;

/// Histogram of idle time of spent in event processing loop
pub static EVENT_PROCESSING_LOOP_IDLE_DURATION_S: Lazy<DurationHistogram> = Lazy::new(|| {
    DurationHistogram::new(
        register_histogram!(
            "simple_onchain_discovery_event_processing_loop_idle_duration_s",
            "Histogram of idle time of spent in event processing loop"
        )
        .unwrap(),
    )
});

/// Histogram of busy time of spent in event processing loop
pub static EVENT_PROCESSING_LOOP_BUSY_DURATION_S: Lazy<DurationHistogram> = Lazy::new(|| {
    DurationHistogram::new(
        register_histogram!(
            "simple_onchain_discovery_event_processing_loop_busy_duration_s",
            "Histogram of busy time of spent in event processing loop"
        )
        .unwrap(),
    )
});

pub static DISCOVERY_COUNTS: Lazy<IntCounterVec> = Lazy::new(|| {
    register_int_counter_vec!(
        "diem_simple_onchain_discovery_counts",
        "Histogram of busy time of spent in event processing loop",
        &["role_type", "network_id", "peer_id", "metric"]
    )
    .unwrap()
});

/// Listener which converts published  updates from the OnChainConfig to ConnectivityRequests
/// for the ConnectivityManager.
pub struct ConfigurationChangeListener {
    network_context: Arc<NetworkContext>,
    encryptor: Encryptor,
    conn_mgr_reqs_tx: channel::Sender<ConnectivityRequest>,
    reconfig_events: diem_channel::Receiver<(), OnChainConfigPayload>,
}

pub fn gen_simple_discovery_reconfig_subscription(
) -> (ReconfigSubscription, Receiver<(), OnChainConfigPayload>) {
    ReconfigSubscription::subscribe_all("network", ON_CHAIN_CONFIG_REGISTRY.to_vec(), vec![])
}

/// Extracts a set of ConnectivityRequests from a ValidatorSet which are appropriate for a network with type role.
fn extract_updates(
    network_context: Arc<NetworkContext>,
    encryptor: &Encryptor,
    node_set: ValidatorSet,
) -> Vec<ConnectivityRequest> {
    let role = network_context.role();

    // Decode addresses while ignoring bad addresses
    let new_peer_addrs: HashMap<_, _> = node_set
        .into_iter()
        .map(|info| {
            let peer_id = *info.account_address();
            let config = info.into_config();

            let addrs = match role {
                RoleType::Validator => {
                    let result = encryptor.decrypt(&config.validator_network_addresses, peer_id);
                    if let Err(EncryptorError::StorageError(_)) = result {
                        panic!(format!(
                            "Unable to initialize validator network addresses: {:?}",
                            result
                        ));
                    }
                    result.map_err(anyhow::Error::from)
                }
                RoleType::FullNode => config
                    .fullnode_network_addresses()
                    .map_err(anyhow::Error::from),
            }
            .map_err(|err| {
                inc_by_with_context(&DISCOVERY_COUNTS, &network_context, "read_failure", 1);

                warn!(
                    NetworkSchema::new(&network_context),
                    "OnChainDiscovery: Failed to parse any network address: peer: {}, err: {}",
                    peer_id,
                    err
                )
            })
            .unwrap_or_default();

            (peer_id, addrs)
        })
        .collect();

    // Retrieve public keys from addresses
    let new_peer_pubkeys: HashMap<_, _> = new_peer_addrs
        .iter()
        .map(|(peer_id, addrs)| {
            let pubkeys: HashSet<x25519::PublicKey> = addrs
                .iter()
                .filter_map(NetworkAddress::find_noise_proto)
                .collect();
            (*peer_id, pubkeys)
        })
        .collect();

    vec![
        ConnectivityRequest::UpdateAddresses(DiscoverySource::OnChain, new_peer_addrs),
        ConnectivityRequest::UpdateEligibleNodes(DiscoverySource::OnChain, new_peer_pubkeys),
    ]
}

impl ConfigurationChangeListener {
    /// Creates a new ConfigurationListener
    pub fn new(
        network_context: Arc<NetworkContext>,
        encryptor: Encryptor,
        conn_mgr_reqs_tx: channel::Sender<ConnectivityRequest>,
        reconfig_events: diem_channel::Receiver<(), OnChainConfigPayload>,
    ) -> Self {
        Self {
            network_context,
            encryptor,
            conn_mgr_reqs_tx,
            reconfig_events,
        }
    }

    /// Processes a received OnChainConfigPayload.  Depending on role (Validator or FullNode), parses
    /// the appropriate configuration changes and passes it to the ConnectionManager channel.
    async fn process_payload(&mut self, payload: OnChainConfigPayload) {
        let node_set: ValidatorSet = payload
            .get()
            .expect("failed to get ValidatorSet from payload");

        let updates = extract_updates(self.network_context.clone(), &self.encryptor, node_set);

        inc_by_with_context(
            &DISCOVERY_COUNTS,
            &self.network_context,
            "new_nodes",
            updates.len() as u64,
        );
        info!(
            NetworkSchema::new(&self.network_context),
            "Update {} Network about new Node IDs",
            self.network_context.role()
        );

        for update in updates {
            match self.conn_mgr_reqs_tx.send(update).await {
                Ok(()) => (),
                Err(e) => {
                    inc_by_with_context(
                        &DISCOVERY_COUNTS,
                        &self.network_context,
                        "send_failure",
                        1,
                    );
                    warn!(
                        NetworkSchema::new(&self.network_context),
                        "Failed to send update to ConnectivityManager {}", e
                    )
                }
            }
        }
    }

    /// Starts the listener to wait on reconfiguration events.  Creates an infinite loop.
    pub async fn start(mut self) {
        info!(
            NetworkSchema::new(&self.network_context),
            "{} Starting OnChain Discovery actor", self.network_context
        );
        loop {
            let start_idle_time = Instant::now();
            let payload = self.reconfig_events.select_next_some().await;
            let idle_duration = start_idle_time.elapsed();
            let start_process_time = Instant::now();
            self.process_payload(payload).await;
            let process_duration = start_process_time.elapsed();

            EVENT_PROCESSING_LOOP_IDLE_DURATION_S.observe_duration(idle_duration);
            EVENT_PROCESSING_LOOP_BUSY_DURATION_S.observe_duration(process_duration);
        }
    }
}
<commit_msg>[network] Fix incorrect usage of select_next_some in onchain discovery<commit_after>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use channel::diem_channel::{self, Receiver};
use diem_config::{config::RoleType, network_id::NetworkContext};
use diem_crypto::x25519;
use diem_logger::prelude::*;
use diem_metrics::{
    register_histogram, register_int_counter_vec, DurationHistogram, IntCounterVec,
};
use diem_network_address::NetworkAddress;
use diem_network_address_encryption::{Encryptor, Error as EncryptorError};
use diem_types::on_chain_config::{OnChainConfigPayload, ValidatorSet, ON_CHAIN_CONFIG_REGISTRY};
use futures::{sink::SinkExt, StreamExt};
use network::{
    connectivity_manager::{ConnectivityRequest, DiscoverySource},
    counters::inc_by_with_context,
    logging::NetworkSchema,
};
use once_cell::sync::Lazy;
use std::{
    collections::{HashMap, HashSet},
    sync::Arc,
};
use subscription_service::ReconfigSubscription;

pub mod builder;

/// Histogram of idle time of spent in event processing loop
pub static EVENT_PROCESSING_LOOP_IDLE_DURATION_S: Lazy<DurationHistogram> = Lazy::new(|| {
    DurationHistogram::new(
        register_histogram!(
            "simple_onchain_discovery_event_processing_loop_idle_duration_s",
            "Histogram of idle time of spent in event processing loop"
        )
        .unwrap(),
    )
});

/// Histogram of busy time of spent in event processing loop
pub static EVENT_PROCESSING_LOOP_BUSY_DURATION_S: Lazy<DurationHistogram> = Lazy::new(|| {
    DurationHistogram::new(
        register_histogram!(
            "simple_onchain_discovery_event_processing_loop_busy_duration_s",
            "Histogram of busy time of spent in event processing loop"
        )
        .unwrap(),
    )
});

pub static DISCOVERY_COUNTS: Lazy<IntCounterVec> = Lazy::new(|| {
    register_int_counter_vec!(
        "diem_simple_onchain_discovery_counts",
        "Histogram of busy time of spent in event processing loop",
        &["role_type", "network_id", "peer_id", "metric"]
    )
    .unwrap()
});

/// Listener which converts published  updates from the OnChainConfig to ConnectivityRequests
/// for the ConnectivityManager.
pub struct ConfigurationChangeListener {
    network_context: Arc<NetworkContext>,
    encryptor: Encryptor,
    conn_mgr_reqs_tx: channel::Sender<ConnectivityRequest>,
    reconfig_events: diem_channel::Receiver<(), OnChainConfigPayload>,
}

pub fn gen_simple_discovery_reconfig_subscription(
) -> (ReconfigSubscription, Receiver<(), OnChainConfigPayload>) {
    ReconfigSubscription::subscribe_all("network", ON_CHAIN_CONFIG_REGISTRY.to_vec(), vec![])
}

/// Extracts a set of ConnectivityRequests from a ValidatorSet which are appropriate for a network with type role.
fn extract_updates(
    network_context: Arc<NetworkContext>,
    encryptor: &Encryptor,
    node_set: ValidatorSet,
) -> Vec<ConnectivityRequest> {
    let role = network_context.role();

    // Decode addresses while ignoring bad addresses
    let new_peer_addrs: HashMap<_, _> = node_set
        .into_iter()
        .map(|info| {
            let peer_id = *info.account_address();
            let config = info.into_config();

            let addrs = match role {
                RoleType::Validator => {
                    let result = encryptor.decrypt(&config.validator_network_addresses, peer_id);
                    if let Err(EncryptorError::StorageError(_)) = result {
                        panic!(format!(
                            "Unable to initialize validator network addresses: {:?}",
                            result
                        ));
                    }
                    result.map_err(anyhow::Error::from)
                }
                RoleType::FullNode => config
                    .fullnode_network_addresses()
                    .map_err(anyhow::Error::from),
            }
            .map_err(|err| {
                inc_by_with_context(&DISCOVERY_COUNTS, &network_context, "read_failure", 1);

                warn!(
                    NetworkSchema::new(&network_context),
                    "OnChainDiscovery: Failed to parse any network address: peer: {}, err: {}",
                    peer_id,
                    err
                )
            })
            .unwrap_or_default();

            (peer_id, addrs)
        })
        .collect();

    // Retrieve public keys from addresses
    let new_peer_pubkeys: HashMap<_, _> = new_peer_addrs
        .iter()
        .map(|(peer_id, addrs)| {
            let pubkeys: HashSet<x25519::PublicKey> = addrs
                .iter()
                .filter_map(NetworkAddress::find_noise_proto)
                .collect();
            (*peer_id, pubkeys)
        })
        .collect();

    vec![
        ConnectivityRequest::UpdateAddresses(DiscoverySource::OnChain, new_peer_addrs),
        ConnectivityRequest::UpdateEligibleNodes(DiscoverySource::OnChain, new_peer_pubkeys),
    ]
}

impl ConfigurationChangeListener {
    /// Creates a new ConfigurationChangeListener
    pub fn new(
        network_context: Arc<NetworkContext>,
        encryptor: Encryptor,
        conn_mgr_reqs_tx: channel::Sender<ConnectivityRequest>,
        reconfig_events: diem_channel::Receiver<(), OnChainConfigPayload>,
    ) -> Self {
        Self {
            network_context,
            encryptor,
            conn_mgr_reqs_tx,
            reconfig_events,
        }
    }

    async fn next_reconfig_event(&mut self) -> Option<OnChainConfigPayload> {
        let _idle_timer = EVENT_PROCESSING_LOOP_IDLE_DURATION_S.start_timer();
        self.reconfig_events.next().await
    }

    /// Processes a received OnChainConfigPayload. Depending on role (Validator or FullNode), parses
    /// the appropriate configuration changes and passes it to the ConnectionManager channel.
    async fn process_payload(&mut self, payload: OnChainConfigPayload) {
        let _process_timer = EVENT_PROCESSING_LOOP_BUSY_DURATION_S.start_timer();

        let node_set: ValidatorSet = payload
            .get()
            .expect("failed to get ValidatorSet from payload");

        let updates = extract_updates(self.network_context.clone(), &self.encryptor, node_set);

        inc_by_with_context(
            &DISCOVERY_COUNTS,
            &self.network_context,
            "new_nodes",
            updates.len() as u64,
        );
        info!(
            NetworkSchema::new(&self.network_context),
            "Update {} Network about new Node IDs",
            self.network_context.role()
        );

        for update in updates {
            match self.conn_mgr_reqs_tx.send(update).await {
                Ok(()) => (),
                Err(e) => {
                    inc_by_with_context(
                        &DISCOVERY_COUNTS,
                        &self.network_context,
                        "send_failure",
                        1,
                    );
                    warn!(
                        NetworkSchema::new(&self.network_context),
                        "Failed to send update to ConnectivityManager {}", e
                    )
                }
            }
        }
    }

    /// Starts the listener to wait on reconfiguration events.
    pub async fn start(mut self) {
        info!(
            NetworkSchema::new(&self.network_context),
            "{} Starting OnChain Discovery actor", self.network_context
        );

        while let Some(payload) = self.next_reconfig_event().await {
            self.process_payload(payload).await;
        }

        warn!(
            NetworkSchema::new(&self.network_context),
            "{} OnChain Discovery actor terminated", self.network_context,
        );
    }
}
<|endoftext|>