<commit_before>use std::io::Write;

use libzakopane::config::Config;
use libzakopane::snapshot::Snapshot;
use libzakopane::structs::CompareCliOptions;
use libzakopane::structs::ZakopaneError;

use clap::{App, Arg, ArgMatches};

const DEFAULT_POLICY_ARG_NAME: &'static str = "default-policy";
const CONFIG_FILE_ARG_NAME: &'static str = "config";
const OLD_SNAPSHOT_PATH_ARG_NAME: &'static str = "old-snapshot-path";
const NEW_SNAPSHOT_PATH_ARG_NAME: &'static str = "new-snapshot-path";

// Holds one instance of each struct necessary to execute the `compare`
// subcommand.
struct CompareData {
    config: Config,
    old_snapshot: Snapshot,
    new_snapshot: Snapshot,
}

enum SubcommandData {
    Compare(CompareData),
    Checksum(std::path::PathBuf),
}

// Reads parsed command-line arguments and returns the appropriate
// operational data. Can abort the program on error.
fn compare_data_from(matches: &ArgMatches) -> Result<SubcommandData, ZakopaneError> {
    // The two snapshot paths are required, so these are safe to unwrap.
    let old_snapshot_path = matches.value_of(OLD_SNAPSHOT_PATH_ARG_NAME).unwrap();
    let new_snapshot_path = matches.value_of(NEW_SNAPSHOT_PATH_ARG_NAME).unwrap();
    let old_contents = libzakopane::helpers::ingest_file(old_snapshot_path)?;
    let new_contents = libzakopane::helpers::ingest_file(new_snapshot_path)?;

    let options = CompareCliOptions {
        config_path: matches.value_of(CONFIG_FILE_ARG_NAME),
        default_policy: matches.value_of(DEFAULT_POLICY_ARG_NAME),
    };

    Ok(SubcommandData::Compare(CompareData {
        config: Config::new(&options)?,
        old_snapshot: Snapshot::new(&old_contents)?,
        new_snapshot: Snapshot::new(&new_contents)?,
    }))
}

// Begins parsing command-line arguments. Can abort the program on
// error.
fn initialize() -> Result<SubcommandData, ZakopaneError> {
    let matches = App::new("zakopane")
        .version("0.2.0")
        .author("j39m")
        .about("checksums directories")
        .subcommand(
            App::new("compare")
                .about("compares two zakopane snapshots")
                .arg(
                    Arg::with_name(CONFIG_FILE_ARG_NAME)
                        .short("c")
                        .long("config")
                        .value_name("FILE")
                        .help("specifies a zakopane config")
                        .takes_value(true),
                )
                .arg(
                    Arg::with_name(DEFAULT_POLICY_ARG_NAME)
                        .short("d")
                        .long("default-policy")
                        .value_name("POLICY_TOKENS")
                        .help("specifies an explicit default policy")
                        .takes_value(true),
                )
                .arg(
                    Arg::with_name(OLD_SNAPSHOT_PATH_ARG_NAME)
                        .help("path to older snapshot")
                        .index(1)
                        .required(true),
                )
                .arg(
                    Arg::with_name(NEW_SNAPSHOT_PATH_ARG_NAME)
                        .help("path to newer snapshot")
                        .index(2)
                        .required(true),
                ),
        )
        .subcommand(
            App::new("checksum")
                .about("produces checksums for a directory")
                .arg(
                    Arg::with_name("target-path")
                        .help("directory to checksum")
                        .index(1)
                        .required(true),
                ),
        )
        .get_matches();

    if let Some(ref matches) = matches.subcommand_matches("compare") {
        return compare_data_from(&matches);
    }
    if let Some(ref matches) = matches.subcommand_matches("checksum") {
        return Ok(SubcommandData::Checksum(std::path::PathBuf::from(
            matches.value_of("target-path").unwrap(),
        )));
    }
    Err(ZakopaneError::Unknown("not implemented".to_string()))
}

fn do_compare(data: CompareData) {
    let CompareData {
        config,
        new_snapshot,
        old_snapshot,
    } = data;
    assert!(config.rules() > 0);
    let violations = libzakopane::compare(&config, &old_snapshot, &new_snapshot);
    println!("{}", violations);
}

fn generate_snapshot_header(
    path: &std::path::PathBuf,
    start_time: &chrono::DateTime<chrono::offset::Local>,
) -> String {
    let buffer: Vec<String> = vec![
        format!("zakopane: {}", start_time),
        format!("zakopane: {}", path.display()),
        String::new(),
        String::new(),
    ];

    buffer.join("\n")
}

fn do_checksum(path: std::path::PathBuf) {
    if !path.is_dir() {
        eprintln!("``{}'' is not a dir", path.display());
        return;
    }
    let start_time: chrono::DateTime<chrono::offset::Local> = chrono::offset::Local::now();
    println!("checksum ``{}'' at {}", path.display(), start_time);

    let header = generate_snapshot_header(&path, &start_time);
    let checksums = libzakopane::checksum(path);
    let output_basename = format!("{}.txt", start_time.format("%Y-%m-%d-%H%M"));
    let mut output_file = std::fs::File::create(&output_basename).unwrap();

    output_file.write_all(header.as_ref()).unwrap();
    output_file.write_all(checksums.as_ref()).unwrap();
    println!("wrote ``{}''", output_basename);

    let end_time: chrono::DateTime<chrono::offset::Local> = chrono::offset::Local::now();
    println!(
        "finished at {} ({}s elapsed)",
        end_time,
        (end_time - start_time).num_seconds()
    );
}

fn main() {
    let subcommand = match initialize() {
        Ok(data) => data,
        Err(error) => {
            eprintln!("{}", error.to_string());
            std::process::exit(1);
        }
    };
    match subcommand {
        SubcommandData::Compare(compare_data) => return do_compare(compare_data),
        SubcommandData::Checksum(target_path) => return do_checksum(target_path),
    }
}
<commit_msg>zakopane: Mark unreachable code<commit_after>use std::io::Write;

use libzakopane::config::Config;
use libzakopane::snapshot::Snapshot;
use libzakopane::structs::CompareCliOptions;
use libzakopane::structs::ZakopaneError;

use clap::{App, Arg, ArgMatches};

const DEFAULT_POLICY_ARG_NAME: &'static str = "default-policy";
const CONFIG_FILE_ARG_NAME: &'static str = "config";
const OLD_SNAPSHOT_PATH_ARG_NAME: &'static str = "old-snapshot-path";
const NEW_SNAPSHOT_PATH_ARG_NAME: &'static str = "new-snapshot-path";

// Holds one instance of each struct necessary to execute the `compare`
// subcommand.
struct CompareData {
    config: Config,
    old_snapshot: Snapshot,
    new_snapshot: Snapshot,
}

enum SubcommandData {
    Compare(CompareData),
    Checksum(std::path::PathBuf),
}

// Reads parsed command-line arguments and returns the appropriate
// operational data. Can abort the program on error.
fn compare_data_from(matches: &ArgMatches) -> Result<SubcommandData, ZakopaneError> {
    // The two snapshot paths are required, so these are safe to unwrap.
    let old_snapshot_path = matches.value_of(OLD_SNAPSHOT_PATH_ARG_NAME).unwrap();
    let new_snapshot_path = matches.value_of(NEW_SNAPSHOT_PATH_ARG_NAME).unwrap();
    let old_contents = libzakopane::helpers::ingest_file(old_snapshot_path)?;
    let new_contents = libzakopane::helpers::ingest_file(new_snapshot_path)?;

    let options = CompareCliOptions {
        config_path: matches.value_of(CONFIG_FILE_ARG_NAME),
        default_policy: matches.value_of(DEFAULT_POLICY_ARG_NAME),
    };

    Ok(SubcommandData::Compare(CompareData {
        config: Config::new(&options)?,
        old_snapshot: Snapshot::new(&old_contents)?,
        new_snapshot: Snapshot::new(&new_contents)?,
    }))
}

// Begins parsing command-line arguments. Can abort the program on
// error.
fn initialize() -> Result<SubcommandData, ZakopaneError> {
    let matches = App::new("zakopane")
        .version("0.2.0")
        .author("j39m")
        .about("checksums directories")
        .subcommand(
            App::new("compare")
                .about("compares two zakopane snapshots")
                .arg(
                    Arg::with_name(CONFIG_FILE_ARG_NAME)
                        .short("c")
                        .long("config")
                        .value_name("FILE")
                        .help("specifies a zakopane config")
                        .takes_value(true),
                )
                .arg(
                    Arg::with_name(DEFAULT_POLICY_ARG_NAME)
                        .short("d")
                        .long("default-policy")
                        .value_name("POLICY_TOKENS")
                        .help("specifies an explicit default policy")
                        .takes_value(true),
                )
                .arg(
                    Arg::with_name(OLD_SNAPSHOT_PATH_ARG_NAME)
                        .help("path to older snapshot")
                        .index(1)
                        .required(true),
                )
                .arg(
                    Arg::with_name(NEW_SNAPSHOT_PATH_ARG_NAME)
                        .help("path to newer snapshot")
                        .index(2)
                        .required(true),
                ),
        )
        .subcommand(
            App::new("checksum")
                .about("produces checksums for a directory")
                .arg(
                    Arg::with_name("target-path")
                        .help("directory to checksum")
                        .index(1)
                        .required(true),
                ),
        )
        .get_matches();

    if let Some(ref matches) = matches.subcommand_matches("compare") {
        return compare_data_from(&matches);
    }
    if let Some(ref matches) = matches.subcommand_matches("checksum") {
        return Ok(SubcommandData::Checksum(std::path::PathBuf::from(
            matches.value_of("target-path").unwrap(),
        )));
    }
    panic!("BUG: unhandled subcommand");
}

fn do_compare(data: CompareData) {
    let CompareData {
        config,
        new_snapshot,
        old_snapshot,
    } = data;
    assert!(config.rules() > 0);
    let violations = libzakopane::compare(&config, &old_snapshot, &new_snapshot);
    println!("{}", violations);
}

fn generate_snapshot_header(
    path: &std::path::PathBuf,
    start_time: &chrono::DateTime<chrono::offset::Local>,
) -> String {
    let buffer: Vec<String> = vec![
        format!("zakopane: {}", start_time),
        format!("zakopane: {}", path.display()),
        String::new(),
        String::new(),
    ];

    buffer.join("\n")
}

fn do_checksum(path: std::path::PathBuf) {
    if !path.is_dir() {
        eprintln!("``{}'' is not a dir", path.display());
        return;
    }
    let start_time: chrono::DateTime<chrono::offset::Local> = chrono::offset::Local::now();
    println!("checksum ``{}'' at {}", path.display(), start_time);

    let header = generate_snapshot_header(&path, &start_time);
    let checksums = libzakopane::checksum(path);
    let output_basename = format!("{}.txt", start_time.format("%Y-%m-%d-%H%M"));
    let mut output_file = std::fs::File::create(&output_basename).unwrap();

    output_file.write_all(header.as_ref()).unwrap();
    output_file.write_all(checksums.as_ref()).unwrap();
    println!("wrote ``{}''", output_basename);

    let end_time: chrono::DateTime<chrono::offset::Local> = chrono::offset::Local::now();
    println!(
        "finished at {} ({}s elapsed)",
        end_time,
        (end_time - start_time).num_seconds()
    );
}

fn main() {
    let subcommand = match initialize() {
        Ok(data) => data,
        Err(error) => {
            eprintln!("{}", error.to_string());
            std::process::exit(1);
        }
    };
    match subcommand {
        SubcommandData::Compare(compare_data) => return do_compare(compare_data),
        SubcommandData::Checksum(target_path) => return do_checksum(target_path),
    }
}
<|endoftext|>


<commit_before>#![allow(warnings)]
use ffi;
use libc::{c_uint, c_int, c_void};
use num::complex::{Complex, Complex64};

use strided::{MutStrided, MutStride};

use plan::RawPlan;

mod fft_data;

/// Values for which `[Self] -> [Target]` works as a transform.
pub trait FftData<Target> {
    type State;

    #[doc(hidden)]
    unsafe fn plan(in_: MutStride<Self>, out: Option<MutStride<Target>>,
                   meta: &Meta) -> PlanResult<RawPlan>;

    #[doc(hidden)]
    fn secret() -> Secret;
}

/// How much effort FFTW should put into computing the best strategy
/// to use.
///
/// The `FFTW_WISDOM_ONLY` rigor level is replaced by the
pub enum Rigor {
    Estimate,
    Measure,
    Patient,
    Exhaustive,
}
impl Rigor {
    fn flags(self) -> c_uint {
        match self {
            Estimate => ffi::FFTW_ESTIMATE,
            Measure => ffi::FFTW_MEASURE,
            Patient => ffi::FFTW_PATIENT,
            Exhaustive => ffi::FFTW_EXHAUSTIVE,
        }
    }
}

/// The direction of the transform to perform..
pub enum Direction {
    Forward, Backward
}
impl Direction {
    fn sign(self) -> c_int {
        match self {
            Forward => ffi::FFTW_FORWARD,
            Backward => ffi::FFTW_BACKWARD,
        }
    }
}

pub struct Begin;
pub struct Input<I> {
    in_: I
}
pub struct Io<I, O> {
    in_: I,
    out: O
}
pub struct Inplace<I> {
    in_out: I
}

struct R2R;
struct Ready;

#[repr(C)]
#[deriving(Show, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct Dim {
    pub n: uint,
    pub in_stride: uint,
    pub out_stride: uint,
}

impl Dim {
    // TODO: should be num_elems?
    fn size(ds: &[Dim]) -> (uint, uint) {
        assert!(ds.len() > 0)
        let head = ds.slice_to(ds.len() - 1).iter().fold(1, |m, d| m * d.n);
        let last = ds.last().expect("no dims in Dim::num_elems");
        (head * last.n, head * (last.n / 2 + 1))
    }

    // TODO: the checks in fft_size.rs need to use this
    fn array_size(ds: &[Dim], c2r: bool, r2c: bool) -> (uint, uint) {
        debug_assert!(!(c2r && r2c));
        assert!(ds.len() > 0);

        let (in_, out) = ds.slice_to(ds.len() - 1).iter().fold((0, 0), |(a,b), d| {
            (a + d.n * d.in_stride, b + d.n * d.out_stride)
        });
        let last = ds.last().unwrap();
        if c2r {
            (in_ + (last.n / 2 + 1) * last.in_stride,
             out + last.n * last.out_stride)
        } else if r2c {
            (in_ + last.n * last.out_stride,
             out + (last.n / 2 + 1) * last.out_stride)
        } else {
            (in_, out)
        }
    }
}

pub enum R2rKind {
    R2ch,
    Hc2r,
    Dht,
    Dct00,
    Dct01,
    Dct10,
    Dct11,
    Dst00,
    Dst01,
    Dst10,
    Dst11,
}
impl R2rKind {
    fn as_fftw(self) -> c_uint {
        match self {
            R2ch => ffi::FFTW_R2HC,
            Hc2r => ffi::FFTW_HC2R,
            Dht => ffi::FFTW_DHT,
            Dct00 => ffi::FFTW_REDFT00,
            Dct01 => ffi::FFTW_REDFT01,
            Dct10 => ffi::FFTW_REDFT10,
            Dct11 => ffi::FFTW_REDFT11,
            Dst00 => ffi::FFTW_RODFT00,
            Dst01 => ffi::FFTW_RODFT01,
            Dst10 => ffi::FFTW_RODFT10,
            Dst11 => ffi::FFTW_RODFT11,
        }
    }
}

#[doc(hidden)]
pub struct Meta {
    rigor: Rigor,
    wisdom_restriction: bool,
    direction: Direction,

    in_stride: uint,
    out_stride: uint,

    r2r_kinds: Vec<c_uint>,
    overall_dims: Vec<Dim>,
    dims: Vec<Dim>,
    howmany: Vec<Dim>,
}

#[deriving(Show)]
pub enum PlanningError {
    FftwError,
    NoLengthNoDefault,
    BufferTooSmall(uint, uint, Vec<Dim>),
}

pub type PlanResult<T> = Result<T, PlanningError>;

fn do_plan(f: || -> ffi::fftw_plan) -> PlanResult<RawPlan> {
    match RawPlan::new(f) {
        Some(p) => Ok(p),
        None => Err(PlanningError::FftwError),
    }
}

/// This is designed to stop the must-be-public traits from being able
/// to be implemented externally, because that would be rather
/// strange.
#[doc(hidden)]
pub struct Secret(());

pub struct Planner<Stage, State> {
    meta: Meta,
    data: Stage
}

impl Planner<Begin, Begin> {
    pub fn new() -> Planner<Begin, Begin> {
        Planner {
            meta: Meta {
                rigor: Rigor::Estimate,
                wisdom_restriction: false,
                direction: Direction::Forward,

                r2r_kinds: vec![],

                overall_dims: vec![],
                dims: vec![],
                howmany: vec![],

                in_stride: 1,
                out_stride: 1,
            },

            data: Begin,
        }
    }
}

impl<Y> Planner<Begin, Y> {
    pub fn input<T, I: MutStrided<T>>(mut self, in_: I) -> Planner<Input<I>, Y> {
        self.meta.in_stride = in_.stride();

        Planner {
            meta: self.meta,
            data: Input { in_: in_ },
        }
    }
}

impl<X, Y> Planner<X, Y> {
    /// Set the rigor to use for this plan.
    pub fn rigor(mut self, r: Rigor) -> Planner<X, Y> {
        self.meta.rigor = r;
        self
    }
    /// Set whether the planner should only be successfully created if
    /// there exists wisdom created with at least the rigor level set.
    pub fn wisdom_restriction(mut self, wisdom_only: bool) -> Planner<X, Y> {
        self.meta.wisdom_restriction = wisdom_only;
        self
    }

    /// Set the direction of the transform to perform.
    pub fn direction(mut self, direction: Direction) -> Planner<X, Y> {
        self.meta.direction = direction;
        self
    }
}

impl<U, T: FftData<U>, I: MutStrided<T>, Y> Planner<Input<I>, Y> {
    pub fn output<O: MutStrided<U>>(mut self, out: O)
                                    -> Planner<Io<I, O>, <T as FftData<U>>::State> {
        self.meta.out_stride = out.stride();

        Planner {
            meta: self.meta,
            data: Io { in_: self.data.in_, out: out },
        }
    }
}

impl<T: FftData<T>, I: MutStrided<T>> Planner<Input<I>> {
    pub fn inplace(mut self) -> Planner<Inplace<I>, <T as FftData<T>>::State>  {
        self.meta.out_stride = self.meta.in_stride;

        Planner {
            meta: self.meta,
            data: Inplace { in_out: self.data.in_ },
        }
    }
}

impl<T, U, I, J, X, Y> Planner<X, Y>
    where T: FftData<U>, I: MutStrided<T>, J: MutStrided<U>, X: HasInput<I> + HasOutput<J> {

    pub fn _1d(mut self, n: uint) -> Planner<X, Y> {
        self.nd(&[n])
    }

    pub fn _2d(mut self, n0: uint, n1: uint) -> Planner<X, Y> {
        self.nd(&[n0, n1])
    }

    pub fn _3d(mut self, n0: uint, n1: uint, n2: uint) -> Planner<X, Y> {
        self.nd(&[n0, n1, n2])
    }

    pub fn nd(mut self, dims: &[uint]) -> Planner<X, Y> {
        assert!(dims.len() > 0, "Planner.nd: empty dimensions");
        self.meta.dims.clear();
        self.meta.dims.extend(dims.iter().map(|n| Dim { n: *n, in_stride: 0, out_stride: 0 }));

        let mut stride = 1;
        for place in self.meta.dims.iter_mut().rev() {
            place.in_stride = stride;
            place.out_stride = stride;
            stride *= place.n;
        }
        self
    }

    pub fn nd_subarray(mut self, dims: &[(uint, uint, uint)]) -> Planner<X, Y> {
        assert!(dims.len() > 0,
                "Planner.nd_subarray: empty dimensions");

        self.meta.dims.clear();
        self.meta.dims.extend(dims.iter().map(|n| Dim { n: n.0 , in_stride: 0, out_stride: 0 }));

        let mut in_stride = 1;
        let mut out_stride = 1;
        for (place, &(_, in_, out)) in self.meta.dims.iter_mut().zip(dims.iter()).rev() {
            place.in_stride = in_stride;
            place.out_stride = out_stride;

            in_stride *= in_;
            out_stride *= out;
        }
        debug!("dimensions & strides: {}", self.meta.dims);
        self
    }
}

pub trait HasInput<I> {
    fn input(&mut self) -> &mut I;
}
impl<I> HasInput<I> for Input<I> {
    fn input(&mut self) -> &mut I { &mut self.in_ }
}
impl<I> HasInput<I> for Inplace<I> {
    fn input(&mut self) -> &mut I { &mut self.in_out }
}
impl<I, O> HasInput<I> for Io<I, O> {
    fn input(&mut self) -> &mut I { &mut self.in_ }
}

pub trait HasOutput<O> {
    fn output(&mut self) -> &mut O;
}
impl<I> HasOutput<I> for Inplace<I> {
    fn output(&mut self) -> &mut I { &mut self.in_out }
}
impl<I, O> HasOutput<O> for Io<I, O> {
    fn output(&mut self) -> &mut O { &mut self.out }
}

// add associated item here for the target result, to make r2r kind always required.
pub trait FftSpec<From, To> {
    #[doc(hidden)]
    unsafe fn plan(&mut self, meta: &Meta) -> PlanResult<RawPlan>;

    #[doc(hidden)]
    fn secret() -> Secret;
}

impl<X: FftSpec<f64, f64>> Planner<X, R2R> {
    pub fn r2r_kind(mut self, kind: R2rKind) -> Planner<X, Ready> {
        self.r2r_kinds(&[kind])
    }
    pub fn r2r_kinds(mut self, kinds: &[R2rKind]) -> Planner<X, Ready> {
        assert!(kinds.len() > 0, "Planner.r2r_kinds: no kinds");

        self.meta.r2r_kinds.clear();
        self.meta.r2r_kinds.extend(kinds.iter().map(|k| k.as_fftw()));

        self
    }
}

impl<T, U, X: FftSpec<T, U>> Planner<X, Ready> {
    pub fn plan(mut self) -> Result<Plan<X>, PlanningError> {
        // space things out appropriately for the backing array.
        for d in self.meta.dims.iter_mut() {
            d.in_stride *= self.meta.in_stride;
            d.out_stride *= self.meta.out_stride;
        }

        match unsafe {self.data.plan(&self.meta)} {
            Ok(p) => Ok(Plan { planner: self, plan: p }),
            Err(e) => Err(e)
        }
    }
}

pub struct Plan<X> {
    planner: Planner<X>,
    plan: RawPlan,
}

impl<T: FftData<T>, I: MutStrided<T>> Plan<Inplace<I>> {
    pub fn in_out(&mut self) -> &mut I {
        &mut self.planner.data.in_out
    }
}
impl<I, O> Plan<Io<I, O>> {
    pub fn input(&mut self) -> &mut I {
        &mut self.planner.data.in_
    }
    pub fn output(&mut self) -> &mut O {
        &mut self.planner.data.out
    }
}

impl<X> Plan<X> {
    pub fn execute(&mut self) {
        unsafe {
            self.plan.execute()
        }
    }

    pub fn debug_print(&self) {
        self.plan.debug_print()
    }
}
<commit_msg>Use the implicit associated types syntax.<commit_after>#![allow(warnings)]
use ffi;
use libc::{c_uint, c_int, c_void};
use num::complex::{Complex, Complex64};

use strided::{MutStrided, MutStride};

use plan::RawPlan;

mod fft_data;

/// Values for which `[Self] -> [Target]` works as a transform.
pub trait FftData<Target> {
    type State;

    #[doc(hidden)]
    unsafe fn plan(in_: MutStride<Self>, out: Option<MutStride<Target>>,
                   meta: &Meta) -> PlanResult<RawPlan>;

    #[doc(hidden)]
    fn secret() -> Secret;
}

/// How much effort FFTW should put into computing the best strategy
/// to use.
///
/// The `FFTW_WISDOM_ONLY` rigor level is replaced by the
pub enum Rigor {
    Estimate,
    Measure,
    Patient,
    Exhaustive,
}
impl Rigor {
    fn flags(self) -> c_uint {
        match self {
            Estimate => ffi::FFTW_ESTIMATE,
            Measure => ffi::FFTW_MEASURE,
            Patient => ffi::FFTW_PATIENT,
            Exhaustive => ffi::FFTW_EXHAUSTIVE,
        }
    }
}

/// The direction of the transform to perform..
pub enum Direction {
    Forward, Backward
}
impl Direction {
    fn sign(self) -> c_int {
        match self {
            Forward => ffi::FFTW_FORWARD,
            Backward => ffi::FFTW_BACKWARD,
        }
    }
}

pub struct Begin;
pub struct Input<I> {
    in_: I
}
pub struct Io<I, O> {
    in_: I,
    out: O
}
pub struct Inplace<I> {
    in_out: I
}

struct R2R;
struct Ready;

#[repr(C)]
#[deriving(Show, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct Dim {
    pub n: uint,
    pub in_stride: uint,
    pub out_stride: uint,
}

impl Dim {
    // TODO: should be num_elems?
    fn size(ds: &[Dim]) -> (uint, uint) {
        assert!(ds.len() > 0)
        let head = ds.slice_to(ds.len() - 1).iter().fold(1, |m, d| m * d.n);
        let last = ds.last().expect("no dims in Dim::num_elems");
        (head * last.n, head * (last.n / 2 + 1))
    }

    // TODO: the checks in fft_size.rs need to use this
    fn array_size(ds: &[Dim], c2r: bool, r2c: bool) -> (uint, uint) {
        debug_assert!(!(c2r && r2c));
        assert!(ds.len() > 0);

        let (in_, out) = ds.slice_to(ds.len() - 1).iter().fold((0, 0), |(a,b), d| {
            (a + d.n * d.in_stride, b + d.n * d.out_stride)
        });
        let last = ds.last().unwrap();
        if c2r {
            (in_ + (last.n / 2 + 1) * last.in_stride,
             out + last.n * last.out_stride)
        } else if r2c {
            (in_ + last.n * last.out_stride,
             out + (last.n / 2 + 1) * last.out_stride)
        } else {
            (in_, out)
        }
    }
}

pub enum R2rKind {
    R2ch,
    Hc2r,
    Dht,
    Dct00,
    Dct01,
    Dct10,
    Dct11,
    Dst00,
    Dst01,
    Dst10,
    Dst11,
}
impl R2rKind {
    fn as_fftw(self) -> c_uint {
        match self {
            R2ch => ffi::FFTW_R2HC,
            Hc2r => ffi::FFTW_HC2R,
            Dht => ffi::FFTW_DHT,
            Dct00 => ffi::FFTW_REDFT00,
            Dct01 => ffi::FFTW_REDFT01,
            Dct10 => ffi::FFTW_REDFT10,
            Dct11 => ffi::FFTW_REDFT11,
            Dst00 => ffi::FFTW_RODFT00,
            Dst01 => ffi::FFTW_RODFT01,
            Dst10 => ffi::FFTW_RODFT10,
            Dst11 => ffi::FFTW_RODFT11,
        }
    }
}

#[doc(hidden)]
pub struct Meta {
    rigor: Rigor,
    wisdom_restriction: bool,
    direction: Direction,

    in_stride: uint,
    out_stride: uint,

    r2r_kinds: Vec<c_uint>,
    overall_dims: Vec<Dim>,
    dims: Vec<Dim>,
    howmany: Vec<Dim>,
}

#[deriving(Show)]
pub enum PlanningError {
    FftwError,
    NoLengthNoDefault,
    BufferTooSmall(uint, uint, Vec<Dim>),
}

pub type PlanResult<T> = Result<T, PlanningError>;

fn do_plan(f: || -> ffi::fftw_plan) -> PlanResult<RawPlan> {
    match RawPlan::new(f) {
        Some(p) => Ok(p),
        None => Err(PlanningError::FftwError),
    }
}

/// This is designed to stop the must-be-public traits from being able
/// to be implemented externally, because that would be rather
/// strange.
#[doc(hidden)]
pub struct Secret(());

pub struct Planner<Stage, State> {
    meta: Meta,
    data: Stage
}

impl Planner<Begin, Begin> {
    pub fn new() -> Planner<Begin, Begin> {
        Planner {
            meta: Meta {
                rigor: Rigor::Estimate,
                wisdom_restriction: false,
                direction: Direction::Forward,

                r2r_kinds: vec![],

                overall_dims: vec![],
                dims: vec![],
                howmany: vec![],

                in_stride: 1,
                out_stride: 1,
            },

            data: Begin,
        }
    }
}

impl<Y> Planner<Begin, Y> {
    pub fn input<T, I: MutStrided<T>>(mut self, in_: I) -> Planner<Input<I>, Y> {
        self.meta.in_stride = in_.stride();

        Planner {
            meta: self.meta,
            data: Input { in_: in_ },
        }
    }
}

impl<X, Y> Planner<X, Y> {
    /// Set the rigor to use for this plan.
    pub fn rigor(mut self, r: Rigor) -> Planner<X, Y> {
        self.meta.rigor = r;
        self
    }
    /// Set whether the planner should only be successfully created if
    /// there exists wisdom created with at least the rigor level set.
    pub fn wisdom_restriction(mut self, wisdom_only: bool) -> Planner<X, Y> {
        self.meta.wisdom_restriction = wisdom_only;
        self
    }

    /// Set the direction of the transform to perform.
    pub fn direction(mut self, direction: Direction) -> Planner<X, Y> {
        self.meta.direction = direction;
        self
    }
}

impl<U, T: FftData<U>, I: MutStrided<T>, Y> Planner<Input<I>, Y> {
    pub fn output<O: MutStrided<U>>(mut self, out: O)
                                    -> Planner<Io<I, O>, T::State> {
        self.meta.out_stride = out.stride();

        Planner {
            meta: self.meta,
            data: Io { in_: self.data.in_, out: out },
        }
    }
}

impl<T: FftData<T>, I: MutStrided<T>> Planner<Input<I>> {
    pub fn inplace(mut self) -> Planner<Inplace<I>, T::State>  {
        self.meta.out_stride = self.meta.in_stride;

        Planner {
            meta: self.meta,
            data: Inplace { in_out: self.data.in_ },
        }
    }
}

impl<T, U, I, J, X, Y> Planner<X, Y>
    where T: FftData<U>, I: MutStrided<T>, J: MutStrided<U>, X: HasInput<I> + HasOutput<J> {

    pub fn _1d(mut self, n: uint) -> Planner<X, Y> {
        self.nd(&[n])
    }

    pub fn _2d(mut self, n0: uint, n1: uint) -> Planner<X, Y> {
        self.nd(&[n0, n1])
    }

    pub fn _3d(mut self, n0: uint, n1: uint, n2: uint) -> Planner<X, Y> {
        self.nd(&[n0, n1, n2])
    }

    pub fn nd(mut self, dims: &[uint]) -> Planner<X, Y> {
        assert!(dims.len() > 0, "Planner.nd: empty dimensions");
        self.meta.dims.clear();
        self.meta.dims.extend(dims.iter().map(|n| Dim { n: *n, in_stride: 0, out_stride: 0 }));

        let mut stride = 1;
        for place in self.meta.dims.iter_mut().rev() {
            place.in_stride = stride;
            place.out_stride = stride;
            stride *= place.n;
        }
        self
    }

    pub fn nd_subarray(mut self, dims: &[(uint, uint, uint)]) -> Planner<X, Y> {
        assert!(dims.len() > 0,
                "Planner.nd_subarray: empty dimensions");

        self.meta.dims.clear();
        self.meta.dims.extend(dims.iter().map(|n| Dim { n: n.0 , in_stride: 0, out_stride: 0 }));

        let mut in_stride = 1;
        let mut out_stride = 1;
        for (place, &(_, in_, out)) in self.meta.dims.iter_mut().zip(dims.iter()).rev() {
            place.in_stride = in_stride;
            place.out_stride = out_stride;

            in_stride *= in_;
            out_stride *= out;
        }
        debug!("dimensions & strides: {}", self.meta.dims);
        self
    }
}

pub trait HasInput<I> {
    fn input(&mut self) -> &mut I;
}
impl<I> HasInput<I> for Input<I> {
    fn input(&mut self) -> &mut I { &mut self.in_ }
}
impl<I> HasInput<I> for Inplace<I> {
    fn input(&mut self) -> &mut I { &mut self.in_out }
}
impl<I, O> HasInput<I> for Io<I, O> {
    fn input(&mut self) -> &mut I { &mut self.in_ }
}

pub trait HasOutput<O> {
    fn output(&mut self) -> &mut O;
}
impl<I> HasOutput<I> for Inplace<I> {
    fn output(&mut self) -> &mut I { &mut self.in_out }
}
impl<I, O> HasOutput<O> for Io<I, O> {
    fn output(&mut self) -> &mut O { &mut self.out }
}

// add associated item here for the target result, to make r2r kind always required.
pub trait FftSpec<From, To> {
    #[doc(hidden)]
    unsafe fn plan(&mut self, meta: &Meta) -> PlanResult<RawPlan>;

    #[doc(hidden)]
    fn secret() -> Secret;
}

impl<X: FftSpec<f64, f64>> Planner<X, R2R> {
    pub fn r2r_kind(mut self, kind: R2rKind) -> Planner<X, Ready> {
        self.r2r_kinds(&[kind])
    }
    pub fn r2r_kinds(mut self, kinds: &[R2rKind]) -> Planner<X, Ready> {
        assert!(kinds.len() > 0, "Planner.r2r_kinds: no kinds");

        self.meta.r2r_kinds.clear();
        self.meta.r2r_kinds.extend(kinds.iter().map(|k| k.as_fftw()));

        self
    }
}

impl<T, U, X: FftSpec<T, U>> Planner<X, Ready> {
    pub fn plan(mut self) -> Result<Plan<X>, PlanningError> {
        // space things out appropriately for the backing array.
        for d in self.meta.dims.iter_mut() {
            d.in_stride *= self.meta.in_stride;
            d.out_stride *= self.meta.out_stride;
        }

        match unsafe {self.data.plan(&self.meta)} {
            Ok(p) => Ok(Plan { planner: self, plan: p }),
            Err(e) => Err(e)
        }
    }
}

pub struct Plan<X> {
    planner: Planner<X>,
    plan: RawPlan,
}

impl<T: FftData<T>, I: MutStrided<T>> Plan<Inplace<I>> {
    pub fn in_out(&mut self) -> &mut I {
        &mut self.planner.data.in_out
    }
}
impl<I, O> Plan<Io<I, O>> {
    pub fn input(&mut self) -> &mut I {
        &mut self.planner.data.in_
    }
    pub fn output(&mut self) -> &mut O {
        &mut self.planner.data.out
    }
}

impl<X> Plan<X> {
    pub fn execute(&mut self) {
        unsafe {
            self.plan.execute()
        }
    }

    pub fn debug_print(&self) {
        self.plan.debug_print()
    }
}
<|endoftext|>


<commit_before>/// Demonstrates the simultaneous mixing of music and sound effects.

extern crate sdl2;

use std::env;
use std::path::Path;
use sdl2::mixer::{DEFAULT_CHANNELS, INIT_MP3, INIT_FLAC, INIT_MOD, INIT_FLUIDSYNTH, INIT_MODPLUG,
                  INIT_OGG, AUDIO_S16LSB};

fn main() {

    let args: Vec<_> = env::args().collect();

    if args.len() < 2 {
        println!("Usage: ./demo music.[mp3|wav|ogg] [sound-effect.[mp3|wav|ogg]]")
    } else {
        let sound_file = args.get(2).map(|sound_file| Path::new(sound_file));
        demo(Path::new(&args[1]), sound_file);
    }
}

fn demo(music_file: &Path, sound_file: Option<&Path>) {

    println!("linked version: {}", sdl2::mixer::get_linked_version());

    let sdl = sdl2::init().unwrap();
    let _audio = sdl.audio().unwrap();
    let mut timer = sdl.timer().unwrap();
    let _mixer_context = sdl2::mixer::init(INIT_MP3 | INIT_FLAC | INIT_MOD | INIT_FLUIDSYNTH |
                                           INIT_MODPLUG | INIT_OGG)
            .unwrap();

    let frequency = 44_100;
    let format = AUDIO_S16LSB; // signed 16 bit samples, in little-endian byte order
    let channels = DEFAULT_CHANNELS; // Stereo
    let chunk_size = 1_024;
    sdl2::mixer::open_audio(frequency, format, channels, chunk_size).unwrap();

    // Number of mixing channels available for sound effect `Chunk`s to play
    // simultaneously.
    sdl2::mixer::allocate_channels(4);

    {
        let n = sdl2::mixer::get_chunk_decoders_number();
        println!("available chunk(sample) decoders: {}", n);
        for i in 0..n {
            println!("  decoder {} => {}", i, sdl2::mixer::get_chunk_decoder(i));
        }
    }

    {
        let n = sdl2::mixer::get_music_decoders_number();
        println!("available music decoders: {}", n);
        for i in 0..n {
            println!("  decoder {} => {}", i, sdl2::mixer::get_music_decoder(i));
        }
    }

    println!("query spec => {:?}", sdl2::mixer::query_spec());

    let music = sdl2::mixer::Music::from_file(music_file).unwrap();

    fn hook_finished() {
        println!("play ends! from rust cb");
    }

    sdl2::mixer::Music::hook_finished(hook_finished);

    println!("music => {:?}", music);
    println!("music type => {:?}", music.get_type());
    println!("music volume => {:?}", sdl2::mixer::Music::get_volume());
    println!("play => {:?}", music.play(1));

    if let Some(sound_file_path) = sound_file {
        let sound_chunk_res = sdl2::mixer::Chunk::from_file(sound_file_path);

        match sound_chunk_res {
            Ok(sound_chunk) => {
                println!("chunk volume => {:?}", sound_chunk.get_volume());
                println!("playing sound twice");
                let play_res = sdl2::mixer::Channel::all().play(&sound_chunk, 1);

                // This delay is needed because when the `Chunk` goes out of scope,
                // the sound effect stops playing. Delay long enough to hear the
                // sound.
                timer.delay(5_000);

                match play_res {
                    Ok(_) => println!("played sound"),
                    Err(e) => println!("{:?}", e),
                }
            }
            Err(e) => println!("Cannot load sound file: {:?}", e),
        }
    }

    timer.delay(10_000);

    println!("fading out ... {:?}", sdl2::mixer::Music::fade_out(4_000));

    timer.delay(5_000);

    println!("fading in from pos ... {:?}",
             music.fade_in_from_pos(1, 10_000, 100.0));

    timer.delay(5_000);
    sdl2::mixer::Music::halt();
    timer.delay(1_000);

    println!("quitting sdl");
}
<commit_msg>sdl2::mixer::open_audio must be called before other functions<commit_after>/// Demonstrates the simultaneous mixing of music and sound effects.

extern crate sdl2;

use std::env;
use std::path::Path;
use sdl2::mixer::{DEFAULT_CHANNELS, INIT_MP3, INIT_FLAC, INIT_MOD, INIT_FLUIDSYNTH, INIT_MODPLUG,
                  INIT_OGG, AUDIO_S16LSB};

fn main() {

    let args: Vec<_> = env::args().collect();

    if args.len() < 2 {
        println!("Usage: ./demo music.[mp3|wav|ogg] [sound-effect.[mp3|wav|ogg]]")
    } else {
        let sound_file = args.get(2).map(|sound_file| Path::new(sound_file));
        demo(Path::new(&args[1]), sound_file);
    }
}

fn demo(music_file: &Path, sound_file: Option<&Path>) {

    println!("linked version: {}", sdl2::mixer::get_linked_version());

    let sdl = sdl2::init().unwrap();
    let _audio = sdl.audio().unwrap();
    let mut timer = sdl.timer().unwrap();

    let frequency = 44_100;
    let format = AUDIO_S16LSB; // signed 16 bit samples, in little-endian byte order
    let channels = DEFAULT_CHANNELS; // Stereo
    let chunk_size = 1_024;
    sdl2::mixer::open_audio(frequency, format, channels, chunk_size).unwrap();
    let _mixer_context = sdl2::mixer::init(
        INIT_MP3 | INIT_FLAC | INIT_MOD | INIT_FLUIDSYNTH | INIT_MODPLUG | INIT_OGG
    ).unwrap();

    // Number of mixing channels available for sound effect `Chunk`s to play
    // simultaneously.
    sdl2::mixer::allocate_channels(4);

    {
        let n = sdl2::mixer::get_chunk_decoders_number();
        println!("available chunk(sample) decoders: {}", n);
        for i in 0..n {
            println!("  decoder {} => {}", i, sdl2::mixer::get_chunk_decoder(i));
        }
    }

    {
        let n = sdl2::mixer::get_music_decoders_number();
        println!("available music decoders: {}", n);
        for i in 0..n {
            println!("  decoder {} => {}", i, sdl2::mixer::get_music_decoder(i));
        }
    }

    println!("query spec => {:?}", sdl2::mixer::query_spec());

    let music = sdl2::mixer::Music::from_file(music_file).unwrap();

    fn hook_finished() {
        println!("play ends! from rust cb");
    }

    sdl2::mixer::Music::hook_finished(hook_finished);

    println!("music => {:?}", music);
    println!("music type => {:?}", music.get_type());
    println!("music volume => {:?}", sdl2::mixer::Music::get_volume());
    println!("play => {:?}", music.play(1));

    if let Some(sound_file_path) = sound_file {
        let sound_chunk_res = sdl2::mixer::Chunk::from_file(sound_file_path);

        match sound_chunk_res {
            Ok(sound_chunk) => {
                println!("chunk volume => {:?}", sound_chunk.get_volume());
                println!("playing sound twice");
                let play_res = sdl2::mixer::Channel::all().play(&sound_chunk, 1);

                // This delay is needed because when the `Chunk` goes out of scope,
                // the sound effect stops playing. Delay long enough to hear the
                // sound.
                timer.delay(5_000);

                match play_res {
                    Ok(_) => println!("played sound"),
                    Err(e) => println!("{:?}", e),
                }
            }
            Err(e) => println!("Cannot load sound file: {:?}", e),
        }
    }

    timer.delay(10_000);

    println!("fading out ... {:?}", sdl2::mixer::Music::fade_out(4_000));

    timer.delay(5_000);

    println!("fading in from pos ... {:?}",
             music.fade_in_from_pos(1, 10_000, 100.0));

    timer.delay(5_000);
    sdl2::mixer::Music::halt();
    timer.delay(1_000);

    println!("quitting sdl");
}
<|endoftext|>


<commit_before>use std::borrow::Borrow;
use std::env;
use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use std::process::{self, Command};

fn run_command_or_fail<P, S>(dir: &str, cmd: P, args: &[S])
where
    P: AsRef<Path>,
    S: Borrow<str> + AsRef<OsStr>,
{
    let cmd = cmd.as_ref();
    let cmd = if cmd.components().count() > 1 && cmd.is_relative() {
        // If `cmd` is a relative path (and not a bare command that should be
        // looked up in PATH), absolutize it relative to `dir`, as otherwise the
        // behavior of std::process::Command is undefined.
        // https://github.com/rust-lang/rust/issues/37868
        PathBuf::from(dir)
            .join(cmd)
            .canonicalize()
            .expect("canonicalization failed")
    } else {
        PathBuf::from(cmd)
    };
    eprintln!(
        "Running command: \"{} {}\" in dir: {}",
        cmd.display(),
        args.join(" "),
        dir
    );
    let ret = Command::new(cmd).current_dir(dir).args(args).status();
    match ret.map(|status| (status.success(), status.code())) {
        Ok((true, _)) => return,
        Ok((false, Some(c))) => panic!("Command failed with error code {}", c),
        Ok((false, None)) => panic!("Command got killed"),
        Err(e) => panic!("Command failed with error: {}", e),
    }
}

fn main() {
    let librdkafka_version = env!("CARGO_PKG_VERSION")
        .split('-')
        .next()
        .expect("Crate version is not valid");

    if env::var("DEP_OPENSSL_VENDORED").is_ok() {
        let openssl_root = env::var("DEP_OPENSSL_ROOT").expect("DEP_OPENSSL_ROOT is not set");
        env::set_var("CFLAGS", format!("-I{}/include", openssl_root));
        env::set_var("LDFLAGS", format!("-L{}/lib", openssl_root));
    }

    if env::var("CARGO_FEATURE_DYNAMIC_LINKING").is_ok() {
        eprintln!("librdkafka will be linked dynamically");
        let pkg_probe = pkg_config::Config::new()
            .cargo_metadata(true)
            .atleast_version(librdkafka_version)
            .probe("rdkafka");

        match pkg_probe {
            Ok(library) => {
                eprintln!("librdkafka found on the system:");
                eprintln!("  Name: {:?}", library.libs);
                eprintln!("  Path: {:?}", library.link_paths);
                eprintln!("  Version: {}", library.version);
            }
            Err(_) => {
                eprintln!(
                    "librdkafka {} cannot be found on the system",
                    librdkafka_version
                );
                eprintln!("Dynamic linking failed. Exiting.");
                process::exit(1);
            }
        }
    } else {
        if !Path::new("librdkafka/LICENSE").exists() {
            eprintln!("Setting up submodules");
            run_command_or_fail("../", "git", &["submodule", "update", "--init"]);
        }
        eprintln!("Building and linking librdkafka statically");
        build_librdkafka();
    }
}

#[cfg(not(feature = "cmake-build"))]
fn build_librdkafka() {
    let mut configure_flags: Vec<String> = Vec::new();

    let mut cflags = Vec::new();
    if let Ok(var) = env::var("CFLAGS") {
        cflags.push(var);
    }

    let mut ldflags = Vec::new();
    if let Ok(var) = env::var("LDFLAGS") {
        ldflags.push(var);
    }

    if env::var("CARGO_FEATURE_SSL").is_ok() {
        configure_flags.push("--enable-ssl".into());
        if let Ok(openssl_root) = env::var("DEP_OPENSSL_ROOT") {
            cflags.push(format!("-I{}/include", openssl_root));
            ldflags.push(format!("-L{}/lib", openssl_root));
        }
    } else {
        configure_flags.push("--disable-ssl".into());
    }

    if env::var("CARGO_FEATURE_GSSAPI").is_ok() {
        configure_flags.push("--enable-gssapi".into());
        println!("cargo:rustc-link-lib=sasl2");
    } else {
        configure_flags.push("--disable-gssapi".into());
    }

    if env::var("CARGO_FEATURE_LIBZ").is_ok() {
        // There is no --enable-zlib option, but it is enabled by default.
        if let Ok(z_root) = env::var("DEP_Z_ROOT") {
            cflags.push(format!("-I{}/include", z_root));
            ldflags.push(format!("-L{}/build", z_root));
        }
    } else {
        configure_flags.push("--disable-zlib".into());
    }

    if env::var("CARGO_FEATURE_ZSTD").is_ok() {
        configure_flags.push("--enable-zstd".into());
        if let Ok(zstd_root) = env::var("DEP_ZSTD_ROOT") {
            cflags.push(format!("-I{}/include", zstd_root));
            ldflags.push(format!("-L{}", zstd_root));
        }
    } else {
        configure_flags.push("--disable-zstd".into());
    }

    if env::var("CARGO_FEATURE_EXTERNAL_LZ4").is_ok() {
        configure_flags.push("--enable-lz4-ext".into());
        if let Ok(lz4_root) = env::var("DEP_LZ4_ROOT") {
            cflags.push(format!("-I{}/include", lz4_root));
            ldflags.push(format!("-L{}", lz4_root));
        }
    } else {
        configure_flags.push("--disable-lz4-ext".into());
    }

    env::set_var("CFLAGS", cflags.join(" "));
    env::set_var("LDFLAGS", ldflags.join(" "));

    println!("Configuring librdkafka");
    run_command_or_fail("librdkafka", "./configure", configure_flags.as_slice());

    println!("Compiling librdkafka");
    env::set_var("MAKEFLAGS", env::var_os("CARGO_MAKEFLAGS").expect("CARGO_MAKEFLAGS env var missing"));
    run_command_or_fail(
        "librdkafka",
        if cfg!(target_os = "freebsd") { "gmake" } else { "make" },
        &["libs"],
    );

    println!(
        "cargo:rustc-link-search=native={}/librdkafka/src",
        env::current_dir()
            .expect("Can't find current dir")
            .display()
    );
    println!("cargo:rustc-link-lib=static=rdkafka");
    println!("cargo:root={}", env::var("OUT_DIR").expect("OUT_DIR missing"));
}

#[cfg(feature = "cmake-build")]
fn build_librdkafka() {
    let mut config = cmake::Config::new("librdkafka");

    config.define("RDKAFKA_BUILD_STATIC", "1");

    if env::var("CARGO_FEATURE_LIBZ").is_ok() {
        config.define("WITH_ZLIB", "1");
        config.register_dep("z");
        if let Ok(z_root) = env::var("DEP_Z_ROOT") {
            env::set_var("CMAKE_LIBRARY_PATH", format!("{}/build", z_root));
        }
    } else {
        config.define("WITH_ZLIB", "0");
    }

    if env::var("CARGO_FEATURE_SSL").is_ok() {
        config.define("WITH_SSL", "1");
        config.register_dep("openssl");
    } else {
        config.define("WITH_SSL", "0");
    }

    if env::var("CARGO_FEATURE_GSSAPI").is_ok() {
        config.define("WITH_SASL", "1");
        println!("cargo:rustc-link-lib=sasl2");
    } else {
        config.define("WITH_SASL", "0");
    }

    if env::var("CARGO_FEATURE_ZSTD").is_ok() {
        config.define("WITH_ZSTD", "1");
        config.register_dep("zstd");
    } else {
        config.define("WITH_ZSTD", "0");
    }

    if env::var("CARGO_FEATURE_EXTERNAL_LZ4").is_ok() {
        config.define("ENABLE_LZ4_EXT", "1");
        config.register_dep("lz4");
    } else {
        config.define("ENABLE_LZ4_EXT", "0");
    }

    if let Ok(system_name) = env::var("CMAKE_SYSTEM_NAME") {
        config.define("CMAKE_SYSTEM_NAME", system_name);
    }

    // The CMake build will incorrectly use config.h from the non-CMake build,
    // if it exists, so remove it if it does.
    match std::fs::remove_file("librdkafka/config.h") {
        Ok(()) => (),
        Err(err) => match err.kind() {
            std::io::ErrorKind::NotFound => (),
            _ => panic!("Unable to remove config.h from non-CMake build: {}", err),
        }
    }

    println!("Configuring and compiling librdkafka");
    let dst = config.build();

    println!("cargo:rustc-link-search=native={}/lib", dst.display());
    println!("cargo:rustc-link-lib=static=rdkafka");
}
<commit_msg>rdkafka-sys: don't let CMake install to lib64<commit_after>use std::borrow::Borrow;
use std::env;
use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use std::process::{self, Command};

fn run_command_or_fail<P, S>(dir: &str, cmd: P, args: &[S])
where
    P: AsRef<Path>,
    S: Borrow<str> + AsRef<OsStr>,
{
    let cmd = cmd.as_ref();
    let cmd = if cmd.components().count() > 1 && cmd.is_relative() {
        // If `cmd` is a relative path (and not a bare command that should be
        // looked up in PATH), absolutize it relative to `dir`, as otherwise the
        // behavior of std::process::Command is undefined.
        // https://github.com/rust-lang/rust/issues/37868
        PathBuf::from(dir)
            .join(cmd)
            .canonicalize()
            .expect("canonicalization failed")
    } else {
        PathBuf::from(cmd)
    };
    eprintln!(
        "Running command: \"{} {}\" in dir: {}",
        cmd.display(),
        args.join(" "),
        dir
    );
    let ret = Command::new(cmd).current_dir(dir).args(args).status();
    match ret.map(|status| (status.success(), status.code())) {
        Ok((true, _)) => return,
        Ok((false, Some(c))) => panic!("Command failed with error code {}", c),
        Ok((false, None)) => panic!("Command got killed"),
        Err(e) => panic!("Command failed with error: {}", e),
    }
}

fn main() {
    let librdkafka_version = env!("CARGO_PKG_VERSION")
        .split('-')
        .next()
        .expect("Crate version is not valid");

    if env::var("DEP_OPENSSL_VENDORED").is_ok() {
        let openssl_root = env::var("DEP_OPENSSL_ROOT").expect("DEP_OPENSSL_ROOT is not set");
        env::set_var("CFLAGS", format!("-I{}/include", openssl_root));
        env::set_var("LDFLAGS", format!("-L{}/lib", openssl_root));
    }

    if env::var("CARGO_FEATURE_DYNAMIC_LINKING").is_ok() {
        eprintln!("librdkafka will be linked dynamically");
        let pkg_probe = pkg_config::Config::new()
            .cargo_metadata(true)
            .atleast_version(librdkafka_version)
            .probe("rdkafka");

        match pkg_probe {
            Ok(library) => {
                eprintln!("librdkafka found on the system:");
                eprintln!("  Name: {:?}", library.libs);
                eprintln!("  Path: {:?}", library.link_paths);
                eprintln!("  Version: {}", library.version);
            }
            Err(_) => {
                eprintln!(
                    "librdkafka {} cannot be found on the system",
                    librdkafka_version
                );
                eprintln!("Dynamic linking failed. Exiting.");
                process::exit(1);
            }
        }
    } else {
        if !Path::new("librdkafka/LICENSE").exists() {
            eprintln!("Setting up submodules");
            run_command_or_fail("../", "git", &["submodule", "update", "--init"]);
        }
        eprintln!("Building and linking librdkafka statically");
        build_librdkafka();
    }
}

#[cfg(not(feature = "cmake-build"))]
fn build_librdkafka() {
    let mut configure_flags: Vec<String> = Vec::new();

    let mut cflags = Vec::new();
    if let Ok(var) = env::var("CFLAGS") {
        cflags.push(var);
    }

    let mut ldflags = Vec::new();
    if let Ok(var) = env::var("LDFLAGS") {
        ldflags.push(var);
    }

    if env::var("CARGO_FEATURE_SSL").is_ok() {
        configure_flags.push("--enable-ssl".into());
        if let Ok(openssl_root) = env::var("DEP_OPENSSL_ROOT") {
            cflags.push(format!("-I{}/include", openssl_root));
            ldflags.push(format!("-L{}/lib", openssl_root));
        }
    } else {
        configure_flags.push("--disable-ssl".into());
    }

    if env::var("CARGO_FEATURE_GSSAPI").is_ok() {
        configure_flags.push("--enable-gssapi".into());
        println!("cargo:rustc-link-lib=sasl2");
    } else {
        configure_flags.push("--disable-gssapi".into());
    }

    if env::var("CARGO_FEATURE_LIBZ").is_ok() {
        // There is no --enable-zlib option, but it is enabled by default.
        if let Ok(z_root) = env::var("DEP_Z_ROOT") {
            cflags.push(format!("-I{}/include", z_root));
            ldflags.push(format!("-L{}/build", z_root));
        }
    } else {
        configure_flags.push("--disable-zlib".into());
    }

    if env::var("CARGO_FEATURE_ZSTD").is_ok() {
        configure_flags.push("--enable-zstd".into());
        if let Ok(zstd_root) = env::var("DEP_ZSTD_ROOT") {
            cflags.push(format!("-I{}/include", zstd_root));
            ldflags.push(format!("-L{}", zstd_root));
        }
    } else {
        configure_flags.push("--disable-zstd".into());
    }

    if env::var("CARGO_FEATURE_EXTERNAL_LZ4").is_ok() {
        configure_flags.push("--enable-lz4-ext".into());
        if let Ok(lz4_root) = env::var("DEP_LZ4_ROOT") {
            cflags.push(format!("-I{}/include", lz4_root));
            ldflags.push(format!("-L{}", lz4_root));
        }
    } else {
        configure_flags.push("--disable-lz4-ext".into());
    }

    env::set_var("CFLAGS", cflags.join(" "));
    env::set_var("LDFLAGS", ldflags.join(" "));

    println!("Configuring librdkafka");
    run_command_or_fail("librdkafka", "./configure", configure_flags.as_slice());

    println!("Compiling librdkafka");
    env::set_var("MAKEFLAGS", env::var_os("CARGO_MAKEFLAGS").expect("CARGO_MAKEFLAGS env var missing"));
    run_command_or_fail(
        "librdkafka",
        if cfg!(target_os = "freebsd") { "gmake" } else { "make" },
        &["libs"],
    );

    println!(
        "cargo:rustc-link-search=native={}/librdkafka/src",
        env::current_dir()
            .expect("Can't find current dir")
            .display()
    );
    println!("cargo:rustc-link-lib=static=rdkafka");
    println!("cargo:root={}", env::var("OUT_DIR").expect("OUT_DIR missing"));
}

#[cfg(feature = "cmake-build")]
fn build_librdkafka() {
    let mut config = cmake::Config::new("librdkafka");

    config
        .define("RDKAFKA_BUILD_STATIC", "1")
        .define("RDKAFKA_BUILD_TESTS", "0")
        .define("RDKAFKA_BUILD_EXAMPLES", "0")
        // CMAKE_INSTALL_LIBDIR is inferred as "lib64" on some platforms, but we
        // want a stable location that we can add to the linker search path.
        // Since we're not actually installing to /usr or /usr/local, there's no
        // harm to always using "lib" here.
        .define("CMAKE_INSTALL_LIBDIR", "lib");

    if env::var("CARGO_FEATURE_LIBZ").is_ok() {
        config.define("WITH_ZLIB", "1");
        config.register_dep("z");
        if let Ok(z_root) = env::var("DEP_Z_ROOT") {
            env::set_var("CMAKE_LIBRARY_PATH", format!("{}/build", z_root));
        }
    } else {
        config.define("WITH_ZLIB", "0");
    }

    if env::var("CARGO_FEATURE_SSL").is_ok() {
        config.define("WITH_SSL", "1");
        config.register_dep("openssl");
    } else {
        config.define("WITH_SSL", "0");
    }

    if env::var("CARGO_FEATURE_GSSAPI").is_ok() {
        config.define("WITH_SASL", "1");
        println!("cargo:rustc-link-lib=sasl2");
    } else {
        config.define("WITH_SASL", "0");
    }

    if env::var("CARGO_FEATURE_ZSTD").is_ok() {
        config.define("WITH_ZSTD", "1");
        config.register_dep("zstd");
    } else {
        config.define("WITH_ZSTD", "0");
    }

    if env::var("CARGO_FEATURE_EXTERNAL_LZ4").is_ok() {
        config.define("ENABLE_LZ4_EXT", "1");
        config.register_dep("lz4");
    } else {
        config.define("ENABLE_LZ4_EXT", "0");
    }

    if let Ok(system_name) = env::var("CMAKE_SYSTEM_NAME") {
        config.define("CMAKE_SYSTEM_NAME", system_name);
    }

    // The CMake build will incorrectly use config.h from the non-CMake build,
    // if it exists, so remove it if it does.
    match std::fs::remove_file("librdkafka/config.h") {
        Ok(()) => (),
        Err(err) => match err.kind() {
            std::io::ErrorKind::NotFound => (),
            _ => panic!("Unable to remove config.h from non-CMake build: {}", err),
        }
    }

    println!("Configuring and compiling librdkafka");
    let dst = config.build();

    println!("cargo:rustc-link-search=native={}/lib", dst.display());
    println!("cargo:rustc-link-lib=static=rdkafka");
}
<|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

#![forbid(unsafe_code)]

use crate::{
    components::{
        apply_chunk_output::{ensure_no_discard, ensure_no_retry},
        chunk_commit_queue::ChunkCommitQueue,
        chunk_output::ChunkOutput,
    },
    logging::{LogEntry, LogSchema},
    metrics::{
        APTOS_EXECUTOR_APPLY_CHUNK_SECONDS, APTOS_EXECUTOR_COMMIT_CHUNK_SECONDS,
        APTOS_EXECUTOR_EXECUTE_CHUNK_SECONDS, APTOS_EXECUTOR_VM_EXECUTE_CHUNK_SECONDS,
    },
};
use anyhow::Result;
use aptos_infallible::{Mutex, RwLock};
use aptos_logger::prelude::*;
use aptos_state_view::StateViewId;
use aptos_types::{
    ledger_info::LedgerInfoWithSignatures,
    transaction::{
        Transaction, TransactionInfo, TransactionListWithProof, TransactionOutputListWithProof,
    },
};
use aptos_vm::VMExecutor;
use executor_types::{
    ChunkCommitNotification, ChunkExecutorTrait, ExecutedChunk, TransactionReplayer,
};
use fail::fail_point;
use std::{marker::PhantomData, sync::Arc};
use storage_interface::{
    cached_state_view::CachedStateView, sync_proof_fetcher::SyncProofFetcher, DbReaderWriter,
    ExecutedTrees,
};

pub struct ChunkExecutor<V> {
    db: DbReaderWriter,
    inner: RwLock<Option<ChunkExecutorInner<V>>>,
}

impl<V: VMExecutor> ChunkExecutor<V> {
    pub fn new(db: DbReaderWriter) -> Self {
        Self {
            db,
            inner: RwLock::new(None),
        }
    }

    fn maybe_initialize(&self) -> Result<()> {
        if self.inner.read().is_none() {
            self.reset()?;
        }
        Ok(())
    }
}

impl<V: VMExecutor> ChunkExecutorTrait for ChunkExecutor<V> {
    fn execute_chunk(
        &self,
        txn_list_with_proof: TransactionListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<()> {
        self.maybe_initialize()?;
        self.inner
            .read()
            .as_ref()
            .expect("not reset")
            .execute_chunk(txn_list_with_proof, verified_target_li, epoch_change_li)
    }

    fn apply_chunk(
        &self,
        txn_output_list_with_proof: TransactionOutputListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<()> {
        self.inner.read().as_ref().expect("not reset").apply_chunk(
            txn_output_list_with_proof,
            verified_target_li,
            epoch_change_li,
        )
    }

    fn commit_chunk(&self) -> Result<ChunkCommitNotification> {
        self.inner
            .read()
            .as_ref()
            .expect("not reset")
            .commit_chunk()
    }

    fn execute_and_commit_chunk(
        &self,
        txn_list_with_proof: TransactionListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<ChunkCommitNotification> {
        // Re-sync with DB, make sure the queue is empty.
        self.reset()?;
        self.inner
            .read()
            .as_ref()
            .expect("not reset")
            .execute_and_commit_chunk(txn_list_with_proof, verified_target_li, epoch_change_li)
    }

    fn apply_and_commit_chunk(
        &self,
        txn_output_list_with_proof: TransactionOutputListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<ChunkCommitNotification> {
        // Re-sync with DB, make sure the queue is empty.
        self.reset()?;
        self.inner
            .read()
            .as_ref()
            .expect("not reset")
            .apply_and_commit_chunk(
                txn_output_list_with_proof,
                verified_target_li,
                epoch_change_li,
            )
    }

    fn reset(&self) -> Result<()> {
        *self.inner.write() = Some(ChunkExecutorInner::new(self.db.clone())?);
        Ok(())
    }

    fn finish(&self) {
        *self.inner.write() = None;
    }
}

struct ChunkExecutorInner<V> {
    db: DbReaderWriter,
    commit_queue: Mutex<ChunkCommitQueue>,
    _phantom: PhantomData<V>,
}

impl<V: VMExecutor> ChunkExecutorInner<V> {
    pub fn new(db: DbReaderWriter) -> Result<Self> {
        let commit_queue = Mutex::new(ChunkCommitQueue::new_from_db(&db.reader)?);
        Ok(Self {
            db,
            commit_queue,
            _phantom: PhantomData,
        })
    }

    fn state_view(&self, latest_view: &ExecutedTrees) -> Result<CachedStateView> {
        latest_view.verified_state_view(
            StateViewId::ChunkExecution {
                first_version: latest_view.txn_accumulator().num_leaves(),
            },
            Arc::clone(&self.db.reader),
            Arc::new(SyncProofFetcher::new(self.db.reader.clone())),
        )
    }

    fn apply_chunk_output_for_state_sync(
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
        latest_view: &ExecutedTrees,
        chunk_output: ChunkOutput,
        transaction_infos: &[TransactionInfo],
    ) -> Result<ExecutedChunk> {
        let (mut executed_chunk, to_discard, to_retry) =
            chunk_output.apply_to_ledger(latest_view)?;
        ensure_no_discard(to_discard)?;
        ensure_no_retry(to_retry)?;
        executed_chunk.ledger_info = executed_chunk
            .maybe_select_chunk_ending_ledger_info(verified_target_li, epoch_change_li)?;
        executed_chunk.ensure_transaction_infos_match(transaction_infos)?;

        Ok(executed_chunk)
    }

    fn commit_chunk_impl(&self) -> Result<Arc<ExecutedChunk>> {
        let (base_view, to_commit) = self.commit_queue.lock().next_chunk_to_commit()?;
        let txns_to_commit = to_commit.transactions_to_commit()?;
        let ledger_info = to_commit.ledger_info.as_ref();
        if ledger_info.is_some() || !txns_to_commit.is_empty() {
            fail_point!("executor::commit_chunk", |_| {
                Err(anyhow::anyhow!("Injected error in commit_chunk"))
            });
            self.db.writer.save_transactions(
                &txns_to_commit,
                base_view.txn_accumulator().num_leaves(),
                base_view.state().base_version,
                ledger_info,
                false, /* sync_commit */
                to_commit.result_view.state().clone(),
            )?;
        }

        self.commit_queue.lock().dequeue()?;
        Ok(to_commit)
    }

    // ************************* Block Executor Implementation *************************
    fn execute_chunk(
        &self,
        txn_list_with_proof: TransactionListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<()> {
        let _timer = APTOS_EXECUTOR_EXECUTE_CHUNK_SECONDS.start_timer();

        let num_txns = txn_list_with_proof.transactions.len();
        let first_version_in_request = txn_list_with_proof.first_transaction_version;
        let (_persisted_view, latest_view) = self.commit_queue.lock().persisted_and_latest_view();

        // Verify input transaction list.
        txn_list_with_proof.verify(verified_target_li.ledger_info(), first_version_in_request)?;

        // Skip transactions already in ledger.
        let txns_to_skip = txn_list_with_proof.proof.verify_extends_ledger(
            latest_view.txn_accumulator().num_leaves(),
            latest_view.txn_accumulator().root_hash(),
            first_version_in_request,
        )?;
        let mut transactions = txn_list_with_proof.transactions;
        transactions.drain(..txns_to_skip as usize);
        if txns_to_skip == num_txns {
            info!(
                "Skipping all transactions in the given chunk! Num transactions: {:?}",
                num_txns
            );
        }

        // Execute transactions.
        let state_view = self.state_view(&latest_view)?;
        let chunk_output = {
            let _timer = APTOS_EXECUTOR_VM_EXECUTE_CHUNK_SECONDS.start_timer();
            ChunkOutput::by_transaction_execution::<V>(transactions, state_view)?
        };
        let executed_chunk = Self::apply_chunk_output_for_state_sync(
            verified_target_li,
            epoch_change_li,
            &latest_view,
            chunk_output,
            &txn_list_with_proof.proof.transaction_infos[txns_to_skip..],
        )?;

        // Add result to commit queue.
        self.commit_queue.lock().enqueue(executed_chunk);

        info!(
            LogSchema::new(LogEntry::ChunkExecutor)
                .local_synced_version(latest_view.version().unwrap_or(0))
                .first_version_in_request(first_version_in_request)
                .num_txns_in_request(num_txns),
            "Executed transaction chunk!",
        );

        Ok(())
    }

    fn apply_chunk(
        &self,
        txn_output_list_with_proof: TransactionOutputListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<()> {
        let _timer = APTOS_EXECUTOR_APPLY_CHUNK_SECONDS.start_timer();

        let num_txns = txn_output_list_with_proof.transactions_and_outputs.len();
        let first_version_in_request = txn_output_list_with_proof.first_transaction_output_version;
        let (_persisted_view, latest_view) = self.commit_queue.lock().persisted_and_latest_view();

        // Verify input transaction list.
        txn_output_list_with_proof
            .verify(verified_target_li.ledger_info(), first_version_in_request)?;

        // Skip transactions already in ledger.
        let txns_to_skip = txn_output_list_with_proof.proof.verify_extends_ledger(
            latest_view.txn_accumulator().num_leaves(),
            latest_view.txn_accumulator().root_hash(),
            first_version_in_request,
        )?;
        let mut txns_and_outputs = txn_output_list_with_proof.transactions_and_outputs;
        txns_and_outputs.drain(..txns_to_skip as usize);

        // Apply transaction outputs.
        let state_view = self.state_view(&latest_view)?;
        let chunk_output = ChunkOutput::by_transaction_output(txns_and_outputs, state_view)?;
        let executed_chunk = Self::apply_chunk_output_for_state_sync(
            verified_target_li,
            epoch_change_li,
            &latest_view,
            chunk_output,
            &txn_output_list_with_proof.proof.transaction_infos[txns_to_skip..],
        )?;

        // Add result to commit queue.
        self.commit_queue.lock().enqueue(executed_chunk);

        info!(
            LogSchema::new(LogEntry::ChunkExecutor)
                .local_synced_version(latest_view.version().unwrap_or(0))
                .first_version_in_request(first_version_in_request)
                .num_txns_in_request(num_txns),
            "Applied transaction output chunk!",
        );

        Ok(())
    }

    fn commit_chunk(&self) -> Result<ChunkCommitNotification> {
        let _timer = APTOS_EXECUTOR_COMMIT_CHUNK_SECONDS.start_timer();
        let executed_chunk = self.commit_chunk_impl()?;
        Ok(ChunkCommitNotification {
            committed_events: executed_chunk.events_to_commit(),
            committed_transactions: executed_chunk.transactions(),
            reconfiguration_occurred: executed_chunk.has_reconfiguration(),
        })
    }

    fn execute_and_commit_chunk(
        &self,
        txn_list_with_proof: TransactionListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<ChunkCommitNotification> {
        self.execute_chunk(txn_list_with_proof, verified_target_li, epoch_change_li)?;
        self.commit_chunk()
    }

    fn apply_and_commit_chunk(
        &self,
        txn_output_list_with_proof: TransactionOutputListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<ChunkCommitNotification> {
        self.apply_chunk(
            txn_output_list_with_proof,
            verified_target_li,
            epoch_change_li,
        )?;
        self.commit_chunk()
    }
}

impl<V: VMExecutor> TransactionReplayer for ChunkExecutor<V> {
    fn replay(
        &self,
        transactions: Vec<Transaction>,
        transaction_infos: Vec<TransactionInfo>,
    ) -> Result<()> {
        self.maybe_initialize()?;
        self.inner
            .read()
            .as_ref()
            .expect("not reset")
            .replay(transactions, transaction_infos)
    }

    fn commit(&self) -> Result<Arc<ExecutedChunk>> {
        self.inner.read().as_ref().expect("not reset").commit()
    }
}

impl<V: VMExecutor> TransactionReplayer for ChunkExecutorInner<V> {
    fn replay(
        &self,
        transactions: Vec<Transaction>,
        mut transaction_infos: Vec<TransactionInfo>,
    ) -> Result<()> {
        let (_persisted_view, mut latest_view) =
            self.commit_queue.lock().persisted_and_latest_view();

        let mut executed_chunk = ExecutedChunk::default();
        let mut to_run = Some(transactions);
        while !to_run.as_ref().unwrap().is_empty() {
            // Execute transactions.
            let state_view = self.state_view(&latest_view)?;
            let txns = to_run.take().unwrap();
            let (executed, to_discard, to_retry) =
                ChunkOutput::by_transaction_execution::<V>(txns, state_view)?
                    .apply_to_ledger(&latest_view)?;

            // Accumulate result and deal with retry
            ensure_no_discard(to_discard)?;
            let n = executed.to_commit.len();
            executed.ensure_transaction_infos_match(&transaction_infos[..n])?;
            transaction_infos.drain(..n);

            to_run = Some(to_retry);
            executed_chunk = executed_chunk.combine(executed)?;
            latest_view = executed_chunk.result_view.clone();
        }

        // Add result to commit queue.
        self.commit_queue.lock().enqueue(executed_chunk);

        Ok(())
    }

    fn commit(&self) -> Result<Arc<ExecutedChunk>> {
        self.commit_chunk_impl()
    }
}
<commit_msg>[executor] commit_chunk panics if error is hit after writing to DB<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

#![forbid(unsafe_code)]

use crate::{
    components::{
        apply_chunk_output::{ensure_no_discard, ensure_no_retry},
        chunk_commit_queue::ChunkCommitQueue,
        chunk_output::ChunkOutput,
    },
    logging::{LogEntry, LogSchema},
    metrics::{
        APTOS_EXECUTOR_APPLY_CHUNK_SECONDS, APTOS_EXECUTOR_COMMIT_CHUNK_SECONDS,
        APTOS_EXECUTOR_EXECUTE_CHUNK_SECONDS, APTOS_EXECUTOR_VM_EXECUTE_CHUNK_SECONDS,
    },
};
use anyhow::Result;
use aptos_infallible::{Mutex, RwLock};
use aptos_logger::prelude::*;
use aptos_state_view::StateViewId;
use aptos_types::{
    ledger_info::LedgerInfoWithSignatures,
    transaction::{
        Transaction, TransactionInfo, TransactionListWithProof, TransactionOutputListWithProof,
    },
};
use aptos_vm::VMExecutor;
use executor_types::{
    ChunkCommitNotification, ChunkExecutorTrait, ExecutedChunk, TransactionReplayer,
};
use fail::fail_point;
use std::{marker::PhantomData, sync::Arc};
use storage_interface::{
    cached_state_view::CachedStateView, sync_proof_fetcher::SyncProofFetcher, DbReaderWriter,
    ExecutedTrees,
};

pub struct ChunkExecutor<V> {
    db: DbReaderWriter,
    inner: RwLock<Option<ChunkExecutorInner<V>>>,
}

impl<V: VMExecutor> ChunkExecutor<V> {
    pub fn new(db: DbReaderWriter) -> Self {
        Self {
            db,
            inner: RwLock::new(None),
        }
    }

    fn maybe_initialize(&self) -> Result<()> {
        if self.inner.read().is_none() {
            self.reset()?;
        }
        Ok(())
    }
}

impl<V: VMExecutor> ChunkExecutorTrait for ChunkExecutor<V> {
    fn execute_chunk(
        &self,
        txn_list_with_proof: TransactionListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<()> {
        self.maybe_initialize()?;
        self.inner
            .read()
            .as_ref()
            .expect("not reset")
            .execute_chunk(txn_list_with_proof, verified_target_li, epoch_change_li)
    }

    fn apply_chunk(
        &self,
        txn_output_list_with_proof: TransactionOutputListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<()> {
        self.inner.read().as_ref().expect("not reset").apply_chunk(
            txn_output_list_with_proof,
            verified_target_li,
            epoch_change_li,
        )
    }

    fn commit_chunk(&self) -> Result<ChunkCommitNotification> {
        self.inner
            .read()
            .as_ref()
            .expect("not reset")
            .commit_chunk()
    }

    fn execute_and_commit_chunk(
        &self,
        txn_list_with_proof: TransactionListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<ChunkCommitNotification> {
        // Re-sync with DB, make sure the queue is empty.
        self.reset()?;
        self.inner
            .read()
            .as_ref()
            .expect("not reset")
            .execute_and_commit_chunk(txn_list_with_proof, verified_target_li, epoch_change_li)
    }

    fn apply_and_commit_chunk(
        &self,
        txn_output_list_with_proof: TransactionOutputListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<ChunkCommitNotification> {
        // Re-sync with DB, make sure the queue is empty.
        self.reset()?;
        self.inner
            .read()
            .as_ref()
            .expect("not reset")
            .apply_and_commit_chunk(
                txn_output_list_with_proof,
                verified_target_li,
                epoch_change_li,
            )
    }

    fn reset(&self) -> Result<()> {
        *self.inner.write() = Some(ChunkExecutorInner::new(self.db.clone())?);
        Ok(())
    }

    fn finish(&self) {
        *self.inner.write() = None;
    }
}

struct ChunkExecutorInner<V> {
    db: DbReaderWriter,
    commit_queue: Mutex<ChunkCommitQueue>,
    _phantom: PhantomData<V>,
}

impl<V: VMExecutor> ChunkExecutorInner<V> {
    pub fn new(db: DbReaderWriter) -> Result<Self> {
        let commit_queue = Mutex::new(ChunkCommitQueue::new_from_db(&db.reader)?);
        Ok(Self {
            db,
            commit_queue,
            _phantom: PhantomData,
        })
    }

    fn state_view(&self, latest_view: &ExecutedTrees) -> Result<CachedStateView> {
        latest_view.verified_state_view(
            StateViewId::ChunkExecution {
                first_version: latest_view.txn_accumulator().num_leaves(),
            },
            Arc::clone(&self.db.reader),
            Arc::new(SyncProofFetcher::new(self.db.reader.clone())),
        )
    }

    fn apply_chunk_output_for_state_sync(
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
        latest_view: &ExecutedTrees,
        chunk_output: ChunkOutput,
        transaction_infos: &[TransactionInfo],
    ) -> Result<ExecutedChunk> {
        let (mut executed_chunk, to_discard, to_retry) =
            chunk_output.apply_to_ledger(latest_view)?;
        ensure_no_discard(to_discard)?;
        ensure_no_retry(to_retry)?;
        executed_chunk.ledger_info = executed_chunk
            .maybe_select_chunk_ending_ledger_info(verified_target_li, epoch_change_li)?;
        executed_chunk.ensure_transaction_infos_match(transaction_infos)?;

        Ok(executed_chunk)
    }

    fn commit_chunk_impl(&self) -> Result<Arc<ExecutedChunk>> {
        let (base_view, to_commit) = self.commit_queue.lock().next_chunk_to_commit()?;
        let txns_to_commit = to_commit.transactions_to_commit()?;
        let ledger_info = to_commit.ledger_info.as_ref();
        if ledger_info.is_some() || !txns_to_commit.is_empty() {
            fail_point!("executor::commit_chunk", |_| {
                Err(anyhow::anyhow!("Injected error in commit_chunk"))
            });
            self.db.writer.save_transactions(
                &txns_to_commit,
                base_view.txn_accumulator().num_leaves(),
                base_view.state().base_version,
                ledger_info,
                false, /* sync_commit */
                to_commit.result_view.state().clone(),
            )?;
        }

        self.commit_queue
            .lock()
            .dequeue()
            .expect("commit_queue.deque() failed.");
        Ok(to_commit)
    }

    // ************************* Block Executor Implementation *************************
    fn execute_chunk(
        &self,
        txn_list_with_proof: TransactionListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<()> {
        let _timer = APTOS_EXECUTOR_EXECUTE_CHUNK_SECONDS.start_timer();

        let num_txns = txn_list_with_proof.transactions.len();
        let first_version_in_request = txn_list_with_proof.first_transaction_version;
        let (_persisted_view, latest_view) = self.commit_queue.lock().persisted_and_latest_view();

        // Verify input transaction list.
        txn_list_with_proof.verify(verified_target_li.ledger_info(), first_version_in_request)?;

        // Skip transactions already in ledger.
        let txns_to_skip = txn_list_with_proof.proof.verify_extends_ledger(
            latest_view.txn_accumulator().num_leaves(),
            latest_view.txn_accumulator().root_hash(),
            first_version_in_request,
        )?;
        let mut transactions = txn_list_with_proof.transactions;
        transactions.drain(..txns_to_skip as usize);
        if txns_to_skip == num_txns {
            info!(
                "Skipping all transactions in the given chunk! Num transactions: {:?}",
                num_txns
            );
        }

        // Execute transactions.
        let state_view = self.state_view(&latest_view)?;
        let chunk_output = {
            let _timer = APTOS_EXECUTOR_VM_EXECUTE_CHUNK_SECONDS.start_timer();
            ChunkOutput::by_transaction_execution::<V>(transactions, state_view)?
        };
        let executed_chunk = Self::apply_chunk_output_for_state_sync(
            verified_target_li,
            epoch_change_li,
            &latest_view,
            chunk_output,
            &txn_list_with_proof.proof.transaction_infos[txns_to_skip..],
        )?;

        // Add result to commit queue.
        self.commit_queue.lock().enqueue(executed_chunk);

        info!(
            LogSchema::new(LogEntry::ChunkExecutor)
                .local_synced_version(latest_view.version().unwrap_or(0))
                .first_version_in_request(first_version_in_request)
                .num_txns_in_request(num_txns),
            "Executed transaction chunk!",
        );

        Ok(())
    }

    fn apply_chunk(
        &self,
        txn_output_list_with_proof: TransactionOutputListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<()> {
        let _timer = APTOS_EXECUTOR_APPLY_CHUNK_SECONDS.start_timer();

        let num_txns = txn_output_list_with_proof.transactions_and_outputs.len();
        let first_version_in_request = txn_output_list_with_proof.first_transaction_output_version;
        let (_persisted_view, latest_view) = self.commit_queue.lock().persisted_and_latest_view();

        // Verify input transaction list.
        txn_output_list_with_proof
            .verify(verified_target_li.ledger_info(), first_version_in_request)?;

        // Skip transactions already in ledger.
        let txns_to_skip = txn_output_list_with_proof.proof.verify_extends_ledger(
            latest_view.txn_accumulator().num_leaves(),
            latest_view.txn_accumulator().root_hash(),
            first_version_in_request,
        )?;
        let mut txns_and_outputs = txn_output_list_with_proof.transactions_and_outputs;
        txns_and_outputs.drain(..txns_to_skip as usize);

        // Apply transaction outputs.
        let state_view = self.state_view(&latest_view)?;
        let chunk_output = ChunkOutput::by_transaction_output(txns_and_outputs, state_view)?;
        let executed_chunk = Self::apply_chunk_output_for_state_sync(
            verified_target_li,
            epoch_change_li,
            &latest_view,
            chunk_output,
            &txn_output_list_with_proof.proof.transaction_infos[txns_to_skip..],
        )?;

        // Add result to commit queue.
        self.commit_queue.lock().enqueue(executed_chunk);

        info!(
            LogSchema::new(LogEntry::ChunkExecutor)
                .local_synced_version(latest_view.version().unwrap_or(0))
                .first_version_in_request(first_version_in_request)
                .num_txns_in_request(num_txns),
            "Applied transaction output chunk!",
        );

        Ok(())
    }

    fn commit_chunk(&self) -> Result<ChunkCommitNotification> {
        let _timer = APTOS_EXECUTOR_COMMIT_CHUNK_SECONDS.start_timer();
        let executed_chunk = self.commit_chunk_impl()?;
        Ok(ChunkCommitNotification {
            committed_events: executed_chunk.events_to_commit(),
            committed_transactions: executed_chunk.transactions(),
            reconfiguration_occurred: executed_chunk.has_reconfiguration(),
        })
    }

    fn execute_and_commit_chunk(
        &self,
        txn_list_with_proof: TransactionListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<ChunkCommitNotification> {
        self.execute_chunk(txn_list_with_proof, verified_target_li, epoch_change_li)?;
        self.commit_chunk()
    }

    fn apply_and_commit_chunk(
        &self,
        txn_output_list_with_proof: TransactionOutputListWithProof,
        verified_target_li: &LedgerInfoWithSignatures,
        epoch_change_li: Option<&LedgerInfoWithSignatures>,
    ) -> Result<ChunkCommitNotification> {
        self.apply_chunk(
            txn_output_list_with_proof,
            verified_target_li,
            epoch_change_li,
        )?;
        self.commit_chunk()
    }
}

impl<V: VMExecutor> TransactionReplayer for ChunkExecutor<V> {
    fn replay(
        &self,
        transactions: Vec<Transaction>,
        transaction_infos: Vec<TransactionInfo>,
    ) -> Result<()> {
        self.maybe_initialize()?;
        self.inner
            .read()
            .as_ref()
            .expect("not reset")
            .replay(transactions, transaction_infos)
    }

    fn commit(&self) -> Result<Arc<ExecutedChunk>> {
        self.inner.read().as_ref().expect("not reset").commit()
    }
}

impl<V: VMExecutor> TransactionReplayer for ChunkExecutorInner<V> {
    fn replay(
        &self,
        transactions: Vec<Transaction>,
        mut transaction_infos: Vec<TransactionInfo>,
    ) -> Result<()> {
        let (_persisted_view, mut latest_view) =
            self.commit_queue.lock().persisted_and_latest_view();

        let mut executed_chunk = ExecutedChunk::default();
        let mut to_run = Some(transactions);
        while !to_run.as_ref().unwrap().is_empty() {
            // Execute transactions.
            let state_view = self.state_view(&latest_view)?;
            let txns = to_run.take().unwrap();
            let (executed, to_discard, to_retry) =
                ChunkOutput::by_transaction_execution::<V>(txns, state_view)?
                    .apply_to_ledger(&latest_view)?;

            // Accumulate result and deal with retry
            ensure_no_discard(to_discard)?;
            let n = executed.to_commit.len();
            executed.ensure_transaction_infos_match(&transaction_infos[..n])?;
            transaction_infos.drain(..n);

            to_run = Some(to_retry);
            executed_chunk = executed_chunk.combine(executed)?;
            latest_view = executed_chunk.result_view.clone();
        }

        // Add result to commit queue.
        self.commit_queue.lock().enqueue(executed_chunk);

        Ok(())
    }

    fn commit(&self) -> Result<Arc<ExecutedChunk>> {
        self.commit_chunk_impl()
    }
}
<|endoftext|>