<commit_before>//! # C API for rav1e
//!
//! [rav1e](https://github.com/xiph/rav1e/) is an [AV1](https://aomediacodec.github.io/av1-spec/)
//! encoder written in [Rust](https://rust-lang.org)
//!
//! This is the C-compatible API

extern crate rav1e;
extern crate libc;

use std::slice;
use std::sync::Arc;

use std::ffi::CStr;
use std::ffi::CString;
use std::os::raw::c_char;
use std::os::raw::c_int;

use libc::size_t;
use libc::ptrdiff_t;

/// Raw video Frame
///
/// It can be allocated throught rav1e_frame_new(), populated using rav1e_frame_fill_plane()
/// and freed using rav1e_frame_unref().
pub struct Frame(Arc<rav1e::Frame<u16>>);

#[repr(C)]
#[derive(Copy, Clone, Debug)]
pub enum EncoderStatus {
    /// Normal operation
    Success = 0,
    /// The encoder needs more data to produce an output Packet
    /// May be emitted by `Context::receive_packet`  when frame reordering is enabled.
    NeedMoreData,
    /// There are enough Frames queue
    /// May be emitted by `Context::send_frame` when the input queue is constrained
    EnoughData,
    /// The encoder already produced the number of frames requested
    /// May be emitted by `Context::receive_packet` after a flush request had been processed
    /// or the frame limit had been reached.
    LimitReached,
    /// Generic fatal error
    Failure = -1,
}

impl From<Option<rav1e::EncoderStatus>> for EncoderStatus {
    fn from(status: Option<rav1e::EncoderStatus>) -> Self {
        match status {
            None => EncoderStatus::Success,
            Some(s) => match s {
                rav1e::EncoderStatus::NeedMoreData => EncoderStatus::NeedMoreData,
                rav1e::EncoderStatus::EnoughData => EncoderStatus::EnoughData,
                rav1e::EncoderStatus::LimitReached => EncoderStatus::LimitReached,
                rav1e::EncoderStatus::Failure => EncoderStatus::Failure,
            }
        }
    }
}

/// Encoder configuration
///
/// Instantiate it using rav1e_config_default() and fine-tune it using
/// rav1e_config_parse().
///
/// Use rav1e_config_unref() to free its memory.
pub struct Config {
    cfg: rav1e::Config,
}

/// Encoder context
///
/// Contains the encoding state, it is created by rav1e_context_new() using an
/// Encoder configuration.
///
/// Use rav1e_context_unref() to free its memory.
pub struct Context {
    ctx: rav1e::Context<u16>,
    last_err: Option<rav1e::EncoderStatus>,
}

type FrameType = rav1e::FrameType;

/// Encoded Packet
///
/// The encoded packets are retrieved using rav1e_receive_packet().
///
/// Use rav1e_packet_unref() to free its memory.
#[repr(C)]
pub struct Packet {
    /// Encoded data buffer
    pub data: *const u8,
    /// Encoded data buffer size
    pub len: size_t,
    /// Frame sequence number
    pub number: u64,
    /// Frame type
    pub frame_type: FrameType,
}

type ChromaSamplePosition=rav1e::ChromaSamplePosition;
type ChromaSampling=rav1e::ChromaSampling;
type MatrixCoefficients=rav1e::MatrixCoefficients;
type ColorPrimaries=rav1e::ColorPrimaries;
type TransferCharacteristics=rav1e::TransferCharacteristics;
type Rational=rav1e::Rational;

#[no_mangle]
pub unsafe extern "C" fn rav1e_config_default() -> *mut Config {
    let cfg = rav1e::Config {
        enc: rav1e::EncoderConfig::default(),
        threads: 0,
    };

    let c = Box::new(Config {
        cfg,
    });

    Box::into_raw(c)
}

/// Set color properties of the stream.
///
/// Supported values are defined by the enum types
/// RaMatrixCoefficients, RaColorPrimaries, and RaTransferCharacteristics
/// respectively.
///
/// Return a negative value on error or 0.
#[no_mangle]
pub unsafe extern "C" fn rav1e_config_set_color_description(cfg: *mut Config,
                                                            matrix: MatrixCoefficients,
                                                            primaries: ColorPrimaries,
                                                            transfer: TransferCharacteristics
) -> c_int {
    (*cfg).cfg.enc.color_description = Some(rav1e::ColorDescription {
        matrix_coefficients: matrix,
        color_primaries: primaries,
        transfer_characteristics: transfer,
    });

    if (*cfg).cfg.enc.color_description.is_some() { 0 } else { -1 }
}

/// Set the content light level information for HDR10 streams.
///
/// Return a negative value on error or 0.
#[no_mangle]
pub unsafe extern "C" fn rav1e_config_set_content_light(cfg: *mut Config,
                                                        max_content_light_level: u16,
                                                        max_frame_average_light_level: u16
) -> c_int {
    (*cfg).cfg.enc.content_light = Some(rav1e::ContentLight {
        max_content_light_level,
        max_frame_average_light_level,
    });

    if (*cfg).cfg.enc.content_light.is_some() { 0 } else { -1 }
}

/// Set the mastering display information for HDR10 streams.
///
/// primaries and white_point arguments are RaPoint, containing 0.16 fixed point values.
/// max_luminance is a 24.8 fixed point value.
/// min_luminance is a 18.14 fixed point value.
///
/// Returns a negative value on error or 0.
#[no_mangle]
pub unsafe extern "C" fn rav1e_config_set_mastering_display(cfg: *mut Config,
                                                            primaries: [rav1e::Point; 3],
                                                            white_point: rav1e::Point,
                                                            max_luminance: u32,
                                                            min_luminance: u32
) -> c_int {
    (*cfg).cfg.enc.mastering_display = Some(rav1e::MasteringDisplay {
        primaries,
        white_point,
        max_luminance,
        min_luminance,
    });

    if (*cfg).cfg.enc.mastering_display.is_some() { 0 } else { -1 }
}

#[no_mangle]
pub unsafe extern "C" fn rav1e_config_unref(cfg: *mut Config) {
    if !cfg.is_null() {
        let _ = Box::from_raw(cfg);
    }
}

unsafe fn option_match(
    cfg: *mut Config,
    key: *const c_char,
    value: *const c_char
) -> Result<(), ()> {
    let key = CStr::from_ptr(key).to_str().map_err(|_| ())?;
    let value = CStr::from_ptr(value).to_str().map_err(|_| ())?;
    let enc = &mut(*cfg).cfg.enc;

    match key {
        "width" => enc.width = value.parse().map_err(|_| ())?,
        "height" => enc.height = value.parse().map_err(|_| ())?,
        "speed" => enc.speed_settings = rav1e::SpeedSettings::from_preset(value.parse().map_err(|_| ())?),

        "threads" => (*cfg).cfg.threads = value.parse().map_err(|_| ())?,

        "tune" => enc.tune = value.parse().map_err(|_| ())?,
        "quantizer" => enc.quantizer = value.parse().map_err(|_| ())?,

        "key_frame_interval" => enc.max_key_frame_interval = value.parse().map_err(|_| ())?,
        "min_key_frame_interval" => enc.min_key_frame_interval = value.parse().map_err(|_| ())?,
        "low_latency" => enc.low_latency = value.parse().map_err(|_| ())?,

        _ => return Err(())
    }

    Ok(())
}

/// Set a configuration parameter using its key and value as string.
///
/// Available keys and values
/// - "quantizer": 0-255, default 100
/// - "speed": 0-10, default 3
/// - "tune": "psnr"-"psychovisual", default "psnr"
///
/// Return a negative value on error or 0.
#[no_mangle]
pub unsafe extern "C" fn rav1e_config_parse(
    cfg: *mut Config,
    key: *const c_char,
    value: *const c_char,
) -> c_int {
    if option_match(cfg, key, value) == Ok(()) { 0 } else { -1 }
}

/// Set a configuration parameter using its key and value as integer.
///
/// Available keys and values are the same as rav1e_config_parse()
///
/// Return a negative value on error or 0.
#[no_mangle]
pub unsafe extern "C" fn rav1e_config_parse_int(
    cfg: *mut Config,
    key: *const c_char,
    value: c_int,
) -> c_int {
    let val = CString::new(value.to_string()).unwrap();
    if option_match(cfg, key, val.as_ptr()) == Ok(()) { 0 } else { -1 }
}

/// Generate a new encoding context from a populated encoder configuration
///
/// Multiple contexts can be generated through it.
#[no_mangle]
pub unsafe extern "C" fn rav1e_context_new(cfg: *const Config) -> *mut Context {
    let ctx = Context {
        ctx: (*cfg).cfg.new_context(),
        last_err: None,
    };

    Box::into_raw(Box::new(ctx))
}

#[no_mangle]
pub unsafe extern "C" fn rav1e_context_unref(ctx: *mut Context) {
    if !ctx.is_null() {
        let _ = Box::from_raw(ctx);
    }
}

/// Produce a new frame from the encoding context
///
/// It must be populated using rav1e_frame_fill_plane().
///
/// The frame is reference counted and must be released passing it to rav1e_frame_unref(),
/// see rav1e_send_frame().
#[no_mangle]
pub unsafe extern "C" fn rav1e_frame_new(ctx: *const Context) -> *mut Frame {
    let f = (*ctx).ctx.new_frame();
    let frame = Box::new(Frame(f));

    Box::into_raw(frame)
}

#[no_mangle]
pub unsafe extern "C" fn rav1e_frame_unref(frame: *mut Frame) {
    if !frame.is_null() {
        let _ = Box::from_raw(frame);
    }
}

/// Send the frame for encoding
///
/// The function increases the frame internal reference count and it can be passed multiple
/// times to different rav1e_send_frame().
///
/// Returns:
/// - `0` on success,
/// - `> 0` if the input queue is full
/// - `< 0` on unrecoverable failure
#[no_mangle]
pub unsafe extern "C" fn rav1e_send_frame(ctx: *mut Context, frame: *const Frame) -> EncoderStatus {
    let frame = if frame.is_null() {
        None
    } else {
        Some((*frame).0.clone())
    };

    let ret = (*ctx)
        .ctx
        .send_frame(frame)
        .map(|_v| {
            None
        }).unwrap_or_else(|e| {
            Some(e)
        });

    (*ctx).last_err = ret;

    ret.into()
}

/// Return the last encoder status
#[no_mangle]
pub unsafe extern "C" fn rav1e_last_status(ctx: *const Context) -> EncoderStatus {
    (*ctx).last_err.into()
}

/// Return a string matching the EncooderStatus variant.
#[no_mangle]
pub unsafe extern "C" fn rav1e_status_to_str(status: EncoderStatus) -> *mut c_char {
    let status = format!("{:?}", status);
    let cptr = CString::new(status).unwrap().as_ptr();

    libc::strdup(cptr)
}

/// Receive encoded data
///
/// Returns:
/// - `0` on success
/// - `> 0` if additional frame data is required
/// - `< 0` on unrecoverable failure
#[no_mangle]
pub unsafe extern "C" fn rav1e_receive_packet(
    ctx: *mut Context,
    pkt: *mut *mut Packet,
) -> EncoderStatus {
    let ret = (*ctx)
        .ctx
        .receive_packet()
        .map(|p| {
            let rav1e::Packet { data, number, frame_type, .. } = p;
            let len  = data.len();
            let data = Box::into_raw(data.into_boxed_slice()) as *const u8;
            let packet = Packet {
                data,
                len,
                number,
                frame_type,
            };
            *pkt = Box::into_raw(Box::new(packet));
            None
        }).unwrap_or_else(|e| {
            Some(e)
        });

    (*ctx).last_err = ret;

    ret.into()
}

#[no_mangle]
pub unsafe extern fn rav1e_packet_unref(pkt: *mut Packet) {
    if !pkt.is_null() {
        let pkt = Box::from_raw(pkt);
        let _ = Box::from_raw(pkt.data as *mut u8);
    }
}

/// Produce a sequence header matching the current encoding context
///
/// Its format is compatible with the AV1 Matroska and ISOBMFF specification.
///
/// Use rav1e_container_sequence_header_unref() to free it.
#[no_mangle]
pub unsafe extern fn rav1e_container_sequence_header(ctx: *mut Context, buf_size: *mut size_t) -> *mut u8 {
    let buf = (*ctx).ctx.container_sequence_header();

    *buf_size = buf.len();
    Box::into_raw(buf.into_boxed_slice()) as *mut u8
}

#[no_mangle]
pub unsafe extern fn rav1e_container_sequence_header_unref(sequence: *mut u8) {
    if !sequence.is_null() {
        let _ = Box::from_raw(sequence);
    }
}

/// Fill a frame plane
///
/// Currently the frame contains 3 planes, the first is luminance followed by
/// chrominance.
///
/// The data is copied and this function has to be called for each plane.
///
/// frame: A frame provided by rav1e_frame_new()
/// plane: The index of the plane starting from 0
/// data: The data to be copied
/// data_len: Lenght of the buffer
/// stride: Plane line in bytes, including padding
/// bytewidth: Number of bytes per component, either 1 or 2
#[no_mangle]
pub unsafe extern "C" fn rav1e_frame_fill_plane(
    frame: *mut Frame,
    plane: c_int,
    data: *const u8,
    data_len: size_t,
    stride: ptrdiff_t,
    bytewidth: c_int,
) {
    let f = &mut (*frame).0;
    let input = Arc::make_mut(f);
    let data_slice = slice::from_raw_parts(data, data_len as usize);

    input.planes[plane as usize].copy_from_raw_u8(data_slice, stride as usize, bytewidth as usize);
}
<commit_msg>crav1e: Add bitrate option<commit_after>//! # C API for rav1e
//!
//! [rav1e](https://github.com/xiph/rav1e/) is an [AV1](https://aomediacodec.github.io/av1-spec/)
//! encoder written in [Rust](https://rust-lang.org)
//!
//! This is the C-compatible API

extern crate rav1e;
extern crate libc;

use std::slice;
use std::sync::Arc;

use std::ffi::CStr;
use std::ffi::CString;
use std::os::raw::c_char;
use std::os::raw::c_int;

use libc::size_t;
use libc::ptrdiff_t;

/// Raw video Frame
///
/// It can be allocated throught rav1e_frame_new(), populated using rav1e_frame_fill_plane()
/// and freed using rav1e_frame_unref().
pub struct Frame(Arc<rav1e::Frame<u16>>);

#[repr(C)]
#[derive(Copy, Clone, Debug)]
pub enum EncoderStatus {
    /// Normal operation
    Success = 0,
    /// The encoder needs more data to produce an output Packet
    /// May be emitted by `Context::receive_packet`  when frame reordering is enabled.
    NeedMoreData,
    /// There are enough Frames queue
    /// May be emitted by `Context::send_frame` when the input queue is constrained
    EnoughData,
    /// The encoder already produced the number of frames requested
    /// May be emitted by `Context::receive_packet` after a flush request had been processed
    /// or the frame limit had been reached.
    LimitReached,
    /// Generic fatal error
    Failure = -1,
}

impl From<Option<rav1e::EncoderStatus>> for EncoderStatus {
    fn from(status: Option<rav1e::EncoderStatus>) -> Self {
        match status {
            None => EncoderStatus::Success,
            Some(s) => match s {
                rav1e::EncoderStatus::NeedMoreData => EncoderStatus::NeedMoreData,
                rav1e::EncoderStatus::EnoughData => EncoderStatus::EnoughData,
                rav1e::EncoderStatus::LimitReached => EncoderStatus::LimitReached,
                rav1e::EncoderStatus::Failure => EncoderStatus::Failure,
            }
        }
    }
}

/// Encoder configuration
///
/// Instantiate it using rav1e_config_default() and fine-tune it using
/// rav1e_config_parse().
///
/// Use rav1e_config_unref() to free its memory.
pub struct Config {
    cfg: rav1e::Config,
}

/// Encoder context
///
/// Contains the encoding state, it is created by rav1e_context_new() using an
/// Encoder configuration.
///
/// Use rav1e_context_unref() to free its memory.
pub struct Context {
    ctx: rav1e::Context<u16>,
    last_err: Option<rav1e::EncoderStatus>,
}

type FrameType = rav1e::FrameType;

/// Encoded Packet
///
/// The encoded packets are retrieved using rav1e_receive_packet().
///
/// Use rav1e_packet_unref() to free its memory.
#[repr(C)]
pub struct Packet {
    /// Encoded data buffer
    pub data: *const u8,
    /// Encoded data buffer size
    pub len: size_t,
    /// Frame sequence number
    pub number: u64,
    /// Frame type
    pub frame_type: FrameType,
}

type ChromaSamplePosition=rav1e::ChromaSamplePosition;
type ChromaSampling=rav1e::ChromaSampling;
type MatrixCoefficients=rav1e::MatrixCoefficients;
type ColorPrimaries=rav1e::ColorPrimaries;
type TransferCharacteristics=rav1e::TransferCharacteristics;
type Rational=rav1e::Rational;

#[no_mangle]
pub unsafe extern "C" fn rav1e_config_default() -> *mut Config {
    let cfg = rav1e::Config {
        enc: rav1e::EncoderConfig::default(),
        threads: 0,
    };

    let c = Box::new(Config {
        cfg,
    });

    Box::into_raw(c)
}

/// Set color properties of the stream.
///
/// Supported values are defined by the enum types
/// RaMatrixCoefficients, RaColorPrimaries, and RaTransferCharacteristics
/// respectively.
///
/// Return a negative value on error or 0.
#[no_mangle]
pub unsafe extern "C" fn rav1e_config_set_color_description(cfg: *mut Config,
                                                            matrix: MatrixCoefficients,
                                                            primaries: ColorPrimaries,
                                                            transfer: TransferCharacteristics
) -> c_int {
    (*cfg).cfg.enc.color_description = Some(rav1e::ColorDescription {
        matrix_coefficients: matrix,
        color_primaries: primaries,
        transfer_characteristics: transfer,
    });

    if (*cfg).cfg.enc.color_description.is_some() { 0 } else { -1 }
}

/// Set the content light level information for HDR10 streams.
///
/// Return a negative value on error or 0.
#[no_mangle]
pub unsafe extern "C" fn rav1e_config_set_content_light(cfg: *mut Config,
                                                        max_content_light_level: u16,
                                                        max_frame_average_light_level: u16
) -> c_int {
    (*cfg).cfg.enc.content_light = Some(rav1e::ContentLight {
        max_content_light_level,
        max_frame_average_light_level,
    });

    if (*cfg).cfg.enc.content_light.is_some() { 0 } else { -1 }
}

/// Set the mastering display information for HDR10 streams.
///
/// primaries and white_point arguments are RaPoint, containing 0.16 fixed point values.
/// max_luminance is a 24.8 fixed point value.
/// min_luminance is a 18.14 fixed point value.
///
/// Returns a negative value on error or 0.
#[no_mangle]
pub unsafe extern "C" fn rav1e_config_set_mastering_display(cfg: *mut Config,
                                                            primaries: [rav1e::Point; 3],
                                                            white_point: rav1e::Point,
                                                            max_luminance: u32,
                                                            min_luminance: u32
) -> c_int {
    (*cfg).cfg.enc.mastering_display = Some(rav1e::MasteringDisplay {
        primaries,
        white_point,
        max_luminance,
        min_luminance,
    });

    if (*cfg).cfg.enc.mastering_display.is_some() { 0 } else { -1 }
}

#[no_mangle]
pub unsafe extern "C" fn rav1e_config_unref(cfg: *mut Config) {
    if !cfg.is_null() {
        let _ = Box::from_raw(cfg);
    }
}

unsafe fn option_match(
    cfg: *mut Config,
    key: *const c_char,
    value: *const c_char
) -> Result<(), ()> {
    let key = CStr::from_ptr(key).to_str().map_err(|_| ())?;
    let value = CStr::from_ptr(value).to_str().map_err(|_| ())?;
    let enc = &mut(*cfg).cfg.enc;

    match key {
        "width" => enc.width = value.parse().map_err(|_| ())?,
        "height" => enc.height = value.parse().map_err(|_| ())?,
        "speed" => enc.speed_settings = rav1e::SpeedSettings::from_preset(value.parse().map_err(|_| ())?),

        "threads" => (*cfg).cfg.threads = value.parse().map_err(|_| ())?,

        "tune" => enc.tune = value.parse().map_err(|_| ())?,
        "quantizer" => enc.quantizer = value.parse().map_err(|_| ())?,
        "bitrate" => enc.bitrate = value.parse().map_err(|_| ())?,

        "key_frame_interval" => enc.max_key_frame_interval = value.parse().map_err(|_| ())?,
        "min_key_frame_interval" => enc.min_key_frame_interval = value.parse().map_err(|_| ())?,
        "low_latency" => enc.low_latency = value.parse().map_err(|_| ())?,

        _ => return Err(())
    }

    Ok(())
}

/// Set a configuration parameter using its key and value as string.
///
/// Available keys and values
/// - "quantizer": 0-255, default 100
/// - "speed": 0-10, default 3
/// - "tune": "psnr"-"psychovisual", default "psnr"
///
/// Return a negative value on error or 0.
#[no_mangle]
pub unsafe extern "C" fn rav1e_config_parse(
    cfg: *mut Config,
    key: *const c_char,
    value: *const c_char,
) -> c_int {
    if option_match(cfg, key, value) == Ok(()) { 0 } else { -1 }
}

/// Set a configuration parameter using its key and value as integer.
///
/// Available keys and values are the same as rav1e_config_parse()
///
/// Return a negative value on error or 0.
#[no_mangle]
pub unsafe extern "C" fn rav1e_config_parse_int(
    cfg: *mut Config,
    key: *const c_char,
    value: c_int,
) -> c_int {
    let val = CString::new(value.to_string()).unwrap();
    if option_match(cfg, key, val.as_ptr()) == Ok(()) { 0 } else { -1 }
}

/// Generate a new encoding context from a populated encoder configuration
///
/// Multiple contexts can be generated through it.
#[no_mangle]
pub unsafe extern "C" fn rav1e_context_new(cfg: *const Config) -> *mut Context {
    let ctx = Context {
        ctx: (*cfg).cfg.new_context(),
        last_err: None,
    };

    Box::into_raw(Box::new(ctx))
}

#[no_mangle]
pub unsafe extern "C" fn rav1e_context_unref(ctx: *mut Context) {
    if !ctx.is_null() {
        let _ = Box::from_raw(ctx);
    }
}

/// Produce a new frame from the encoding context
///
/// It must be populated using rav1e_frame_fill_plane().
///
/// The frame is reference counted and must be released passing it to rav1e_frame_unref(),
/// see rav1e_send_frame().
#[no_mangle]
pub unsafe extern "C" fn rav1e_frame_new(ctx: *const Context) -> *mut Frame {
    let f = (*ctx).ctx.new_frame();
    let frame = Box::new(Frame(f));

    Box::into_raw(frame)
}

#[no_mangle]
pub unsafe extern "C" fn rav1e_frame_unref(frame: *mut Frame) {
    if !frame.is_null() {
        let _ = Box::from_raw(frame);
    }
}

/// Send the frame for encoding
///
/// The function increases the frame internal reference count and it can be passed multiple
/// times to different rav1e_send_frame().
///
/// Returns:
/// - `0` on success,
/// - `> 0` if the input queue is full
/// - `< 0` on unrecoverable failure
#[no_mangle]
pub unsafe extern "C" fn rav1e_send_frame(ctx: *mut Context, frame: *const Frame) -> EncoderStatus {
    let frame = if frame.is_null() {
        None
    } else {
        Some((*frame).0.clone())
    };

    let ret = (*ctx)
        .ctx
        .send_frame(frame)
        .map(|_v| {
            None
        }).unwrap_or_else(|e| {
            Some(e)
        });

    (*ctx).last_err = ret;

    ret.into()
}

/// Return the last encoder status
#[no_mangle]
pub unsafe extern "C" fn rav1e_last_status(ctx: *const Context) -> EncoderStatus {
    (*ctx).last_err.into()
}

/// Return a string matching the EncooderStatus variant.
#[no_mangle]
pub unsafe extern "C" fn rav1e_status_to_str(status: EncoderStatus) -> *mut c_char {
    let status = format!("{:?}", status);
    let cptr = CString::new(status).unwrap().as_ptr();

    libc::strdup(cptr)
}

/// Receive encoded data
///
/// Returns:
/// - `0` on success
/// - `> 0` if additional frame data is required
/// - `< 0` on unrecoverable failure
#[no_mangle]
pub unsafe extern "C" fn rav1e_receive_packet(
    ctx: *mut Context,
    pkt: *mut *mut Packet,
) -> EncoderStatus {
    let ret = (*ctx)
        .ctx
        .receive_packet()
        .map(|p| {
            let rav1e::Packet { data, number, frame_type, .. } = p;
            let len  = data.len();
            let data = Box::into_raw(data.into_boxed_slice()) as *const u8;
            let packet = Packet {
                data,
                len,
                number,
                frame_type,
            };
            *pkt = Box::into_raw(Box::new(packet));
            None
        }).unwrap_or_else(|e| {
            Some(e)
        });

    (*ctx).last_err = ret;

    ret.into()
}

#[no_mangle]
pub unsafe extern fn rav1e_packet_unref(pkt: *mut Packet) {
    if !pkt.is_null() {
        let pkt = Box::from_raw(pkt);
        let _ = Box::from_raw(pkt.data as *mut u8);
    }
}

/// Produce a sequence header matching the current encoding context
///
/// Its format is compatible with the AV1 Matroska and ISOBMFF specification.
///
/// Use rav1e_container_sequence_header_unref() to free it.
#[no_mangle]
pub unsafe extern fn rav1e_container_sequence_header(ctx: *mut Context, buf_size: *mut size_t) -> *mut u8 {
    let buf = (*ctx).ctx.container_sequence_header();

    *buf_size = buf.len();
    Box::into_raw(buf.into_boxed_slice()) as *mut u8
}

#[no_mangle]
pub unsafe extern fn rav1e_container_sequence_header_unref(sequence: *mut u8) {
    if !sequence.is_null() {
        let _ = Box::from_raw(sequence);
    }
}

/// Fill a frame plane
///
/// Currently the frame contains 3 planes, the first is luminance followed by
/// chrominance.
///
/// The data is copied and this function has to be called for each plane.
///
/// frame: A frame provided by rav1e_frame_new()
/// plane: The index of the plane starting from 0
/// data: The data to be copied
/// data_len: Lenght of the buffer
/// stride: Plane line in bytes, including padding
/// bytewidth: Number of bytes per component, either 1 or 2
#[no_mangle]
pub unsafe extern "C" fn rav1e_frame_fill_plane(
    frame: *mut Frame,
    plane: c_int,
    data: *const u8,
    data_len: size_t,
    stride: ptrdiff_t,
    bytewidth: c_int,
) {
    let f = &mut (*frame).0;
    let input = Arc::make_mut(f);
    let data_slice = slice::from_raw_parts(data, data_len as usize);

    input.planes[plane as usize].copy_from_raw_u8(data_slice, stride as usize, bytewidth as usize);
}
<|endoftext|>


<commit_before>//! Common utilities for demoing hash types
//!
//! NOTE: not considered part of the crate's stable API
#![allow(missing_docs)]

use image::DynamicImage;

use std::{env, fs, process};

pub struct DemoConfig {
    pub image: DynamicImage,
    pub output_dir: String,
    pub width: u32,
    pub height: u32,
}

#[macro_export]
macro_rules! explain {
    ($($arg:tt)*) => { |e| format!("{}: {}", format_args!($($arg)*), e) }
}

impl DemoConfig {
    pub fn init(name: &str, alg: &str) -> Result<DemoConfig, String> {
        let args = env::args().collect::<Vec<_>>();

        if args.len() != 5 {
            return Err(format!("\
                usage: {name} [FILE] [OUTPUT-DIR] [WIDTH] [HEIGHT]\n\
                demos `{alg}` for FILE, exporting gifs of each step to OUTPUT-DIR\n\
                each gif will be WIDTH x HEIGHT in size; the hash config is fixed\n\
          ", name = name, alg = alg));
        }

        let file = &args[1];
        let output = &args[2];
        let width = args[3].parse::<u32>().map_err(explain!("could not parse WIDTH: {}", args[3]))?;
        let height = args[4].parse::<u32>().map_err(explain!("could not parse HEIGHT: {}", args[4]))?;

        let image = image::open(file).map_err(explain!("failed to open {}", file))?;

        let output_dir = fs::create_dir_all(output)
            .map_err(explain!("failed to create output dir {}", output));
    }

    pub fn open_output_file(&self, name: &str) -> Result<fs::File, String> {

    }
}
<commit_msg>expand demo API<commit_after>//! Common utilities for demoing hash types
//!
//! NOTE: not considered part of the crate's stable API
#![allow(missing_docs)]

use image::{DynamicImage, gif};

use std::env;
use std::fs::{self, File};
use std::path::PathBuf;

pub struct DemoCtxt {
    pub image: DynamicImage,
    pub output_dir: PathBuf,
    pub width: u32,
    pub height: u32,
}

#[macro_export]
macro_rules! explain {
    ($($arg:tt)*) => { |e| format!("{}: {}", format_args!($($arg)*), e) }
}

impl DemoCtxt {
    pub fn init(name: &str, alg: &str) -> Result<DemoCtxt, String> {
        let args = env::args().collect::<Vec<_>>();

        if args.len() != 5 {
            return Err(format!("\
                usage: {name} [FILE] [OUTPUT-DIR] [WIDTH] [HEIGHT]\n\
                demos `{alg}` for FILE, exporting gifs of each step to OUTPUT-DIR\n\
                each gif will be WIDTH x HEIGHT in size; the hash config is fixed\n\
          ", name = name, alg = alg));
        }

        let file = &args[1];
        let output = &args[2];
        let width = args[3].parse::<u32>().map_err(explain!("could not parse WIDTH: {}", args[3]))?;
        let height = args[4].parse::<u32>().map_err(explain!("could not parse HEIGHT: {}", args[4]))?;

        let image = image::open(file).map_err(explain!("failed to open {}", file))?;

        fs::create_dir_all(output).map_err(explain!("failed to create output dir {}", output));

        Ok(Self {
            image,
            output_dir: output.into(),
            width,
            height
        })
    }

    /// Save the frame set as a gif with the given name, without extension
    pub fn save_gif(&self, name: &str, frames: image::Frames) -> Result<(), String> {
        let path = self.output_dir.join(name).with_extension(".gif");
        let file = fs::File::create(&path).map_err(explain!("failed to create {}", path.display()));

        let mut encoder = gif::Encoder::new(file);
        encoder.encode_frames(frames)
            .map_err(explain!("failed to write gif frames to {}", path.display()))
    }
}
<|endoftext|>


<commit_before>#![feature(inclusive_range, inclusive_range_fields, range_contains)]

extern crate dotenv;
extern crate itertools;
#[macro_use]
extern crate lazy_static;
extern crate rand;
extern crate serenity;

use std::collections::BTreeMap;
use std::env;
use std::ops::RangeInclusive;

use dotenv::dotenv;
use itertools::Itertools;
use rand::Rng;
use serenity::prelude::*;
use serenity::model::prelude::*;

trait Command {
    fn required_arg_count(&self) -> RangeInclusive<usize>; //impl Range<usize>;
    fn exec(&self, ctx: &Context, msg: &Message, args: &[&str]);
}
type CommandMap = BTreeMap<&'static str, Box<Command + Sync>>;

lazy_static! {
    static ref COMMAND_MAP: CommandMap = {
        let mut commands = CommandMap::new();
        commands.insert("help", Box::new(Help));
        commands.insert("flip", Box::new(Flip));
        commands.insert("roll", Box::new(Roll));
        commands
    };
}

fn main() {
    let _ = dotenv();

    let creds = env::var("Z_CREDENTIALS")
        .expect("Unspecified credentials: set Z_CREDENTIALS in the environment (or in `.env`).");
    let mut client = Client::new(&creds, Handler).unwrap();
    client.start().unwrap();
}

struct Handler;
impl EventHandler for Handler {
    fn ready(&self, ctx: Context, _: Ready) {
        ctx.set_presence(Some(Game::playing("Rust")), OnlineStatus::Online)
    }

    fn message(&self, ctx: Context, msg: Message) {
        if msg.content.chars().next() == Some('!') {
            let mut tokens = msg.content[1..].split_whitespace();
            let maybe_command = tokens.next();
            let args = tokens.collect::<Vec<_>>();

            if let Some(command) = maybe_command {
                if let Some(command_handler) = COMMAND_MAP.get(command) {
                    let valid_arg_range = command_handler.required_arg_count();
                    if valid_arg_range.contains(args.len()) {
                        command_handler.exec(&ctx, &msg, &*args);
                    } else {
                        msg.reply(&format!(
                            "This command requires {} to {} arguments.",
                            valid_arg_range.start, valid_arg_range.end
                        )).unwrap();
                    }
                } else {
                    msg.reply(&format!("Unknown command: {}", command)).unwrap();
                }
            } else {
                msg.reply("No command specified.").unwrap();
            }
        }
    }
}

struct Help;
impl Command for Help {
    fn required_arg_count(&self) -> RangeInclusive<usize> {
        0..=0
    }

    fn exec(&self, _ctx: &Context, msg: &Message, _args: &[&str]) {
        let help_output = COMMAND_MAP
            .keys()
            .intersperse(&", !")
            .fold("Available commands:\n!".to_owned(), |output, command| {
                output + command
            });

        msg.channel_id.say(&help_output).unwrap();
    }
}

struct Flip;
impl Command for Flip {
    fn required_arg_count(&self) -> RangeInclusive<usize> {
        0..=0
    }

    fn exec(&self, _ctx: &Context, msg: &Message, _args: &[&str]) {
        if rand::thread_rng().gen() {
            msg.reply("Heads!").unwrap();
        } else {
            msg.reply("Tails!").unwrap();
        }
    }
}

struct Roll;
impl Command for Roll {
    fn required_arg_count(&self) -> RangeInclusive<usize> {
        0..=1
    }

    fn exec(&self, _ctx: &Context, msg: &Message, args: &[&str]) {
        let upper_bound = if args.len() == 0 {
            6
        } else if args.len() == 1 {
            if let Ok(num) = args[0].parse() {
                num
            } else {
                msg.reply("That is not a number.").unwrap();
                return;
            }
        } else {
            unreachable!();
        };

        if upper_bound <= 0 {
            msg.reply("The upper bound must be a positive number.")
                .unwrap();
        } else {
            let result = rand::thread_rng().gen_range(0, upper_bound) + 1;
            msg.reply(&format!("{}", result)).unwrap();
        }
    }
}
<commit_msg>Inclusive ranges are stable, their fields aren't yet.<commit_after>#![feature(inclusive_range_fields, range_contains)]

extern crate dotenv;
extern crate itertools;
#[macro_use]
extern crate lazy_static;
extern crate rand;
extern crate serenity;

use std::collections::BTreeMap;
use std::env;
use std::ops::RangeInclusive;

use dotenv::dotenv;
use itertools::Itertools;
use rand::Rng;
use serenity::prelude::*;
use serenity::model::prelude::*;

trait Command {
    fn required_arg_count(&self) -> RangeInclusive<usize>; //impl Range<usize>;
    fn exec(&self, ctx: &Context, msg: &Message, args: &[&str]);
}
type CommandMap = BTreeMap<&'static str, Box<Command + Sync>>;

lazy_static! {
    static ref COMMAND_MAP: CommandMap = {
        let mut commands = CommandMap::new();
        commands.insert("help", Box::new(Help));
        commands.insert("flip", Box::new(Flip));
        commands.insert("roll", Box::new(Roll));
        commands
    };
}

fn main() {
    let _ = dotenv();

    let creds = env::var("Z_CREDENTIALS")
        .expect("Unspecified credentials: set Z_CREDENTIALS in the environment (or in `.env`).");
    let mut client = Client::new(&creds, Handler).unwrap();
    client.start().unwrap();
}

struct Handler;
impl EventHandler for Handler {
    fn ready(&self, ctx: Context, _: Ready) {
        ctx.set_presence(Some(Game::playing("Rust")), OnlineStatus::Online)
    }

    fn message(&self, ctx: Context, msg: Message) {
        if msg.content.chars().next() == Some('!') {
            let mut tokens = msg.content[1..].split_whitespace();
            let maybe_command = tokens.next();
            let args = tokens.collect::<Vec<_>>();

            if let Some(command) = maybe_command {
                if let Some(command_handler) = COMMAND_MAP.get(command) {
                    let valid_arg_range = command_handler.required_arg_count();
                    if valid_arg_range.contains(args.len()) {
                        command_handler.exec(&ctx, &msg, &*args);
                    } else {
                        msg.reply(&format!(
                            "This command requires {} to {} arguments.",
                            valid_arg_range.start, valid_arg_range.end
                        )).unwrap();
                    }
                } else {
                    msg.reply(&format!("Unknown command: {}", command)).unwrap();
                }
            } else {
                msg.reply("No command specified.").unwrap();
            }
        }
    }
}

struct Help;
impl Command for Help {
    fn required_arg_count(&self) -> RangeInclusive<usize> {
        0..=0
    }

    fn exec(&self, _ctx: &Context, msg: &Message, _args: &[&str]) {
        let help_output = COMMAND_MAP
            .keys()
            .intersperse(&", !")
            .fold("Available commands:\n!".to_owned(), |output, command| {
                output + command
            });

        msg.channel_id.say(&help_output).unwrap();
    }
}

struct Flip;
impl Command for Flip {
    fn required_arg_count(&self) -> RangeInclusive<usize> {
        0..=0
    }

    fn exec(&self, _ctx: &Context, msg: &Message, _args: &[&str]) {
        if rand::thread_rng().gen() {
            msg.reply("Heads!").unwrap();
        } else {
            msg.reply("Tails!").unwrap();
        }
    }
}

struct Roll;
impl Command for Roll {
    fn required_arg_count(&self) -> RangeInclusive<usize> {
        0..=1
    }

    fn exec(&self, _ctx: &Context, msg: &Message, args: &[&str]) {
        let upper_bound = if args.len() == 0 {
            6
        } else if args.len() == 1 {
            if let Ok(num) = args[0].parse() {
                num
            } else {
                msg.reply("That is not a number.").unwrap();
                return;
            }
        } else {
            unreachable!();
        };

        if upper_bound <= 0 {
            msg.reply("The upper bound must be a positive number.")
                .unwrap();
        } else {
            let result = rand::thread_rng().gen_range(0, upper_bound) + 1;
            msg.reply(&format!("{}", result)).unwrap();
        }
    }
}
<|endoftext|>


<commit_before>extern crate mmap;
extern crate time;
extern crate timely;
extern crate docopt;

use docopt::Docopt;

use std::fs::File;
use std::thread;
use std::io::{Read, BufRead, BufReader};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_shared::*;
use timely::example_shared::operators::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;

use timely::drain::DrainExt;

mod typedrw;
mod graphmap;
use graphmap::GraphMMap;

mod sorting;
use sorting::{SegmentList, radix_sort_32};

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -a <arg>, --aggregation <arg>  the aggregation strategy, \"worker\" or \"process\" [default: worker]
    -h <arg>, --hosts <arg>        file with list of host:port for workers
    -n <arg>, --processes <arg>    number of processes involved  [default: 1]
    -p <arg>, --processid <arg>    identity of this process      [default: 0]
    -w <arg>, --workers <arg>      number of workers per process [default: 1]
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    let source = args.get_str("<source>").to_owned();

    let agg_strategy = args.get_str("-a").to_owned();

    if agg_strategy != "process" && agg_strategy != "worker" {
        panic!("invalid setting for --aggregation: {}", args.get_str("-a"));
    }

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);
    println!("\taggregation:\t{}", agg_strategy);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let addresses: Vec<_> = if hosts != "" {
            let reader = BufReader::new(File::open(hosts).unwrap());
            reader.lines().take(processes as usize).map(|x| x.unwrap()).collect()
        }
        else {
            (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect()
        };

        if addresses.len() != processes as usize { panic!("only {} hosts for -p: {}", addresses.len(), processes); }

        let communicators = initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking");

        pagerank_spawn(communicators, source, agg_strategy);
    }
    else if workers > 1 { pagerank_spawn(ProcessCommunicator::new_vector(workers), source, agg_strategy); }
    else { pagerank_spawn(vec![ThreadCommunicator], source, agg_strategy); };
}

fn pagerank_spawn<C>(communicators: Vec<C>, filename: String, agg_strategy: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    let process_agg = agg_strategy == "process";
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank_thread(communicator, filename, workers, process_agg))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

// returns [src/peers] degrees, (dst, deg) pairs, and a list of [src/peers] endpoints
fn transpose(mut edges: Vec<Vec<(u32, u32)>>, peers: usize, nodes: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let mut deg = vec![0; (nodes as usize / peers) + 1];
    for list in &edges {
        for &(s, _) in list {
            deg[s as usize / peers] += 1;
        }
    }

    radix_sort_32(&mut edges, &mut Vec::new(), &|&(_,d)| d);

    let mut rev = Vec::<(u32,u32)>::with_capacity(deg.len());
    let mut trn = Vec::with_capacity(edges.len() * 1024);
    for list in edges {
        for (s,d) in list {
            let len = rev.len();
            if (len == 0) || (rev[len-1].0 < d) {
                rev.push((d, 0u32));
            }

            let len = rev.len();
            rev[len-1].1 += 1;
            trn.push(s / peers as u32);
        }
    }

    return (deg, rev, trn);
}

// pagerank dataflow graph has a set of edges as input, and a binary vertex that for each epoch of
// received edges initiates an iterative subcomputation to compute the pagerank.

fn pagerank_thread<C>(communicator: C, filename: String, _workers: usize, _process_aggregate: bool)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    let start = time::precise_time_s();

    let nodes = GraphMMap::new(&filename).nodes();

    let mut segments = SegmentList::new(1024); // list of edge segments

    let mut src = vec![];   // holds ranks
    let mut deg = vec![];   // holds source degrees
    let mut rev = vec![];   // holds (dst, deg) pairs
    let mut trn = vec![];   // holds transposed sources

    let mut going = start;

    let mut input = root.subcomputation(|builder| {

        let (input, edges) = builder.new_input::<(u32, u32)>();
        let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let ranks = edges.binary_notify(&ranks,
                            Exchange::new(|x: &(u32,u32)| x.0 as u64),
                            Exchange::new(|x: &(u32,f32)| x.0 as u64),
                            format!("pagerank"),
                            vec![RootTimestamp::new(0)],
                            move |input1, input2, output, notificator| {

            // receive incoming edges (should only be iter 0)
            while let Some((_index, data)) = input1.pull() {
                segments.push(data.drain_temp());
            }

            // all inputs received for iter, commence multiplication
            while let Some((iter, _)) = notificator.next() {

                let now = time::now();

                if index == 0 { println!("{}:{}:{}.{} starting iteration {}", now.tm_hour, now.tm_min, now.tm_sec, now.tm_nsec, iter.inner); }

                // if the very first iteration, prepare some stuff.
                // specifically, transpose edges and sort by destination.
                if iter.inner == 0 {
                    let (a, b, c) = transpose(segments.finalize(), peers, nodes);
                    deg = a; rev = b; trn = c;
                    src = vec![0.0f32; deg.len()];
                }

                // record some timings in order to estimate per-iteration times
                if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                // prepare src for transmitting to destinations
                for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }

                // wander through destinations
                let mut trn_slice = &trn[..];
                let mut rev_slice = &rev[..];
                while rev_slice.len() > 0 {
                    // TODO: session should flush automatically...
                    let mut session = output.session(&iter);
                    let next = std::cmp::min(200_000, rev_slice.len());
                    for &(dst, deg) in &rev_slice[..next] {
                        let mut accum = 0.0;
                        for &s in &trn_slice[..deg as usize] {
                            accum += src[s as usize];
                        }
                        trn_slice = &trn_slice[deg as usize..];
                        session.give((dst, accum));
                    }
                    rev_slice = &rev_slice[next..];
                }

                // clean out src to accumulate into
                for s in &mut src { *s = 0.0; }
            }

            // receive data from workers, accumulate in src
            while let Some((iter, data)) = input2.pull() {
                notificator.notify_at(&iter);
                for x in data.drain_temp() {
                    src[x.0 as usize / peers] += x.1;
                }
            }
        });

        // optionally, do process-local accumulation
        if _process_aggregate {
            let local_base = _workers * (index / _workers);
            let local_index = index % _workers;
            let mut acc = vec![0.0; (nodes / _workers) + 1];   // holds ranks
            let ranks = ranks.unary_notify(
                Exchange::new(move |x: &(u32,f32)| (local_base as u64 + (x.0 as u64 % _workers as u64))),
                format!("Aggregation"),
                vec![],
                move |input, output, iterator| {
                    while let Some((iter, data)) = input.pull() {
                        iterator.notify_at(&iter);
                        for x in data.drain_temp() {
                            acc[x.0 as usize / _workers] += x.1;
                        }
                    }

                    while let Some((item, _)) = iterator.next() {
                        output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
                                                 .map(|(u,f)| ((u * _workers + local_index) as u32, f)));

                        for _ in 0..(1 + (nodes/_workers)) { acc.push(0.0); }
                    }
                }
            );
        }

        ranks.connect_loop(cycle);

        input
    });

    // introduce edges into the computation;
    // allow mmaped file to drop
    {
        let graph = GraphMMap::new(&filename);

        let mut edges = Vec::new();
        for node in 0..graph.nodes() {
            if node % peers == index {
                for dst in graph.edges(node) {
                    edges.push((node as u32, *dst as u32));
                }
                if edges.len() > 100_000 {
                    input.send_at(0, edges.drain_temp());
                    root.step();
                }
            }
        }

        input.send_at(0, edges.drain_temp());
    }
    input.close();
    while root.step() { }

    if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
}
<commit_msg>fixed up aggregation logic<commit_after>extern crate mmap;
extern crate time;
extern crate timely;
extern crate docopt;

use docopt::Docopt;

use std::fs::File;
use std::thread;
use std::io::{Read, BufRead, BufReader};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_shared::*;
use timely::example_shared::operators::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;

use timely::drain::DrainExt;

mod typedrw;
mod graphmap;
use graphmap::GraphMMap;

mod sorting;
use sorting::{SegmentList, radix_sort_32};

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -a <arg>, --aggregation <arg>  the aggregation strategy, \"worker\" or \"process\" [default: worker]
    -h <arg>, --hosts <arg>        file with list of host:port for workers
    -n <arg>, --processes <arg>    number of processes involved  [default: 1]
    -p <arg>, --processid <arg>    identity of this process      [default: 0]
    -w <arg>, --workers <arg>      number of workers per process [default: 1]
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    let source = args.get_str("<source>").to_owned();

    let agg_strategy = args.get_str("-a").to_owned();

    if agg_strategy != "process" && agg_strategy != "worker" {
        panic!("invalid setting for --aggregation: {}", args.get_str("-a"));
    }

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);
    println!("\taggregation:\t{}", agg_strategy);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let addresses: Vec<_> = if hosts != "" {
            let reader = BufReader::new(File::open(hosts).unwrap());
            reader.lines().take(processes as usize).map(|x| x.unwrap()).collect()
        }
        else {
            (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect()
        };

        if addresses.len() != processes as usize { panic!("only {} hosts for -p: {}", addresses.len(), processes); }

        let communicators = initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking");

        pagerank_spawn(communicators, source, agg_strategy);
    }
    else if workers > 1 { pagerank_spawn(ProcessCommunicator::new_vector(workers), source, agg_strategy); }
    else { pagerank_spawn(vec![ThreadCommunicator], source, agg_strategy); };
}

fn pagerank_spawn<C>(communicators: Vec<C>, filename: String, agg_strategy: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    let process_agg = agg_strategy == "process";
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank_thread(communicator, filename, workers, process_agg))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

// returns [src/peers] degrees, (dst, deg) pairs, and a list of [src/peers] endpoints
fn transpose(mut edges: Vec<Vec<(u32, u32)>>, peers: usize, nodes: usize) -> (Vec<u32>, Vec<(u32, u32)>, Vec<u32>)  {

    let mut deg = vec![0; (nodes as usize / peers) + 1];
    for list in &edges {
        for &(s, _) in list {
            deg[s as usize / peers] += 1;
        }
    }

    radix_sort_32(&mut edges, &mut Vec::new(), &|&(_,d)| d);

    let mut rev = Vec::<(u32,u32)>::with_capacity(deg.len());
    let mut trn = Vec::with_capacity(edges.len() * 1024);
    for list in edges {
        for (s,d) in list {
            let len = rev.len();
            if (len == 0) || (rev[len-1].0 < d) {
                rev.push((d, 0u32));
            }

            let len = rev.len();
            rev[len-1].1 += 1;
            trn.push(s / peers as u32);
        }
    }

    return (deg, rev, trn);
}

// pagerank dataflow graph has a set of edges as input, and a binary vertex that for each epoch of
// received edges initiates an iterative subcomputation to compute the pagerank.

fn pagerank_thread<C>(communicator: C, filename: String, _workers: usize, _process_aggregate: bool)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    let start = time::precise_time_s();

    let nodes = GraphMMap::new(&filename).nodes();

    let mut segments = SegmentList::new(1024); // list of edge segments

    let mut src = vec![];   // holds ranks
    let mut deg = vec![];   // holds source degrees
    let mut rev = vec![];   // holds (dst, deg) pairs
    let mut trn = vec![];   // holds transposed sources

    let mut going = start;

    let mut input = root.subcomputation(|builder| {

        let (input, edges) = builder.new_input::<(u32, u32)>();
        let (cycle, ranks) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let mut ranks = edges.binary_notify(&ranks,
                            Exchange::new(|x: &(u32,u32)| x.0 as u64),
                            Exchange::new(|x: &(u32,f32)| x.0 as u64),
                            format!("pagerank"),
                            vec![RootTimestamp::new(0)],
                            move |input1, input2, output, notificator| {

            // receive incoming edges (should only be iter 0)
            while let Some((_index, data)) = input1.pull() {
                segments.push(data.drain_temp());
            }

            // all inputs received for iter, commence multiplication
            while let Some((iter, _)) = notificator.next() {

                let now = time::now();

                if index == 0 { println!("{}:{}:{}.{} starting iteration {}", now.tm_hour, now.tm_min, now.tm_sec, now.tm_nsec, iter.inner); }

                // if the very first iteration, prepare some stuff.
                // specifically, transpose edges and sort by destination.
                if iter.inner == 0 {
                    let (a, b, c) = transpose(segments.finalize(), peers, nodes);
                    deg = a; rev = b; trn = c;
                    src = vec![0.0f32; deg.len()];
                }

                // record some timings in order to estimate per-iteration times
                if iter.inner == 0  && index == 0 { println!("src: {}, dst: {}, edges: {}", src.len(), rev.len(), trn.len()); }
                if iter.inner == 10 && index == 0 { going = time::precise_time_s(); }
                if iter.inner == 20 && index == 0 { println!("average: {}", (time::precise_time_s() - going) / 10.0 ); }

                // prepare src for transmitting to destinations
                for s in 0..src.len() { src[s] = 0.15 + 0.85 * src[s] / deg[s] as f32; }

                // wander through destinations
                let mut trn_slice = &trn[..];
                let mut rev_slice = &rev[..];
                while rev_slice.len() > 0 {
                    // TODO: session should flush automatically...
                    let mut session = output.session(&iter);
                    let next = std::cmp::min(200_000, rev_slice.len());
                    for &(dst, deg) in &rev_slice[..next] {
                        let mut accum = 0.0;
                        for &s in &trn_slice[..deg as usize] {
                            accum += src[s as usize];
                        }
                        trn_slice = &trn_slice[deg as usize..];
                        session.give((dst, accum));
                    }
                    rev_slice = &rev_slice[next..];
                }

                // clean out src to accumulate into
                for s in &mut src { *s = 0.0; }
            }

            // receive data from workers, accumulate in src
            while let Some((iter, data)) = input2.pull() {
                notificator.notify_at(&iter);
                for x in data.drain_temp() {
                    src[x.0 as usize / peers] += x.1;
                }
            }
        });

        // optionally, do process-local accumulation
        if _process_aggregate {
            let local_base = _workers * (index / _workers);
            let local_index = index % _workers;
            let mut acc = vec![0.0; (nodes / _workers) + 1];   // holds ranks
            ranks = ranks.unary_notify(
                Exchange::new(move |x: &(u32,f32)| (local_base as u64 + (x.0 as u64 % _workers as u64))),
                format!("Aggregation"),
                vec![],
                move |input, output, iterator| {
                    while let Some((iter, data)) = input.pull() {
                        iterator.notify_at(&iter);
                        for x in data.drain_temp() {
                            acc[x.0 as usize / _workers] += x.1;
                        }
                    }

                    while let Some((item, _)) = iterator.next() {
                        output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
                                                 .map(|(u,f)| ((u * _workers + local_index) as u32, f)));

                        for _ in 0..(1 + (nodes/_workers)) { acc.push(0.0); }
                    }
                }
            );
        }

        ranks.connect_loop(cycle);

        input
    });

    // introduce edges into the computation;
    // allow mmaped file to drop
    {
        let graph = GraphMMap::new(&filename);

        let mut edges = Vec::new();
        for node in 0..graph.nodes() {
            if node % peers == index {
                for dst in graph.edges(node) {
                    edges.push((node as u32, *dst as u32));
                }
                if edges.len() > 100_000 {
                    input.send_at(0, edges.drain_temp());
                    root.step();
                }
            }
        }

        input.send_at(0, edges.drain_temp());
    }
    input.close();
    while root.step() { }

    if index == 0 { println!("elapsed: {}", time::precise_time_s() - start); }
}
<|endoftext|>


<commit_before>// comment_units V0.1 (2017.05.12), written by Willi Kappler
//
// Licensed under the MIT License
//
// A tool to check the physical units described in comments
//

// External crates:
extern crate clap;
extern crate walkdir;

// Std modules
use std::env;

// External modules:
use clap::{App, Arg};

// Local modules
mod file_util;
mod language_rust;
mod language_fortran;
mod language_matlab;

use file_util::{process_folder, extract_language_extensions, consider_file, process_file,
        supported_languages};

fn main() {
    let matches = App::new("My Super Program")
        .version("0.1")
        .author("Willi Kappler. <grandor@gmx.de>")
        .about("Check physical units in comments")
        .arg(Arg::with_name("directory")
            .short("d")
            .long("directory")
            .value_name("DIRECTORY")
            .help("The input directory to check for source files recursively. Default is current directory")
            .takes_value(true)
            .required(false))
        .arg(Arg::with_name("file")
            .short("f")
            .long("file")
            .value_name("FILE")
            .help("The input file to check, if you just want to check only one source file")
            .takes_value(true)
            .required(false))
        .arg(Arg::with_name("languages")
            .short("l")
            .long("languages")
            .value_name("LANGUAGES")
            .help("The programming language(s) of the input source file(s).\n\
                  Default is to check for all supported languages.\n\
                  Otherwise only the provided soure files will be checked, based on the file extension.\n\
                  You can provie multiple languages (separated by comma), these are currently supported:\n\
                  rust, fortran, matlab"
              )
            .takes_value(true)
            .required(false))
        .get_matches();

    let languages = match matches.value_of("languages") {
        Some(languages) => extract_language_extensions(languages),
        None => supported_languages()
    };

    match matches.value_of("file") {
        Some(input_file) => {
            println!("single input file: '{}''", input_file);
            if let Some(language) = consider_file(input_file, &languages) {
                process_file(input_file, language)
            }
        },
        None => {
            match matches.value_of("directory") {
                Some(input_directory) => {
                    println!("input directory: '{}'", input_directory);
                    process_folder(input_directory, languages);
                }
                None => {
                    println!("current directory");
                    let current_dir = env::current_dir().unwrap();
                    process_folder(current_dir.to_str().unwrap(), languages);
                }
            }
        }
    }

}
<commit_msg>better help message with examples<commit_after>// comment_units V0.1 (2017.05.12), written by Willi Kappler
//
// Licensed under the MIT License
//
// A tool to check the physical units described in comments
//

// External crates:
extern crate clap;
extern crate walkdir;

// Std modules
use std::env;

// External modules:
use clap::{App, Arg};

// Local modules
mod file_util;
mod language_rust;
mod language_fortran;
mod language_matlab;

use file_util::{process_folder, extract_language_extensions, consider_file, process_file,
        supported_languages};

fn main() {
    let matches = App::new("comment_units")
        .version("0.1")
        .author("Willi Kappler. <grandor@gmx.de>")
        .about("Check physical units in comments")
        .arg(Arg::with_name("directory")
            .short("d")
            .long("directory")
            .value_name("DIRECTORY")
            .help("The input directory to check for source files recursively. Default is current directory")
            .takes_value(true)
            .required(false))
        .arg(Arg::with_name("file")
            .short("f")
            .long("file")
            .value_name("FILE")
            .help("The input file to check, if you just want to check only one source file")
            .takes_value(true)
            .required(false))
        .arg(Arg::with_name("languages")
            .short("l")
            .long("languages")
            .value_name("LANGUAGES")
            .help("The programming language(s) of the input source file(s).\n\
                  Default is to check for all supported languages.\n\
                  Otherwise only the provided soure files will be checked, based on the file extension.\n\
                  You can provie multiple languages (separated by comma), these are currently supported:\n\
                  rs (Rust), f90 (Fortran), m (Matlab)")
            .takes_value(true)
            .required(false))
        .after_help(
              "Examples:\n\n\
              comment_units                 # process all supported source files recursively in the current directory.\n\
              \n\
              comment_units -d source       # process all supported source files recursively in the directory 'source'.\n\
              \n\
              comment_units -f main.f90     # process only the file 'main.f90'.\n\
              \n\
              comment_units -d build -l rs  # process all Rust (rs) files recursively in the directory 'build'.\n\
              \n\
              comment_units -l rs,m         # process all Rust and Matlab (rs,m) files recursively in the current directory.\n\
              "
          )
        .get_matches();

    let languages = match matches.value_of("languages") {
        Some(languages) => extract_language_extensions(languages),
        None => supported_languages()
    };

    match matches.value_of("file") {
        Some(input_file) => {
            println!("single input file: '{}''", input_file);
            if let Some(language) = consider_file(input_file, &languages) {
                process_file(input_file, language)
            }
        },
        None => {
            match matches.value_of("directory") {
                Some(input_directory) => {
                    println!("input directory: '{}'", input_directory);
                    process_folder(input_directory, languages);
                }
                None => {
                    println!("current directory");
                    let current_dir = env::current_dir().unwrap();
                    process_folder(current_dir.to_str().unwrap(), languages);
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>extern crate ppbert;
#[macro_use] extern crate clap;

use std::io::{self, Read, Write};
use std::fs;
use std::process::exit;
use std::time::{Duration, Instant};

use clap::{Arg, App};

use ppbert::bertterm::BertTerm;
use ppbert::error::Result;
use ppbert::parser;

const DEFAULT_INDENT_WIDTH: &str = "2";
const DEFAULT_MAX_TERMS_PER_LINE: &str = "5";
const PROG_NAME: &str = "ppbert";

fn main() {
    let matches = App::new(PROG_NAME)
        .version(crate_version!())
        .author("Vincent Foley")
        .about("Pretty print structure encoded in Erlang's External Term Format")
        .arg(Arg::with_name("input_files")
             .value_name("FILES")
             .multiple(true))
        .arg(Arg::with_name("indent_width")
             .help("Indents with <num> spaces")
             .value_name("num")
             .short("i")
             .long("indent-width")
             .default_value(DEFAULT_INDENT_WIDTH)
             .takes_value(true))
        .arg(Arg::with_name("max_per_line")
             .help("Prints at most <num> basic terms per line")
             .value_name("num")
             .short("m")
             .long("max-terms-per-line")
             .default_value(DEFAULT_MAX_TERMS_PER_LINE)
             .takes_value(true))
        .arg(Arg::with_name("verbose")
             .help("Enables verbose mode")
             .short("v")
             .long("verbose"))
        .arg(Arg::with_name("parse")
             .help("Parses the input, doesn't pretty print it")
             .short("p")
             .long("parse"))
        .arg(Arg::with_name("bert2")
             .help("Parses .bert2 files")
             .conflicts_with("disk_log")
             .short("2")
             .long("bert2"))
        .arg(Arg::with_name("disk_log")
             .help("Parses disk_log files")
             .conflicts_with("bert2")
             .short("d")
             .long("disk-log"))
        .arg(Arg::with_name("json")
             .help("Outputs in JSON")
             .short("j")
             .long("json"))
        .arg(Arg::with_name("transform-proplists")
             .help("Transforms proplists into JSON objects (only valid with --json)")
             .short("t")
             .long("transform-proplists"))
        .get_matches();

    let files: Vec<&str> = match matches.values_of("input_files") {
        Some(files) => files.collect(),
        None => vec!["-"]
    };
    let indent_level = value_t!(matches, "indent_width", usize).unwrap();
    let max_per_line = value_t!(matches, "max_per_line", usize).unwrap();
    let verbose = matches.is_present("verbose");
    let parse_only = matches.is_present("parse");
    let json = matches.is_present("json");
    let transform_proplists = matches.is_present("transform-proplists");

    let parse_fn =
        if matches.is_present("bert2") {
            parser::Parser::bert2_next
        } else if matches.is_present("disk_log") {
            parser::Parser::disk_log_next
        } else {
            parser::Parser::bert_next
        };
    let output_fn = match (json, transform_proplists) {
        (true, false)  => pp_json,
        (true, true)   => pp_json_proplist,
        (false, false) => pp_bert,
        (false, true)  => {
            eprintln!("{}: --transform-proplists is only valid with the --json flag", PROG_NAME);
            pp_bert
        }
    };

    let mut return_code = 0;
    for file in files {
        let res = handle_file(file, parse_only, verbose,
                              indent_level, max_per_line,
                              parse_fn, output_fn);
        match res {
            Ok(()) => (),
            Err(e) => {
                return_code = 1;
                eprintln!("{}: {}: {}", PROG_NAME, file, e);
            }
        }
    }
    exit(return_code);
}

fn handle_file(
    file: &str,
    parse_only: bool,
    verbose: bool,
    indent: usize,
    terms_per_line: usize,
    parse_fn: fn(&mut parser::Parser) -> Option<Result<BertTerm>>,
    pp_fn: fn(BertTerm, usize, usize) -> ()
) -> Result<()> {

    // Read file or stdin into buffer
    let now = Instant::now();
    let buf = if file == "-" {
        let stdin = io::stdin();
        let mut stdin = stdin.lock();
        let mut buf: Vec<u8> = Vec::with_capacity(4096);
        stdin.read_to_end(&mut buf)?;
        buf
    } else {
        fs::read(file)?
    };
    let read_dur = now.elapsed();

    let mut parser = parser::Parser::new(buf);
    let mut parse_dur = Duration::new(0, 0);
    let mut pp_dur = Duration::new(0, 0);

    loop {
        let now = Instant::now();
        let next_item = parse_fn(&mut parser);
        parse_dur += now.elapsed();

        match next_item {
            None => { break; }
            Some(Err(e)) => { return Err(e); }
            Some(Ok(t)) => {
                if !parse_only {
                    let now = Instant::now();
                    pp_fn(t, indent, terms_per_line);
                    pp_dur += now.elapsed();
                }
            }
        }
    }

    if verbose {
        eprintln!("{}: read time: {}.{:09}", PROG_NAME, read_dur.as_secs(), read_dur.subsec_nanos());
        eprintln!("{}: parse time: {}.{:09}", PROG_NAME, parse_dur.as_secs(), parse_dur.subsec_nanos());
        if !parse_only {
            eprintln!("{}: print time: {}.{:09}", PROG_NAME, pp_dur.as_secs(), pp_dur.subsec_nanos());
        }
    }

    return Ok(());
}

/// Outputs a BertTerm to stdout.
fn pp_bert(term: BertTerm, indent_width: usize, terms_per_line: usize) {
    let stdout = io::stdout();
    let mut stdout = stdout.lock();
    let _ = term.write_as_erlang(&mut stdout, indent_width, terms_per_line);
    let _ = writeln!(&mut stdout, ""); // newline, ignore SIGPIPE
}

/// Outputs a BertTerm as JSON to stdout.
fn pp_json(term: BertTerm, _: usize, _: usize) {
    let stdout = io::stdout();
    let mut stdout = stdout.lock();
    let _ = term.write_as_json(&mut stdout, false);
    let _ = writeln!(&mut stdout, "");
}

/// Outputs a BertTerm as JSON to stdout;
/// Erlang proplists are converted to JSON objects.
fn pp_json_proplist(term: BertTerm, _: usize, _: usize) {
    let stdout = io::stdout();
    let mut stdout = stdout.lock();
    let _ = term.write_as_json(&mut stdout, true);
    let _ = writeln!(&mut stdout, "");
}
<commit_msg>Add more details to diagnostic messages<commit_after>extern crate ppbert;
#[macro_use] extern crate clap;

use std::io::{self, Read, Write};
use std::fs;
use std::process::exit;
use std::time::{Duration, Instant};

use clap::{Arg, App};

use ppbert::bertterm::BertTerm;
use ppbert::error::Result;
use ppbert::parser;

const DEFAULT_INDENT_WIDTH: &str = "2";
const DEFAULT_MAX_TERMS_PER_LINE: &str = "5";
const PROG_NAME: &str = "ppbert";

fn main() {
    let matches = App::new(PROG_NAME)
        .version(crate_version!())
        .author("Vincent Foley")
        .about("Pretty print structure encoded in Erlang's External Term Format")
        .arg(Arg::with_name("input_files")
             .value_name("FILES")
             .multiple(true))
        .arg(Arg::with_name("indent_width")
             .help("Indents with <num> spaces")
             .value_name("num")
             .short("i")
             .long("indent-width")
             .default_value(DEFAULT_INDENT_WIDTH)
             .takes_value(true))
        .arg(Arg::with_name("max_per_line")
             .help("Prints at most <num> basic terms per line")
             .value_name("num")
             .short("m")
             .long("max-terms-per-line")
             .default_value(DEFAULT_MAX_TERMS_PER_LINE)
             .takes_value(true))
        .arg(Arg::with_name("verbose")
             .help("Enables verbose mode")
             .short("v")
             .long("verbose"))
        .arg(Arg::with_name("parse")
             .help("Parses the input, doesn't pretty print it")
             .short("p")
             .long("parse"))
        .arg(Arg::with_name("bert2")
             .help("Parses .bert2 files")
             .conflicts_with("disk_log")
             .short("2")
             .long("bert2"))
        .arg(Arg::with_name("disk_log")
             .help("Parses disk_log files")
             .conflicts_with("bert2")
             .short("d")
             .long("disk-log"))
        .arg(Arg::with_name("json")
             .help("Outputs in JSON")
             .short("j")
             .long("json"))
        .arg(Arg::with_name("transform-proplists")
             .help("Transforms proplists into JSON objects (only valid with --json)")
             .short("t")
             .long("transform-proplists"))
        .get_matches();

    let files: Vec<&str> = match matches.values_of("input_files") {
        Some(files) => files.collect(),
        None => vec!["-"]
    };
    let indent_level = value_t!(matches, "indent_width", usize).unwrap();
    let max_per_line = value_t!(matches, "max_per_line", usize).unwrap();
    let verbose = matches.is_present("verbose");
    let parse_only = matches.is_present("parse");
    let json = matches.is_present("json");
    let transform_proplists = matches.is_present("transform-proplists");

    let parse_fn =
        if matches.is_present("bert2") {
            parser::Parser::bert2_next
        } else if matches.is_present("disk_log") {
            parser::Parser::disk_log_next
        } else {
            parser::Parser::bert_next
        };
    let output_fn = match (json, transform_proplists) {
        (true, false)  => pp_json,
        (true, true)   => pp_json_proplist,
        (false, false) => pp_bert,
        (false, true)  => {
            eprintln!("{}: --transform-proplists is only valid with the --json flag", PROG_NAME);
            pp_bert
        }
    };

    let mut return_code = 0;
    for file in files {
        let res = handle_file(file, parse_only, verbose,
                              indent_level, max_per_line,
                              parse_fn, output_fn);
        match res {
            Ok(()) => (),
            Err(e) => {
                return_code = 1;
                eprintln!("{}: {}: {}", PROG_NAME, file, e);
            }
        }
    }
    exit(return_code);
}

fn handle_file(
    file: &str,
    parse_only: bool,
    verbose: bool,
    indent: usize,
    terms_per_line: usize,
    parse_fn: fn(&mut parser::Parser) -> Option<Result<BertTerm>>,
    pp_fn: fn(BertTerm, usize, usize) -> ()
) -> Result<()> {

    // Read file or stdin into buffer
    let now = Instant::now();
    let buf = if file == "-" {
        let stdin = io::stdin();
        let mut stdin = stdin.lock();
        let mut buf: Vec<u8> = Vec::with_capacity(4096);
        stdin.read_to_end(&mut buf)?;
        buf
    } else {
        fs::read(file)?
    };
    let read_dur = now.elapsed();

    let mut parser = parser::Parser::new(buf);
    let mut parse_dur = Duration::new(0, 0);
    let mut pp_dur = Duration::new(0, 0);

    loop {
        let now = Instant::now();
        let next_item = parse_fn(&mut parser);
        parse_dur += now.elapsed();

        match next_item {
            None => { break; }
            Some(Err(e)) => { return Err(e); }
            Some(Ok(t)) => {
                if !parse_only {
                    let now = Instant::now();
                    pp_fn(t, indent, terms_per_line);
                    pp_dur += now.elapsed();
                }
            }
        }
    }

    if verbose {
        eprintln!("{}: {} read time: {}.{:06} seconds", PROG_NAME, file, read_dur.as_secs(), read_dur.subsec_micros());
        eprintln!("{}: {} parse time: {}.{:06} seconds", PROG_NAME, file, parse_dur.as_secs(), parse_dur.subsec_micros());
        if !parse_only {
            eprintln!("{}: {} print time: {}.{:06} seconds", PROG_NAME, file, pp_dur.as_secs(), pp_dur.subsec_micros());
        }
    }

    return Ok(());
}

/// Outputs a BertTerm to stdout.
fn pp_bert(term: BertTerm, indent_width: usize, terms_per_line: usize) {
    let stdout = io::stdout();
    let mut stdout = stdout.lock();
    let _ = term.write_as_erlang(&mut stdout, indent_width, terms_per_line);
    let _ = writeln!(&mut stdout, ""); // newline, ignore SIGPIPE
}

/// Outputs a BertTerm as JSON to stdout.
fn pp_json(term: BertTerm, _: usize, _: usize) {
    let stdout = io::stdout();
    let mut stdout = stdout.lock();
    let _ = term.write_as_json(&mut stdout, false);
    let _ = writeln!(&mut stdout, "");
}

/// Outputs a BertTerm as JSON to stdout;
/// Erlang proplists are converted to JSON objects.
fn pp_json_proplist(term: BertTerm, _: usize, _: usize) {
    let stdout = io::stdout();
    let mut stdout = stdout.lock();
    let _ = term.write_as_json(&mut stdout, true);
    let _ = writeln!(&mut stdout, "");
}
<|endoftext|>


<commit_before>#![cfg_attr(feature = "deny-warnings", deny(warnings))]

use rustc_tools_util::VersionInfo;

const CARGO_CLIPPY_HELP: &str = r#"Checks a package to catch common mistakes and improve your Rust code.

Usage:
    cargo clippy [options] [--] [<opts>...]

Common options:
    -h, --help               Print this message
    -V, --version            Print version info and exit

Other options are the same as `cargo check`.

To allow or deny a lint from the command line you can use `cargo clippy --`
with:

    -W --warn OPT       Set lint warnings
    -A --allow OPT      Set lint allowed
    -D --deny OPT       Set lint denied
    -F --forbid OPT     Set lint forbidden

You can use tool lints to allow or deny lints from your code, eg.:

    #[allow(clippy::needless_lifetimes)]
"#;

fn show_help() {
    println!("{}", CARGO_CLIPPY_HELP);
}

fn show_version() {
    let version_info = rustc_tools_util::get_version_info!();
    println!("{}", version_info);
}

pub fn main() {
    // Check for version and help flags even when invoked as 'cargo-clippy'
    if std::env::args().any(|a| a == "--help" || a == "-h") {
        show_help();
        return;
    }

    if std::env::args().any(|a| a == "--version" || a == "-V") {
        show_version();
        return;
    }

    if let Err(code) = process(std::env::args().skip(2)) {
        std::process::exit(code);
    }
}

fn process<I>(mut old_args: I) -> Result<(), i32>
where
    I: Iterator<Item = String>,
{
    let mut args = vec!["check".to_owned()];

    let mut fix = false;
    let mut unstable_options = false;

    for arg in old_args.by_ref() {
        match arg.as_str() {
            "--fix" => {
                fix = true;
                continue;
            }
            "--" => break,
            // Cover -Zunstable-options and -Z unstable-options
            s if s.ends_with("unstable-options") => unstable_options = true,
            _ => {}
        }

        args.push(arg);
    }

    if fix {
        if !unstable_options {
            panic!("Usage of `--fix` requires `-Z unstable-options`");
        } else {
            args[0] = "fix".to_owned();
        }
    }

    let path_env = if unstable_options {
        "RUSTC_WORKSPACE_WRAPPER"
    } else {
        "RUSTC_WRAPPER"
    };

    let clippy_args: String = old_args.map(|arg| format!("{}__CLIPPY_HACKERY__", arg)).collect();

    let mut path = std::env::current_exe()
        .expect("current executable path invalid")
        .with_file_name("clippy-driver");
    if cfg!(windows) {
        path.set_extension("exe");
    }

    let target_dir = std::env::var_os("CLIPPY_DOGFOOD")
        .map(|_| {
            std::env::var_os("CARGO_MANIFEST_DIR").map_or_else(
                || std::ffi::OsString::from("clippy_dogfood"),
                |d| {
                    std::path::PathBuf::from(d)
                        .join("target")
                        .join("dogfood")
                        .into_os_string()
                },
            )
        })
        .map(|p| ("CARGO_TARGET_DIR", p));

    // Run the dogfood tests directly on nightly cargo. This is required due
    // to a bug in rustup.rs when running cargo on custom toolchains. See issue #3118.
    if std::env::var_os("CLIPPY_DOGFOOD").is_some() && cfg!(windows) {
        args.insert(0, "+nightly".to_string());
    }

    let exit_status = std::process::Command::new("cargo")
        .args(&args)
        .env(path_env, path)
        .env("CLIPPY_ARGS", clippy_args)
        .envs(target_dir)
        .spawn()
        .expect("could not run cargo")
        .wait()
        .expect("failed to wait for cargo?");

    if exit_status.success() {
        Ok(())
    } else {
        Err(exit_status.code().unwrap_or(-1))
    }
}
<commit_msg>fix rustfmt issue<commit_after>#![cfg_attr(feature = "deny-warnings", deny(warnings))]

use rustc_tools_util::VersionInfo;

const CARGO_CLIPPY_HELP: &str = r#"Checks a package to catch common mistakes and improve your Rust code.

Usage:
    cargo clippy [options] [--] [<opts>...]

Common options:
    -h, --help               Print this message
    -V, --version            Print version info and exit

Other options are the same as `cargo check`.

To allow or deny a lint from the command line you can use `cargo clippy --`
with:

    -W --warn OPT       Set lint warnings
    -A --allow OPT      Set lint allowed
    -D --deny OPT       Set lint denied
    -F --forbid OPT     Set lint forbidden

You can use tool lints to allow or deny lints from your code, eg.:

    #[allow(clippy::needless_lifetimes)]
"#;

fn show_help() {
    println!("{}", CARGO_CLIPPY_HELP);
}

fn show_version() {
    let version_info = rustc_tools_util::get_version_info!();
    println!("{}", version_info);
}

pub fn main() {
    // Check for version and help flags even when invoked as 'cargo-clippy'
    if std::env::args().any(|a| a == "--help" || a == "-h") {
        show_help();
        return;
    }

    if std::env::args().any(|a| a == "--version" || a == "-V") {
        show_version();
        return;
    }

    if let Err(code) = process(std::env::args().skip(2)) {
        std::process::exit(code);
    }
}

fn process<I>(mut old_args: I) -> Result<(), i32>
where
    I: Iterator<Item = String>,
{
    let mut args = vec!["check".to_owned()];

    let mut fix = false;
    let mut unstable_options = false;

    for arg in old_args.by_ref() {
        match arg.as_str() {
            "--fix" => {
                fix = true;
                continue;
            },
            "--" => break,
            // Cover -Zunstable-options and -Z unstable-options
            s if s.ends_with("unstable-options") => unstable_options = true,
            _ => {},
        }

        args.push(arg);
    }

    if fix {
        if !unstable_options {
            panic!("Usage of `--fix` requires `-Z unstable-options`");
        } else {
            args[0] = "fix".to_owned();
        }
    }

    let path_env = if unstable_options {
        "RUSTC_WORKSPACE_WRAPPER"
    } else {
        "RUSTC_WRAPPER"
    };

    let clippy_args: String = old_args.map(|arg| format!("{}__CLIPPY_HACKERY__", arg)).collect();

    let mut path = std::env::current_exe()
        .expect("current executable path invalid")
        .with_file_name("clippy-driver");
    if cfg!(windows) {
        path.set_extension("exe");
    }

    let target_dir = std::env::var_os("CLIPPY_DOGFOOD")
        .map(|_| {
            std::env::var_os("CARGO_MANIFEST_DIR").map_or_else(
                || std::ffi::OsString::from("clippy_dogfood"),
                |d| {
                    std::path::PathBuf::from(d)
                        .join("target")
                        .join("dogfood")
                        .into_os_string()
                },
            )
        })
        .map(|p| ("CARGO_TARGET_DIR", p));

    // Run the dogfood tests directly on nightly cargo. This is required due
    // to a bug in rustup.rs when running cargo on custom toolchains. See issue #3118.
    if std::env::var_os("CLIPPY_DOGFOOD").is_some() && cfg!(windows) {
        args.insert(0, "+nightly".to_string());
    }

    let exit_status = std::process::Command::new("cargo")
        .args(&args)
        .env(path_env, path)
        .env("CLIPPY_ARGS", clippy_args)
        .envs(target_dir)
        .spawn()
        .expect("could not run cargo")
        .wait()
        .expect("failed to wait for cargo?");

    if exit_status.success() {
        Ok(())
    } else {
        Err(exit_status.code().unwrap_or(-1))
    }
}
<|endoftext|>


<commit_before>#![feature(box_syntax)]
extern crate curl;
extern crate serde;
extern crate num_cpus;
extern crate serde_json;
extern crate getopts;
extern crate crossbeam;
extern crate define;
use define::dictionaries::{Dictionary, Definition, wordnik};
use define::thesaureses::Thesaurus;
use std::collections::HashMap;
use std::sync::Arc;
use std::env;
use std::sync::mpsc;
use getopts::{Matches, Options};

static KEY: &'static str = "a2a73e7b926c924fad7001ca3111acd55af2ffabf50eb4ae5";
static UKEY: &'static str = "ub2JDDg9Iumsh1HfdO3a3HQbZi0up1qe8LkjsnWQvyVvQLFn1q";
const THREAD_ENABLED: bool = false;
struct Config {
    max_definitions: i16,
}

fn parse_args() -> (Options, Matches) {
    let argv: Vec<String> = { let mut args: Vec<String> = env::args().collect(); args.dedup(); args };
    let mut parser = Options::new();
    parser.optflag("t", "thesaurus", "Finds synonyms for word");
    parser.optflag("u", "urban", "Urban Dictionary");
    parser.optopt("", "source", "Set dictionary source", "SOURCE");
    parser.optopt("",
                  "thesaurus-source",
                  "Set thesaurus source (not functional yet)",
                  "SOURCE");
    let parsed = parser.parse(&argv[1..]).unwrap();
    (parser, parsed)
}

fn get_sources()
    -> (HashMap<String, Box<Dictionary + Send>>,
        HashMap<String, Box<Thesaurus>>)
{
    // insert your dictionaries here
    let mut dictionaries: HashMap<String, Box<Dictionary + Send>> = HashMap::new();
    let wordnik = wordnik::Wordnik::new(KEY);
    dictionaries.insert(String::from("wordnik"), box wordnik.clone());
    dictionaries.insert(String::from("example"),
                        box define::dictionaries::example::ExampleDictionary);
    dictionaries.insert(String::from("urban"),
                        box define::dictionaries::urban::Urban::new(UKEY));
    let mut thesaureses: HashMap<String, Box<Thesaurus>> = HashMap::new();
    thesaureses.insert(String::from("wordnik"), box wordnik.clone());
    (dictionaries, thesaureses)
}

fn print_definition<'a, T: Dictionary + ?Sized>(dict: &'a mut Box<T>,
                                                word: &str,
                                                max_definitions: Option<usize>)
                                                -> Result<(), &'a str> {
    let definitions = try!(dict.get_definitions(word));
    let max_definitions = max_definitions.unwrap_or(1);
    if max_definitions > 1 {
        for (wordnumber, word) in definitions.iter().take(max_definitions).enumerate() {
            println!("{}. {}", wordnumber + 1, word.text);
        }
    } else {
        println!("{}", definitions[0].text);
    }
    Ok(())
}

fn print_synonyms<'a, T: Thesaurus + ?Sized>(thes: &'a mut Box<T>,
                                             word: &str)
                                             -> Result<(), &'a str> {
    let synonyms = try!(thes.get_synonyms(word));
    println!("{}", synonyms.join(", "));
    Ok(())
}

fn get_source<'a, T: Dictionary + ?Sized, K: Thesaurus + ?Sized>
    (dictionaries: &'a mut HashMap<String, Box<T>>,
     thesaureses: &'a mut HashMap<String, Box<K>>,
     args: &Matches)
     -> (&'a mut Box<T>, &'a mut Box<K>) {

    let mut source: String = "wordnik".to_owned();
    let tsource: String;
    tsource = args.opt_str("thesaurus-source").unwrap_or("wordnik".to_owned()).to_owned();
    if !args.opt_present("source") {
        for dictionary_source in dictionaries.keys() {
            if args.opt_defined(&dictionary_source) && args.opt_present(&dictionary_source) {
                source = dictionary_source.clone();
            }
        }
    }
    else if args.opt_present("source") {
        source = args.opt_str("source").unwrap_or("wordnik".to_owned()).to_owned();
    }
    let dict: Option<&mut Box<T>> = dictionaries.get_mut(&source);
    let thes: Option<&mut Box<K>> = thesaureses.get_mut(&tsource);
    (dict.unwrap(), thes.unwrap())
}

fn main() {
    let (mut dictionaries, mut thesaureses) = get_sources();
    let (opts, args) = parse_args(); //we needs opts too for printing brief description
    let (dictionary, thesaurus) = get_source(&mut dictionaries, &mut thesaureses, &args);
    if args.free.is_empty() {
        println!("{}", opts.usage("USAGE: define WORD"));
    }
    if ! THREAD_ENABLED {
        for word in &args.free {
            println!("{}:", word.to_uppercase());
            print_definition(dictionary, &word.to_lowercase(), Some(3)).unwrap_or_else(|err| println!("{}", err));
            if args.opt_present("t") {
                println!("SYNONYMS:");
                print_synonyms(thesaurus, word).unwrap_or_else(|err| println!("{}", err));
            }
        }
    }
    else {
        let free_clone = args.free.clone();
        let dict_arc = Arc::new(dictionary.clone_to_box());
        //chunks.map(|chunk| chunks_vec.push(chunk));
        let (tx, rx) = mpsc::channel::<Vec<Definition>>();
        let mut words: Vec<&[String]> = Vec::new();
        let threads = num_cpus::get();;
        {
            let threads = num_cpus::get();
            for chunk in free_clone.chunks((free_clone.len() + threads - 1) / threads) {
                words.push(chunk);
            }
        }
        let words_arc = Arc::new(words);
        for num in 0..words_arc.len() {
            let dict = dict_arc.clone();
            let tx = tx.clone();
            let words = words_arc.clone()[num];
            unsafe {
            crossbeam::spawn_unsafe(move || {
                let mut dictionary = dict.clone_to_box();
                let mut definitions = Vec::new();
                for word in words.iter() {
                    let defs = dictionary.get_definitions(word).unwrap_or(Vec::new());
                    for definition in &defs {
                        definitions.push(definition.clone());
                    }
                }
                tx.send(definitions);
            });
            }
        }
        let mut definitions: HashMap<String, Vec<Definition>> = HashMap::new();
        for _ in 0..words_arc.len() {
            let thread_definitions = rx.recv().unwrap();
            for definition in thread_definitions {
                if definitions.contains_key(&definition.word) {
                    definitions.get_mut(&definition.word).unwrap().push(definition);
                }

                else {
                    definitions.insert(definition.word.clone().to_lowercase(), vec![definition]); //lowercase is necessary because some dictionaries return the definitions in different cases
                }
            }
        }
        for (word, definitions) in definitions.iter() {
                println!("{}:", word.to_uppercase());
                for (number, definition) in definitions.iter().take(3).enumerate() {
                    println!("{}. {}", number + 1, definition.text);
                }
            }
    }
}
<commit_msg>Remove threading for now<commit_after>#![feature(box_syntax)]
extern crate curl;
extern crate serde;
extern crate num_cpus;
extern crate serde_json;
extern crate getopts;
extern crate crossbeam;
extern crate define;
use define::dictionaries::{Dictionary, Definition, wordnik};
use define::thesaureses::Thesaurus;
use std::collections::HashMap;
use std::sync::Arc;
use std::env;
use std::sync::mpsc;
use getopts::{Matches, Options};

static KEY: &'static str = "a2a73e7b926c924fad7001ca3111acd55af2ffabf50eb4ae5";
static UKEY: &'static str = "ub2JDDg9Iumsh1HfdO3a3HQbZi0up1qe8LkjsnWQvyVvQLFn1q";
//const THREAD_ENABLED: bool = true;
struct Config {
    max_definitions: i16,
}

fn parse_args() -> (Options, Matches) {
    let argv: Vec<String> = { let mut args: Vec<String> = env::args().collect(); args.dedup(); args };
    let mut parser = Options::new();
    parser.optflag("t", "thesaurus", "Finds synonyms for word");
    parser.optflag("u", "urban", "Urban Dictionary");
    parser.optopt("", "source", "Set dictionary source", "SOURCE");
    parser.optopt("",
                  "thesaurus-source",
                  "Set thesaurus source (not functional yet)",
                  "SOURCE");
    let parsed = parser.parse(&argv[1..]).unwrap();
    (parser, parsed)
}

fn get_sources()
    -> (HashMap<String, Box<Dictionary + Send>>,
        HashMap<String, Box<Thesaurus>>)
{
    // insert your dictionaries here
    let mut dictionaries: HashMap<String, Box<Dictionary + Send>> = HashMap::new();
    let wordnik = wordnik::Wordnik::new(KEY);
    dictionaries.insert(String::from("wordnik"), box wordnik.clone());
    dictionaries.insert(String::from("example"),
                        box define::dictionaries::example::ExampleDictionary);
    dictionaries.insert(String::from("urban"),
                        box define::dictionaries::urban::Urban::new(UKEY));
    let mut thesaureses: HashMap<String, Box<Thesaurus>> = HashMap::new();
    thesaureses.insert(String::from("wordnik"), box wordnik.clone());
    (dictionaries, thesaureses)
}

fn print_definition<'a, T: Dictionary + ?Sized>(dict: &'a mut Box<T>,
                                                word: &str,
                                                max_definitions: Option<usize>)
                                                -> Result<(), &'a str> {
    let definitions = try!(dict.get_definitions(word));
    let max_definitions = max_definitions.unwrap_or(1);
    if max_definitions > 1 {
        for (wordnumber, word) in definitions.iter().take(max_definitions).enumerate() {
            println!("{}. {}", wordnumber + 1, word.text);
        }
    } else {
        println!("{}", definitions[0].text);
    }
    Ok(())
}

fn print_synonyms<'a, T: Thesaurus + ?Sized>(thes: &'a mut Box<T>,
                                             word: &str)
                                             -> Result<(), &'a str> {
    let synonyms = try!(thes.get_synonyms(word));
    println!("{}", synonyms.join(", "));
    Ok(())
}

fn get_source<'a, T: Dictionary + ?Sized, K: Thesaurus + ?Sized>
    (dictionaries: &'a mut HashMap<String, Box<T>>,
     thesaureses: &'a mut HashMap<String, Box<K>>,
     args: &Matches)
     -> (&'a mut Box<T>, &'a mut Box<K>) {

    let mut source: String = "wordnik".to_owned();
    let tsource: String;
    tsource = args.opt_str("thesaurus-source").unwrap_or("wordnik".to_owned()).to_owned();
    if !args.opt_present("source") {
        for dictionary_source in dictionaries.keys() {
            if args.opt_defined(&dictionary_source) && args.opt_present(&dictionary_source) {
                source = dictionary_source.clone();
            }
        }
    }
    else if args.opt_present("source") {
        source = args.opt_str("source").unwrap_or("wordnik".to_owned()).to_owned();
    }
    let dict: Option<&mut Box<T>> = dictionaries.get_mut(&source);
    let thes: Option<&mut Box<K>> = thesaureses.get_mut(&tsource);
    (dict.unwrap(), thes.unwrap())
}

fn main() {
    let (mut dictionaries, mut thesaureses) = get_sources();
    let (opts, args) = parse_args(); //we needs opts too for printing brief description
    let (dictionary, thesaurus) = get_source(&mut dictionaries, &mut thesaureses, &args);
    if args.free.is_empty() {
        println!("{}", opts.usage("USAGE: define WORD"));
    }
    for word in &args.free {
        println!("{}:", word.to_uppercase());
        print_definition(dictionary, &word.to_lowercase(), Some(3)).unwrap_or_else(|err| println!("{}", err));
        if args.opt_present("t") {
            println!("SYNONYMS:");
            print_synonyms(thesaurus, word).unwrap_or_else(|err| println!("{}", err));
        }
    }
}
<|endoftext|>