<commit_before>//! Legalization of heaps.
//!
//! This module exports the `expand_heap_addr` function which transforms a `heap_addr`
//! instruction into code that depends on the kind of heap referenced.

use cursor::{Cursor, FuncCursor};
use flowgraph::ControlFlowGraph;
use ir::condcodes::IntCC;
use ir::{self, InstBuilder};
use isa::TargetIsa;

/// Expand a `heap_addr` instruction according to the definition of the heap.
pub fn expand_heap_addr(
    inst: ir::Inst,
    func: &mut ir::Function,
    cfg: &mut ControlFlowGraph,
    _isa: &TargetIsa,
) {
    // Unpack the instruction.
    let (heap, offset, size) = match func.dfg[inst] {
        ir::InstructionData::HeapAddr {
            opcode,
            heap,
            arg,
            imm,
        } => {
            debug_assert_eq!(opcode, ir::Opcode::HeapAddr);
            (heap, arg, imm.into())
        }
        _ => panic!("Wanted heap_addr: {}", func.dfg.display_inst(inst, None)),
    };

    match func.heaps[heap].style {
        ir::HeapStyle::Dynamic { bound_gv } => {
            dynamic_addr(inst, heap, offset, size, bound_gv, func)
        }
        ir::HeapStyle::Static { bound } => {
            static_addr(inst, heap, offset, size, bound.into(), func, cfg)
        }
    }
}

/// Expand a `heap_addr` for a dynamic heap.
fn dynamic_addr(
    inst: ir::Inst,
    heap: ir::Heap,
    offset: ir::Value,
    size: u32,
    bound_gv: ir::GlobalValue,
    func: &mut ir::Function,
) {
    let size = i64::from(size);
    let offset_ty = func.dfg.value_type(offset);
    let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
    let min_size = func.heaps[heap].min_size.into();
    let mut pos = FuncCursor::new(func).at_inst(inst);
    pos.use_srcloc(inst);

    // Start with the bounds check. Trap if `offset + size > bound`.
    let bound = pos.ins().global_value(addr_ty, bound_gv);
    let oob;
    if size == 1 {
        // `offset > bound - 1` is the same as `offset >= bound`.
        oob = pos.ins()
            .icmp(IntCC::UnsignedGreaterThanOrEqual, offset, bound);
    } else if size <= min_size {
        // We know that bound >= min_size, so here we can compare `offset > bound - size` without
        // wrapping.
        let adj_bound = pos.ins().iadd_imm(bound, -size);
        oob = pos.ins()
            .icmp(IntCC::UnsignedGreaterThan, offset, adj_bound);
    } else {
        // We need an overflow check for the adjusted offset.
        let size_val = pos.ins().iconst(offset_ty, size);
        let (adj_offset, overflow) = pos.ins().iadd_cout(offset, size_val);
        pos.ins().trapnz(overflow, ir::TrapCode::HeapOutOfBounds);
        oob = pos.ins()
            .icmp(IntCC::UnsignedGreaterThan, adj_offset, bound);
    }
    pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);

    offset_addr(inst, heap, addr_ty, offset, offset_ty, pos.func);
}

/// Expand a `heap_addr` for a static heap.
fn static_addr(
    inst: ir::Inst,
    heap: ir::Heap,
    offset: ir::Value,
    size: u32,
    bound: i64,
    func: &mut ir::Function,
    cfg: &mut ControlFlowGraph,
) {
    let size = i64::from(size);
    let offset_ty = func.dfg.value_type(offset);
    let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
    let mut pos = FuncCursor::new(func).at_inst(inst);
    pos.use_srcloc(inst);

    // Start with the bounds check. Trap if `offset + size > bound`.
    if size > bound {
        // This will simply always trap since `offset >= 0`.
        pos.ins().trap(ir::TrapCode::HeapOutOfBounds);
        pos.func.dfg.replace(inst).iconst(addr_ty, 0);

        // Split Ebb, as the trap is a terminator instruction.
        let curr_ebb = pos.current_ebb().expect("Cursor is not in an ebb");
        let new_ebb = pos.func.dfg.make_ebb();
        pos.insert_ebb(new_ebb);
        cfg.recompute_ebb(pos.func, curr_ebb);
        cfg.recompute_ebb(pos.func, new_ebb);
        return;
    }

    // Check `offset > limit` which is now known non-negative.
    let limit = bound - size;

    // We may be able to omit the check entirely for 32-bit offsets if the heap bound is 4 GB or
    // more.
    if offset_ty != ir::types::I32 || limit < 0xffff_ffff {
        let oob = if limit & 1 == 1 {
            // Prefer testing `offset >= limit - 1` when limit is odd because an even number is
            // likely to be a convenient constant on ARM and other RISC architectures.
            pos.ins()
                .icmp_imm(IntCC::UnsignedGreaterThanOrEqual, offset, limit - 1)
        } else {
            pos.ins()
                .icmp_imm(IntCC::UnsignedGreaterThan, offset, limit)
        };
        pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
    }

    offset_addr(inst, heap, addr_ty, offset, offset_ty, pos.func);
}

/// Emit code for the base address computation of a `heap_addr` instruction.
///
///
fn offset_addr(
    inst: ir::Inst,
    heap: ir::Heap,
    addr_ty: ir::Type,
    mut offset: ir::Value,
    offset_ty: ir::Type,
    func: &mut ir::Function,
) {
    let mut pos = FuncCursor::new(func).at_inst(inst);
    pos.use_srcloc(inst);

    // Convert `offset` to `addr_ty`.
    if offset_ty != addr_ty {
        offset = pos.ins().uextend(addr_ty, offset);
    }

    // Add the heap base address base
    match pos.func.heaps[heap].base {
        ir::HeapBase::ReservedReg => unimplemented!(),
        ir::HeapBase::GlobalValue(base_gv) => {
            let base = pos.ins().global_value(addr_ty, base_gv);
            pos.func.dfg.replace(inst).iadd(base, offset);
        }
    }
}
<commit_msg>Rename local variables to clarify their purpose.<commit_after>//! Legalization of heaps.
//!
//! This module exports the `expand_heap_addr` function which transforms a `heap_addr`
//! instruction into code that depends on the kind of heap referenced.

use cursor::{Cursor, FuncCursor};
use flowgraph::ControlFlowGraph;
use ir::condcodes::IntCC;
use ir::{self, InstBuilder};
use isa::TargetIsa;

/// Expand a `heap_addr` instruction according to the definition of the heap.
pub fn expand_heap_addr(
    inst: ir::Inst,
    func: &mut ir::Function,
    cfg: &mut ControlFlowGraph,
    _isa: &TargetIsa,
) {
    // Unpack the instruction.
    let (heap, offset, access_size) = match func.dfg[inst] {
        ir::InstructionData::HeapAddr {
            opcode,
            heap,
            arg,
            imm,
        } => {
            debug_assert_eq!(opcode, ir::Opcode::HeapAddr);
            (heap, arg, imm.into())
        }
        _ => panic!("Wanted heap_addr: {}", func.dfg.display_inst(inst, None)),
    };

    match func.heaps[heap].style {
        ir::HeapStyle::Dynamic { bound_gv } => {
            dynamic_addr(inst, heap, offset, access_size, bound_gv, func)
        }
        ir::HeapStyle::Static { bound } => {
            static_addr(inst, heap, offset, access_size, bound.into(), func, cfg)
        }
    }
}

/// Expand a `heap_addr` for a dynamic heap.
fn dynamic_addr(
    inst: ir::Inst,
    heap: ir::Heap,
    offset: ir::Value,
    access_size: u32,
    bound_gv: ir::GlobalValue,
    func: &mut ir::Function,
) {
    let access_size = i64::from(access_size);
    let offset_ty = func.dfg.value_type(offset);
    let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
    let min_size = func.heaps[heap].min_size.into();
    let mut pos = FuncCursor::new(func).at_inst(inst);
    pos.use_srcloc(inst);

    // Start with the bounds check. Trap if `offset + access_size > bound`.
    let bound = pos.ins().global_value(addr_ty, bound_gv);
    let oob;
    if access_size == 1 {
        // `offset > bound - 1` is the same as `offset >= bound`.
        oob = pos.ins()
            .icmp(IntCC::UnsignedGreaterThanOrEqual, offset, bound);
    } else if access_size <= min_size {
        // We know that bound >= min_size, so here we can compare `offset > bound - access_size` without
        // wrapping.
        let adj_bound = pos.ins().iadd_imm(bound, -access_size);
        oob = pos.ins()
            .icmp(IntCC::UnsignedGreaterThan, offset, adj_bound);
    } else {
        // We need an overflow check for the adjusted offset.
        let access_size_val = pos.ins().iconst(offset_ty, access_size);
        let (adj_offset, overflow) = pos.ins().iadd_cout(offset, access_size_val);
        pos.ins().trapnz(overflow, ir::TrapCode::HeapOutOfBounds);
        oob = pos.ins()
            .icmp(IntCC::UnsignedGreaterThan, adj_offset, bound);
    }
    pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);

    offset_addr(inst, heap, addr_ty, offset, offset_ty, pos.func);
}

/// Expand a `heap_addr` for a static heap.
fn static_addr(
    inst: ir::Inst,
    heap: ir::Heap,
    offset: ir::Value,
    access_size: u32,
    bound: i64,
    func: &mut ir::Function,
    cfg: &mut ControlFlowGraph,
) {
    let access_size = i64::from(access_size);
    let offset_ty = func.dfg.value_type(offset);
    let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
    let mut pos = FuncCursor::new(func).at_inst(inst);
    pos.use_srcloc(inst);

    // Start with the bounds check. Trap if `offset + access_size > bound`.
    if access_size > bound {
        // This will simply always trap since `offset >= 0`.
        pos.ins().trap(ir::TrapCode::HeapOutOfBounds);
        pos.func.dfg.replace(inst).iconst(addr_ty, 0);

        // Split Ebb, as the trap is a terminator instruction.
        let curr_ebb = pos.current_ebb().expect("Cursor is not in an ebb");
        let new_ebb = pos.func.dfg.make_ebb();
        pos.insert_ebb(new_ebb);
        cfg.recompute_ebb(pos.func, curr_ebb);
        cfg.recompute_ebb(pos.func, new_ebb);
        return;
    }

    // Check `offset > limit` which is now known non-negative.
    let limit = bound - access_size;

    // We may be able to omit the check entirely for 32-bit offsets if the heap bound is 4 GB or
    // more.
    if offset_ty != ir::types::I32 || limit < 0xffff_ffff {
        let oob = if limit & 1 == 1 {
            // Prefer testing `offset >= limit - 1` when limit is odd because an even number is
            // likely to be a convenient constant on ARM and other RISC architectures.
            pos.ins()
                .icmp_imm(IntCC::UnsignedGreaterThanOrEqual, offset, limit - 1)
        } else {
            pos.ins()
                .icmp_imm(IntCC::UnsignedGreaterThan, offset, limit)
        };
        pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
    }

    offset_addr(inst, heap, addr_ty, offset, offset_ty, pos.func);
}

/// Emit code for the base address computation of a `heap_addr` instruction.
///
///
fn offset_addr(
    inst: ir::Inst,
    heap: ir::Heap,
    addr_ty: ir::Type,
    mut offset: ir::Value,
    offset_ty: ir::Type,
    func: &mut ir::Function,
) {
    let mut pos = FuncCursor::new(func).at_inst(inst);
    pos.use_srcloc(inst);

    // Convert `offset` to `addr_ty`.
    if offset_ty != addr_ty {
        offset = pos.ins().uextend(addr_ty, offset);
    }

    // Add the heap base address base
    match pos.func.heaps[heap].base {
        ir::HeapBase::ReservedReg => unimplemented!(),
        ir::HeapBase::GlobalValue(base_gv) => {
            let base = pos.ins().global_value(addr_ty, base_gv);
            pos.func.dfg.replace(inst).iadd(base, offset);
        }
    }
}
<|endoftext|>


<commit_before>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use core::prelude::*;

use io;
use io::{ReaderUtil, WriterUtil};
use libc;
use libc::{c_int, pid_t};
use os;
use run;
use run::spawn_process;
use pipes;
use str;
use task;

export run;

#[cfg(target_os = "win32")]
fn target_env(lib_path: ~str, prog: ~str) -> ~[(~str,~str)] {

    let mut env = os::env();

    // Make sure we include the aux directory in the path
    assert prog.ends_with(~".exe");
    let aux_path = prog.slice(0u, prog.len() - 4u) + ~".libaux";

    env = do vec::map(env) |pair| {
        let (k,v) = *pair;
        if k == ~"PATH" { (~"PATH", v + ~";" + lib_path + ~";" + aux_path) }
        else { (k,v) }
    };
    if str::ends_with(prog, ~"rustc.exe") {
        env.push((~"RUST_THREADS", ~"1"));
    }
    return env;
}

#[cfg(target_os = "linux")]
#[cfg(target_os = "macos")]
#[cfg(target_os = "freebsd")]
fn target_env(_lib_path: ~str, _prog: ~str) -> ~[(~str,~str)] {
    ~[]
}

// FIXME (#2659): This code is duplicated in core::run::program_output
fn run(lib_path: ~str,
       prog: ~str,
       args: ~[~str],
       env: ~[(~str, ~str)],
       input: Option<~str>) -> {status: int, out: ~str, err: ~str} {

    let pipe_in = os::pipe();
    let pipe_out = os::pipe();
    let pipe_err = os::pipe();
    let pid = spawn_process(prog, args,
                            &Some(env + target_env(lib_path, prog)),
                            &None, pipe_in.in, pipe_out.out, pipe_err.out);

    os::close(pipe_in.in);
    os::close(pipe_out.out);
    os::close(pipe_err.out);
    if pid == -1i32 {
        os::close(pipe_in.out);
        os::close(pipe_out.in);
        os::close(pipe_err.in);
        fail;
    }

    writeclose(pipe_in.out, input);
    let p = pipes::PortSet();
    let ch = p.chan();
    do task::spawn_sched(task::SingleThreaded) |move ch| {
        let errput = readclose(pipe_err.in);
        ch.send((2, errput));
    }
    let ch = p.chan();
    do task::spawn_sched(task::SingleThreaded) |move ch| {
        let output = readclose(pipe_out.in);
        ch.send((1, output));
    }
    let status = run::waitpid(pid);
    let mut errs = ~"";
    let mut outs = ~"";
    let mut count = 2;
    while count > 0 {
        match p.recv() {
          (1, s) => {
            outs = s;
          }
          (2, s) => {
            errs = s;
          }
          _ => { fail }
        };
        count -= 1;
    };
    return {status: status, out: outs, err: errs};
}

fn writeclose(fd: c_int, s: Option<~str>) {
    if s.is_some() {
        let writer = io::fd_writer(fd, false);
        writer.write_str(s.get());
    }

    os::close(fd);
}

fn readclose(fd: c_int) -> ~str {
    // Copied from run::program_output
    let file = os::fdopen(fd);
    let reader = io::FILE_reader(file, false);
    let mut buf = ~"";
    while !reader.eof() {
        let bytes = reader.read_bytes(4096u);
        str::push_str(&mut buf, str::from_bytes(bytes));
    }
    os::fclose(file);
    return buf;
}
<commit_msg>compiletest: more win32 fallout from 2db3abd<commit_after>// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use core::prelude::*;

use io;
use io::{ReaderUtil, WriterUtil};
use libc;
use libc::{c_int, pid_t};
use os;
use run;
use run::spawn_process;
use pipes;
use str;
use task;
use vec;

export run;

#[cfg(target_os = "win32")]
fn target_env(lib_path: ~str, prog: ~str) -> ~[(~str,~str)] {

    let mut env = os::env();

    // Make sure we include the aux directory in the path
    assert prog.ends_with(~".exe");
    let aux_path = prog.slice(0u, prog.len() - 4u) + ~".libaux";

    env = do vec::map(env) |pair| {
        let (k,v) = *pair;
        if k == ~"PATH" { (~"PATH", v + ~";" + lib_path + ~";" + aux_path) }
        else { (k,v) }
    };
    if str::ends_with(prog, ~"rustc.exe") {
        env.push((~"RUST_THREADS", ~"1"));
    }
    return env;
}

#[cfg(target_os = "linux")]
#[cfg(target_os = "macos")]
#[cfg(target_os = "freebsd")]
fn target_env(_lib_path: ~str, _prog: ~str) -> ~[(~str,~str)] {
    ~[]
}

// FIXME (#2659): This code is duplicated in core::run::program_output
fn run(lib_path: ~str,
       prog: ~str,
       args: ~[~str],
       env: ~[(~str, ~str)],
       input: Option<~str>) -> {status: int, out: ~str, err: ~str} {

    let pipe_in = os::pipe();
    let pipe_out = os::pipe();
    let pipe_err = os::pipe();
    let pid = spawn_process(prog, args,
                            &Some(env + target_env(lib_path, prog)),
                            &None, pipe_in.in, pipe_out.out, pipe_err.out);

    os::close(pipe_in.in);
    os::close(pipe_out.out);
    os::close(pipe_err.out);
    if pid == -1i32 {
        os::close(pipe_in.out);
        os::close(pipe_out.in);
        os::close(pipe_err.in);
        fail;
    }

    writeclose(pipe_in.out, input);
    let p = pipes::PortSet();
    let ch = p.chan();
    do task::spawn_sched(task::SingleThreaded) |move ch| {
        let errput = readclose(pipe_err.in);
        ch.send((2, errput));
    }
    let ch = p.chan();
    do task::spawn_sched(task::SingleThreaded) |move ch| {
        let output = readclose(pipe_out.in);
        ch.send((1, output));
    }
    let status = run::waitpid(pid);
    let mut errs = ~"";
    let mut outs = ~"";
    let mut count = 2;
    while count > 0 {
        match p.recv() {
          (1, s) => {
            outs = s;
          }
          (2, s) => {
            errs = s;
          }
          _ => { fail }
        };
        count -= 1;
    };
    return {status: status, out: outs, err: errs};
}

fn writeclose(fd: c_int, s: Option<~str>) {
    if s.is_some() {
        let writer = io::fd_writer(fd, false);
        writer.write_str(s.get());
    }

    os::close(fd);
}

fn readclose(fd: c_int) -> ~str {
    // Copied from run::program_output
    let file = os::fdopen(fd);
    let reader = io::FILE_reader(file, false);
    let mut buf = ~"";
    while !reader.eof() {
        let bytes = reader.read_bytes(4096u);
        str::push_str(&mut buf, str::from_bytes(bytes));
    }
    os::fclose(file);
    return buf;
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#[crate_id = "github.com/mozilla/servo#macros:0.1"];
#[crate_type = "lib"];

#[feature(macro_rules)];

// Spawn a task, capturing the listed variables in a way that avoids the
// move-from-closure error.  This is sugar around the function spawn_with,
// taking care of building a tuple and a lambda.

#[macro_export]
macro_rules! bitfield(
    ($bitfieldname:ident, $getter:ident, $setter:ident, $value:expr) => (
        impl $bitfieldname {
            #[inline]
            pub fn $getter(self) -> bool {
                let $bitfieldname(this) = self;
                (this & $value) != 0
            }

            #[inline]
            pub fn $setter(&mut self, value: bool) {
                let $bitfieldname(this) = *self;
                *self = $bitfieldname((this & !$value) | (if value { $value } else { 0 }))
            }
        }
    )
)

<commit_msg>auto merge of #2021 : SimonSapin/servo/patch-1, r=jdm<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#[crate_id = "github.com/mozilla/servo#macros:0.1"];
#[crate_type = "lib"];

#[feature(macro_rules)];

#[macro_export]
macro_rules! bitfield(
    ($bitfieldname:ident, $getter:ident, $setter:ident, $value:expr) => (
        impl $bitfieldname {
            #[inline]
            pub fn $getter(self) -> bool {
                let $bitfieldname(this) = self;
                (this & $value) != 0
            }

            #[inline]
            pub fn $setter(&mut self, value: bool) {
                let $bitfieldname(this) = *self;
                *self = $bitfieldname((this & !$value) | (if value { $value } else { 0 }))
            }
        }
    )
)

<|endoftext|>


<commit_before>//! [GET /_matrix/client/versions](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-versions)

use std::{collections::BTreeMap, convert::TryInto as _};

use ruma_api::{ruma_api, MatrixVersion};

ruma_api! {
    metadata: {
        description: "Get the versions of the client-server API supported by this homeserver.",
        method: GET,
        name: "api_versions",
        path: "/_matrix/client/versions",
        rate_limited: false,
        authentication: None,
    }

    #[derive(Default)]
    request: {}

    response: {
        /// A list of Matrix client API protocol versions supported by the homeserver.
        pub versions: Vec<String>,

        /// Experimental features supported by the server.
        #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
        pub unstable_features: BTreeMap<String, bool>,
    }

    error: crate::Error
}

impl Request {
    /// Creates an empty `Request`.
    pub fn new() -> Self {
        Self {}
    }
}

impl Response {
    /// Creates a new `Response` with the given `versions`.
    pub fn new(versions: Vec<String>) -> Self {
        Self { versions, unstable_features: BTreeMap::new() }
    }

    /// Extracts known Matrix versions from this response.
    ///
    /// Matrix versions that Ruma cannot parse, or does not know about, are discarded.
    pub fn known_versions(&self) -> Vec<MatrixVersion> {
        let mut versions = vec![];
        for s in &self.versions {
            if let Ok(ver) = s.as_str().try_into() {
                if !versions.contains(&ver) {
                    versions.push(ver)
                }
            }
        }
        versions
    }
}
<commit_msg>client-api: Sort versions in get_supported_versions::Response::known_versions<commit_after>//! [GET /_matrix/client/versions](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-versions)

use std::collections::BTreeMap;

use ruma_api::{ruma_api, MatrixVersion};

ruma_api! {
    metadata: {
        description: "Get the versions of the client-server API supported by this homeserver.",
        method: GET,
        name: "api_versions",
        path: "/_matrix/client/versions",
        rate_limited: false,
        authentication: None,
    }

    #[derive(Default)]
    request: {}

    response: {
        /// A list of Matrix client API protocol versions supported by the homeserver.
        pub versions: Vec<String>,

        /// Experimental features supported by the server.
        #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
        pub unstable_features: BTreeMap<String, bool>,
    }

    error: crate::Error
}

impl Request {
    /// Creates an empty `Request`.
    pub fn new() -> Self {
        Self {}
    }
}

impl Response {
    /// Creates a new `Response` with the given `versions`.
    pub fn new(versions: Vec<String>) -> Self {
        Self { versions, unstable_features: BTreeMap::new() }
    }

    /// Extracts known Matrix versions from this response.
    ///
    /// Matrix versions that Ruma cannot parse, or does not know about, are discarded.
    ///
    /// The versions returned will be sorted from oldest to latest. Use [`.find()`][Iterator::find]
    /// or [`.rfind()`][DoubleEndedIterator::rfind] to look for a minimum or maximum version to use
    /// given some constraint.
    pub fn known_versions(&self) -> impl Iterator<Item = MatrixVersion> + DoubleEndedIterator {
        self.versions
            .iter()
            // Parse, discard unknown versions
            .flat_map(|s| s.parse::<MatrixVersion>())
            // Map to key-value pairs where the key is the major-minor representation
            // (which can be used as a BTreeMap unlike MatrixVersion itself)
            .map(|v| (v.into_parts(), v))
            // Collect to BTreeMap
            .collect::<BTreeMap<_, _>>()
            // Return an iterator over just the values (`MatrixVersion`s)
            .into_values()
    }
}

#[cfg(test)]
mod tests {
    use ruma_api::MatrixVersion;

    use super::Response;

    #[test]
    fn known_versions() {
        let none = Response::new(vec![]);
        assert_eq!(none.known_versions().next(), None);

        let single_known = Response::new(vec!["r0.6.0".to_owned()]);
        assert_eq!(single_known.known_versions().collect::<Vec<_>>(), vec![MatrixVersion::V1_0]);

        let single_unknown = Response::new(vec!["v0.0".to_owned()]);
        assert_eq!(single_unknown.known_versions().next(), None);
    }

    #[test]
    fn known_versions_order() {
        let sorted = Response::new(vec![
            "r0.0.1".to_owned(),
            "r0.5.0".to_owned(),
            "r0.6.0".to_owned(),
            "r0.6.1".to_owned(),
            "v1.1".to_owned(),
            "v1.2".to_owned(),
        ]);
        assert_eq!(
            sorted.known_versions().collect::<Vec<_>>(),
            vec![MatrixVersion::V1_0, MatrixVersion::V1_1, MatrixVersion::V1_2],
        );

        let sorted_reverse = Response::new(vec![
            "v1.2".to_owned(),
            "v1.1".to_owned(),
            "r0.6.1".to_owned(),
            "r0.6.0".to_owned(),
            "r0.5.0".to_owned(),
            "r0.0.1".to_owned(),
        ]);
        assert_eq!(
            sorted_reverse.known_versions().collect::<Vec<_>>(),
            vec![MatrixVersion::V1_0, MatrixVersion::V1_1, MatrixVersion::V1_2],
        );

        let random_order = Response::new(vec![
            "v1.1".to_owned(),
            "r0.6.1".to_owned(),
            "r0.5.0".to_owned(),
            "r0.6.0".to_owned(),
            "r0.0.1".to_owned(),
            "v1.2".to_owned(),
        ]);
        assert_eq!(
            random_order.known_versions().collect::<Vec<_>>(),
            vec![MatrixVersion::V1_0, MatrixVersion::V1_1, MatrixVersion::V1_2],
        );
    }
}
<|endoftext|>


<commit_before>pub mod de;
pub mod doc;
pub mod ser;

pub use lib::clone::Clone;
pub use lib::convert::{From, Into};
pub use lib::default::Default;
pub use lib::fmt::{self, Formatter};
pub use lib::marker::PhantomData;
pub use lib::option::Option::{self, None, Some};
pub use lib::result::Result::{self, Err, Ok};

pub use self::string::from_utf8_lossy;

#[cfg(any(feature = "alloc", feature = "std"))]
pub use lib::{ToString, Vec};

#[cfg(core_try_from)]
pub use lib::convert::TryFrom;

mod string {
    use lib::*;

    #[cfg(any(feature = "std", feature = "alloc"))]
    pub fn from_utf8_lossy(bytes: &[u8]) -> Cow<str> {
        String::from_utf8_lossy(bytes)
    }

    // The generated code calls this like:
    //
    //     let value = &_serde::__private::from_utf8_lossy(bytes);
    //     Err(_serde::de::Error::unknown_variant(value, VARIANTS))
    //
    // so it is okay for the return type to be different from the std case as long
    // as the above works.
    #[cfg(not(any(feature = "std", feature = "alloc")))]
    pub fn from_utf8_lossy(bytes: &[u8]) -> &str {
        // Three unicode replacement characters if it fails. They look like a
        // white-on-black question mark. The user will recognize it as invalid
        // UTF-8.
        str::from_utf8(bytes).unwrap_or("\u{fffd}\u{fffd}\u{fffd}")
    }
}
<commit_msg>Add link to rust-lang/rust#67295<commit_after>pub mod de;
pub mod ser;

// FIXME: #[cfg(doctest)] once https://github.com/rust-lang/rust/issues/67295 is fixed.
pub mod doc;

pub use lib::clone::Clone;
pub use lib::convert::{From, Into};
pub use lib::default::Default;
pub use lib::fmt::{self, Formatter};
pub use lib::marker::PhantomData;
pub use lib::option::Option::{self, None, Some};
pub use lib::result::Result::{self, Err, Ok};

pub use self::string::from_utf8_lossy;

#[cfg(any(feature = "alloc", feature = "std"))]
pub use lib::{ToString, Vec};

#[cfg(core_try_from)]
pub use lib::convert::TryFrom;

mod string {
    use lib::*;

    #[cfg(any(feature = "std", feature = "alloc"))]
    pub fn from_utf8_lossy(bytes: &[u8]) -> Cow<str> {
        String::from_utf8_lossy(bytes)
    }

    // The generated code calls this like:
    //
    //     let value = &_serde::__private::from_utf8_lossy(bytes);
    //     Err(_serde::de::Error::unknown_variant(value, VARIANTS))
    //
    // so it is okay for the return type to be different from the std case as long
    // as the above works.
    #[cfg(not(any(feature = "std", feature = "alloc")))]
    pub fn from_utf8_lossy(bytes: &[u8]) -> &str {
        // Three unicode replacement characters if it fails. They look like a
        // white-on-black question mark. The user will recognize it as invalid
        // UTF-8.
        str::from_utf8(bytes).unwrap_or("\u{fffd}\u{fffd}\u{fffd}")
    }
}
<|endoftext|>


<commit_before>//! The master module provides the Master struct which instructs Workers which
//! portions of the image they should render and collects their results to combine
//! into the final image.

use std::path::PathBuf;
use std::io::prelude::*;
use std::collections::HashMap;
use std::net::ToSocketAddrs;
use std::iter;

use bincode::SizeLimit;
use bincode::rustc_serialize::{encode, decode};
use image;
use mio::tcp::{TcpStream, Shutdown};
use mio::*;

use film::Image;
use exec::Config;
use exec::distrib::{worker, Instructions};
use sampler::BlockQueue;

#[derive(Debug)]
enum DistributedFrame {
    InProgress {
        frame: usize,
        // The number of workers who have reported results for this
        // frame so far
        num_reporting: usize,
        render: Image,
    },
    Completed,
}

impl DistributedFrame {
    pub fn start(frame_num: usize, img_dim: (usize, usize)) -> DistributedFrame {
        DistributedFrame::InProgress { frame: frame_num, num_reporting: 0, render: Image::new(img_dim) }
    }
}

#[derive(Clone, Debug)]
struct WorkerBuffer {
    pub buf: Vec<u8>,
    pub expected_size: usize,
    pub currently_read: usize,
}

impl WorkerBuffer {
    pub fn new() -> WorkerBuffer {
        WorkerBuffer { buf: Vec::new(), expected_size: 8, currently_read: 0 }
    }
}

pub struct Master {
    // Hostnames of the workers to send work too
    workers: Vec<String>,
    connections: Vec<TcpStream>,
    // Temporary buffers to store worker results in as they're
    // read in over TCP
    worker_buffers: Vec<WorkerBuffer>,
    config: Config,
    frames: HashMap<usize, DistributedFrame>,
    img_dim: (usize, usize),
    blocks_per_worker: usize,
    blocks_remainder: usize,
}

impl Master {
    /// Create a new master that will contact the worker nodes passed and
    /// send instructions on what parts of the scene to start rendering
    pub fn start_workers(workers: Vec<String>, config: Config, img_dim: (usize, usize))
                         -> (Master, EventLoop<Master>) {
        // Figure out how many blocks we have for this image and assign them to our workers
        let queue = BlockQueue::new((img_dim.0 as u32, img_dim.1 as u32), (8, 8), (0, 0));
        let blocks_per_worker = queue.len() / workers.len();
        let blocks_remainder = queue.len() % workers.len();
        println!("Have {} workers, each worker does {} blocks w/ {} remainder",
                 workers.len(), blocks_per_worker, blocks_remainder);

        let mut event_loop = EventLoop::<Master>::new().unwrap();
        let mut connections = Vec::new();

        // Connect to each worker and send instructions on what to render
        // I guess this should also be queue'd up and the writes actually performed asynchronously?
        // Otherwise we need to loop on write_all while waiting for the connection to open
        for (i, host) in workers.iter().enumerate() {
            let addr = (&host[..], worker::PORT).to_socket_addrs().unwrap().next().unwrap();
            println!("Connecting too {:?}", addr);
            match TcpStream::connect(&addr) {
                Ok(stream) => {
                    println!("Connected and writing");
                    println!("Adding stream to event loop");
                    // Each worker is identified in the event loop by their index in the vec
                    match event_loop.register(&stream, Token(i), EventSet::all(), PollOpt::level()){
                        Err(e) => println!("Error registering stream from {}: {}", host, e),
                        Ok(_) => {},
                    }
                    connections.push(stream);
                },
                Err(e) => println!("Failed to contact worker {}: {:?}", host, e),
            }
        }
        let worker_buffers: Vec<_> = iter::repeat(WorkerBuffer::new()).take(workers.len()).collect();
        let master = Master { workers: workers, connections: connections,
                              worker_buffers: worker_buffers, config: config,
                              frames: HashMap::new(),
                              img_dim: img_dim,
                              blocks_per_worker: blocks_per_worker,
                              blocks_remainder: blocks_remainder };
        (master, event_loop)
    }
    /// Read a result frame from a worker and save it into the list of frames we're collecting from
    /// all workers. Will save out the final render if all workers have reported results for this
    /// frame.
    fn save_results(&mut self, frame: worker::Frame) {
        // Find the frame this worker is referring to and add its results
        let frame_num = frame.frame as usize;
        let img_dim = self.img_dim;
        println!("Worker reporting frame {}", frame_num);
        let mut df = self.frames.entry(frame_num).or_insert_with(|| {
                println!("This is a new frame");
                DistributedFrame::start(frame_num, img_dim)
            });
        let mut finished = false;
        match df {
            &mut DistributedFrame::InProgress { frame: _, ref mut num_reporting, ref mut render } => {
                render.add_blocks(frame.block_size, &frame.blocks, &frame.pixels);
                *num_reporting += 1;
                println!("We have parts of this frame already");
                if *num_reporting == self.workers.len() {
                    println!("We have completed frame {}", frame_num);
                    let out_file = match self.config.out_path.extension() {
                        Some(_) => self.config.out_path.clone(),
                        None => self.config.out_path.join(
                            PathBuf::from(format!("frame{:05}.png", frame_num))),
                    };
                    println!("Frame {}: rendered to '{}'\n--------------------",
                             frame_num, out_file.display());
                    let img = render.get_srgb8();
                    let dim = render.dimensions();
                    match image::save_buffer(&out_file.as_path(), &img[..], dim.0 as u32,
                    dim.1 as u32, image::RGB(8)) {
                        Ok(_) => {},
                        Err(e) => println!("Error saving image, {}", e),
                    };
                    finished = true;
                }
            },
            &mut DistributedFrame::Completed => println!("Worker reporting on completed frame!?"),
        }
        // This is also a bit awkward, since we borrow the df in the match
        // we can't mark it finished in there
        if finished {
            *df = DistributedFrame::Completed;
        }
    }
    fn read_worker_buffer(&mut self, worker: usize) -> bool {
        let mut buf = &mut self.worker_buffers[worker];
        // If we haven't read the size of data being sent, read that now
        if buf.currently_read < 8 {
            // First 8 bytes are a u64 specifying the number of bytes being sent
            buf.buf.extend(iter::repeat(0u8).take(8));
            println!("Reading size header...");
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
            println!("Read {} bytes so far, must read {} in total", buf.currently_read, buf.expected_size);
            if buf.currently_read == buf.expected_size {
                println!("Got size header");
                // How many bytes we expect to get from the worker for a frame
                buf.expected_size = decode(&buf.buf[..]).unwrap();
                println!("Expecting {} bytes", buf.expected_size);
                // Extend the Vec so we've got enough room for the remaning bytes, minus the 8 for the
                // encoded size header
                buf.buf.extend(iter::repeat(0u8).take(buf.expected_size - 8));
            }
        }
        // If we've finished reading the size header we can now start reading the frame data
        if buf.currently_read >= 8 {
            println!("Reading from worker");
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
            println!("Read {} bytes so far, must read {} in total", buf.currently_read, buf.expected_size);
        }
        buf.currently_read == buf.expected_size
    }
}

impl Handler for Master {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self, event_loop: &mut EventLoop<Master>, token: Token, event: EventSet) {
        let worker = token.as_usize();
        println!("Event from {}: {:?}", worker, event);
        if event.is_error() {
            println!("Error connecting too {}", self.workers[worker]);
            match self.connections[worker].shutdown(Shutdown::Both) {
                Err(e) => println!("Error shutting down worker {}: {}", worker, e),
                Ok(_) => {},
            }
            // Remove the connection from the event loop
            match event_loop.deregister(&self.connections[worker]) {
                Err(e) => println!("Error deregistering worker {}: {}", worker, e),
                Ok(_) => {},
            }
        }
        // A worker is ready to receive instructions from us
        if event.is_writable() {
            let b_start = worker * self.blocks_per_worker;
            let b_count =
                if worker == self.workers.len() - 1 {
                    self.blocks_per_worker + self.blocks_remainder
                } else {
                    self.blocks_per_worker
                };
            let instr = Instructions::new(&self.config.scene_file,
                                          (self.config.frame_info.start, self.config.frame_info.end),
                                          b_start, b_count);
            println!("Sending instructions {:?} to {}", instr, self.workers[worker]);
            let bytes = encode(&instr, SizeLimit::Infinite).unwrap();
            println!("Encoded instructions size: {}", instr.encoded_size);
            // Loop until we successfully write the byes
            match self.connections[worker].write_all(&bytes[..]) {
                Err(e) => println!("Failed to send instructions to {}: {:?}", self.workers[worker], e),
                Ok(_) => println!("Instructions sent"),
            }
            // Register that we no longer care about writable events on this connection
            event_loop.reregister(&self.connections[worker], token,
                                  EventSet::readable() | EventSet::error() | EventSet::hup(),
                                  PollOpt::level()).expect("Re-registering failed");
        }
        // Some results are available from a worker
        if event.is_readable() {
            println!("Readable event from worker {}", worker);
            if self.read_worker_buffer(worker) {
                println!("Got all frame data, now parsing");
                let frame: worker::Frame = decode(&self.worker_buffers[worker].buf[..]).unwrap();
                println!("Saving frame results");
                self.save_results(frame);
                // Clean up our read buffer
                self.worker_buffers[worker].buf.clear();
                self.worker_buffers[worker].expected_size = 8;
                self.worker_buffers[worker].currently_read = 0;
            }
        }
        // If the worker has terminated, shutdown the read end of the connection
        if event.is_hup() {
            println!("Worker {} has hung up", worker);
            match self.connections[worker].shutdown(Shutdown::Read) {
                Err(e) => println!("Error shutting down worker {}: {}", worker, e),
                Ok(_) => {},
            }
            // Remove the connection from the event loop
            match event_loop.deregister(&self.connections[worker]) {
                Err(e) => println!("Error deregistering worker {}: {}", worker, e),
                Ok(_) => {},
            }
        }
        // After getting results from the worker we check if we've completed all our frames
        // and exit if so
        let all_complete = self.frames.values().fold(true,
                                |all, v| {
                                    match v {
                                        &DistributedFrame::Completed => true && all,
                                        _ => false,
                                    }
                                });
        // The frame start/end range is inclusive, so we must add 1 here
        let num_frames = self.config.frame_info.end - self.config.frame_info.start + 1;
        println!("Rendering {} frames have {}, all_complete? {}",
                 num_frames, self.frames.len(), all_complete);
        // TODO: We can't use frame_info.frames as this is not the `subset` of frames
        // we're rendering
        if self.frames.len() == num_frames && all_complete {
            println!("All frames have been rendered, master exiting");
            event_loop.shutdown();
        }
    }
}

<commit_msg>Some more minor clean up<commit_after>//! The master module provides the Master struct which instructs Workers which
//! portions of the image they should render and collects their results to combine
//! into the final image.

use std::path::PathBuf;
use std::io::prelude::*;
use std::collections::HashMap;
use std::net::ToSocketAddrs;
use std::iter;

use bincode::SizeLimit;
use bincode::rustc_serialize::{encode, decode};
use image;
use mio::tcp::{TcpStream, Shutdown};
use mio::*;

use film::Image;
use exec::Config;
use exec::distrib::{worker, Instructions};
use sampler::BlockQueue;

#[derive(Debug)]
enum DistributedFrame {
    InProgress {
        frame: usize,
        // The number of workers who have reported results for this
        // frame so far
        num_reporting: usize,
        render: Image,
    },
    Completed,
}

impl DistributedFrame {
    pub fn start(frame_num: usize, img_dim: (usize, usize)) -> DistributedFrame {
        DistributedFrame::InProgress { frame: frame_num, num_reporting: 0, render: Image::new(img_dim) }
    }
}

#[derive(Clone, Debug)]
struct WorkerBuffer {
    pub buf: Vec<u8>,
    pub expected_size: usize,
    pub currently_read: usize,
}

impl WorkerBuffer {
    pub fn new() -> WorkerBuffer {
        WorkerBuffer { buf: Vec::new(), expected_size: 8, currently_read: 0 }
    }
}

pub struct Master {
    // Hostnames of the workers to send work too
    workers: Vec<String>,
    connections: Vec<TcpStream>,
    // Temporary buffers to store worker results in as they're
    // read in over TCP
    worker_buffers: Vec<WorkerBuffer>,
    config: Config,
    frames: HashMap<usize, DistributedFrame>,
    img_dim: (usize, usize),
    blocks_per_worker: usize,
    blocks_remainder: usize,
}

impl Master {
    /// Create a new master that will contact the worker nodes passed and
    /// send instructions on what parts of the scene to start rendering
    pub fn start_workers(workers: Vec<String>, config: Config, img_dim: (usize, usize))
                         -> (Master, EventLoop<Master>) {
        // Figure out how many blocks we have for this image and assign them to our workers
        let queue = BlockQueue::new((img_dim.0 as u32, img_dim.1 as u32), (8, 8), (0, 0));
        let blocks_per_worker = queue.len() / workers.len();
        let blocks_remainder = queue.len() % workers.len();
        println!("Have {} workers, each worker does {} blocks w/ {} remainder",
                 workers.len(), blocks_per_worker, blocks_remainder);

        let mut event_loop = EventLoop::<Master>::new().unwrap();
        let mut connections = Vec::new();

        // Connect to each worker and send instructions on what to render
        // I guess this should also be queue'd up and the writes actually performed asynchronously?
        // Otherwise we need to loop on write_all while waiting for the connection to open
        for (i, host) in workers.iter().enumerate() {
            let addr = (&host[..], worker::PORT).to_socket_addrs().unwrap().next().unwrap();
            println!("Connecting too {:?}", addr);
            match TcpStream::connect(&addr) {
                Ok(stream) => {
                    println!("Connected and writing");
                    println!("Adding stream to event loop");
                    // Each worker is identified in the event loop by their index in the vec
                    match event_loop.register(&stream, Token(i), EventSet::all(), PollOpt::level()){
                        Err(e) => println!("Error registering stream from {}: {}", host, e),
                        Ok(_) => {},
                    }
                    connections.push(stream);
                },
                Err(e) => println!("Failed to contact worker {}: {:?}", host, e),
            }
        }
        let worker_buffers: Vec<_> = iter::repeat(WorkerBuffer::new()).take(workers.len()).collect();
        let master = Master { workers: workers, connections: connections,
                              worker_buffers: worker_buffers, config: config,
                              frames: HashMap::new(),
                              img_dim: img_dim,
                              blocks_per_worker: blocks_per_worker,
                              blocks_remainder: blocks_remainder };
        (master, event_loop)
    }
    /// Read a result frame from a worker and save it into the list of frames we're collecting from
    /// all workers. Will save out the final render if all workers have reported results for this
    /// frame.
    fn save_results(&mut self, frame: worker::Frame) {
        // Find the frame this worker is referring to and add its results
        let frame_num = frame.frame as usize;
        let img_dim = self.img_dim;
        println!("Worker reporting frame {}", frame_num);
        let mut df = self.frames.entry(frame_num).or_insert_with(|| {
                        println!("This is a new frame");
                        DistributedFrame::start(frame_num, img_dim)
                    });
        let mut finished = false;
        match df {
            &mut DistributedFrame::InProgress { frame: _, ref mut num_reporting, ref mut render } => {
                render.add_blocks(frame.block_size, &frame.blocks, &frame.pixels);
                *num_reporting += 1;
                if *num_reporting == self.workers.len() {
                    println!("We have completed frame {}", frame_num);
                    let out_file = match self.config.out_path.extension() {
                        Some(_) => self.config.out_path.clone(),
                        None => self.config.out_path.join(
                            PathBuf::from(format!("frame{:05}.png", frame_num))),
                    };
                    println!("Frame {}: rendered to '{}'\n--------------------",
                             frame_num, out_file.display());
                    let img = render.get_srgb8();
                    let dim = render.dimensions();
                    match image::save_buffer(&out_file.as_path(), &img[..], dim.0 as u32,
                    dim.1 as u32, image::RGB(8)) {
                        Ok(_) => {},
                        Err(e) => println!("Error saving image, {}", e),
                    };
                    finished = true;
                }
            },
            &mut DistributedFrame::Completed => println!("Worker reporting on completed frame!?"),
        }
        // This is also a bit awkward, since we borrow the df in the match
        // we can't mark it finished in there
        if finished {
            *df = DistributedFrame::Completed;
        }
    }
    fn read_worker_buffer(&mut self, worker: usize) -> bool {
        let mut buf = &mut self.worker_buffers[worker];
        // If we haven't read the size of data being sent, read that now
        if buf.currently_read < 8 {
            // First 8 bytes are a u64 specifying the number of bytes being sent
            buf.buf.extend(iter::repeat(0u8).take(8));
            println!("Reading size header...");
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
            println!("Read {} bytes so far, must read {} in total", buf.currently_read, buf.expected_size);
            if buf.currently_read == buf.expected_size {
                println!("Got size header");
                // How many bytes we expect to get from the worker for a frame
                buf.expected_size = decode(&buf.buf[..]).unwrap();
                println!("Expecting {} bytes", buf.expected_size);
                // Extend the Vec so we've got enough room for the remaning bytes, minus the 8 for the
                // encoded size header
                buf.buf.extend(iter::repeat(0u8).take(buf.expected_size - 8));
            }
        }
        // If we've finished reading the size header we can now start reading the frame data
        if buf.currently_read >= 8 {
            println!("Reading from worker");
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
            println!("Read {} bytes so far, must read {} in total", buf.currently_read, buf.expected_size);
        }
        buf.currently_read == buf.expected_size
    }
}

impl Handler for Master {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self, event_loop: &mut EventLoop<Master>, token: Token, event: EventSet) {
        let worker = token.as_usize();
        println!("Event from {}: {:?}", worker, event);
        if event.is_error() {
            println!("Error connecting too {}", self.workers[worker]);
            match self.connections[worker].shutdown(Shutdown::Both) {
                Err(e) => println!("Error shutting down worker {}: {}", worker, e),
                Ok(_) => {},
            }
            // Remove the connection from the event loop
            match event_loop.deregister(&self.connections[worker]) {
                Err(e) => println!("Error deregistering worker {}: {}", worker, e),
                Ok(_) => {},
            }
        }
        // If the worker has terminated, shutdown the read end of the connection
        if event.is_hup() {
            println!("Worker {} has hung up", worker);
            match self.connections[worker].shutdown(Shutdown::Both) {
                Err(e) => println!("Error shutting down worker {}: {}", worker, e),
                Ok(_) => {},
            }
            // Remove the connection from the event loop
            match event_loop.deregister(&self.connections[worker]) {
                Err(e) => println!("Error deregistering worker {}: {}", worker, e),
                Ok(_) => {},
            }
        }
        // A worker is ready to receive instructions from us
        if event.is_writable() {
            let b_start = worker * self.blocks_per_worker;
            let b_count =
                if worker == self.workers.len() - 1 {
                    self.blocks_per_worker + self.blocks_remainder
                } else {
                    self.blocks_per_worker
                };
            let instr = Instructions::new(&self.config.scene_file,
                                          (self.config.frame_info.start, self.config.frame_info.end),
                                          b_start, b_count);
            println!("Sending instructions {:?} to {}", instr, self.workers[worker]);
            let bytes = encode(&instr, SizeLimit::Infinite).unwrap();
            println!("Encoded instructions size: {}", instr.encoded_size);
            // Loop until we successfully write the byes
            match self.connections[worker].write_all(&bytes[..]) {
                Err(e) => println!("Failed to send instructions to {}: {:?}", self.workers[worker], e),
                Ok(_) => println!("Instructions sent"),
            }
            // Register that we no longer care about writable events on this connection
            event_loop.reregister(&self.connections[worker], token,
                                  EventSet::readable() | EventSet::error() | EventSet::hup(),
                                  PollOpt::level()).expect("Re-registering failed");
        }
        // Some results are available from a worker
        if event.is_readable() {
            println!("Readable event from worker {}", worker);
            if self.read_worker_buffer(worker) {
                println!("Got all frame data, now parsing");
                let frame: worker::Frame = decode(&self.worker_buffers[worker].buf[..]).unwrap();
                println!("Saving frame results");
                self.save_results(frame);
                // Clean up our read buffer
                self.worker_buffers[worker].buf.clear();
                self.worker_buffers[worker].expected_size = 8;
                self.worker_buffers[worker].currently_read = 0;
            }
        }
        // After getting results from the worker we check if we've completed all our frames
        // and exit if so
        let all_complete = self.frames.values().fold(true,
                                |all, v| {
                                    match v {
                                        &DistributedFrame::Completed => true && all,
                                        _ => false,
                                    }
                                });
        // The frame start/end range is inclusive, so we must add 1 here
        let num_frames = self.config.frame_info.end - self.config.frame_info.start + 1;
        println!("Rendering {} frames have {}, all_complete? {}",
                 num_frames, self.frames.len(), all_complete);
        if self.frames.len() == num_frames && all_complete {
            println!("All frames have been rendered, master exiting");
            event_loop.shutdown();
        }
    }
}

<|endoftext|>


<commit_before>//! The master module provides the Master struct which instructs Workers which
//! portions of the image they should render and collects their results to combine
//! into the final image.

use std::path::PathBuf;
use std::io::prelude::*;
use std::collections::HashMap;
use std::net::ToSocketAddrs;
use std::iter;
use std::time::SystemTime;

use bincode::{Infinite, serialize, deserialize};
use image;
use mio::tcp::{TcpStream, Shutdown};
use mio::*;

use film::Image;
use exec::Config;
use exec::distrib::{worker, Instructions, Frame};
use sampler::BlockQueue;

/// Stores distributed rendering status. The frame is either `InProgress` and contains
/// partially rendered results from the workers who've reported the frame or is `Completed`
/// and has been saved out to disk.
#[derive(Debug)]
enum DistributedFrame {
    InProgress {
        // The number of workers who have reported results for this
        // frame so far
        num_reporting: usize,
        render: Image,
        // Start time of this frame, when we got the first tiles in from a worker
        first_tile_recv: SystemTime,
    },
    Completed,
}

impl DistributedFrame {
    pub fn start(img_dim: (usize, usize)) -> DistributedFrame {
        DistributedFrame::InProgress {
            num_reporting: 0,
            render: Image::new(img_dim),
            first_tile_recv: SystemTime::now(),
        }
    }
}

/// Buffer for collecting results from a worker asynchronously. The buffer is filled
/// as we get readable events from the workers until it reaches the expected size.
/// After this the Frame is decoded and accumulated in the appropriate `DistributedFrame`
#[derive(Clone, Debug)]
struct WorkerBuffer {
    pub buf: Vec<u8>,
    pub expected_size: usize,
    pub currently_read: usize,
}

impl WorkerBuffer {
    pub fn new() -> WorkerBuffer {
        WorkerBuffer { buf: Vec::new(), expected_size: 8, currently_read: 0 }
    }
}

/// The Master organizes the set of Worker processes and instructions them what parts
/// of the scene to render. As workers report results the master collects them and
/// saves out the PNG once all workers have reported the frame.
pub struct Master {
    /// Hostnames of the workers to send work too
    workers: Vec<String>,
    connections: Vec<TcpStream>,
    /// Temporary buffers to store worker results in as they're
    /// read in over TCP
    worker_buffers: Vec<WorkerBuffer>,
    config: Config,
    /// List of the frames we're collecting or have completed
    frames: HashMap<usize, DistributedFrame>,
    img_dim: (usize, usize),
    /// Number of 8x8 blocks we're assigning per worker
    blocks_per_worker: usize,
    /// Remainder of blocks that will be tacked on to the last
    /// worker's assignment
    blocks_remainder: usize,
}

impl Master {
    /// Create a new master that will contact the worker nodes passed and
    /// send instructions on what parts of the scene to start rendering
    pub fn start_workers(workers: Vec<String>, config: Config, img_dim: (usize, usize))
                         -> (Master, EventLoop<Master>) {
        // Figure out how many blocks we have for this image and assign them to our workers
        let queue = BlockQueue::new((img_dim.0 as u32, img_dim.1 as u32), (8, 8), (0, 0));
        let blocks_per_worker = queue.len() / workers.len();
        let blocks_remainder = queue.len() % workers.len();

        let mut event_loop = EventLoop::<Master>::new().unwrap();
        let mut connections = Vec::new();

        // Connect to each worker and add them to the event loop
        for (i, host) in workers.iter().enumerate() {
            let addr = (&host[..], worker::PORT).to_socket_addrs().unwrap().next().unwrap();
            match TcpStream::connect(&addr) {
                Ok(stream) => {
                    // Each worker is identified in the event loop by their index in the vec
                    if let Err(e) = event_loop.register(&stream, Token(i), EventSet::all(), PollOpt::level()) {
                        panic!("Error registering stream from {}: {}", host, e);
                    }
                    connections.push(stream);
                },
                Err(e) => panic!("Failed to contact worker {}: {:?}", host, e),
            }
        }
        let worker_buffers: Vec<_> = iter::repeat(WorkerBuffer::new()).take(workers.len()).collect();
        let master = Master { workers: workers, connections: connections,
                              worker_buffers: worker_buffers, config: config,
                              frames: HashMap::new(),
                              img_dim: img_dim,
                              blocks_per_worker: blocks_per_worker,
                              blocks_remainder: blocks_remainder };
        (master, event_loop)
    }
    /// Read a result frame from a worker and save it into the list of frames we're collecting from
    /// all workers. Will save out the final render if all workers have reported results for this
    /// frame.
    fn save_results(&mut self, frame: Frame) {
        let frame_num = frame.frame as usize;
        let img_dim = self.img_dim;
        // Find the frame being reported and create it if we haven't received parts of this frame yet
        let mut df = self.frames.entry(frame_num).or_insert_with(|| DistributedFrame::start(img_dim));

        let mut finished = false;
        match *df {
            DistributedFrame::InProgress { ref mut num_reporting, ref mut render, ref first_tile_recv } => {
                // Collect results from the worker and see if we've finished the frame and can save
                // it out
                render.add_blocks(frame.block_size, &frame.blocks, &frame.pixels);
                *num_reporting += 1;
                if *num_reporting == self.workers.len() {
                    let render_time = first_tile_recv.elapsed().expect("Failed to get rendering time?");
                    let out_file = match self.config.out_path.extension() {
                        Some(_) => self.config.out_path.clone(),
                        None => self.config.out_path.join(
                            PathBuf::from(format!("frame{:05}.png", frame_num))),
                    };
                    let img = render.get_srgb8();
                    let dim = render.dimensions();
                    match image::save_buffer(&out_file.as_path(), &img[..], dim.0 as u32,
                    dim.1 as u32, image::RGB(8)) {
                        Ok(_) => {},
                        Err(e) => println!("Error saving image, {}", e),
                    };
                    println!("Frame {}: rendering took {:4}s\nFrame {}: rendered to '{}'\n--------------------",
                             frame_num, render_time.as_secs() as f64 + render_time.subsec_nanos() as f64 * 1e-9,
                             frame_num, out_file.display());
                    finished = true;
                }
            },
            DistributedFrame::Completed => println!("Worker reporting on completed frame {}?", frame_num),
        }
        // This is a bit awkward, since we borrow df in the match we can't mark it finished in there
        if finished {
            *df = DistributedFrame::Completed;
        }
    }
    /// Read results from a worker and accumulate this data in its worker buffer. Returns true if
    /// we've read the data being sent and can decode the buffer
    fn read_worker_buffer(&mut self, worker: usize) -> bool {
        let buf = &mut self.worker_buffers[worker];
        // If we haven't read the size of data being sent, read that now
        if buf.currently_read < 8 {
            // First 8 bytes are a u64 specifying the number of bytes being sent
            buf.buf.extend(iter::repeat(0u8).take(8));
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
            if buf.currently_read == buf.expected_size {
                // How many bytes we expect to get from the worker for a frame
                buf.expected_size = deserialize(&buf.buf[..]).unwrap();
                // Extend the Vec so we've got enough room for the remaning bytes, minus the 8 for the
                // encoded size header
                buf.buf.extend(iter::repeat(0u8).take(buf.expected_size - 8));
            }
        }
        // If we've finished reading the size header we can now start reading the frame data
        if buf.currently_read >= 8 {
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
        }
        buf.currently_read == buf.expected_size
    }
}

impl Handler for Master {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self, event_loop: &mut EventLoop<Master>, token: Token, event: EventSet) {
        let worker = token.as_usize();
        if event.is_error() {
            // We don't do distributed error handling so should abort if we fail to
            // connect for now
            panic!("Error connecting to {}", self.workers[worker]);
        }
        // If the worker has terminated, shutdown the read end of the connection
        if event.is_hup() {
            if let Err(e) = self.connections[worker].shutdown(Shutdown::Both) {
                println!("Error shutting down worker {}: {}", worker, e);
            }
            // Remove the connection from the event loop
            if let Err(e) = event_loop.deregister(&self.connections[worker]) {
                println!("Error deregistering worker {}: {}", worker, e);
            }
        }
        // A worker is ready to receive instructions from us
        if event.is_writable() {
            let b_start = worker * self.blocks_per_worker;
            let b_count =
                if worker == self.workers.len() - 1 {
                    self.blocks_per_worker + self.blocks_remainder
                } else {
                    self.blocks_per_worker
                };
            let instr = Instructions::new(&self.config.scene_file,
                                          (self.config.frame_info.start, self.config.frame_info.end),
                                          b_start, b_count);
            // Encode and send our instructions to the worker
            let bytes = serialize(&instr, Infinite).unwrap();
            if let Err(e) = self.connections[worker].write_all(&bytes[..]) {
                println!("Failed to send instructions to {}: {:?}", self.workers[worker], e);
            }
            // Register that we no longer care about writable events on this connection
            event_loop.reregister(&self.connections[worker], token,
                                  EventSet::readable() | EventSet::error() | EventSet::hup(),
                                  PollOpt::level()).expect("Re-registering failed");
        }
        // Some results are available from a worker
        // Read results from the worker, if we've accumulated all the data being sent
        // decode and accumulate the frame
        if event.is_readable() && self.read_worker_buffer(worker) {
            let frame = deserialize(&self.worker_buffers[worker].buf[..]).unwrap();
            self.save_results(frame);
            // Clean up the worker buffer for the next frame
            self.worker_buffers[worker].buf.clear();
            self.worker_buffers[worker].expected_size = 8;
            self.worker_buffers[worker].currently_read = 0;
        }
        // After getting results from the worker we check if we've completed all our frames
        // and exit if so
        let all_complete = self.frames.values().fold(true,
                                |all, v| {
                                    match *v {
                                        DistributedFrame::Completed => true && all,
                                        _ => false,
                                    }
                                });
        // The frame start/end range is inclusive, so we must add 1 here
        let num_frames = self.config.frame_info.end - self.config.frame_info.start + 1;
        if self.frames.len() == num_frames && all_complete {
            event_loop.shutdown();
        }
    }
}

<commit_msg>The master doesn't know the rneder time, just time between first/last<commit_after>//! The master module provides the Master struct which instructs Workers which
//! portions of the image they should render and collects their results to combine
//! into the final image.

use std::path::PathBuf;
use std::io::prelude::*;
use std::collections::HashMap;
use std::net::ToSocketAddrs;
use std::iter;
use std::time::SystemTime;

use bincode::{Infinite, serialize, deserialize};
use image;
use mio::tcp::{TcpStream, Shutdown};
use mio::*;

use film::Image;
use exec::Config;
use exec::distrib::{worker, Instructions, Frame};
use sampler::BlockQueue;

/// Stores distributed rendering status. The frame is either `InProgress` and contains
/// partially rendered results from the workers who've reported the frame or is `Completed`
/// and has been saved out to disk.
#[derive(Debug)]
enum DistributedFrame {
    InProgress {
        // The number of workers who have reported results for this
        // frame so far
        num_reporting: usize,
        render: Image,
        // Start time of this frame, when we got the first tiles in from a worker
        first_tile_recv: SystemTime,
    },
    Completed,
}

impl DistributedFrame {
    pub fn start(img_dim: (usize, usize)) -> DistributedFrame {
        DistributedFrame::InProgress {
            num_reporting: 0,
            render: Image::new(img_dim),
            first_tile_recv: SystemTime::now(),
        }
    }
}

/// Buffer for collecting results from a worker asynchronously. The buffer is filled
/// as we get readable events from the workers until it reaches the expected size.
/// After this the Frame is decoded and accumulated in the appropriate `DistributedFrame`
#[derive(Clone, Debug)]
struct WorkerBuffer {
    pub buf: Vec<u8>,
    pub expected_size: usize,
    pub currently_read: usize,
}

impl WorkerBuffer {
    pub fn new() -> WorkerBuffer {
        WorkerBuffer { buf: Vec::new(), expected_size: 8, currently_read: 0 }
    }
}

/// The Master organizes the set of Worker processes and instructions them what parts
/// of the scene to render. As workers report results the master collects them and
/// saves out the PNG once all workers have reported the frame.
pub struct Master {
    /// Hostnames of the workers to send work too
    workers: Vec<String>,
    connections: Vec<TcpStream>,
    /// Temporary buffers to store worker results in as they're
    /// read in over TCP
    worker_buffers: Vec<WorkerBuffer>,
    config: Config,
    /// List of the frames we're collecting or have completed
    frames: HashMap<usize, DistributedFrame>,
    img_dim: (usize, usize),
    /// Number of 8x8 blocks we're assigning per worker
    blocks_per_worker: usize,
    /// Remainder of blocks that will be tacked on to the last
    /// worker's assignment
    blocks_remainder: usize,
}

impl Master {
    /// Create a new master that will contact the worker nodes passed and
    /// send instructions on what parts of the scene to start rendering
    pub fn start_workers(workers: Vec<String>, config: Config, img_dim: (usize, usize))
                         -> (Master, EventLoop<Master>) {
        // Figure out how many blocks we have for this image and assign them to our workers
        let queue = BlockQueue::new((img_dim.0 as u32, img_dim.1 as u32), (8, 8), (0, 0));
        let blocks_per_worker = queue.len() / workers.len();
        let blocks_remainder = queue.len() % workers.len();

        let mut event_loop = EventLoop::<Master>::new().unwrap();
        let mut connections = Vec::new();

        // Connect to each worker and add them to the event loop
        for (i, host) in workers.iter().enumerate() {
            let addr = (&host[..], worker::PORT).to_socket_addrs().unwrap().next().unwrap();
            match TcpStream::connect(&addr) {
                Ok(stream) => {
                    // Each worker is identified in the event loop by their index in the vec
                    if let Err(e) = event_loop.register(&stream, Token(i), EventSet::all(), PollOpt::level()) {
                        panic!("Error registering stream from {}: {}", host, e);
                    }
                    connections.push(stream);
                },
                Err(e) => panic!("Failed to contact worker {}: {:?}", host, e),
            }
        }
        let worker_buffers: Vec<_> = iter::repeat(WorkerBuffer::new()).take(workers.len()).collect();
        let master = Master { workers: workers, connections: connections,
                              worker_buffers: worker_buffers, config: config,
                              frames: HashMap::new(),
                              img_dim: img_dim,
                              blocks_per_worker: blocks_per_worker,
                              blocks_remainder: blocks_remainder };
        (master, event_loop)
    }
    /// Read a result frame from a worker and save it into the list of frames we're collecting from
    /// all workers. Will save out the final render if all workers have reported results for this
    /// frame.
    fn save_results(&mut self, frame: Frame) {
        let frame_num = frame.frame as usize;
        let img_dim = self.img_dim;
        // Find the frame being reported and create it if we haven't received parts of this frame yet
        let mut df = self.frames.entry(frame_num).or_insert_with(|| DistributedFrame::start(img_dim));

        let mut finished = false;
        match *df {
            DistributedFrame::InProgress { ref mut num_reporting, ref mut render, ref first_tile_recv } => {
                // Collect results from the worker and see if we've finished the frame and can save
                // it out
                render.add_blocks(frame.block_size, &frame.blocks, &frame.pixels);
                *num_reporting += 1;
                if *num_reporting == self.workers.len() {
                    let render_time = first_tile_recv.elapsed().expect("Failed to get rendering time?");
                    let out_file = match self.config.out_path.extension() {
                        Some(_) => self.config.out_path.clone(),
                        None => self.config.out_path.join(
                            PathBuf::from(format!("frame{:05}.png", frame_num))),
                    };
                    let img = render.get_srgb8();
                    let dim = render.dimensions();
                    match image::save_buffer(&out_file.as_path(), &img[..], dim.0 as u32,
                    dim.1 as u32, image::RGB(8)) {
                        Ok(_) => {},
                        Err(e) => println!("Error saving image, {}", e),
                    };
                    println!("Frame {}: time between receiving first and last tile {:4}s",
                             frame_num, render_time.as_secs() as f64 + render_time.subsec_nanos() as f64 * 1e-9);
                    println!("Frame {}: rendered to '{}'\n--------------------", frame_num, out_file.display());
                    finished = true;
                }
            },
            DistributedFrame::Completed => println!("Worker reporting on completed frame {}?", frame_num),
        }
        // This is a bit awkward, since we borrow df in the match we can't mark it finished in there
        if finished {
            *df = DistributedFrame::Completed;
        }
    }
    /// Read results from a worker and accumulate this data in its worker buffer. Returns true if
    /// we've read the data being sent and can decode the buffer
    fn read_worker_buffer(&mut self, worker: usize) -> bool {
        let buf = &mut self.worker_buffers[worker];
        // If we haven't read the size of data being sent, read that now
        if buf.currently_read < 8 {
            // First 8 bytes are a u64 specifying the number of bytes being sent
            buf.buf.extend(iter::repeat(0u8).take(8));
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
            if buf.currently_read == buf.expected_size {
                // How many bytes we expect to get from the worker for a frame
                buf.expected_size = deserialize(&buf.buf[..]).unwrap();
                // Extend the Vec so we've got enough room for the remaning bytes, minus the 8 for the
                // encoded size header
                buf.buf.extend(iter::repeat(0u8).take(buf.expected_size - 8));
            }
        }
        // If we've finished reading the size header we can now start reading the frame data
        if buf.currently_read >= 8 {
            match self.connections[worker].read(&mut buf.buf[buf.currently_read..]) {
                Ok(n) => buf.currently_read += n,
                Err(e) => println!("Error reading results from worker {}: {}", self.workers[worker], e),
            }
        }
        buf.currently_read == buf.expected_size
    }
}

impl Handler for Master {
    type Timeout = ();
    type Message = ();

    fn ready(&mut self, event_loop: &mut EventLoop<Master>, token: Token, event: EventSet) {
        let worker = token.as_usize();
        if event.is_error() {
            // We don't do distributed error handling so should abort if we fail to
            // connect for now
            panic!("Error connecting to {}", self.workers[worker]);
        }
        // If the worker has terminated, shutdown the read end of the connection
        if event.is_hup() {
            if let Err(e) = self.connections[worker].shutdown(Shutdown::Both) {
                println!("Error shutting down worker {}: {}", worker, e);
            }
            // Remove the connection from the event loop
            if let Err(e) = event_loop.deregister(&self.connections[worker]) {
                println!("Error deregistering worker {}: {}", worker, e);
            }
        }
        // A worker is ready to receive instructions from us
        if event.is_writable() {
            let b_start = worker * self.blocks_per_worker;
            let b_count =
                if worker == self.workers.len() - 1 {
                    self.blocks_per_worker + self.blocks_remainder
                } else {
                    self.blocks_per_worker
                };
            let instr = Instructions::new(&self.config.scene_file,
                                          (self.config.frame_info.start, self.config.frame_info.end),
                                          b_start, b_count);
            // Encode and send our instructions to the worker
            let bytes = serialize(&instr, Infinite).unwrap();
            if let Err(e) = self.connections[worker].write_all(&bytes[..]) {
                println!("Failed to send instructions to {}: {:?}", self.workers[worker], e);
            }
            // Register that we no longer care about writable events on this connection
            event_loop.reregister(&self.connections[worker], token,
                                  EventSet::readable() | EventSet::error() | EventSet::hup(),
                                  PollOpt::level()).expect("Re-registering failed");
        }
        // Some results are available from a worker
        // Read results from the worker, if we've accumulated all the data being sent
        // decode and accumulate the frame
        if event.is_readable() && self.read_worker_buffer(worker) {
            let frame = deserialize(&self.worker_buffers[worker].buf[..]).unwrap();
            self.save_results(frame);
            // Clean up the worker buffer for the next frame
            self.worker_buffers[worker].buf.clear();
            self.worker_buffers[worker].expected_size = 8;
            self.worker_buffers[worker].currently_read = 0;
        }
        // After getting results from the worker we check if we've completed all our frames
        // and exit if so
        let all_complete = self.frames.values().fold(true,
                                |all, v| {
                                    match *v {
                                        DistributedFrame::Completed => true && all,
                                        _ => false,
                                    }
                                });
        // The frame start/end range is inclusive, so we must add 1 here
        let num_frames = self.config.frame_info.end - self.config.frame_info.start + 1;
        if self.frames.len() == num_frames && all_complete {
            event_loop.shutdown();
        }
    }
}

<|endoftext|>


<commit_before>//! An information element is a portion of a SBD message.
//!
//! Information elements come after the SBD header. They come in many types, including more
//! header-type information and the actual data payload.

use std::io::Read;

use byteorder::{ReadBytesExt, BigEndian};

use {Error, Result};

/// An information element, or IE.
///
/// These are the building blocks of a SBD message. There are several types, generally divided into
/// MO and MT IE's. This general structure just holds the basic IE data, when then can be converted
/// into a specific type of IE.
#[derive(Debug, Default)]
pub struct InformationElement {
    id: u8,
    length: u16,
    contents: Vec<u8>,
}

impl InformationElement {

    /// Reads an information element from something that implements `Read`.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::io::{Seek, SeekFrom};
    /// use std::fs::File;
    /// use sbd::information_element::InformationElement;
    /// let mut file = File::open("data/0-mo.sbd").unwrap();
    /// file.seek(SeekFrom::Start(3));
    /// let readable = file.take(28);
    /// InformationElement::read_from(readable).unwrap();
    /// ```
    pub fn read_from<R: Read>(mut readable: R) -> Result<InformationElement> {
        let mut information_element: InformationElement = Default::default();
        information_element.id = try!(readable.read_u8());
        information_element.length = try!(readable.read_u16::<BigEndian>());
        let content_length = information_element.length - 3;
        let bytes_read = try!(readable.take(content_length as u64)
                              .read_to_end(&mut information_element.contents));
        assert!(!(bytes_read > content_length as usize));
        if bytes_read < content_length as usize {
            return Err(Error::Undersized(bytes_read + 3));
        }
        Ok(information_element)
    }

    /// Returns the length of the information element.
    ///
    /// # Examples
    ///
    /// ```
    /// # use std::io::{Seek, SeekFrom};
    /// # use std::fs::File;
    /// # use sbd::information_element::InformationElement;
    /// # let mut file = File::open("data/0-mo.sbd").unwrap();
    /// # file.seek(SeekFrom::Start(3));
    /// # let readable = file.take(28);
    /// let information_element = InformationElement::read_from(readable).unwrap();
    /// assert_eq!(28, information_element.len());
    /// ```
    pub fn len(&self) -> u16 {
        self.length
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::io::{Read, Seek, SeekFrom};
    use std::fs::File;

    #[test]
    fn read_from() {
        let mut file = File::open("data/0-mo.sbd").unwrap();
        file.seek(SeekFrom::Start(3)).unwrap();
        let readable = file.take(28);
        InformationElement::read_from(readable).unwrap();
    }

    #[test]
    fn len() {
        let mut file = File::open("data/0-mo.sbd").unwrap();
        file.seek(SeekFrom::Start(3)).unwrap();
        let readable = file.take(28);
        let ie = InformationElement::read_from(readable).unwrap();
        assert_eq!(28, ie.len());
    }

    #[test]
    fn undersized() {
        let mut file = File::open("data/0-mo.sbd").unwrap();
        file.seek(SeekFrom::Start(3)).unwrap();
        let readable = file.take(27);
        assert!(InformationElement::read_from(readable).is_err());
    }
}
<commit_msg>Okay, we're closer, but unexpected eof<commit_after>//! An information element is a portion of a SBD message.
//!
//! Information elements come after the SBD header. They come in many types, including more
//! header-type information and the actual data payload.

use std::io::Read;

use byteorder::{ReadBytesExt, BigEndian};

use {Error, Result};

/// An information element, or IE.
///
/// These are the building blocks of a SBD message. There are several types, generally divided into
/// MO and MT IE's. This general structure just holds the basic IE data, when then can be converted
/// into a specific type of IE.
#[derive(Debug, Default)]
pub struct InformationElement {
    id: u8,
    length: u16,
    contents: Vec<u8>,
}

impl InformationElement {

    /// Reads an information element from something that implements `Read`.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::io::{Seek, SeekFrom};
    /// use std::fs::File;
    /// use sbd::information_element::InformationElement;
    /// let mut file = File::open("data/0-mo.sbd").unwrap();
    /// file.seek(SeekFrom::Start(3));
    /// let readable = file.take(28);
    /// InformationElement::read_from(readable).unwrap();
    /// ```
    pub fn read_from<R: Read>(mut readable: R) -> Result<InformationElement> {
        let mut information_element: InformationElement = Default::default();
        information_element.id = try!(readable.read_u8());
        information_element.length = try!(readable.read_u16::<BigEndian>());
        let bytes_read = try!(readable.take(information_element.length as u64)
                              .read_to_end(&mut information_element.contents));
        assert!(!(bytes_read > information_element.length as usize));
        if bytes_read < information_element.length as usize {
            return Err(Error::Undersized(bytes_read + 3));
        }
        Ok(information_element)
    }

    /// Returns the length of the information element.
    ///
    /// # Examples
    ///
    /// ```
    /// # use std::io::{Seek, SeekFrom};
    /// # use std::fs::File;
    /// # use sbd::information_element::InformationElement;
    /// # let mut file = File::open("data/0-mo.sbd").unwrap();
    /// # file.seek(SeekFrom::Start(3));
    /// # let readable = file.take(28);
    /// let information_element = InformationElement::read_from(readable).unwrap();
    /// assert_eq!(28, information_element.len());
    /// ```
    pub fn len(&self) -> u16 {
        self.length
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::io::{Read, Seek, SeekFrom};
    use std::fs::File;

    #[test]
    fn read_from() {
        let mut file = File::open("data/0-mo.sbd").unwrap();
        file.seek(SeekFrom::Start(3)).unwrap();
        let readable = file.take(31);
        InformationElement::read_from(readable).unwrap();
    }

    #[test]
    fn len() {
        let mut file = File::open("data/0-mo.sbd").unwrap();
        file.seek(SeekFrom::Start(3)).unwrap();
        let readable = file.take(31);
        let ie = InformationElement::read_from(readable).unwrap();
        assert_eq!(28, ie.len());
    }

    #[test]
    fn undersized() {
        let mut file = File::open("data/0-mo.sbd").unwrap();
        file.seek(SeekFrom::Start(3)).unwrap();
        let readable = file.take(30);
        assert!(InformationElement::read_from(readable).is_err());
    }
}
<|endoftext|>


<commit_before>#![crate_id="cargo-verify-project"]

extern crate toml;
extern crate getopts;

use std::io::File;
use std::os::{args, set_exit_status};
use getopts::{reqopt, getopts};

/**
    cargo-verify-project --manifest=LOCATION
*/

fn main() {
    let arguments = args();

    let opts = vec!(
        reqopt("m", "manifest", "the location of the manifest", "MANIFEST")
    );

    let matches = match getopts(arguments.tail(), opts.as_slice()) {
        Ok(m) => m,
        Err(_) => {
            fail("missing-argument", "manifest");
            return;
        }
    };

    if !matches.opt_present("m") {
        fail("missing-argument", "manifest");
        return;
    }

    let manifest = matches.opt_str("m").unwrap();
    let file = Path::new(manifest);
    let contents = match File::open(&file).read_to_str() {
        Ok(s) => s,
        Err(e) => return fail("invalid", format!("error reading file: {}",
                                                 e).as_slice())
    };
    match toml::Parser::new(contents.as_slice()).parse() {
        None => {
            fail("invalid", "invalid-format");
            return;
        },
        Some(..) => {}
    };

    println!("{}", "{ \"success\": \"true\" }");
}

fn fail(reason: &str, value: &str) {
    println!(r#"{{ "{:s}", "{:s}" }}"#, reason, value);
    set_exit_status(1);
}
<commit_msg>s/,/:/ in what appears to be JSON.<commit_after>#![crate_id="cargo-verify-project"]

extern crate toml;
extern crate getopts;

use std::io::File;
use std::os::{args, set_exit_status};
use getopts::{reqopt, getopts};

/**
    cargo-verify-project --manifest=LOCATION
*/

fn main() {
    let arguments = args();

    let opts = vec!(
        reqopt("m", "manifest", "the location of the manifest", "MANIFEST")
    );

    let matches = match getopts(arguments.tail(), opts.as_slice()) {
        Ok(m) => m,
        Err(_) => {
            fail("missing-argument", "manifest");
            return;
        }
    };

    if !matches.opt_present("m") {
        fail("missing-argument", "manifest");
        return;
    }

    let manifest = matches.opt_str("m").unwrap();
    let file = Path::new(manifest);
    let contents = match File::open(&file).read_to_str() {
        Ok(s) => s,
        Err(e) => return fail("invalid", format!("error reading file: {}",
                                                 e).as_slice())
    };
    match toml::Parser::new(contents.as_slice()).parse() {
        None => {
            fail("invalid", "invalid-format");
            return;
        },
        Some(..) => {}
    };

    println!("{}", "{ \"success\": \"true\" }");
}

fn fail(reason: &str, value: &str) {
    println!(r#"{{ "{:s}": "{:s}" }}"#, reason, value);
    set_exit_status(1);
}
<|endoftext|>