<commit_before>//! Intrinsics and other functions that the miri engine executes without
//! looking at their MIR. Intrinsics/functions supported here are shared by CTFE
//! and miri.

use syntax::symbol::Symbol;
use rustc::ty;
use rustc::ty::layout::{LayoutOf, Primitive, Size};
use rustc::mir::BinOp;
use rustc::mir::interpret::{
    EvalResult, EvalErrorKind, Scalar,
};

use super::{
    Machine, PlaceTy, OpTy, EvalContext,
};

fn numeric_intrinsic<'tcx, Tag>(
    name: &str,
    bits: u128,
    kind: Primitive,
) -> EvalResult<'tcx, Scalar<Tag>> {
    let size = match kind {
        Primitive::Int(integer, _) => integer.size(),
        _ => bug!("invalid `{}` argument: {:?}", name, bits),
    };
    let extra = 128 - size.bits() as u128;
    let bits_out = match name {
        "ctpop" => bits.count_ones() as u128,
        "ctlz" => bits.leading_zeros() as u128 - extra,
        "cttz" => (bits << extra).trailing_zeros() as u128 - extra,
        "bswap" => (bits << extra).swap_bytes(),
        "bitreverse" => (bits << extra).reverse_bits(),
        _ => bug!("not a numeric intrinsic: {}", name),
    };
    Ok(Scalar::from_uint(bits_out, size))
}

impl<'a, 'mir, 'tcx, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
    /// Returns `true` if emulation happened.
    pub fn emulate_intrinsic(
        &mut self,
        instance: ty::Instance<'tcx>,
        args: &[OpTy<'tcx, M::PointerTag>],
        dest: PlaceTy<'tcx, M::PointerTag>,
    ) -> EvalResult<'tcx, bool> {
        let substs = instance.substs;

        let intrinsic_name = &self.tcx.item_name(instance.def_id()).as_str()[..];
        match intrinsic_name {
            "min_align_of" => {
                let elem_ty = substs.type_at(0);
                let elem_align = self.layout_of(elem_ty)?.align.abi.bytes();
                let align_val = Scalar::from_uint(elem_align, dest.layout.size);
                self.write_scalar(align_val, dest)?;
            }

            "needs_drop" => {
                let ty = substs.type_at(0);
                let ty_needs_drop = ty.needs_drop(self.tcx.tcx, self.param_env);
                let val = Scalar::from_bool(ty_needs_drop);
                self.write_scalar(val, dest)?;
            }

            "size_of" => {
                let ty = substs.type_at(0);
                let size = self.layout_of(ty)?.size.bytes() as u128;
                let size_val = Scalar::from_uint(size, dest.layout.size);
                self.write_scalar(size_val, dest)?;
            }

            "type_id" => {
                let ty = substs.type_at(0);
                let type_id = self.tcx.type_id_hash(ty) as u128;
                let id_val = Scalar::from_uint(type_id, dest.layout.size);
                self.write_scalar(id_val, dest)?;
            }
            | "ctpop"
            | "cttz"
            | "cttz_nonzero"
            | "ctlz"
            | "ctlz_nonzero"
            | "bswap"
            | "bitreverse" => {
                let ty = substs.type_at(0);
                let layout_of = self.layout_of(ty)?;
                let bits = self.read_scalar(args[0])?.to_bits(layout_of.size)?;
                let kind = match layout_of.abi {
                    ty::layout::Abi::Scalar(ref scalar) => scalar.value,
                    _ => Err(::rustc::mir::interpret::EvalErrorKind::TypeNotPrimitive(ty))?,
                };
                let out_val = if intrinsic_name.ends_with("_nonzero") {
                    if bits == 0 {
                        return err!(Intrinsic(format!("{} called on 0", intrinsic_name)));
                    }
                    numeric_intrinsic(intrinsic_name.trim_end_matches("_nonzero"), bits, kind)?
                } else {
                    numeric_intrinsic(intrinsic_name, bits, kind)?
                };
                self.write_scalar(out_val, dest)?;
            }
            | "overflowing_add"
            | "overflowing_sub"
            | "overflowing_mul"
            | "add_with_overflow"
            | "sub_with_overflow"
            | "mul_with_overflow" => {
                let lhs = self.read_immediate(args[0])?;
                let rhs = self.read_immediate(args[1])?;
                let (bin_op, ignore_overflow) = match intrinsic_name {
                    "overflowing_add" => (BinOp::Add, true),
                    "overflowing_sub" => (BinOp::Sub, true),
                    "overflowing_mul" => (BinOp::Mul, true),
                    "add_with_overflow" => (BinOp::Add, false),
                    "sub_with_overflow" => (BinOp::Sub, false),
                    "mul_with_overflow" => (BinOp::Mul, false),
                    _ => bug!("Already checked for int ops")
                };
                if ignore_overflow {
                    self.binop_ignore_overflow(bin_op, lhs, rhs, dest)?;
                } else {
                    self.binop_with_overflow(bin_op, lhs, rhs, dest)?;
                }
            }
            "saturating_add" | "saturating_sub" => {
                let l = self.read_immediate(args[0])?;
                let r = self.read_immediate(args[1])?;
                let is_add = intrinsic_name == "saturating_add";
                let (val, overflowed) = self.binary_op_imm(if is_add {
                    BinOp::Add
                } else {
                    BinOp::Sub
                }, l, r)?;
                let val = if overflowed {
                    let num_bits = l.layout.size.bits();
                    if l.layout.abi.is_signed() {
                        // For signed ints the saturated value depends on the sign of the first
                        // term since the sign of the second term can be inferred from this and
                        // the fact that the operation has overflowed (if either is 0 no
                        // overflow can occur)
                        let first_term: u128 = l.to_scalar()?.to_bits(l.layout.size)?;
                        let first_term_positive = first_term & (1 << (num_bits-1)) == 0;
                        if first_term_positive {
                            // Negative overflow not possible since the positive first term
                            // can only increase an (in range) negative term for addition
                            // or corresponding negated positive term for subtraction
                            Scalar::from_uint((1u128 << (num_bits - 1)) - 1,  // max positive
                                Size::from_bits(num_bits))
                        } else {
                            // Positive overflow not possible for similar reason
                            // max negative
                            Scalar::from_uint(1u128 << (num_bits - 1), Size::from_bits(num_bits))
                        }
                    } else {  // unsigned
                        if is_add {
                            // max unsigned
                            Scalar::from_uint(u128::max_value() >> (128 - num_bits),
                                Size::from_bits(num_bits))
                        } else {  // underflow to 0
                            Scalar::from_uint(0u128, Size::from_bits(num_bits))
                        }
                    }
                } else {
                    val
                };
                self.write_scalar(val, dest)?;
            }
            "unchecked_shl" | "unchecked_shr" => {
                let l = self.read_immediate(args[0])?;
                let r = self.read_immediate(args[1])?;
                let bin_op = match intrinsic_name {
                    "unchecked_shl" => BinOp::Shl,
                    "unchecked_shr" => BinOp::Shr,
                    _ => bug!("Already checked for int ops")
                };
                let (val, overflowed) = self.binary_op(bin_op, l, r)?;
                if overflowed {
                    let layout = self.layout_of(substs.type_at(0))?;
                    let r_val =  r.to_scalar()?.to_bits(layout.size)?;
                    return err!(Intrinsic(
                        format!("Overflowing shift by {} in {}", r_val, intrinsic_name),
                    ));
                }
                self.write_scalar(val, dest)?;
            }
            "rotate_left" | "rotate_right" => {
                // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
                // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
                let layout = self.layout_of(substs.type_at(0))?;
                let val_bits = self.read_scalar(args[0])?.to_bits(layout.size)?;
                let raw_shift_bits = self.read_scalar(args[1])?.to_bits(layout.size)?;
                let width_bits = layout.size.bits() as u128;
                let shift_bits = raw_shift_bits % width_bits;
                let inv_shift_bits = (width_bits - raw_shift_bits) % width_bits;
                let result_bits = if intrinsic_name == "rotate_left" {
                    (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
                } else {
                    (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
                };
                let truncated_bits = self.truncate(result_bits, layout);
                let result = Scalar::from_uint(truncated_bits, layout.size);
                self.write_scalar(result, dest)?;
            }
            "transmute" => {
                self.copy_op_transmute(args[0], dest)?;
            }

            _ => return Ok(false),
        }

        Ok(true)
    }

    /// "Intercept" a function call because we have something special to do for it.
    /// Returns `true` if an intercept happened.
    pub fn hook_fn(
        &mut self,
        instance: ty::Instance<'tcx>,
        args: &[OpTy<'tcx, M::PointerTag>],
        dest: Option<PlaceTy<'tcx, M::PointerTag>>,
    ) -> EvalResult<'tcx, bool> {
        let def_id = instance.def_id();
        // Some fn calls are actually BinOp intrinsics
        if let Some((op, oflo)) = self.tcx.is_binop_lang_item(def_id) {
            let dest = dest.expect("128 lowerings can't diverge");
            let l = self.read_immediate(args[0])?;
            let r = self.read_immediate(args[1])?;
            if oflo {
                self.binop_with_overflow(op, l, r, dest)?;
            } else {
                self.binop_ignore_overflow(op, l, r, dest)?;
            }
            return Ok(true);
        } else if Some(def_id) == self.tcx.lang_items().panic_fn() {
            assert!(args.len() == 1);
            // &(&'static str, &'static str, u32, u32)
            let place = self.deref_operand(args[0])?;
            let (msg, file, line, col) = (
                self.mplace_field(place, 0)?,
                self.mplace_field(place, 1)?,
                self.mplace_field(place, 2)?,
                self.mplace_field(place, 3)?,
            );

            let msg_place = self.deref_operand(msg.into())?;
            let msg = Symbol::intern(self.read_str(msg_place)?);
            let file_place = self.deref_operand(file.into())?;
            let file = Symbol::intern(self.read_str(file_place)?);
            let line = self.read_scalar(line.into())?.to_u32()?;
            let col = self.read_scalar(col.into())?.to_u32()?;
            return Err(EvalErrorKind::Panic { msg, file, line, col }.into());
        } else if Some(def_id) == self.tcx.lang_items().begin_panic_fn() {
            assert!(args.len() == 2);
            // &'static str, &(&'static str, u32, u32)
            let msg = args[0];
            let place = self.deref_operand(args[1])?;
            let (file, line, col) = (
                self.mplace_field(place, 0)?,
                self.mplace_field(place, 1)?,
                self.mplace_field(place, 2)?,
            );

            let msg_place = self.deref_operand(msg.into())?;
            let msg = Symbol::intern(self.read_str(msg_place)?);
            let file_place = self.deref_operand(file.into())?;
            let file = Symbol::intern(self.read_str(file_place)?);
            let line = self.read_scalar(line.into())?.to_u32()?;
            let col = self.read_scalar(col.into())?.to_u32()?;
            return Err(EvalErrorKind::Panic { msg, file, line, col }.into());
        } else {
            return Ok(false);
        }
    }
}
<commit_msg>fix rebase fallout<commit_after>//! Intrinsics and other functions that the miri engine executes without
//! looking at their MIR. Intrinsics/functions supported here are shared by CTFE
//! and miri.

use syntax::symbol::Symbol;
use rustc::ty;
use rustc::ty::layout::{LayoutOf, Primitive, Size};
use rustc::mir::BinOp;
use rustc::mir::interpret::{
    EvalResult, EvalErrorKind, Scalar,
};

use super::{
    Machine, PlaceTy, OpTy, EvalContext,
};

fn numeric_intrinsic<'tcx, Tag>(
    name: &str,
    bits: u128,
    kind: Primitive,
) -> EvalResult<'tcx, Scalar<Tag>> {
    let size = match kind {
        Primitive::Int(integer, _) => integer.size(),
        _ => bug!("invalid `{}` argument: {:?}", name, bits),
    };
    let extra = 128 - size.bits() as u128;
    let bits_out = match name {
        "ctpop" => bits.count_ones() as u128,
        "ctlz" => bits.leading_zeros() as u128 - extra,
        "cttz" => (bits << extra).trailing_zeros() as u128 - extra,
        "bswap" => (bits << extra).swap_bytes(),
        "bitreverse" => (bits << extra).reverse_bits(),
        _ => bug!("not a numeric intrinsic: {}", name),
    };
    Ok(Scalar::from_uint(bits_out, size))
}

impl<'a, 'mir, 'tcx, M: Machine<'a, 'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
    /// Returns `true` if emulation happened.
    pub fn emulate_intrinsic(
        &mut self,
        instance: ty::Instance<'tcx>,
        args: &[OpTy<'tcx, M::PointerTag>],
        dest: PlaceTy<'tcx, M::PointerTag>,
    ) -> EvalResult<'tcx, bool> {
        let substs = instance.substs;

        let intrinsic_name = &self.tcx.item_name(instance.def_id()).as_str()[..];
        match intrinsic_name {
            "min_align_of" => {
                let elem_ty = substs.type_at(0);
                let elem_align = self.layout_of(elem_ty)?.align.abi.bytes();
                let align_val = Scalar::from_uint(elem_align, dest.layout.size);
                self.write_scalar(align_val, dest)?;
            }

            "needs_drop" => {
                let ty = substs.type_at(0);
                let ty_needs_drop = ty.needs_drop(self.tcx.tcx, self.param_env);
                let val = Scalar::from_bool(ty_needs_drop);
                self.write_scalar(val, dest)?;
            }

            "size_of" => {
                let ty = substs.type_at(0);
                let size = self.layout_of(ty)?.size.bytes() as u128;
                let size_val = Scalar::from_uint(size, dest.layout.size);
                self.write_scalar(size_val, dest)?;
            }

            "type_id" => {
                let ty = substs.type_at(0);
                let type_id = self.tcx.type_id_hash(ty) as u128;
                let id_val = Scalar::from_uint(type_id, dest.layout.size);
                self.write_scalar(id_val, dest)?;
            }
            | "ctpop"
            | "cttz"
            | "cttz_nonzero"
            | "ctlz"
            | "ctlz_nonzero"
            | "bswap"
            | "bitreverse" => {
                let ty = substs.type_at(0);
                let layout_of = self.layout_of(ty)?;
                let bits = self.read_scalar(args[0])?.to_bits(layout_of.size)?;
                let kind = match layout_of.abi {
                    ty::layout::Abi::Scalar(ref scalar) => scalar.value,
                    _ => Err(::rustc::mir::interpret::EvalErrorKind::TypeNotPrimitive(ty))?,
                };
                let out_val = if intrinsic_name.ends_with("_nonzero") {
                    if bits == 0 {
                        return err!(Intrinsic(format!("{} called on 0", intrinsic_name)));
                    }
                    numeric_intrinsic(intrinsic_name.trim_end_matches("_nonzero"), bits, kind)?
                } else {
                    numeric_intrinsic(intrinsic_name, bits, kind)?
                };
                self.write_scalar(out_val, dest)?;
            }
            | "overflowing_add"
            | "overflowing_sub"
            | "overflowing_mul"
            | "add_with_overflow"
            | "sub_with_overflow"
            | "mul_with_overflow" => {
                let lhs = self.read_immediate(args[0])?;
                let rhs = self.read_immediate(args[1])?;
                let (bin_op, ignore_overflow) = match intrinsic_name {
                    "overflowing_add" => (BinOp::Add, true),
                    "overflowing_sub" => (BinOp::Sub, true),
                    "overflowing_mul" => (BinOp::Mul, true),
                    "add_with_overflow" => (BinOp::Add, false),
                    "sub_with_overflow" => (BinOp::Sub, false),
                    "mul_with_overflow" => (BinOp::Mul, false),
                    _ => bug!("Already checked for int ops")
                };
                if ignore_overflow {
                    self.binop_ignore_overflow(bin_op, lhs, rhs, dest)?;
                } else {
                    self.binop_with_overflow(bin_op, lhs, rhs, dest)?;
                }
            }
            "saturating_add" | "saturating_sub" => {
                let l = self.read_immediate(args[0])?;
                let r = self.read_immediate(args[1])?;
                let is_add = intrinsic_name == "saturating_add";
                let (val, overflowed) = self.binary_op(if is_add {
                    BinOp::Add
                } else {
                    BinOp::Sub
                }, l, r)?;
                let val = if overflowed {
                    let num_bits = l.layout.size.bits();
                    if l.layout.abi.is_signed() {
                        // For signed ints the saturated value depends on the sign of the first
                        // term since the sign of the second term can be inferred from this and
                        // the fact that the operation has overflowed (if either is 0 no
                        // overflow can occur)
                        let first_term: u128 = l.to_scalar()?.to_bits(l.layout.size)?;
                        let first_term_positive = first_term & (1 << (num_bits-1)) == 0;
                        if first_term_positive {
                            // Negative overflow not possible since the positive first term
                            // can only increase an (in range) negative term for addition
                            // or corresponding negated positive term for subtraction
                            Scalar::from_uint((1u128 << (num_bits - 1)) - 1,  // max positive
                                Size::from_bits(num_bits))
                        } else {
                            // Positive overflow not possible for similar reason
                            // max negative
                            Scalar::from_uint(1u128 << (num_bits - 1), Size::from_bits(num_bits))
                        }
                    } else {  // unsigned
                        if is_add {
                            // max unsigned
                            Scalar::from_uint(u128::max_value() >> (128 - num_bits),
                                Size::from_bits(num_bits))
                        } else {  // underflow to 0
                            Scalar::from_uint(0u128, Size::from_bits(num_bits))
                        }
                    }
                } else {
                    val
                };
                self.write_scalar(val, dest)?;
            }
            "unchecked_shl" | "unchecked_shr" => {
                let l = self.read_immediate(args[0])?;
                let r = self.read_immediate(args[1])?;
                let bin_op = match intrinsic_name {
                    "unchecked_shl" => BinOp::Shl,
                    "unchecked_shr" => BinOp::Shr,
                    _ => bug!("Already checked for int ops")
                };
                let (val, overflowed) = self.binary_op(bin_op, l, r)?;
                if overflowed {
                    let layout = self.layout_of(substs.type_at(0))?;
                    let r_val =  r.to_scalar()?.to_bits(layout.size)?;
                    return err!(Intrinsic(
                        format!("Overflowing shift by {} in {}", r_val, intrinsic_name),
                    ));
                }
                self.write_scalar(val, dest)?;
            }
            "rotate_left" | "rotate_right" => {
                // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW))
                // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW))
                let layout = self.layout_of(substs.type_at(0))?;
                let val_bits = self.read_scalar(args[0])?.to_bits(layout.size)?;
                let raw_shift_bits = self.read_scalar(args[1])?.to_bits(layout.size)?;
                let width_bits = layout.size.bits() as u128;
                let shift_bits = raw_shift_bits % width_bits;
                let inv_shift_bits = (width_bits - raw_shift_bits) % width_bits;
                let result_bits = if intrinsic_name == "rotate_left" {
                    (val_bits << shift_bits) | (val_bits >> inv_shift_bits)
                } else {
                    (val_bits >> shift_bits) | (val_bits << inv_shift_bits)
                };
                let truncated_bits = self.truncate(result_bits, layout);
                let result = Scalar::from_uint(truncated_bits, layout.size);
                self.write_scalar(result, dest)?;
            }
            "transmute" => {
                self.copy_op_transmute(args[0], dest)?;
            }

            _ => return Ok(false),
        }

        Ok(true)
    }

    /// "Intercept" a function call because we have something special to do for it.
    /// Returns `true` if an intercept happened.
    pub fn hook_fn(
        &mut self,
        instance: ty::Instance<'tcx>,
        args: &[OpTy<'tcx, M::PointerTag>],
        dest: Option<PlaceTy<'tcx, M::PointerTag>>,
    ) -> EvalResult<'tcx, bool> {
        let def_id = instance.def_id();
        // Some fn calls are actually BinOp intrinsics
        if let Some((op, oflo)) = self.tcx.is_binop_lang_item(def_id) {
            let dest = dest.expect("128 lowerings can't diverge");
            let l = self.read_immediate(args[0])?;
            let r = self.read_immediate(args[1])?;
            if oflo {
                self.binop_with_overflow(op, l, r, dest)?;
            } else {
                self.binop_ignore_overflow(op, l, r, dest)?;
            }
            return Ok(true);
        } else if Some(def_id) == self.tcx.lang_items().panic_fn() {
            assert!(args.len() == 1);
            // &(&'static str, &'static str, u32, u32)
            let place = self.deref_operand(args[0])?;
            let (msg, file, line, col) = (
                self.mplace_field(place, 0)?,
                self.mplace_field(place, 1)?,
                self.mplace_field(place, 2)?,
                self.mplace_field(place, 3)?,
            );

            let msg_place = self.deref_operand(msg.into())?;
            let msg = Symbol::intern(self.read_str(msg_place)?);
            let file_place = self.deref_operand(file.into())?;
            let file = Symbol::intern(self.read_str(file_place)?);
            let line = self.read_scalar(line.into())?.to_u32()?;
            let col = self.read_scalar(col.into())?.to_u32()?;
            return Err(EvalErrorKind::Panic { msg, file, line, col }.into());
        } else if Some(def_id) == self.tcx.lang_items().begin_panic_fn() {
            assert!(args.len() == 2);
            // &'static str, &(&'static str, u32, u32)
            let msg = args[0];
            let place = self.deref_operand(args[1])?;
            let (file, line, col) = (
                self.mplace_field(place, 0)?,
                self.mplace_field(place, 1)?,
                self.mplace_field(place, 2)?,
            );

            let msg_place = self.deref_operand(msg.into())?;
            let msg = Symbol::intern(self.read_str(msg_place)?);
            let file_place = self.deref_operand(file.into())?;
            let file = Symbol::intern(self.read_str(file_place)?);
            let line = self.read_scalar(line.into())?.to_u32()?;
            let col = self.read_scalar(col.into())?.to_u32()?;
            return Err(EvalErrorKind::Panic { msg, file, line, col }.into());
        } else {
            return Ok(false);
        }
    }
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014, David Renshaw (dwrenshaw@gmail.com)
 *
 * See the LICENSE file in the capnproto-rust root directory.
 */

use rpc_capnp::{Message, Return};

use std;
use std::io::Acceptor;
use collections::hashmap::HashMap;
use capnp::{AnyPointer, MessageBuilder, MallocMessageBuilder};
use capnp::capability::{ClientHook, FromClientHook, Server};
use rpc::{Outgoing, RpcConnectionState, RpcEvent, ShutdownEvent, SturdyRefRestorer};
use capability::{LocalClient};

pub struct EzRpcClient {
    rpc_chan : std::comm::Sender<RpcEvent>,
    tcp : std::io::net::tcp::TcpStream,
}

impl Drop for EzRpcClient {
    fn drop(&mut self) {
        self.rpc_chan.send_opt(ShutdownEvent).is_ok();
        self.tcp.close_read().is_ok();
    }
}

impl EzRpcClient {
    pub fn new(server_address : &str) -> std::io::IoResult<EzRpcClient> {
        use std::io::net::{ip, tcp};

        let addr : ip::SocketAddr = std::from_str::FromStr::from_str(server_address).expect("bad server address");

        let tcp = try!(tcp::TcpStream::connect(addr));

        let connection_state = RpcConnectionState::new();

        let chan = connection_state.run(tcp.clone(), tcp.clone(), ());

        return Ok(EzRpcClient { rpc_chan : chan, tcp : tcp });
    }

    pub fn import_cap<T : FromClientHook>(&mut self, name : &str) -> T {
        let mut message = box MallocMessageBuilder::new_default();
        let restore = message.init_root::<Message::Builder>().init_restore();
        restore.init_object_id().set_as_text(name);

        let (outgoing, answer_port, _question_port) = RpcEvent::new_outgoing(message);
        self.rpc_chan.send(Outgoing(outgoing));

        let mut response_hook = answer_port.recv();
        let message : Message::Reader = response_hook.get().get_as_struct();
        let client = match message.which() {
            Some(Message::Return(ret)) => {
                match ret.which() {
                    Some(Return::Results(payload)) => {
                        payload.get_content().get_as_capability::<T>()
                    }
                    _ => { fail!() }
                }
            }
            _ => {fail!()}
        };

        return client;
    }
}

enum ExportEvent {
    ExportEventRestore(~str, std::comm::Sender<Option<Box<ClientHook:Send>>>),
    ExportEventRegister(~str, Box<Server:Send>),
}

struct ExportedCaps {
    objects : HashMap<~str, Box<ClientHook:Send>>,
}

impl ExportedCaps {
    pub fn new() -> std::comm::Sender<ExportEvent> {
        let (chan, port) = std::comm::channel::<ExportEvent>();

        std::task::spawn(proc() {
                let mut vat = ExportedCaps { objects : HashMap::new() };

                loop {
                    match port.recv_opt() {
                        Ok(ExportEventRegister(name, server)) => {
                            vat.objects.insert(name, box LocalClient::new(server) as Box<ClientHook:Send>);
                        }
                        Ok(ExportEventRestore(name, return_chan)) => {
                            return_chan.send(Some((*vat.objects.get(&name)).copy()));
                        }
                        Err(_) => break,
                    }
                }
            });

        chan
    }
}

pub struct Restorer {
    sender : std::comm::Sender<ExportEvent>,
}

impl Restorer {
    fn new(sender : std::comm::Sender<ExportEvent>) -> Restorer {
        Restorer { sender : sender }
    }
}

impl SturdyRefRestorer for Restorer {
    fn restore(&self, obj_id : AnyPointer::Reader) -> Option<Box<ClientHook:Send>> {
        let (tx, rx) = std::comm::channel();
        self.sender.send(ExportEventRestore(obj_id.get_as_text().to_owned(), tx));
        return rx.recv();
    }
}

pub struct EzRpcServer {
    sender : std::comm::Sender<ExportEvent>,
    tcp_acceptor : std::io::net::tcp::TcpAcceptor,
}

impl EzRpcServer {
    pub fn new(bind_address : &str) -> std::io::IoResult<EzRpcServer> {
        use std::io::net::{ip, tcp};
        use std::io::Listener;

        let addr : ip::SocketAddr = std::from_str::FromStr::from_str(bind_address).expect("bad bind address");

        let tcp_listener = try!(tcp::TcpListener::bind(addr));

        let tcp_acceptor = try!(tcp_listener.listen());

        let sender = ExportedCaps::new();

        Ok(EzRpcServer { sender : sender, tcp_acceptor : tcp_acceptor  })
    }

    pub fn export_cap(&self, name : &str, server : Box<Server:Send>) {
        self.sender.send(ExportEventRegister(name.to_owned(), server))
    }

    pub fn serve(self) {
        std::task::spawn(proc() {
            let mut server = self;
            for res in server.incoming() {
                match res {
                    Ok(()) => {}
                    Err(e) => {
                        println!("error: {}", e)
                    }
                }
            }

        });
    }
}

impl std::io::Acceptor<()> for EzRpcServer {
    fn accept(&mut self) -> std::io::IoResult<()> {

        let sender2 = self.sender.clone();
        let tcp = try!(self.tcp_acceptor.accept());
        std::task::spawn(proc() {
            let connection_state = RpcConnectionState::new();
            let _rpc_chan = connection_state.run(tcp.clone(), tcp, Restorer::new(sender2));
        });
        Ok(())
    }
}
<commit_msg>update for new TcpStream API<commit_after>/*
 * Copyright (c) 2014, David Renshaw (dwrenshaw@gmail.com)
 *
 * See the LICENSE file in the capnproto-rust root directory.
 */

use rpc_capnp::{Message, Return};

use std;
use std::io::Acceptor;
use collections::hashmap::HashMap;
use capnp::{AnyPointer, MessageBuilder, MallocMessageBuilder};
use capnp::capability::{ClientHook, FromClientHook, Server};
use rpc::{Outgoing, RpcConnectionState, RpcEvent, ShutdownEvent, SturdyRefRestorer};
use capability::{LocalClient};

pub struct EzRpcClient {
    rpc_chan : std::comm::Sender<RpcEvent>,
    tcp : std::io::net::tcp::TcpStream,
}

impl Drop for EzRpcClient {
    fn drop(&mut self) {
        self.rpc_chan.send_opt(ShutdownEvent).is_ok();
        self.tcp.close_read().is_ok();
    }
}

impl EzRpcClient {
    pub fn new(server_address : &str) -> std::io::IoResult<EzRpcClient> {
        use std::io::net::{ip, tcp};

        let addr : ip::SocketAddr = std::from_str::FromStr::from_str(server_address).expect("bad server address");

        let tcp = try!(tcp::TcpStream::connect(format!("{}", addr.ip),  addr.port));

        let connection_state = RpcConnectionState::new();

        let chan = connection_state.run(tcp.clone(), tcp.clone(), ());

        return Ok(EzRpcClient { rpc_chan : chan, tcp : tcp });
    }

    pub fn import_cap<T : FromClientHook>(&mut self, name : &str) -> T {
        let mut message = box MallocMessageBuilder::new_default();
        let restore = message.init_root::<Message::Builder>().init_restore();
        restore.init_object_id().set_as_text(name);

        let (outgoing, answer_port, _question_port) = RpcEvent::new_outgoing(message);
        self.rpc_chan.send(Outgoing(outgoing));

        let mut response_hook = answer_port.recv();
        let message : Message::Reader = response_hook.get().get_as_struct();
        let client = match message.which() {
            Some(Message::Return(ret)) => {
                match ret.which() {
                    Some(Return::Results(payload)) => {
                        payload.get_content().get_as_capability::<T>()
                    }
                    _ => { fail!() }
                }
            }
            _ => {fail!()}
        };

        return client;
    }
}

enum ExportEvent {
    ExportEventRestore(~str, std::comm::Sender<Option<Box<ClientHook:Send>>>),
    ExportEventRegister(~str, Box<Server:Send>),
}

struct ExportedCaps {
    objects : HashMap<~str, Box<ClientHook:Send>>,
}

impl ExportedCaps {
    pub fn new() -> std::comm::Sender<ExportEvent> {
        let (chan, port) = std::comm::channel::<ExportEvent>();

        std::task::spawn(proc() {
                let mut vat = ExportedCaps { objects : HashMap::new() };

                loop {
                    match port.recv_opt() {
                        Ok(ExportEventRegister(name, server)) => {
                            vat.objects.insert(name, box LocalClient::new(server) as Box<ClientHook:Send>);
                        }
                        Ok(ExportEventRestore(name, return_chan)) => {
                            return_chan.send(Some((*vat.objects.get(&name)).copy()));
                        }
                        Err(_) => break,
                    }
                }
            });

        chan
    }
}

pub struct Restorer {
    sender : std::comm::Sender<ExportEvent>,
}

impl Restorer {
    fn new(sender : std::comm::Sender<ExportEvent>) -> Restorer {
        Restorer { sender : sender }
    }
}

impl SturdyRefRestorer for Restorer {
    fn restore(&self, obj_id : AnyPointer::Reader) -> Option<Box<ClientHook:Send>> {
        let (tx, rx) = std::comm::channel();
        self.sender.send(ExportEventRestore(obj_id.get_as_text().to_owned(), tx));
        return rx.recv();
    }
}

pub struct EzRpcServer {
    sender : std::comm::Sender<ExportEvent>,
    tcp_acceptor : std::io::net::tcp::TcpAcceptor,
}

impl EzRpcServer {
    pub fn new(bind_address : &str) -> std::io::IoResult<EzRpcServer> {
        use std::io::net::{ip, tcp};
        use std::io::Listener;

        let addr : ip::SocketAddr = std::from_str::FromStr::from_str(bind_address).expect("bad bind address");

        let tcp_listener = try!(tcp::TcpListener::bind(format!("{}", addr.ip), addr.port));

        let tcp_acceptor = try!(tcp_listener.listen());

        let sender = ExportedCaps::new();

        Ok(EzRpcServer { sender : sender, tcp_acceptor : tcp_acceptor  })
    }

    pub fn export_cap(&self, name : &str, server : Box<Server:Send>) {
        self.sender.send(ExportEventRegister(name.to_owned(), server))
    }

    pub fn serve(self) {
        std::task::spawn(proc() {
            let mut server = self;
            for res in server.incoming() {
                match res {
                    Ok(()) => {}
                    Err(e) => {
                        println!("error: {}", e)
                    }
                }
            }

        });
    }
}

impl std::io::Acceptor<()> for EzRpcServer {
    fn accept(&mut self) -> std::io::IoResult<()> {

        let sender2 = self.sender.clone();
        let tcp = try!(self.tcp_acceptor.accept());
        std::task::spawn(proc() {
            let connection_state = RpcConnectionState::new();
            let _rpc_chan = connection_state.run(tcp.clone(), tcp, Restorer::new(sender2));
        });
        Ok(())
    }
}
<|endoftext|>


<commit_before>use libc;
use std::thread;
use std::rc::Rc;
use std::sync::{Arc};
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering};
use std::collections::BTreeSet;
use parking_lot::{Mutex, MutexGuard, RwLock, RwLockReadGuard};
use concurrent_hashmap::ConcHashMap;
use ram::schema::Schemas;
use ram::types::Id;
use ram::segs::{Segment, SEGMENT_SIZE};
use ram::cell::{Cell, ReadError, WriteError, Header};
use server::ServerMeta;

pub struct Chunk {
    pub id: usize,
    pub addr: usize,
    pub index: ConcHashMap<u64, Mutex<usize>>,
    pub segs: Vec<Segment>,
    pub seg_round: AtomicUsize,
    pub meta: Arc<ServerMeta>,
    pub backup_storage: Option<String>,
}

pub struct Chunks {
    pub list: Vec<Chunk>,
}

impl Chunk {
    fn new (id: usize, size: usize, meta: Arc<ServerMeta>, back_storage: Option<String>) -> Chunk {
        let mem_ptr = unsafe {libc::malloc(size)} as usize;
        let seg_count = size / SEGMENT_SIZE;
        let mut segments = Vec::<Segment>::new();
        for seg_idx in 0..seg_count {
            let seg_addr = mem_ptr + seg_idx * SEGMENT_SIZE;
            segments.push(Segment {
                addr: seg_addr,
                id: seg_idx,
                bound: seg_addr + SEGMENT_SIZE,
                append_header: AtomicUsize::new(seg_addr),
                lock: RwLock::new(()),
                frags: Mutex::new(BTreeSet::new()),
            });
        }
        debug!("creating chunk at {}, segments {}", mem_ptr, seg_count + 1);
        Chunk {
            id: id,
            addr: mem_ptr,
            index: ConcHashMap::<u64, Mutex<usize>>::new(),
            meta: meta,
            segs: segments,
            seg_round: AtomicUsize::new(0),
            backup_storage: back_storage
        }
    }
    pub fn try_acquire(&self, size: usize) -> Option<(usize, RwLockReadGuard<()>)> {
        let mut retried = 0;
        loop {
            let n = self.seg_round.load(Ordering::Relaxed);
            let seg_id = n % self.segs.len();
            let seg_acquire = self.segs[seg_id].try_acquire(size);
            match seg_acquire {
                None => {
                    if retried > self.segs.len() * 2 {return None;}
                    self.seg_round.fetch_add(1, Ordering::Relaxed);
                    retried += 1;
                },
                _ => {return seg_acquire;}
            }
        }
    }
    fn locate_segment(&self, location: usize) -> &Segment {
        let offset = location - self.addr;
        let seg_id = offset / SEGMENT_SIZE;
        return &self.segs[seg_id];
    }
    pub fn location_of(&self, hash: u64) -> Option<MutexGuard<usize>> {
        match self.index.find(&hash) {
            Some(index) => {
                let index = index.get();
                let index_lock = index.lock();
                if *index_lock == 0 {
                    return None
                }
                return Some(index_lock);
            },
            None => None
        }
    }
    fn put_tombstone(&self, location: usize) {
        let seg = self.locate_segment(location);
        seg.put_cell_tombstone(location);
        seg.put_frag(location);
    }
    fn read_cell(&self, hash: u64) -> Result<Cell, ReadError> {
        match self.location_of(hash) {
            Some(loc) => {
                Cell::from_chunk_raw(*loc, self)
            },
            None => Err(ReadError::CellDoesNotExisted)
        }
    }
    fn write_cell(&self, cell: &mut Cell) -> Result<Header, WriteError> {
        let hash = cell.header.hash;
        if self.location_of(hash).is_some() {
            return Err(WriteError::CellAlreadyExisted);
        } else {
            let written = cell.write_to_chunk(self);
            let need_rollback = AtomicBool::new(false);
            if let Ok(loc) = written {
                self.index.upsert(
                    hash,
                    Mutex::new(loc),
                    &|m| {
                        let mut inserted_loc = m.lock();
                        if *inserted_loc == 0 {
                            *inserted_loc = loc
                        } else {
                            need_rollback.store(true, Ordering::Relaxed);
                        }
                    }
                );
                if need_rollback.load(Ordering::Relaxed) {
                    self.put_tombstone(loc);
                    return Err(WriteError::CellAlreadyExisted)
                }
            }
            return Ok(cell.header)
        }
    }
    fn update_cell(&self, cell: &mut Cell) -> Result<Header, WriteError> {
        let hash = cell.header.hash;
        if let Some(mut cell_location) = self.location_of(hash) {
            let written = cell.write_to_chunk(self);
            if let Ok(new_location) = written {
                let old_location = *cell_location;
                *cell_location = new_location;
                self.put_tombstone(old_location);
            }
            return Ok(cell.header);
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn update_cell_by<U>(&self, hash: u64, update: U) -> Result<Header, WriteError>
        where U: Fn(Cell) -> Option<Cell> {
        if let Some(mut cell_location) = self.location_of(hash) {
            let cell = Cell::from_chunk_raw(*cell_location, self);
            match cell {
                Ok(cell) => {
                    let mut new_cell = update(cell);
                    if let Some(mut new_cell) = new_cell {
                        let written = new_cell.write_to_chunk(self);
                        if let Ok(new_location) = written {
                            let old_location = *cell_location;
                            *cell_location = new_location;
                            self.put_tombstone(old_location);
                        }
                        return Ok(new_cell.header);
                    } else {
                        return Err(WriteError::UserCanceledUpdate);
                    }
                },
                Err(e) => Err(WriteError::ReadError(e))
            }
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn remove_cell(&self, hash: u64) -> Result<(), WriteError> {
        if let Some(cell_location) = self.location_of(hash) {
            self.index.remove(&hash);
            self.put_tombstone(*cell_location);
            return Ok(());
        } else {
            return Err(WriteError::CellDoesNotExisted);
        }
    }
    fn remove_cell_by<P>(&self, hash: u64, predict: P) -> Result<(), WriteError>
        where P: Fn(Cell) -> bool {
        if let Some(cell_location) = self.location_of(hash) {
            let cell = Cell::from_chunk_raw(*cell_location, self);
            match cell {
                Ok(cell) => {
                    if predict(cell) {
                        self.index.remove(&hash);
                        self.put_tombstone(*cell_location);
                        return Ok(());
                    } else {
                        return Err(WriteError::DeletionPredictionFailed);
                    }
                },
                Err(e) => Err(WriteError::ReadError(e))
            }
        } else {
            return Err(WriteError::CellDoesNotExisted);
        }
    }
    fn dispose (&mut self) {
        debug!("disposing chunk at {}", self.addr);
        unsafe {
            libc::free(self.addr as *mut libc::c_void)
        }
    }
}

impl Drop for Chunk {
    fn drop(&mut self) {
        self.dispose();
    }
}

impl Chunks {
    pub fn new (count: usize, size: usize, meta: Arc<ServerMeta>, backup_storage: Option<String>) -> Arc<Chunks> {
        let chunk_size = size / count;
        let mut chunks = Vec::new();
        debug!("Creating {} chunks, total {} bytes", count, size);
        for i in 0..count {
            let backup_storage = match backup_storage {
                Some(ref dir) => Some(format!("{}/data-{}.bak", dir, i)),
                None => None
            };
            chunks.push(Chunk::new(i, chunk_size, meta.clone(), backup_storage));
        }
        Arc::new(Chunks {
            list: chunks
        })
    }
    pub fn new_dummy(count: usize, size: usize) -> Arc<Chunks> {
        Chunks::new(count, size, Arc::<ServerMeta>::new(ServerMeta {
            schemas: Schemas::new(None)
        }), None)
    }
    fn locate_chunk_by_partition(&self, partition: u64) -> &Chunk {
        let chunk_id = partition as usize % self.list.len();
        return &self.list[chunk_id];
    }
    fn locate_chunk_by_key(&self, key: &Id) -> (&Chunk, u64) {
        return (self.locate_chunk_by_partition(key.higher), key.lower);
    }
    pub fn read_cell(&self, key: &Id) -> Result<Cell, ReadError> {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.read_cell(hash);
    }
    pub fn write_cell(&self, cell: &mut Cell) -> Result<Header, WriteError> {
        let chunk = self.locate_chunk_by_partition(cell.header.partition);
        return chunk.write_cell(cell);
    }
    pub fn update_cell(&self, cell: &mut Cell) -> Result<Header, WriteError> {
        let chunk = self.locate_chunk_by_partition(cell.header.partition);
        return chunk.update_cell(cell);
    }
    pub fn update_cell_by<U>(&self, key: &Id, update: U) -> Result<Header, WriteError>
        where U: Fn(Cell) -> Option<Cell>{
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.update_cell_by(hash, update);
    }
    pub fn remove_cell(&self, key: &Id) -> Result<(), WriteError> {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.remove_cell(hash);
    }
    pub fn remove_cell_by<P>(&self, key: &Id, predict: P) -> Result<(), WriteError>
        where P: Fn(Cell) -> bool {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.remove_cell_by(hash, predict);
    }
    pub fn chunk_ptr(&self, key: &Id) -> usize {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return *chunk.location_of(hash).unwrap()
    }
}<commit_msg>return cell for update_cell_by function<commit_after>use libc;
use std::thread;
use std::rc::Rc;
use std::sync::{Arc};
use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering};
use std::collections::BTreeSet;
use parking_lot::{Mutex, MutexGuard, RwLock, RwLockReadGuard};
use concurrent_hashmap::ConcHashMap;
use ram::schema::Schemas;
use ram::types::Id;
use ram::segs::{Segment, SEGMENT_SIZE};
use ram::cell::{Cell, ReadError, WriteError, Header};
use server::ServerMeta;

pub struct Chunk {
    pub id: usize,
    pub addr: usize,
    pub index: ConcHashMap<u64, Mutex<usize>>,
    pub segs: Vec<Segment>,
    pub seg_round: AtomicUsize,
    pub meta: Arc<ServerMeta>,
    pub backup_storage: Option<String>,
}

pub struct Chunks {
    pub list: Vec<Chunk>,
}

impl Chunk {
    fn new (id: usize, size: usize, meta: Arc<ServerMeta>, back_storage: Option<String>) -> Chunk {
        let mem_ptr = unsafe {libc::malloc(size)} as usize;
        let seg_count = size / SEGMENT_SIZE;
        let mut segments = Vec::<Segment>::new();
        for seg_idx in 0..seg_count {
            let seg_addr = mem_ptr + seg_idx * SEGMENT_SIZE;
            segments.push(Segment {
                addr: seg_addr,
                id: seg_idx,
                bound: seg_addr + SEGMENT_SIZE,
                append_header: AtomicUsize::new(seg_addr),
                lock: RwLock::new(()),
                frags: Mutex::new(BTreeSet::new()),
            });
        }
        debug!("creating chunk at {}, segments {}", mem_ptr, seg_count + 1);
        Chunk {
            id: id,
            addr: mem_ptr,
            index: ConcHashMap::<u64, Mutex<usize>>::new(),
            meta: meta,
            segs: segments,
            seg_round: AtomicUsize::new(0),
            backup_storage: back_storage
        }
    }
    pub fn try_acquire(&self, size: usize) -> Option<(usize, RwLockReadGuard<()>)> {
        let mut retried = 0;
        loop {
            let n = self.seg_round.load(Ordering::Relaxed);
            let seg_id = n % self.segs.len();
            let seg_acquire = self.segs[seg_id].try_acquire(size);
            match seg_acquire {
                None => {
                    if retried > self.segs.len() * 2 {return None;}
                    self.seg_round.fetch_add(1, Ordering::Relaxed);
                    retried += 1;
                },
                _ => {return seg_acquire;}
            }
        }
    }
    fn locate_segment(&self, location: usize) -> &Segment {
        let offset = location - self.addr;
        let seg_id = offset / SEGMENT_SIZE;
        return &self.segs[seg_id];
    }
    pub fn location_of(&self, hash: u64) -> Option<MutexGuard<usize>> {
        match self.index.find(&hash) {
            Some(index) => {
                let index = index.get();
                let index_lock = index.lock();
                if *index_lock == 0 {
                    return None
                }
                return Some(index_lock);
            },
            None => None
        }
    }
    fn put_tombstone(&self, location: usize) {
        let seg = self.locate_segment(location);
        seg.put_cell_tombstone(location);
        seg.put_frag(location);
    }
    fn read_cell(&self, hash: u64) -> Result<Cell, ReadError> {
        match self.location_of(hash) {
            Some(loc) => {
                Cell::from_chunk_raw(*loc, self)
            },
            None => Err(ReadError::CellDoesNotExisted)
        }
    }
    fn write_cell(&self, cell: &mut Cell) -> Result<Header, WriteError> {
        let hash = cell.header.hash;
        if self.location_of(hash).is_some() {
            return Err(WriteError::CellAlreadyExisted);
        } else {
            let written = cell.write_to_chunk(self);
            let need_rollback = AtomicBool::new(false);
            if let Ok(loc) = written {
                self.index.upsert(
                    hash,
                    Mutex::new(loc),
                    &|m| {
                        let mut inserted_loc = m.lock();
                        if *inserted_loc == 0 {
                            *inserted_loc = loc
                        } else {
                            need_rollback.store(true, Ordering::Relaxed);
                        }
                    }
                );
                if need_rollback.load(Ordering::Relaxed) {
                    self.put_tombstone(loc);
                    return Err(WriteError::CellAlreadyExisted)
                }
            }
            return Ok(cell.header)
        }
    }
    fn update_cell(&self, cell: &mut Cell) -> Result<Header, WriteError> {
        let hash = cell.header.hash;
        if let Some(mut cell_location) = self.location_of(hash) {
            let written = cell.write_to_chunk(self);
            if let Ok(new_location) = written {
                let old_location = *cell_location;
                *cell_location = new_location;
                self.put_tombstone(old_location);
            }
            return Ok(cell.header);
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn update_cell_by<U>(&self, hash: u64, update: U) -> Result<Cell, WriteError>
        where U: Fn(Cell) -> Option<Cell> {
        if let Some(mut cell_location) = self.location_of(hash) {
            let cell = Cell::from_chunk_raw(*cell_location, self);
            match cell {
                Ok(cell) => {
                    let mut new_cell = update(cell);
                    if let Some(mut new_cell) = new_cell {
                        let written = new_cell.write_to_chunk(self);
                        if let Ok(new_location) = written {
                            let old_location = *cell_location;
                            *cell_location = new_location;
                            self.put_tombstone(old_location);
                        }
                        return Ok(new_cell);
                    } else {
                        return Err(WriteError::UserCanceledUpdate);
                    }
                },
                Err(e) => Err(WriteError::ReadError(e))
            }
        } else {
            return Err(WriteError::CellDoesNotExisted)
        }
    }
    fn remove_cell(&self, hash: u64) -> Result<(), WriteError> {
        if let Some(cell_location) = self.location_of(hash) {
            self.index.remove(&hash);
            self.put_tombstone(*cell_location);
            return Ok(());
        } else {
            return Err(WriteError::CellDoesNotExisted);
        }
    }
    fn remove_cell_by<P>(&self, hash: u64, predict: P) -> Result<(), WriteError>
        where P: Fn(Cell) -> bool {
        if let Some(cell_location) = self.location_of(hash) {
            let cell = Cell::from_chunk_raw(*cell_location, self);
            match cell {
                Ok(cell) => {
                    if predict(cell) {
                        self.index.remove(&hash);
                        self.put_tombstone(*cell_location);
                        return Ok(());
                    } else {
                        return Err(WriteError::DeletionPredictionFailed);
                    }
                },
                Err(e) => Err(WriteError::ReadError(e))
            }
        } else {
            return Err(WriteError::CellDoesNotExisted);
        }
    }
    fn dispose (&mut self) {
        debug!("disposing chunk at {}", self.addr);
        unsafe {
            libc::free(self.addr as *mut libc::c_void)
        }
    }
}

impl Drop for Chunk {
    fn drop(&mut self) {
        self.dispose();
    }
}

impl Chunks {
    pub fn new (count: usize, size: usize, meta: Arc<ServerMeta>, backup_storage: Option<String>) -> Arc<Chunks> {
        let chunk_size = size / count;
        let mut chunks = Vec::new();
        debug!("Creating {} chunks, total {} bytes", count, size);
        for i in 0..count {
            let backup_storage = match backup_storage {
                Some(ref dir) => Some(format!("{}/data-{}.bak", dir, i)),
                None => None
            };
            chunks.push(Chunk::new(i, chunk_size, meta.clone(), backup_storage));
        }
        Arc::new(Chunks {
            list: chunks
        })
    }
    pub fn new_dummy(count: usize, size: usize) -> Arc<Chunks> {
        Chunks::new(count, size, Arc::<ServerMeta>::new(ServerMeta {
            schemas: Schemas::new(None)
        }), None)
    }
    fn locate_chunk_by_partition(&self, partition: u64) -> &Chunk {
        let chunk_id = partition as usize % self.list.len();
        return &self.list[chunk_id];
    }
    fn locate_chunk_by_key(&self, key: &Id) -> (&Chunk, u64) {
        return (self.locate_chunk_by_partition(key.higher), key.lower);
    }
    pub fn read_cell(&self, key: &Id) -> Result<Cell, ReadError> {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.read_cell(hash);
    }
    pub fn write_cell(&self, cell: &mut Cell) -> Result<Header, WriteError> {
        let chunk = self.locate_chunk_by_partition(cell.header.partition);
        return chunk.write_cell(cell);
    }
    pub fn update_cell(&self, cell: &mut Cell) -> Result<Header, WriteError> {
        let chunk = self.locate_chunk_by_partition(cell.header.partition);
        return chunk.update_cell(cell);
    }
    pub fn update_cell_by<U>(&self, key: &Id, update: U) -> Result<Cell, WriteError>
        where U: Fn(Cell) -> Option<Cell>{
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.update_cell_by(hash, update);
    }
    pub fn remove_cell(&self, key: &Id) -> Result<(), WriteError> {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.remove_cell(hash);
    }
    pub fn remove_cell_by<P>(&self, key: &Id, predict: P) -> Result<(), WriteError>
        where P: Fn(Cell) -> bool {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return chunk.remove_cell_by(hash, predict);
    }
    pub fn chunk_ptr(&self, key: &Id) -> usize {
        let (chunk, hash) = self.locate_chunk_by_key(key);
        return *chunk.location_of(hash).unwrap()
    }
}<|endoftext|>


<commit_before>// The Computer Language Benchmarks Game
// http://benchmarksgame.alioth.debian.org/
//
// contributed by the Rust Project Developers
// contributed by BurntSushi
// contributed by TeXitoi

extern crate regex;

use std::io::{self, Read};
use std::sync::Arc;
use std::thread;
use regex::NoExpand;

macro_rules! regex { ($re:expr) => (::regex::Regex::new($re).unwrap()); }

fn main() {
    let mut seq = String::new();
    io::stdin().read_to_string(&mut seq).unwrap();
    let ilen = seq.len();

    seq = regex!(">[^\n]*\n|\n").replace_all(&seq, NoExpand(""));
    let seq_arc = Arc::new(seq.clone()); // copy before it moves
    let clen = seq.len();

    let seqlen = thread::spawn(move|| {
        let substs = vec![
            (regex!("B"), "(c|g|t)"),
            (regex!("D"), "(a|g|t)"),
            (regex!("H"), "(a|c|t)"),
            (regex!("K"), "(g|t)"),
            (regex!("M"), "(a|c)"),
            (regex!("N"), "(a|c|g|t)"),
            (regex!("R"), "(a|g)"),
            (regex!("S"), "(c|g)"),
            (regex!("V"), "(a|c|g)"),
            (regex!("W"), "(a|t)"),
            (regex!("Y"), "(c|t)"),
        ];
        let mut seq = seq;
        for (re, replacement) in substs.into_iter() {
            seq = re.replace_all(&seq, NoExpand(replacement));
        }
        seq.len()
    });

    let variants = vec![
        regex!("agggtaaa|tttaccct"),
        regex!("[cgt]gggtaaa|tttaccc[acg]"),
        regex!("a[act]ggtaaa|tttacc[agt]t"),
        regex!("ag[act]gtaaa|tttac[agt]ct"),
        regex!("agg[act]taaa|ttta[agt]cct"),
        regex!("aggg[acg]aaa|ttt[cgt]ccct"),
        regex!("agggt[cgt]aa|tt[acg]accct"),
        regex!("agggta[cgt]a|t[acg]taccct"),
        regex!("agggtaa[cgt]|[acg]ttaccct"),
    ];
    let (mut variant_strs, mut counts) = (vec!(), vec!());
    for variant in variants.into_iter() {
        let seq_arc_copy = seq_arc.clone();
        variant_strs.push(variant.to_string());
        counts.push(thread::spawn(move|| {
            variant.find_iter(&seq_arc_copy).count()
        }));
    }

    let mut olines = Vec::new();
    for (variant, count) in variant_strs.iter().zip(counts.into_iter()) {
        olines.push(format!("{} {}", variant, count.join().unwrap()));
    }
    olines.push("".to_string());
    olines.push(format!("{}", ilen));
    olines.push(format!("{}", clen));
    olines.push(format!("{}", seqlen.join().unwrap()));
    println!("{}", olines.join("\n"));
}
<commit_msg>include BurntSushi's regex-dna version<commit_after>// The Computer Language Benchmarks Game
// http://benchmarksgame.alioth.debian.org/
//
// contributed by the Rust Project Developers
// contributed by TeXitoi
// contributed by BurntSushi

extern crate regex;

use std::io::{self, Read};
use std::sync::Arc;
use std::thread;

macro_rules! regex { ($re:expr) => { ::regex::Regex::new($re).unwrap() } }

fn main() {
    let mut seq = String::with_capacity(50 * (1 << 20));
    io::stdin().read_to_string(&mut seq).unwrap();
    let ilen = seq.len();

    seq = regex!(">[^\n]*\n|\n").replace_all(&seq, "");
    let clen = seq.len();
    let seq_arc = Arc::new(seq.clone());

    let variants = vec![
        regex!("agggtaaa|tttaccct"),
        regex!("[cgt]gggtaaa|tttaccc[acg]"),
        regex!("a[act]ggtaaa|tttacc[agt]t"),
        regex!("ag[act]gtaaa|tttac[agt]ct"),
        regex!("agg[act]taaa|ttta[agt]cct"),
        regex!("aggg[acg]aaa|ttt[cgt]ccct"),
        regex!("agggt[cgt]aa|tt[acg]accct"),
        regex!("agggta[cgt]a|t[acg]taccct"),
        regex!("agggtaa[cgt]|[acg]ttaccct"),
    ];
    let mut counts = vec![];
    for variant in variants {
        let seq = seq_arc.clone();
        let restr = variant.to_string();
        let future = thread::spawn(move || variant.find_iter(&seq).count());
        counts.push((restr, future));
    }

    let substs = vec![
        (regex!("B"), "(c|g|t)"),
        (regex!("D"), "(a|g|t)"),
        (regex!("H"), "(a|c|t)"),
        (regex!("K"), "(g|t)"),
        (regex!("M"), "(a|c)"),
        (regex!("N"), "(a|c|g|t)"),
        (regex!("R"), "(a|g)"),
        (regex!("S"), "(c|g)"),
        (regex!("V"), "(a|c|g)"),
        (regex!("W"), "(a|t)"),
        (regex!("Y"), "(c|t)"),
    ];
    let mut seq = seq;
    for (re, replacement) in substs.into_iter() {
        seq = re.replace_all(&seq, replacement);
    }
    let rlen = seq.len();

    for (variant, count) in counts {
        println!("{} {}", variant, count.join().unwrap());
    }
    println!("\n{}\n{}\n{}", ilen, clen, rlen);
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate webrender_build;

use std::borrow::Cow;
use std::env;
use std::fs::{canonicalize, read_dir, File};
use std::io::prelude::*;
use std::path::{Path, PathBuf};
use std::collections::hash_map::DefaultHasher;
use std::hash::Hasher;
use webrender_build::shader::*;
use webrender_build::shader_features::{ShaderFeatureFlags, get_shader_features};

/// Compute the shader path for insertion into the include_str!() macro.
/// This makes for more compact generated code than inserting the literal
/// shader source into the generated file.
///
/// If someone is building on a network share, I'm sorry.
fn escape_include_path(path: &Path) -> String {
    let full_path = canonicalize(path).unwrap();
    let full_name = full_path.as_os_str().to_str().unwrap();
    let full_name = full_name.replace("\\\\?\\", "");
    let full_name = full_name.replace("\\", "/");

    full_name
}

fn write_unoptimized_shaders(mut glsl_files: Vec<PathBuf>, shader_file: &mut File) -> Result<(), std::io::Error> {
    writeln!(
        shader_file,
        "  pub static ref UNOPTIMIZED_SHADERS: HashMap<&'static str, SourceWithDigest> = {{"
    )?;
    writeln!(shader_file, "    let mut shaders = HashMap::new();")?;

    // Sort the file list so that the shaders.rs file is filled
    // deterministically.
    glsl_files.sort_by(|a, b| a.file_name().cmp(&b.file_name()));

    for glsl in glsl_files {
        // Compute the shader name.
        assert!(glsl.is_file());
        let shader_name = glsl.file_name().unwrap().to_str().unwrap();
        let shader_name = shader_name.replace(".glsl", "");

        // Compute a digest of the #include-expanded shader source. We store
        // this as a literal alongside the source string so that we don't need
        // to hash large strings at runtime.
        let mut hasher = DefaultHasher::new();
        let base = glsl.parent().unwrap();
        assert!(base.is_dir());
        ShaderSourceParser::new().parse(
            Cow::Owned(shader_source_from_file(&glsl)),
            &|f| Cow::Owned(shader_source_from_file(&base.join(&format!("{}.glsl", f)))),
            &mut |s| hasher.write(s.as_bytes()),
        );
        let digest: ProgramSourceDigest = hasher.into();

        writeln!(
            shader_file,
            "    shaders.insert(\"{}\", SourceWithDigest {{ source: include_str!(\"{}\"), digest: \"{}\"}});",
            shader_name,
            escape_include_path(&glsl),
            digest,
        )?;
    }
    writeln!(shader_file, "    shaders")?;
    writeln!(shader_file, "  }};")?;

    Ok(())
}

#[derive(Clone, Debug)]
struct ShaderOptimizationInput {
    shader_name: &'static str,
    config: String,
    gl_version: ShaderVersion,
}

#[derive(Debug)]
struct ShaderOptimizationOutput {
    full_shader_name: String,
    gl_version: ShaderVersion,
    vert_file_path: PathBuf,
    frag_file_path: PathBuf,
    digest: ProgramSourceDigest,
}

#[derive(Debug)]
struct ShaderOptimizationError {
    shader: ShaderOptimizationInput,
    message: String,
}

fn print_shader_source(shader_src: &str) {
    // For some reason the glsl-opt errors are offset by 1 compared
    // to the provided shader source string.
    println!("0\t|");
    for (n, line) in shader_src.split('\n').enumerate() {
        let line_number = n + 1;
        println!("{}\t|{}", line_number, line);
    }
}

fn write_optimized_shaders(shader_dir: &Path, shader_file: &mut File, out_dir: &str) -> Result<(), std::io::Error> {
    writeln!(
        shader_file,
        "  pub static ref OPTIMIZED_SHADERS: HashMap<(ShaderVersion, &'static str), OptimizedSourceWithDigest> = {{"
    )?;
    writeln!(shader_file, "    let mut shaders = HashMap::new();")?;

    // The full set of optimized shaders can be quite large, so only optimize
    // for the GL version we expect to be used on the target platform. If a different GL
    // version is used we will simply fall back to the unoptimized shaders.
    let shader_versions = match env::var("CARGO_CFG_TARGET_OS").as_ref().map(|s| &**s) {
        Ok("android") | Ok("windows") => [ShaderVersion::Gles],
        _ => [ShaderVersion::Gl],
    };

    let mut shaders = Vec::default();
    for &gl_version in &shader_versions {
        let mut flags = ShaderFeatureFlags::all();
        if gl_version != ShaderVersion::Gl {
            flags.remove(ShaderFeatureFlags::GL);
        }
        if gl_version != ShaderVersion::Gles {
            flags.remove(ShaderFeatureFlags::GLES);
            flags.remove(ShaderFeatureFlags::TEXTURE_EXTERNAL);
        }
        if !matches!(env::var("CARGO_CFG_TARGET_OS").as_ref().map(|s| &**s), Ok("android")) {
            flags.remove(ShaderFeatureFlags::TEXTURE_EXTERNAL_ESSL1);
        }
        flags.remove(ShaderFeatureFlags::DITHERING);

        for (shader_name, configs) in get_shader_features(flags) {
            for config in configs {
                shaders.push(ShaderOptimizationInput {
                    shader_name,
                    config,
                    gl_version,
                });
            }
        }
    }

    let outputs = build_parallel::compile_objects(&|shader: &ShaderOptimizationInput| {
        println!("Optimizing shader {:?}", shader);
        let target = match shader.gl_version {
            ShaderVersion::Gl => glslopt::Target::OpenGl,
            ShaderVersion::Gles => glslopt::Target::OpenGles30,
        };
        let glslopt_ctx = glslopt::Context::new(target);

        let features = shader.config.split(",").filter(|f| !f.is_empty()).collect::<Vec<_>>();

        let (vert_src, frag_src) = build_shader_strings(
            shader.gl_version,
            &features,
            shader.shader_name,
            &|f| Cow::Owned(shader_source_from_file(&shader_dir.join(&format!("{}.glsl", f)))),
        );

        let full_shader_name = if shader.config.is_empty() {
            shader.shader_name.to_string()
        } else {
            format!("{}_{}", shader.shader_name, shader.config.replace(",", "_"))
        };

        let vert = glslopt_ctx.optimize(glslopt::ShaderType::Vertex, vert_src.clone());
        if !vert.get_status() {
            print_shader_source(&vert_src);
            return Err(ShaderOptimizationError {
                shader: shader.clone(),
                message: vert.get_log().to_string(),
            });
        }
        let frag = glslopt_ctx.optimize(glslopt::ShaderType::Fragment, frag_src.clone());
        if !frag.get_status() {
            print_shader_source(&frag_src);
            return Err(ShaderOptimizationError {
                shader: shader.clone(),
                message: frag.get_log().to_string(),
            });
        }

        let vert_source = vert.get_output().unwrap();
        let frag_source = frag.get_output().unwrap();

        // Compute a digest of the optimized shader sources. We store this
        // as a literal alongside the source string so that we don't need
        // to hash large strings at runtime.
        let mut hasher = DefaultHasher::new();

        let vert_file_path = Path::new(out_dir)
            .join(format!("{}_{:?}.vert", full_shader_name, shader.gl_version));
        write_optimized_shader_file(&vert_file_path, vert_source, &shader.shader_name, &features, &mut hasher);

        let frag_file_path = vert_file_path.with_extension("frag");
        write_optimized_shader_file(&frag_file_path, frag_source, &shader.shader_name, &features, &mut hasher);

        let digest: ProgramSourceDigest = hasher.into();

        println!("Finished optimizing shader {:?}", shader);

        Ok(ShaderOptimizationOutput {
            full_shader_name,
            gl_version: shader.gl_version,
            vert_file_path,
            frag_file_path,
            digest,
        })
    }, &shaders);

    match outputs {
        Ok(mut outputs) => {
            // Sort the shader list so that the shaders.rs file is filled
            // deterministically.
            outputs.sort_by(|a, b| {
                (a.gl_version, a.full_shader_name.clone()).cmp(&(b.gl_version, b.full_shader_name.clone()))
            });

            for shader in outputs {
                writeln!(
                    shader_file,
                    "    shaders.insert(({}, \"{}\"), OptimizedSourceWithDigest {{",
                    shader.gl_version.variant_name(),
                    shader.full_shader_name,
                )?;
                writeln!(
                    shader_file,
                    "        vert_source: include_str!(\"{}\"),",
                    escape_include_path(&shader.vert_file_path),
                )?;
                writeln!(
                    shader_file,
                    "        frag_source: include_str!(\"{}\"),",
                    escape_include_path(&shader.frag_file_path),
                )?;
                writeln!(shader_file, "        digest: \"{}\",", shader.digest)?;
                writeln!(shader_file, "    }});")?;
            }
        }
        Err(err) => match err {
            build_parallel::Error::BuildError(err) => {
                panic!("Error optimizing shader {:?}: {}", err.shader, err.message)
            }
            _ => panic!("Error optimizing shaders."),
        }
    }

    writeln!(shader_file, "    shaders")?;
    writeln!(shader_file, "  }};")?;

    Ok(())
}

fn write_optimized_shader_file(
    path: &Path,
    source: &str,
    shader_name: &str,
    features: &[&str],
    hasher: &mut DefaultHasher,
) {
    let mut file = File::create(&path).unwrap();
    for (line_number, line) in source.lines().enumerate() {
        // We embed the shader name and features as a comment in the
        // source to make debugging easier.
        // The #version directive must be on the first line so we insert
        // the extra information on the next line.
        if line_number == 1 {
            let prelude = format!(
                "// {}\n// features: {:?}\n\n",
                shader_name, features
            );
            file.write_all(prelude.as_bytes()).unwrap();
            hasher.write(prelude.as_bytes());
        }
        file.write_all(line.as_bytes()).unwrap();
        file.write_all("\n".as_bytes()).unwrap();
        hasher.write(line.as_bytes());
        hasher.write("\n".as_bytes());
    }
}

fn main() -> Result<(), std::io::Error> {
    let out_dir = env::var("OUT_DIR").unwrap_or("out".to_owned());

    let shaders_file_path = Path::new(&out_dir).join("shaders.rs");
    let mut glsl_files = vec![];

    println!("cargo:rerun-if-changed=res");
    let res_dir = Path::new("res");
    for entry in read_dir(res_dir)? {
        let entry = entry?;
        let path = entry.path();

        if entry.file_name().to_str().unwrap().ends_with(".glsl") {
            println!("cargo:rerun-if-changed={}", path.display());
            glsl_files.push(path.to_owned());
        }
    }

    let mut shader_file = File::create(shaders_file_path)?;

    writeln!(shader_file, "/// AUTO GENERATED BY build.rs\n")?;
    writeln!(shader_file, "use std::collections::HashMap;\n")?;
    writeln!(shader_file, "use webrender_build::shader::ShaderVersion;\n")?;
    writeln!(shader_file, "pub struct SourceWithDigest {{")?;
    writeln!(shader_file, "    pub source: &'static str,")?;
    writeln!(shader_file, "    pub digest: &'static str,")?;
    writeln!(shader_file, "}}\n")?;
    writeln!(shader_file, "pub struct OptimizedSourceWithDigest {{")?;
    writeln!(shader_file, "    pub vert_source: &'static str,")?;
    writeln!(shader_file, "    pub frag_source: &'static str,")?;
    writeln!(shader_file, "    pub digest: &'static str,")?;
    writeln!(shader_file, "}}\n")?;
    writeln!(shader_file, "lazy_static! {{")?;

    write_unoptimized_shaders(glsl_files, &mut shader_file)?;
    writeln!(shader_file, "")?;
    write_optimized_shaders(&res_dir, &mut shader_file, &out_dir)?;
    writeln!(shader_file, "}}")?;

    Ok(())
}
<commit_msg>Bug 1700475 - Allow the WebRender build script to leak under ASAN. r=firefox-build-system-reviewers,glandium<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

extern crate webrender_build;

use std::borrow::Cow;
use std::env;
use std::fs::{canonicalize, read_dir, File};
use std::io::prelude::*;
use std::path::{Path, PathBuf};
use std::collections::hash_map::DefaultHasher;
use std::hash::Hasher;
use webrender_build::shader::*;
use webrender_build::shader_features::{ShaderFeatureFlags, get_shader_features};

// glsopt is known to leak, but we don't particularly care.
#[no_mangle]
pub extern "C" fn __lsan_default_options() -> *const u8 {
    b"detect_leaks=0\0".as_ptr()
}

/// Compute the shader path for insertion into the include_str!() macro.
/// This makes for more compact generated code than inserting the literal
/// shader source into the generated file.
///
/// If someone is building on a network share, I'm sorry.
fn escape_include_path(path: &Path) -> String {
    let full_path = canonicalize(path).unwrap();
    let full_name = full_path.as_os_str().to_str().unwrap();
    let full_name = full_name.replace("\\\\?\\", "");
    let full_name = full_name.replace("\\", "/");

    full_name
}

fn write_unoptimized_shaders(mut glsl_files: Vec<PathBuf>, shader_file: &mut File) -> Result<(), std::io::Error> {
    writeln!(
        shader_file,
        "  pub static ref UNOPTIMIZED_SHADERS: HashMap<&'static str, SourceWithDigest> = {{"
    )?;
    writeln!(shader_file, "    let mut shaders = HashMap::new();")?;

    // Sort the file list so that the shaders.rs file is filled
    // deterministically.
    glsl_files.sort_by(|a, b| a.file_name().cmp(&b.file_name()));

    for glsl in glsl_files {
        // Compute the shader name.
        assert!(glsl.is_file());
        let shader_name = glsl.file_name().unwrap().to_str().unwrap();
        let shader_name = shader_name.replace(".glsl", "");

        // Compute a digest of the #include-expanded shader source. We store
        // this as a literal alongside the source string so that we don't need
        // to hash large strings at runtime.
        let mut hasher = DefaultHasher::new();
        let base = glsl.parent().unwrap();
        assert!(base.is_dir());
        ShaderSourceParser::new().parse(
            Cow::Owned(shader_source_from_file(&glsl)),
            &|f| Cow::Owned(shader_source_from_file(&base.join(&format!("{}.glsl", f)))),
            &mut |s| hasher.write(s.as_bytes()),
        );
        let digest: ProgramSourceDigest = hasher.into();

        writeln!(
            shader_file,
            "    shaders.insert(\"{}\", SourceWithDigest {{ source: include_str!(\"{}\"), digest: \"{}\"}});",
            shader_name,
            escape_include_path(&glsl),
            digest,
        )?;
    }
    writeln!(shader_file, "    shaders")?;
    writeln!(shader_file, "  }};")?;

    Ok(())
}

#[derive(Clone, Debug)]
struct ShaderOptimizationInput {
    shader_name: &'static str,
    config: String,
    gl_version: ShaderVersion,
}

#[derive(Debug)]
struct ShaderOptimizationOutput {
    full_shader_name: String,
    gl_version: ShaderVersion,
    vert_file_path: PathBuf,
    frag_file_path: PathBuf,
    digest: ProgramSourceDigest,
}

#[derive(Debug)]
struct ShaderOptimizationError {
    shader: ShaderOptimizationInput,
    message: String,
}

fn print_shader_source(shader_src: &str) {
    // For some reason the glsl-opt errors are offset by 1 compared
    // to the provided shader source string.
    println!("0\t|");
    for (n, line) in shader_src.split('\n').enumerate() {
        let line_number = n + 1;
        println!("{}\t|{}", line_number, line);
    }
}

fn write_optimized_shaders(shader_dir: &Path, shader_file: &mut File, out_dir: &str) -> Result<(), std::io::Error> {
    writeln!(
        shader_file,
        "  pub static ref OPTIMIZED_SHADERS: HashMap<(ShaderVersion, &'static str), OptimizedSourceWithDigest> = {{"
    )?;
    writeln!(shader_file, "    let mut shaders = HashMap::new();")?;

    // The full set of optimized shaders can be quite large, so only optimize
    // for the GL version we expect to be used on the target platform. If a different GL
    // version is used we will simply fall back to the unoptimized shaders.
    let shader_versions = match env::var("CARGO_CFG_TARGET_OS").as_ref().map(|s| &**s) {
        Ok("android") | Ok("windows") => [ShaderVersion::Gles],
        _ => [ShaderVersion::Gl],
    };

    let mut shaders = Vec::default();
    for &gl_version in &shader_versions {
        let mut flags = ShaderFeatureFlags::all();
        if gl_version != ShaderVersion::Gl {
            flags.remove(ShaderFeatureFlags::GL);
        }
        if gl_version != ShaderVersion::Gles {
            flags.remove(ShaderFeatureFlags::GLES);
            flags.remove(ShaderFeatureFlags::TEXTURE_EXTERNAL);
        }
        if !matches!(env::var("CARGO_CFG_TARGET_OS").as_ref().map(|s| &**s), Ok("android")) {
            flags.remove(ShaderFeatureFlags::TEXTURE_EXTERNAL_ESSL1);
        }
        flags.remove(ShaderFeatureFlags::DITHERING);

        for (shader_name, configs) in get_shader_features(flags) {
            for config in configs {
                shaders.push(ShaderOptimizationInput {
                    shader_name,
                    config,
                    gl_version,
                });
            }
        }
    }

    let outputs = build_parallel::compile_objects(&|shader: &ShaderOptimizationInput| {
        println!("Optimizing shader {:?}", shader);
        let target = match shader.gl_version {
            ShaderVersion::Gl => glslopt::Target::OpenGl,
            ShaderVersion::Gles => glslopt::Target::OpenGles30,
        };
        let glslopt_ctx = glslopt::Context::new(target);

        let features = shader.config.split(",").filter(|f| !f.is_empty()).collect::<Vec<_>>();

        let (vert_src, frag_src) = build_shader_strings(
            shader.gl_version,
            &features,
            shader.shader_name,
            &|f| Cow::Owned(shader_source_from_file(&shader_dir.join(&format!("{}.glsl", f)))),
        );

        let full_shader_name = if shader.config.is_empty() {
            shader.shader_name.to_string()
        } else {
            format!("{}_{}", shader.shader_name, shader.config.replace(",", "_"))
        };

        let vert = glslopt_ctx.optimize(glslopt::ShaderType::Vertex, vert_src.clone());
        if !vert.get_status() {
            print_shader_source(&vert_src);
            return Err(ShaderOptimizationError {
                shader: shader.clone(),
                message: vert.get_log().to_string(),
            });
        }
        let frag = glslopt_ctx.optimize(glslopt::ShaderType::Fragment, frag_src.clone());
        if !frag.get_status() {
            print_shader_source(&frag_src);
            return Err(ShaderOptimizationError {
                shader: shader.clone(),
                message: frag.get_log().to_string(),
            });
        }

        let vert_source = vert.get_output().unwrap();
        let frag_source = frag.get_output().unwrap();

        // Compute a digest of the optimized shader sources. We store this
        // as a literal alongside the source string so that we don't need
        // to hash large strings at runtime.
        let mut hasher = DefaultHasher::new();

        let vert_file_path = Path::new(out_dir)
            .join(format!("{}_{:?}.vert", full_shader_name, shader.gl_version));
        write_optimized_shader_file(&vert_file_path, vert_source, &shader.shader_name, &features, &mut hasher);

        let frag_file_path = vert_file_path.with_extension("frag");
        write_optimized_shader_file(&frag_file_path, frag_source, &shader.shader_name, &features, &mut hasher);

        let digest: ProgramSourceDigest = hasher.into();

        println!("Finished optimizing shader {:?}", shader);

        Ok(ShaderOptimizationOutput {
            full_shader_name,
            gl_version: shader.gl_version,
            vert_file_path,
            frag_file_path,
            digest,
        })
    }, &shaders);

    match outputs {
        Ok(mut outputs) => {
            // Sort the shader list so that the shaders.rs file is filled
            // deterministically.
            outputs.sort_by(|a, b| {
                (a.gl_version, a.full_shader_name.clone()).cmp(&(b.gl_version, b.full_shader_name.clone()))
            });

            for shader in outputs {
                writeln!(
                    shader_file,
                    "    shaders.insert(({}, \"{}\"), OptimizedSourceWithDigest {{",
                    shader.gl_version.variant_name(),
                    shader.full_shader_name,
                )?;
                writeln!(
                    shader_file,
                    "        vert_source: include_str!(\"{}\"),",
                    escape_include_path(&shader.vert_file_path),
                )?;
                writeln!(
                    shader_file,
                    "        frag_source: include_str!(\"{}\"),",
                    escape_include_path(&shader.frag_file_path),
                )?;
                writeln!(shader_file, "        digest: \"{}\",", shader.digest)?;
                writeln!(shader_file, "    }});")?;
            }
        }
        Err(err) => match err {
            build_parallel::Error::BuildError(err) => {
                panic!("Error optimizing shader {:?}: {}", err.shader, err.message)
            }
            _ => panic!("Error optimizing shaders."),
        }
    }

    writeln!(shader_file, "    shaders")?;
    writeln!(shader_file, "  }};")?;

    Ok(())
}

fn write_optimized_shader_file(
    path: &Path,
    source: &str,
    shader_name: &str,
    features: &[&str],
    hasher: &mut DefaultHasher,
) {
    let mut file = File::create(&path).unwrap();
    for (line_number, line) in source.lines().enumerate() {
        // We embed the shader name and features as a comment in the
        // source to make debugging easier.
        // The #version directive must be on the first line so we insert
        // the extra information on the next line.
        if line_number == 1 {
            let prelude = format!(
                "// {}\n// features: {:?}\n\n",
                shader_name, features
            );
            file.write_all(prelude.as_bytes()).unwrap();
            hasher.write(prelude.as_bytes());
        }
        file.write_all(line.as_bytes()).unwrap();
        file.write_all("\n".as_bytes()).unwrap();
        hasher.write(line.as_bytes());
        hasher.write("\n".as_bytes());
    }
}

fn main() -> Result<(), std::io::Error> {
    let out_dir = env::var("OUT_DIR").unwrap_or("out".to_owned());

    let shaders_file_path = Path::new(&out_dir).join("shaders.rs");
    let mut glsl_files = vec![];

    println!("cargo:rerun-if-changed=res");
    let res_dir = Path::new("res");
    for entry in read_dir(res_dir)? {
        let entry = entry?;
        let path = entry.path();

        if entry.file_name().to_str().unwrap().ends_with(".glsl") {
            println!("cargo:rerun-if-changed={}", path.display());
            glsl_files.push(path.to_owned());
        }
    }

    let mut shader_file = File::create(shaders_file_path)?;

    writeln!(shader_file, "/// AUTO GENERATED BY build.rs\n")?;
    writeln!(shader_file, "use std::collections::HashMap;\n")?;
    writeln!(shader_file, "use webrender_build::shader::ShaderVersion;\n")?;
    writeln!(shader_file, "pub struct SourceWithDigest {{")?;
    writeln!(shader_file, "    pub source: &'static str,")?;
    writeln!(shader_file, "    pub digest: &'static str,")?;
    writeln!(shader_file, "}}\n")?;
    writeln!(shader_file, "pub struct OptimizedSourceWithDigest {{")?;
    writeln!(shader_file, "    pub vert_source: &'static str,")?;
    writeln!(shader_file, "    pub frag_source: &'static str,")?;
    writeln!(shader_file, "    pub digest: &'static str,")?;
    writeln!(shader_file, "}}\n")?;
    writeln!(shader_file, "lazy_static! {{")?;

    write_unoptimized_shaders(glsl_files, &mut shader_file)?;
    writeln!(shader_file, "")?;
    write_optimized_shaders(&res_dir, &mut shader_file, &out_dir)?;
    writeln!(shader_file, "}}")?;

    Ok(())
}
<|endoftext|>


<commit_before>//
//  SOS: the Stupid Operating System
//  by Hawk Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015 Hawk Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! `x86_64` architecture-specific implementation.
// pub mod cpu;
pub mod drivers;
pub mod interrupts;

#[path = "../x86_all/bda.rs"] pub mod bda;
#[path = "../x86_all/multiboot2.rs"] pub mod multiboot2;

pub const ARCH_BITS: u8 = 64;

use memory::PAddr;

/// Trampoline to ensure we have a correct stack frame for calling [`arch_init`]
///
/// I have no idea why this works, but it does.
///
/// [`arch_init`]: fn.arch_init
#[naked]
#[no_mangle]
pub unsafe extern "C" fn long_mode_init() {
    asm!("movabsq $$(stack_top), %rsp");
    asm!("mov ax, 0
          mov ss, ax
          mov ds, ax
          mov es, ax
          mov fs, ax
          mov gs, ax
          call arch_init"
        :::: "intel");

}

/// Entry point for architecture-specific kernel init
///
/// This expects to be passed the address of a valid
/// Multiboot 2 info struct. It's the bootloader's responsibility to ensure
/// that this is passed in the correct register as expected by the calling
/// convention (`edi` on x86). If this isn't there, you can expect to have a
/// bad problem and not go to space today.
#[no_mangle]
pub extern "C" fn arch_init(multiboot_addr: PAddr) {
    use cpu::{control_regs, msr};
    use elf;
    use params::InitParams;

    kinfoln!(dots: " . ", "Beginning `arch_init()` for x86_64");

    ::io::term::CONSOLE.lock().clear();
    ::logger::initialize()
        .expect("Could not initialize logger!");

    // -- Unpack multiboot tag ------------------------------------------------
    kinfoln!( dots: " . "
            , "trying to unpack multiboot info at {:?}"
            , multiboot_addr);

    // try to interpret the structure at the multiboot address as a multiboot
    // info struct. if it's invalid, fail.
    let boot_info
        = unsafe { multiboot2::Info::from(multiboot_addr)
                    .expect("Could not unpack multiboot2 information!") };

    // Extract the memory map tag from the multiboot info
    let mmap_tag
        = boot_info.mem_map()
                   .expect("Memory map tag required!");

    kinfoln!(dots: " . ", "Detected memory areas:");
    for a in mmap_tag.areas() {
        kinfoln!( dots: " . . ", "start: {:#08x}, end: {:#08x}"
                , a.base, a.length );
        // TODO: add these to a list of memory areas?
        //       - eliza, 1/23/2017
    }

    // Extract ELF sections tag from the multiboot info
    let elf_sections_tag
        = boot_info.elf_sections()
                   .expect("ELF sections tag required!");

    kinfoln!(dots: " . ", "Detecting kernel ELF sections:");

    // Extract kernel ELF sections from  multiboot info
    let mut n_elf_sections = 0;

    let kernel_begin
        = elf_sections_tag.sections()
            .inspect(|s| {
                kinfoln!( dots: " . . "
                        , "address: {:#08x}, size: {:#08x}, flags: {:#08x}"
                        , s.addr()
                        , s.length()
                        , s.flags() );
                n_elf_sections += 1;
                })
            // the ELF section with the lowest address is the kernel start
            // section
            .min_by_key(elf::Section::addr)
            .expect("Could not find kernel start section!\
                    \nSomething is deeply wrong.");

    let kernel_end
        = elf_sections_tag.sections()
            .max_by_key(elf::Section::addr)
            .expect("Could not find kernel end section!\
                    \nSomething is deeply wrong.");

    kinfoln!( dots: " . ", "Detected {} kernel ELF sections.", n_elf_sections);
    kinfoln!( dots: " . . ", "Kernel begins at {:#p} and ends at {:#p}."
            , kernel_begin.addr(), kernel_end.addr() );

    let multiboot_end = multiboot_addr + boot_info.length as u64;

    kinfoln!( dots: " . . ", "Multiboot info begins at {:#x} and ends at {:#x}."
            , multiboot_addr, multiboot_end);

    let params = InitParams { multiboot_start: Some(multiboot_addr)
                            , multiboot_end: Some(multiboot_end)
                            , ..Default::default()
                            };

     // -- enable flags needed for paging ------------------------------------
     unsafe {
         control_regs::cr0::enable_write_protect(true);
         kinfoln!(dots: " . ", "Page write protect ENABED" );

         msr::enable_nxe();
         kinfoln!(dots: " . ", "Page no execute bit ENABLED");
     }

    kinfoln!(dots: " . ", "Transferring to `kernel_init()`.");
    ::kernel_init(params);
}
<commit_msg>[x86_64] skip non-allocated kernel ELF sections<commit_after>//
//  SOS: the Stupid Operating System
//  by Hawk Weisman (hi@hawkweisman.me)
//
//  Copyright (c) 2015 Hawk Weisman
//  Released under the terms of the MIT license. See `LICENSE` in the root
//  directory of this repository for more information.
//
//! `x86_64` architecture-specific implementation.
// pub mod cpu;
pub mod drivers;
pub mod interrupts;

#[path = "../x86_all/bda.rs"] pub mod bda;
#[path = "../x86_all/multiboot2.rs"] pub mod multiboot2;

pub const ARCH_BITS: u8 = 64;

use memory::PAddr;

/// Trampoline to ensure we have a correct stack frame for calling [`arch_init`]
///
/// I have no idea why this works, but it does.
///
/// [`arch_init`]: fn.arch_init
#[naked]
#[no_mangle]
pub unsafe extern "C" fn long_mode_init() {
    asm!("movabsq $$(stack_top), %rsp");
    asm!("mov ax, 0
          mov ss, ax
          mov ds, ax
          mov es, ax
          mov fs, ax
          mov gs, ax
          call arch_init"
        :::: "intel");

}

/// Entry point for architecture-specific kernel init
///
/// This expects to be passed the address of a valid
/// Multiboot 2 info struct. It's the bootloader's responsibility to ensure
/// that this is passed in the correct register as expected by the calling
/// convention (`edi` on x86). If this isn't there, you can expect to have a
/// bad problem and not go to space today.
#[no_mangle]
pub extern "C" fn arch_init(multiboot_addr: PAddr) {
    use cpu::{control_regs, msr};
    use elf;
    use params::InitParams;

    kinfoln!(dots: " . ", "Beginning `arch_init()` for x86_64");

    ::io::term::CONSOLE.lock().clear();
    ::logger::initialize()
        .expect("Could not initialize logger!");

    // -- Unpack multiboot tag ------------------------------------------------
    kinfoln!( dots: " . "
            , "trying to unpack multiboot info at {:?}"
            , multiboot_addr);

    // try to interpret the structure at the multiboot address as a multiboot
    // info struct. if it's invalid, fail.
    let boot_info
        = unsafe { multiboot2::Info::from(multiboot_addr)
                    .expect("Could not unpack multiboot2 information!") };

    // Extract the memory map tag from the multiboot info
    let mmap_tag
        = boot_info.mem_map()
                   .expect("Memory map tag required!");

    kinfoln!(dots: " . ", "Detected memory areas:");
    for a in mmap_tag.areas() {
        kinfoln!( dots: " . . ", "start: {:#08x}, end: {:#08x}"
                , a.base, a.length );
        // TODO: add these to a list of memory areas?
        //       - eliza, 1/23/2017
    }

    // Extract ELF sections tag from the multiboot info
    let elf_sections_tag
        = boot_info.elf_sections()
                   .expect("ELF sections tag required!");

    kinfoln!(dots: " . ", "Detecting kernel ELF sections:");

    // Extract kernel ELF sections from  multiboot info
    let mut n_elf_sections = 0;

    let kernel_begin
        = elf_sections_tag.sections()
            .filter(|s| s.is_allocated())
            .inspect(|s| {
                kinfoln!( dots: " . . "
                        , "address: {:#08x}, size: {:#08x}, flags: {:#08x}"
                        , s.addr()
                        , s.length()
                        , s.flags() );
                n_elf_sections += 1;
                })
            // the ELF section with the lowest address is the kernel start
            // section
            .min_by_key(elf::Section::addr)
            .expect("Could not find kernel start section!\
                    \nSomething is deeply wrong.");

    let kernel_end
        = elf_sections_tag.sections()
            .filter(|s| s.is_allocated())
            .max_by_key(elf::Section::addr)
            .expect("Could not find kernel end section!\
                    \nSomething is deeply wrong.");

    kinfoln!( dots: " . ", "Detected {} kernel ELF sections.", n_elf_sections);
    kinfoln!( dots: " . . ", "Kernel begins at {:#p} and ends at {:#p}."
            , kernel_begin.addr(), kernel_end.addr() );

    let multiboot_end = multiboot_addr + boot_info.length as u64;

    kinfoln!( dots: " . . ", "Multiboot info begins at {:#x} and ends at {:#x}."
            , multiboot_addr, multiboot_end);

    let params = InitParams { multiboot_start: Some(multiboot_addr)
                            , multiboot_end: Some(multiboot_end)
                            , ..Default::default()
                            };

     // -- enable flags needed for paging ------------------------------------
     unsafe {
         control_regs::cr0::enable_write_protect(true);
         kinfoln!(dots: " . ", "Page write protect ENABED" );

         msr::enable_nxe();
         kinfoln!(dots: " . ", "Page no execute bit ENABLED");
     }

    kinfoln!(dots: " . ", "Transferring to `kernel_init()`.");
    ::kernel_init(params);
}
<|endoftext|>