<commit_before>use std::mem;
use std::ffi::CString;

use cbox::{CSemiBox, DisposeRef};
use ffi::prelude::{LLVMBuilderRef, LLVMValueRef};
use ffi::{core, LLVMBuilder, LLVMRealPredicate, LLVMIntPredicate};
use libc::{c_char, c_uint};

use context::Context;
use block::BasicBlock;
use ty::Type;
use value::{Function, Value, Predicate};
use phi::PhiNode;

static NULL_NAME:[c_char; 1] = [0];

/// This provides a uniform API for creating instructions and inserting them into a basic block.
pub struct Builder;
native_ref!(&Builder = LLVMBuilderRef);

macro_rules! bin_op (
  ($name:ident, $func:ident) => (
    pub fn $name(&self, left: &Value, right: &Value) -> &Value
    {
      unsafe { core::$func(self.into(), left.into(), right.into(), NULL_NAME.as_ptr()) }.into()
    }
  );
  ($name:ident, $ifunc:ident, $ffunc:ident) => (
    pub fn $name(&self, left: &Value, right: &Value) -> &Value {
      let ty = left.get_type();
      unsafe {
        (if ty.is_integer() {
           core::$ifunc
         } else {
           core::$ffunc
         })(self.into(), left.into(), right.into(), NULL_NAME.as_ptr()).into()
      }
    }
  );
);

macro_rules! unary_op (
  ($name:ident, $func:ident) => (
    pub fn $name(&self, value: &Value) -> &Value {
      unsafe { core::$func(self.into(), value.into(), NULL_NAME.as_ptr() as *const c_char) }.into()
    }
  );
);

impl Builder
{
  /// Create a new builder in the context given.
  pub fn new(context: &Context) -> CSemiBox<Builder>
  {
    CSemiBox::new(unsafe { core::LLVMCreateBuilderInContext(context.into()) }.into())
  }

  pub fn get_insert_block(&self) -> &BasicBlock {
    unsafe { core::LLVMGetInsertBlock(self.into()) }.into()
  }

  /// Position the builder at `instr` within `block`.
  pub fn position_at(&self, block: &BasicBlock, instr: &Value) {
    unsafe { core::LLVMPositionBuilder(self.into(), block.into(), instr.into()) }
  }

  /// Position the builder at the end of `block`.
  pub fn position_at_end(&self, block: &BasicBlock)
  {
    unsafe { core::LLVMPositionBuilderAtEnd(self.into(), block.into()) }
  }

  /// Build an instruction that returns from the function with void.
  pub fn create_ret_void(&self) -> &Value
  {
    unsafe { core::LLVMBuildRetVoid(self.into()) }.into()
  }

  /// Build an instruction that returns from the function with `value`.
  pub fn create_ret(&self, value: &Value) -> &Value
  {
    unsafe { core::LLVMBuildRet(self.into(), value.into()) }.into()
  }

  /// Build an instruction that allocates an array with the element type `elem` and the size `size`.
  ///
  /// The size of this array will be the size of `elem` times `size`.
  pub fn build_array_alloca(&self, elem: &Type, size: &Value) -> &Value
  {
    unsafe {
    	core::LLVMBuildArrayAlloca(self.into(),
    														 elem.into(),
    														 size.into(),
    														 NULL_NAME.as_ptr() as *const c_char)
    }.into()
  }

  /// Build an instruction that allocates a pointer to fit the size of `ty` then returns this pointer.
  ///
  /// Make sure to call `build_free` with the pointer value when you're done with it, or you're
  /// gonna have a bad time.
  pub fn create_alloca(&self, ty: &Type) -> &Value
  {
    unsafe {
    	core::LLVMBuildAlloca(self.into(),
    												ty.into(),
    												NULL_NAME.as_ptr() as *const c_char)
    }.into()
  }

  /// Build an instruction that frees the `val`, which _MUST_ be a pointer that was returned
  /// from `build_alloca`.
  pub fn create_free(&self, val: &Value) -> &Value
  {
    unsafe { core::LLVMBuildFree(self.into(), val.into()) }.into()
  }

  /// Build an instruction that store the value `val` in the pointer `ptr`.
  pub fn create_store(&self, val: &Value, ptr: &Value) -> &Value
  {
    unsafe { core::LLVMBuildStore(self.into(), val.into(), ptr.into()) }.into()
  }

  /// Build an instruction that branches to the block `dest`.
  pub fn create_br(&self, dest: &BasicBlock) -> &Value
  {
    unsafe { core::LLVMBuildBr(self.into(), dest.into()).into() }
  }

  /// Build an instruction that branches to `if_block` if `cond` evaluates to true, and `else_block` otherwise.
  pub fn create_cond_br(&self, cond: &Value,
  	                    if_block: &BasicBlock, else_block: Option<&BasicBlock>) -> &Value
  {
    unsafe {
    	core::LLVMBuildCondBr(self.into(),
    		                    cond.into(),
    		                    if_block.into(),
    		                    mem::transmute(else_block)).into()
    }
  }

  /// Build an instruction that calls the function `func` with the arguments `args`.
  ///
  /// This will return the return value of the function.
  pub fn create_call(&self, func: &Function, args: &[&Value]) -> &Value
  {
    unsafe {
        let call = core::LLVMBuildCall(self.into(),
        	                             func.into(), args.as_ptr() as *mut LLVMValueRef,
        	                             args.len() as c_uint,
        	                             NULL_NAME.as_ptr());
        core::LLVMSetTailCall(call, 0);
        call.into()
    }
  }

  /// Build an instruction that calls the function `func` with the arguments `args`.
  ///
  /// This will return the return value of the function.
  pub fn create_tail_call(&self, func: &Function, args: &[&Value]) -> &Value
  {
    unsafe {
        let call = core::LLVMBuildCall(self.into(),
        	                             func.into(), args.as_ptr() as *mut LLVMValueRef,
        	                             args.len() as c_uint,
        	                             NULL_NAME.as_ptr());
        core::LLVMSetTailCall(call, 1);
        call.into()
    }
  }
  /// Build an instruction that yields to `true_val` if `cond` is equal to `1`, and `false_val` otherwise.
  pub fn create_select(&self, cond: &Value, true_val: &Value, false_val: &Value) -> &Value
  {
    unsafe {
    	core::LLVMBuildSelect(self.into(),
    		                    cond.into(),
    		                    true_val.into(),
    		                    false_val.into(),
    		                    NULL_NAME.as_ptr()).into() }
  }

  /// Build an instruction that casts a value into a certain type.
  pub fn create_bit_cast(&self, value: &Value, dest: &Type) -> &Value
  {
    unsafe { core::LLVMBuildBitCast(
    		self.into(),
    		value.into(),
    		dest.into(),
    		NULL_NAME.as_ptr()).into()
    }
  }

  /// Build an instruction that inserts a value into an aggregate data value.
  pub fn create_insert_value(&self, agg: &Value, elem: &Value, index: usize) -> &Value
  {
    unsafe {
    	core::LLVMBuildInsertValue(self.into(),
    		                         agg.into(),
    		                         elem.into(),
    		                         index as c_uint,
    		                         NULL_NAME.as_ptr()).into()
    }
  }

  /// Build an instruction that computes the address of a subelement of an aggregate data structure.
  ///
  /// Basically type-safe pointer arithmetic.
  pub fn create_gep(&self, pointer: &Value, indices: &[&Value]) -> &Value
  {
    unsafe {
    	core::LLVMBuildInBoundsGEP(self.into(),
    	                           pointer.into(),
    	                           indices.as_ptr() as *mut LLVMValueRef,
    	                           indices.len() as c_uint,
    	                           NULL_NAME.as_ptr()).into()
    }
  }

     /// Build an instruction to select a value depending on the predecessor of the current block.
  pub fn create_phi(&self, ty: &Type, name: &str) -> &PhiNode
  {
	  unsafe {
	  	core::LLVMBuildPhi(self.into(), ty.into(), CString::new(name).unwrap().as_ptr())
  	}.into()
  }

  /// Build an instruction that runs whichever block matches the value, or `default` if none of them matched it.
  pub fn create_switch(&self,
  	                   value: &Value,
  	                   default: &BasicBlock,
  	                   cases: &[(&Value, &BasicBlock)]) -> &Value
  {
    unsafe {
      let switch = core::LLVMBuildSwitch(self.into(),
      	                                 value.into(),
      	                                 default.into(),
      	                                 cases.len() as c_uint);
      for case in cases {
        core::LLVMAddCase(switch, case.0.into(), case.1.into());
      }
      switch.into()
    }
  }

  unary_op!{create_load, LLVMBuildLoad}
  unary_op!{create_neg, LLVMBuildNeg}
  unary_op!{create_not, LLVMBuildNot}

  bin_op!{create_add, LLVMBuildAdd, LLVMBuildFAdd}
  bin_op!{create_sub, LLVMBuildSub, LLVMBuildFSub}
  bin_op!{create_mul, LLVMBuildMul, LLVMBuildFMul}
  bin_op!{create_div, LLVMBuildSDiv, LLVMBuildFDiv}
  bin_op!{create_shl, LLVMBuildShl}
  bin_op!{create_ashr, LLVMBuildAShr}
  bin_op!{create_and, LLVMBuildAnd}
  bin_op!{create_or, LLVMBuildOr}
  bin_op!{create_xor, LLVMBuildXor}

  /// Build an instruction to compare two values with the predicate given.
  pub fn create_cmp(&self, a: &Value, b: &Value, pred: Predicate) -> &Value
  {
    let (at, bt) = (a.get_type(), b.get_type());
    assert_eq!(at, bt);

    if at.is_integer() {
      let pred = match pred {
          Predicate::Equal => LLVMIntPredicate::LLVMIntEQ,
          Predicate::NotEqual => LLVMIntPredicate::LLVMIntNE,
          Predicate::GreaterThan => LLVMIntPredicate::LLVMIntSGT,
          Predicate::GreaterThanOrEqual => LLVMIntPredicate::LLVMIntSGE,
          Predicate::LessThan => LLVMIntPredicate::LLVMIntSLT,
          Predicate::LessThanOrEqual => LLVMIntPredicate::LLVMIntSLE
      };
      unsafe {
      	core::LLVMBuildICmp(self.into(),
      		                  pred,
      		                  a.into(), b.into(),
      		                  NULL_NAME.as_ptr())
      }.into()

    } else if at.is_float() {
      let pred = match pred {
          Predicate::Equal => LLVMRealPredicate::LLVMRealOEQ,
          Predicate::NotEqual => LLVMRealPredicate::LLVMRealONE,
          Predicate::GreaterThan => LLVMRealPredicate::LLVMRealOGT,
          Predicate::GreaterThanOrEqual => LLVMRealPredicate::LLVMRealOGE,
          Predicate::LessThan => LLVMRealPredicate::LLVMRealOLT,
          Predicate::LessThanOrEqual => LLVMRealPredicate::LLVMRealOLE
      };
      unsafe {
      	core::LLVMBuildFCmp(self.into(),
      		                 pred, a.into(), b.into(),
      		                 NULL_NAME.as_ptr())
      }.into()

    } else {
        panic!("expected numbers, got {:?}", at)
    }
  }
}

impl DisposeRef for Builder {
  type RefTo = LLVMBuilder;
  #[inline(always)]
  unsafe fn dispose(ptr: LLVMBuilderRef) {
      core::LLVMDisposeBuilder(ptr)
  }
}
<commit_msg>Add support for building a remainder instruction.<commit_after>use std::mem;
use std::ffi::CString;

use cbox::{CSemiBox, DisposeRef};
use ffi::prelude::{LLVMBuilderRef, LLVMValueRef};
use ffi::{core, LLVMBuilder, LLVMRealPredicate, LLVMIntPredicate};
use libc::{c_char, c_uint};

use context::Context;
use block::BasicBlock;
use ty::Type;
use value::{Function, Value, Predicate};
use phi::PhiNode;

static NULL_NAME:[c_char; 1] = [0];

/// This provides a uniform API for creating instructions and inserting them into a basic block.
pub struct Builder;
native_ref!(&Builder = LLVMBuilderRef);

macro_rules! bin_op (
  ($name:ident, $func:ident) => (
    pub fn $name(&self, left: &Value, right: &Value) -> &Value
    {
      unsafe { core::$func(self.into(), left.into(), right.into(), NULL_NAME.as_ptr()) }.into()
    }
  );
  ($name:ident, $ifunc:ident, $ffunc:ident) => (
    pub fn $name(&self, left: &Value, right: &Value) -> &Value {
      let ty = left.get_type();
      unsafe {
        (if ty.is_integer() {
           core::$ifunc
         } else {
           core::$ffunc
         })(self.into(), left.into(), right.into(), NULL_NAME.as_ptr()).into()
      }
    }
  );
);

macro_rules! unary_op (
  ($name:ident, $func:ident) => (
    pub fn $name(&self, value: &Value) -> &Value {
      unsafe { core::$func(self.into(), value.into(), NULL_NAME.as_ptr() as *const c_char) }.into()
    }
  );
);

impl Builder
{
  /// Create a new builder in the context given.
  pub fn new(context: &Context) -> CSemiBox<Builder>
  {
    CSemiBox::new(unsafe { core::LLVMCreateBuilderInContext(context.into()) }.into())
  }

  pub fn get_insert_block(&self) -> &BasicBlock {
    unsafe { core::LLVMGetInsertBlock(self.into()) }.into()
  }

  /// Position the builder at `instr` within `block`.
  pub fn position_at(&self, block: &BasicBlock, instr: &Value) {
    unsafe { core::LLVMPositionBuilder(self.into(), block.into(), instr.into()) }
  }

  /// Position the builder at the end of `block`.
  pub fn position_at_end(&self, block: &BasicBlock)
  {
    unsafe { core::LLVMPositionBuilderAtEnd(self.into(), block.into()) }
  }

  /// Build an instruction that returns from the function with void.
  pub fn create_ret_void(&self) -> &Value
  {
    unsafe { core::LLVMBuildRetVoid(self.into()) }.into()
  }

  /// Build an instruction that returns from the function with `value`.
  pub fn create_ret(&self, value: &Value) -> &Value
  {
    unsafe { core::LLVMBuildRet(self.into(), value.into()) }.into()
  }

  /// Build an instruction that allocates an array with the element type `elem` and the size `size`.
  ///
  /// The size of this array will be the size of `elem` times `size`.
  pub fn build_array_alloca(&self, elem: &Type, size: &Value) -> &Value
  {
    unsafe {
    	core::LLVMBuildArrayAlloca(self.into(),
    														 elem.into(),
    														 size.into(),
    														 NULL_NAME.as_ptr() as *const c_char)
    }.into()
  }

  /// Build an instruction that allocates a pointer to fit the size of `ty` then returns this pointer.
  ///
  /// Make sure to call `build_free` with the pointer value when you're done with it, or you're
  /// gonna have a bad time.
  pub fn create_alloca(&self, ty: &Type) -> &Value
  {
    unsafe {
    	core::LLVMBuildAlloca(self.into(),
    												ty.into(),
    												NULL_NAME.as_ptr() as *const c_char)
    }.into()
  }

  /// Build an instruction that frees the `val`, which _MUST_ be a pointer that was returned
  /// from `build_alloca`.
  pub fn create_free(&self, val: &Value) -> &Value
  {
    unsafe { core::LLVMBuildFree(self.into(), val.into()) }.into()
  }

  /// Build an instruction that store the value `val` in the pointer `ptr`.
  pub fn create_store(&self, val: &Value, ptr: &Value) -> &Value
  {
    unsafe { core::LLVMBuildStore(self.into(), val.into(), ptr.into()) }.into()
  }

  /// Build an instruction that branches to the block `dest`.
  pub fn create_br(&self, dest: &BasicBlock) -> &Value
  {
    unsafe { core::LLVMBuildBr(self.into(), dest.into()).into() }
  }

  /// Build an instruction that branches to `if_block` if `cond` evaluates to true, and `else_block` otherwise.
  pub fn create_cond_br(&self, cond: &Value,
  	                    if_block: &BasicBlock, else_block: Option<&BasicBlock>) -> &Value
  {
    unsafe {
    	core::LLVMBuildCondBr(self.into(),
    		                    cond.into(),
    		                    if_block.into(),
    		                    mem::transmute(else_block)).into()
    }
  }

  /// Build an instruction that calls the function `func` with the arguments `args`.
  ///
  /// This will return the return value of the function.
  pub fn create_call(&self, func: &Function, args: &[&Value]) -> &Value
  {
    unsafe {
        let call = core::LLVMBuildCall(self.into(),
        	                             func.into(), args.as_ptr() as *mut LLVMValueRef,
        	                             args.len() as c_uint,
        	                             NULL_NAME.as_ptr());
        core::LLVMSetTailCall(call, 0);
        call.into()
    }
  }

  /// Build an instruction that calls the function `func` with the arguments `args`.
  ///
  /// This will return the return value of the function.
  pub fn create_tail_call(&self, func: &Function, args: &[&Value]) -> &Value
  {
    unsafe {
        let call = core::LLVMBuildCall(self.into(),
        	                             func.into(), args.as_ptr() as *mut LLVMValueRef,
        	                             args.len() as c_uint,
        	                             NULL_NAME.as_ptr());
        core::LLVMSetTailCall(call, 1);
        call.into()
    }
  }
  /// Build an instruction that yields to `true_val` if `cond` is equal to `1`, and `false_val` otherwise.
  pub fn create_select(&self, cond: &Value, true_val: &Value, false_val: &Value) -> &Value
  {
    unsafe {
    	core::LLVMBuildSelect(self.into(),
    		                    cond.into(),
    		                    true_val.into(),
    		                    false_val.into(),
    		                    NULL_NAME.as_ptr()).into() }
  }

  /// Build an instruction that casts a value into a certain type.
  pub fn create_bit_cast(&self, value: &Value, dest: &Type) -> &Value
  {
    unsafe { core::LLVMBuildBitCast(
    		self.into(),
    		value.into(),
    		dest.into(),
    		NULL_NAME.as_ptr()).into()
    }
  }

  /// Build an instruction that inserts a value into an aggregate data value.
  pub fn create_insert_value(&self, agg: &Value, elem: &Value, index: usize) -> &Value
  {
    unsafe {
    	core::LLVMBuildInsertValue(self.into(),
    		                         agg.into(),
    		                         elem.into(),
    		                         index as c_uint,
    		                         NULL_NAME.as_ptr()).into()
    }
  }

  /// Build an instruction that computes the address of a subelement of an aggregate data structure.
  ///
  /// Basically type-safe pointer arithmetic.
  pub fn create_gep(&self, pointer: &Value, indices: &[&Value]) -> &Value
  {
    unsafe {
    	core::LLVMBuildInBoundsGEP(self.into(),
    	                           pointer.into(),
    	                           indices.as_ptr() as *mut LLVMValueRef,
    	                           indices.len() as c_uint,
    	                           NULL_NAME.as_ptr()).into()
    }
  }

     /// Build an instruction to select a value depending on the predecessor of the current block.
  pub fn create_phi(&self, ty: &Type, name: &str) -> &PhiNode
  {
	  unsafe {
	  	core::LLVMBuildPhi(self.into(), ty.into(), CString::new(name).unwrap().as_ptr())
  	}.into()
  }

  /// Build an instruction that runs whichever block matches the value, or `default` if none of them matched it.
  pub fn create_switch(&self,
  	                   value: &Value,
  	                   default: &BasicBlock,
  	                   cases: &[(&Value, &BasicBlock)]) -> &Value
  {
    unsafe {
      let switch = core::LLVMBuildSwitch(self.into(),
      	                                 value.into(),
      	                                 default.into(),
      	                                 cases.len() as c_uint);
      for case in cases {
        core::LLVMAddCase(switch, case.0.into(), case.1.into());
      }
      switch.into()
    }
  }

  unary_op!{create_load, LLVMBuildLoad}
  unary_op!{create_neg, LLVMBuildNeg}
  unary_op!{create_not, LLVMBuildNot}

  bin_op!{create_add, LLVMBuildAdd, LLVMBuildFAdd}
  bin_op!{create_sub, LLVMBuildSub, LLVMBuildFSub}
  bin_op!{create_mul, LLVMBuildMul, LLVMBuildFMul}
  bin_op!{create_div, LLVMBuildSDiv, LLVMBuildFDiv}
  bin_op!{create_rem, LLVMBuildSRem, LLVMBuildFRem}
  bin_op!{create_shl, LLVMBuildShl}
  bin_op!{create_ashr, LLVMBuildAShr}
  bin_op!{create_and, LLVMBuildAnd}
  bin_op!{create_or, LLVMBuildOr}
  bin_op!{create_xor, LLVMBuildXor}

  /// Build an instruction to compare two values with the predicate given.
  pub fn create_cmp(&self, a: &Value, b: &Value, pred: Predicate) -> &Value
  {
    let (at, bt) = (a.get_type(), b.get_type());
    assert_eq!(at, bt);

    if at.is_integer() {
      let pred = match pred {
          Predicate::Equal => LLVMIntPredicate::LLVMIntEQ,
          Predicate::NotEqual => LLVMIntPredicate::LLVMIntNE,
          Predicate::GreaterThan => LLVMIntPredicate::LLVMIntSGT,
          Predicate::GreaterThanOrEqual => LLVMIntPredicate::LLVMIntSGE,
          Predicate::LessThan => LLVMIntPredicate::LLVMIntSLT,
          Predicate::LessThanOrEqual => LLVMIntPredicate::LLVMIntSLE
      };
      unsafe {
      	core::LLVMBuildICmp(self.into(),
      		                  pred,
      		                  a.into(), b.into(),
      		                  NULL_NAME.as_ptr())
      }.into()

    } else if at.is_float() {
      let pred = match pred {
          Predicate::Equal => LLVMRealPredicate::LLVMRealOEQ,
          Predicate::NotEqual => LLVMRealPredicate::LLVMRealONE,
          Predicate::GreaterThan => LLVMRealPredicate::LLVMRealOGT,
          Predicate::GreaterThanOrEqual => LLVMRealPredicate::LLVMRealOGE,
          Predicate::LessThan => LLVMRealPredicate::LLVMRealOLT,
          Predicate::LessThanOrEqual => LLVMRealPredicate::LLVMRealOLE
      };
      unsafe {
      	core::LLVMBuildFCmp(self.into(),
      		                 pred, a.into(), b.into(),
      		                 NULL_NAME.as_ptr())
      }.into()

    } else {
        panic!("expected numbers, got {:?}", at)
    }
  }
}

impl DisposeRef for Builder {
  type RefTo = LLVMBuilder;
  #[inline(always)]
  unsafe fn dispose(ptr: LLVMBuilderRef) {
      core::LLVMDisposeBuilder(ptr)
  }
}
<|endoftext|>


<commit_before>use criterion::{black_box, criterion_group, criterion_main, Criterion};
use lettre::{Message, SmtpTransport, Transport};

fn bench_simple_send(c: &mut Criterion) {
    let sender = SmtpTransport::builder("127.0.0.1").port(2525).build();

    c.bench_function("send email", move |b| {
        b.iter(|| {
            let email = Message::builder()
                .from("NoBody <nobody@domain.tld>".parse().unwrap())
                .reply_to("Yuin <yuin@domain.tld>".parse().unwrap())
                .to("Hei <hei@domain.tld>".parse().unwrap())
                .subject("Happy new year")
                .body("Be happy!")
                .unwrap();
            let result = black_box(sender.send(&email));
            assert!(result.is_ok());
        })
    });
}

fn bench_reuse_send(c: &mut Criterion) {
    let sender = SmtpTransport::builder("127.0.0.1").port(2525).build();
    c.bench_function("send email with connection reuse", move |b| {
        b.iter(|| {
            let email = Message::builder()
                .from("NoBody <nobody@domain.tld>".parse().unwrap())
                .reply_to("Yuin <yuin@domain.tld>".parse().unwrap())
                .to("Hei <hei@domain.tld>".parse().unwrap())
                .subject("Happy new year")
                .body("Be happy!")
                .unwrap();
            let result = black_box(sender.send(&email));
            assert!(result.is_ok());
        })
    });
}

criterion_group!(benches, bench_simple_send, bench_reuse_send);
criterion_main!(benches);
<commit_msg>chore: fix benches<commit_after>use criterion::{black_box, criterion_group, criterion_main, Criterion};
use lettre::{Message, SmtpTransport, Transport};

fn bench_simple_send(c: &mut Criterion) {
    let sender = SmtpTransport::builder_dangerous("127.0.0.1")
        .port(2525)
        .build();

    c.bench_function("send email", move |b| {
        b.iter(|| {
            let email = Message::builder()
                .from("NoBody <nobody@domain.tld>".parse().unwrap())
                .reply_to("Yuin <yuin@domain.tld>".parse().unwrap())
                .to("Hei <hei@domain.tld>".parse().unwrap())
                .subject("Happy new year")
                .body("Be happy!")
                .unwrap();
            let result = black_box(sender.send(&email));
            assert!(result.is_ok());
        })
    });
}

fn bench_reuse_send(c: &mut Criterion) {
    let sender = SmtpTransport::builder_dangerous("127.0.0.1")
        .port(2525)
        .build();
    c.bench_function("send email with connection reuse", move |b| {
        b.iter(|| {
            let email = Message::builder()
                .from("NoBody <nobody@domain.tld>".parse().unwrap())
                .reply_to("Yuin <yuin@domain.tld>".parse().unwrap())
                .to("Hei <hei@domain.tld>".parse().unwrap())
                .subject("Happy new year")
                .body("Be happy!")
                .unwrap();
            let result = black_box(sender.send(&email));
            assert!(result.is_ok());
        })
    });
}

criterion_group!(benches, bench_simple_send, bench_reuse_send);
criterion_main!(benches);
<|endoftext|>


<commit_before>use libc::*;

use std::io::process::{Command, ProcessOutput, ProcessExit, Process};
use std::io::process::StdioContainer::*;
use std::io::{IoError, IoErrorKind, IoResult};
use std::io::IoErrorKind::*;
use std::collections::VecMap;

use controls::*;
use constants::*;
use termios::*;
use signal::*;

// start off as null pointer
static mut uglobal_term:*mut TermState = 0 as *mut TermState;

unsafe extern fn term_sigint(_:c_int, _:*const SigInfo,
                             _:*const c_void) {
    let term:&mut TermState = match uglobal_term.as_mut() {
        Some(v) => v,
        None => {
            // this handler shouldn't be called when Term isn't active
            panic!("Term signal interrupt called when Term not active");
        }
    };
    term.controls.outs("\nInterrupt\n");
    // pass on to foreground job, if there is one
    match term.fg_job {
        None => {/* nothing */},
        Some(id) => {
            match term.interrupt_job(&id) {
                Err(e) => {
                    term.controls.errf(format_args!("Could not interrupt job: {}", e));
                },
                Ok(_) => {/* nothing */}
            }
        }
    }
}

pub struct TermState {
    pub controls: Controls,
    tios: Termios,
    old_tios: Termios,
    jobs: VecMap<(String, Process)>,
    fg_job: Option<usize>
}

impl TermState {
    pub fn new() -> TermState {
        let mut controls = Controls::new();
        let mut tios = match Termios::get() {
            Some(t) => t,
            None => {
                controls.err("Warning: Could not get terminal mode\n");
                Termios::new()
            }
        };
        let old_tios = tios.clone();
        tios.fdisable(0, 0, ICANON|ECHO, 0);

        return TermState {
            controls: controls,
            tios: tios,
            old_tios: old_tios,
            jobs: VecMap::new(),
            fg_job: None
        };
    }

    pub fn update_terminal(&mut self) {
        if !Termios::set(&self.tios) {
            self.controls.err("Warning: Could not set terminal mode\n");
        }
    }

    pub fn restore_terminal(&mut self) {
        if !Termios::set(&self.old_tios) {
            self.controls.err("Warning: Could not set terminal mode\n");
        }
    }

    fn set_pointer(&mut self) {
        unsafe {
            if !uglobal_term.is_null() {
                panic!("Tried to set Term location twice");
            }
            uglobal_term = self as *mut TermState;
        }
    }

    fn unset_pointer(&mut self) {
        unsafe {
            if uglobal_term.is_null() {
                panic!("Tried to unset Term location twice");
            }
            uglobal_term = 0 as *mut TermState;
        }
    }

    fn handle_sigint(&mut self) {
        self.set_pointer();
        let mut sa = SigAction {
            handler: term_sigint,
            mask: [0; SIGSET_NWORDS],
            flags: SA_RESTART | SA_SIGINFO,
            restorer: 0 // null pointer
        };
        let mask = full_sigset().expect("Could not get a full sigset");
        sa.mask = mask;
        if !signal_handle(SIGINT, &sa) {
            self.controls.err("Warning: could not set handler for SIGINT\n");
        }
    }

    fn unhandle_sigint(&mut self) {
        self.unset_pointer();
        if !signal_ignore(SIGINT) {
            self.controls.err("Warning: could not unset handler for SIGINT\n");
        }
    }

    pub fn interrupt_job(&mut self, id:&usize) -> IoResult<()> {
        match self.jobs.get_mut(id) {
            None => return Err(IoError {
                kind: OtherIoError,
                desc: "Job not found",
                detail: None
            }),
            Some(&mut (_, ref mut job)) => {
                return job.signal(SIGINT as isize);
            }
        }
    }

    fn find_jobs_hole(&self) -> usize {
        // find a hole in the job map
        if self.jobs.len() < 20 {
            // with less than twenty jobs just be lazy
            return self.jobs.len() + 1;
        }
        let mut last = 0;
        for key in self.jobs.keys() {
            if key - last != 1 {
                // we've found a hole
                return key - 1;
            } else {
                last = key;
            }
        }
        // job list is full
        return last + 1;
    }

    pub fn run_job(&mut self, name:&String, args:&Vec<String>) -> Result<(usize, String), String> {
        let mut process = Command::new(name);
        process.args(args.as_slice());
        // running as job means no stdin handle
        process.stdin(Ignored);
        // all others are redirected
        let mut child = match process.spawn() {
            Err(e) => {
                return Err(format!("Couldn't spawn {}: {}", name, e));
            },
            Ok(v) => v
        };
        // set wait timeout to zero so we can check for process exit quickly
        child.set_timeout(Some(0));
        let id = self.find_jobs_hole();
        // panic if we've overwritten a job
        match self.jobs.insert(id.clone(), (name.clone(), child)) {
            Some(_) => panic!("Overwrote job"),
            _ => {/* nothing */}
        }
        return Ok((id, name.clone()));
    }

    pub fn clean_jobs(&mut self) -> Vec<(usize, String, IoResult<ProcessExit>)> {
        let mut out = vec![];
        let mut remove = vec![];
        for (id, &mut (ref mut name, ref mut child)) in self.jobs.iter_mut() {
             match child.wait() {
                 Err(IoError{kind: IoErrorKind::TimedOut, desc: _, detail: _}) => {
                     // this is expected, do nothing
                 },
                 v => {
                     // all other outputs mean the job is done
                     // if it isn't it'll be cleaned up in drop
                     child.set_timeout(None);
                     remove.push(id);
                     out.push((id, name.clone(), v));
                 }
             }
        }
        for id in remove.iter() {
            self.jobs.remove(id);
        }
        return out;
    }

    pub fn run_command(&mut self, name:&String, args:&Vec<String>) -> Result<ProcessExit, String> {
        let mut process = Command::new(name);
        process.args(args.as_slice());
        process.stdout(InheritFd(STDOUT));
        process.stdin(InheritFd(STDIN));
        process.stderr(InheritFd(STDERR));
        // set terminal settings for process
        self.restore_terminal();
        // push job into jobs
        let id = self.find_jobs_hole();
        // handle interrupts
        self.handle_sigint();
        let val = (name.clone(), match process.spawn() {
            Err(e) => {
                self.update_terminal();
                return Err(format!("Couldn't spawn {}: {}", name, e));
            },
            Ok(v) => v
        });
        // insert job
        self.jobs.insert(id, val);
        // set forground job
        self.fg_job = Some(id);
        let out = match self.jobs.get_mut(&id).unwrap().1.wait() {
            Err(e) => {
                // unset foreground job
                self.fg_job = None;
                // remove job
                self.jobs.remove(&id);
                self.update_terminal();
                return Err(format!("Couldn't wait for child to exit: {}", e));
            },
            Ok(v) => v
        };
        // unset forground job
        self.fg_job = None;
        // remove job
        self.jobs.remove(&id);
        // unhandle sigint
        self.unhandle_sigint();
        // restore settings for Wash
        self.update_terminal();
        return Ok(out);
    }

    pub fn run_command_directed(&mut self, name:&String,
                                args:&Vec<String>) -> Result<ProcessOutput, String> {
        let mut process = Command::new(name);
        process.args(args.as_slice());
        let id = self.find_jobs_hole();
        // handle sigint
        self.handle_sigint();
        let val = (name.clone(), match process.spawn() {
            Err(e) => {
                self.update_terminal();
                return Err(format!("Couldn't spawn {}: {}", name, e));
            },
            Ok(v) => v
        });
        // put job in jobs list
        self.jobs.insert(id, val);
        // set foreground job
        self.fg_job = Some(id);
        let out = match self.jobs.get_mut(&id).unwrap().1.stdout.as_mut().unwrap().read_to_end() {
            Ok(v) => v,
            Err(e) => {
                // unset foreground job
                self.fg_job = None;
                // remove job
                self.jobs.remove(&id);
                return Err(format!("Couldn't get stdout: {}", e));
            }
        };
        let err = match self.jobs.get_mut(&id).unwrap().1.stderr.as_mut().unwrap().read_to_end() {
            Ok(v) => v,
            Err(e) => {
                // unset foreground job
                self.fg_job = None;
                // remove job
                self.jobs.remove(&id);
                return Err(format!("Couldn't get stderr: {}", e));
            }
        };
        let output = match self.jobs.get_mut(&id).unwrap().1.wait() {
            Err(e) => {
                // unset foreground job
                self.fg_job = None;
                // remove job
                self.jobs.remove(&id);
                self.update_terminal();
                return Err(format!("Couldn't wait for child to exit: {}", e));
            },
            Ok(v) => ProcessOutput {
                status: v,
                output: out,
                error: err
            }
        };
        // unset foreground job
        self.fg_job = None;
        // remove job
        self.jobs.remove(&id);
        // unhandle sigint
        self.unhandle_sigint();
        return Ok(output);
    }
}

<commit_msg>Background jobs get no standard input/output<commit_after>use libc::*;

use std::io::process::{Command, ProcessOutput, ProcessExit, Process};
use std::io::process::StdioContainer::*;
use std::io::{IoError, IoErrorKind, IoResult};
use std::io::IoErrorKind::*;
use std::collections::VecMap;

use controls::*;
use constants::*;
use termios::*;
use signal::*;

// start off as null pointer
static mut uglobal_term:*mut TermState = 0 as *mut TermState;

unsafe extern fn term_sigint(_:c_int, _:*const SigInfo,
                             _:*const c_void) {
    let term:&mut TermState = match uglobal_term.as_mut() {
        Some(v) => v,
        None => {
            // this handler shouldn't be called when Term isn't active
            panic!("Term signal interrupt called when Term not active");
        }
    };
    term.controls.outs("\nInterrupt\n");
    // pass on to foreground job, if there is one
    match term.fg_job {
        None => {/* nothing */},
        Some(id) => {
            match term.interrupt_job(&id) {
                Err(e) => {
                    term.controls.errf(format_args!("Could not interrupt job: {}", e));
                },
                Ok(_) => {/* nothing */}
            }
        }
    }
}

pub struct TermState {
    pub controls: Controls,
    tios: Termios,
    old_tios: Termios,
    jobs: VecMap<(String, Process)>,
    fg_job: Option<usize>
}

impl TermState {
    pub fn new() -> TermState {
        let mut controls = Controls::new();
        let mut tios = match Termios::get() {
            Some(t) => t,
            None => {
                controls.err("Warning: Could not get terminal mode\n");
                Termios::new()
            }
        };
        let old_tios = tios.clone();
        tios.fdisable(0, 0, ICANON|ECHO, 0);

        return TermState {
            controls: controls,
            tios: tios,
            old_tios: old_tios,
            jobs: VecMap::new(),
            fg_job: None
        };
    }

    pub fn update_terminal(&mut self) {
        if !Termios::set(&self.tios) {
            self.controls.err("Warning: Could not set terminal mode\n");
        }
    }

    pub fn restore_terminal(&mut self) {
        if !Termios::set(&self.old_tios) {
            self.controls.err("Warning: Could not set terminal mode\n");
        }
    }

    fn set_pointer(&mut self) {
        unsafe {
            if !uglobal_term.is_null() {
                panic!("Tried to set Term location twice");
            }
            uglobal_term = self as *mut TermState;
        }
    }

    fn unset_pointer(&mut self) {
        unsafe {
            if uglobal_term.is_null() {
                panic!("Tried to unset Term location twice");
            }
            uglobal_term = 0 as *mut TermState;
        }
    }

    fn handle_sigint(&mut self) {
        self.set_pointer();
        let mut sa = SigAction {
            handler: term_sigint,
            mask: [0; SIGSET_NWORDS],
            flags: SA_RESTART | SA_SIGINFO,
            restorer: 0 // null pointer
        };
        let mask = full_sigset().expect("Could not get a full sigset");
        sa.mask = mask;
        if !signal_handle(SIGINT, &sa) {
            self.controls.err("Warning: could not set handler for SIGINT\n");
        }
    }

    fn unhandle_sigint(&mut self) {
        self.unset_pointer();
        if !signal_ignore(SIGINT) {
            self.controls.err("Warning: could not unset handler for SIGINT\n");
        }
    }

    pub fn interrupt_job(&mut self, id:&usize) -> IoResult<()> {
        match self.jobs.get_mut(id) {
            None => return Err(IoError {
                kind: OtherIoError,
                desc: "Job not found",
                detail: None
            }),
            Some(&mut (_, ref mut job)) => {
                return job.signal(SIGINT as isize);
            }
        }
    }

    fn find_jobs_hole(&self) -> usize {
        // find a hole in the job map
        if self.jobs.len() < 20 {
            // with less than twenty jobs just be lazy
            return self.jobs.len() + 1;
        }
        let mut last = 0;
        for key in self.jobs.keys() {
            if key - last != 1 {
                // we've found a hole
                return key - 1;
            } else {
                last = key;
            }
        }
        // job list is full
        return last + 1;
    }

    pub fn run_job(&mut self, name:&String, args:&Vec<String>) -> Result<(usize, String), String> {
        let mut process = Command::new(name);
        process.args(args.as_slice());
        // running as job means no input/output handles
        process.stdin(Ignored);
        process.stdout(Ignored);
        process.stderr(Ignored);
        // all others are redirected
        let mut child = match process.spawn() {
            Err(e) => {
                return Err(format!("Couldn't spawn {}: {}", name, e));
            },
            Ok(v) => v
        };
        // set wait timeout to zero so we can check for process exit quickly
        child.set_timeout(Some(0));
        let id = self.find_jobs_hole();
        // panic if we've overwritten a job
        match self.jobs.insert(id.clone(), (name.clone(), child)) {
            Some(_) => panic!("Overwrote job"),
            _ => {/* nothing */}
        }
        return Ok((id, name.clone()));
    }

    pub fn clean_jobs(&mut self) -> Vec<(usize, String, IoResult<ProcessExit>)> {
        let mut out = vec![];
        let mut remove = vec![];
        for (id, &mut (ref mut name, ref mut child)) in self.jobs.iter_mut() {
             match child.wait() {
                 Err(IoError{kind: IoErrorKind::TimedOut, desc: _, detail: _}) => {
                     // this is expected, do nothing
                 },
                 v => {
                     // all other outputs mean the job is done
                     // if it isn't it'll be cleaned up in drop
                     child.set_timeout(None);
                     remove.push(id);
                     out.push((id, name.clone(), v));
                 }
             }
        }
        for id in remove.iter() {
            self.jobs.remove(id);
        }
        return out;
    }

    pub fn run_command(&mut self, name:&String, args:&Vec<String>) -> Result<ProcessExit, String> {
        let mut process = Command::new(name);
        process.args(args.as_slice());
        process.stdout(InheritFd(STDOUT));
        process.stdin(InheritFd(STDIN));
        process.stderr(InheritFd(STDERR));
        // set terminal settings for process
        self.restore_terminal();
        // push job into jobs
        let id = self.find_jobs_hole();
        // handle interrupts
        self.handle_sigint();
        let val = (name.clone(), match process.spawn() {
            Err(e) => {
                self.update_terminal();
                return Err(format!("Couldn't spawn {}: {}", name, e));
            },
            Ok(v) => v
        });
        // insert job
        self.jobs.insert(id, val);
        // set forground job
        self.fg_job = Some(id);
        let out = match self.jobs.get_mut(&id).unwrap().1.wait() {
            Err(e) => {
                // unset foreground job
                self.fg_job = None;
                // remove job
                self.jobs.remove(&id);
                self.update_terminal();
                return Err(format!("Couldn't wait for child to exit: {}", e));
            },
            Ok(v) => v
        };
        // unset forground job
        self.fg_job = None;
        // remove job
        self.jobs.remove(&id);
        // unhandle sigint
        self.unhandle_sigint();
        // restore settings for Wash
        self.update_terminal();
        return Ok(out);
    }

    pub fn run_command_directed(&mut self, name:&String,
                                args:&Vec<String>) -> Result<ProcessOutput, String> {
        let mut process = Command::new(name);
        process.args(args.as_slice());
        let id = self.find_jobs_hole();
        // handle sigint
        self.handle_sigint();
        let val = (name.clone(), match process.spawn() {
            Err(e) => {
                self.update_terminal();
                return Err(format!("Couldn't spawn {}: {}", name, e));
            },
            Ok(v) => v
        });
        // put job in jobs list
        self.jobs.insert(id, val);
        // set foreground job
        self.fg_job = Some(id);
        let out = match self.jobs.get_mut(&id).unwrap().1.stdout.as_mut().unwrap().read_to_end() {
            Ok(v) => v,
            Err(e) => {
                // unset foreground job
                self.fg_job = None;
                // remove job
                self.jobs.remove(&id);
                return Err(format!("Couldn't get stdout: {}", e));
            }
        };
        let err = match self.jobs.get_mut(&id).unwrap().1.stderr.as_mut().unwrap().read_to_end() {
            Ok(v) => v,
            Err(e) => {
                // unset foreground job
                self.fg_job = None;
                // remove job
                self.jobs.remove(&id);
                return Err(format!("Couldn't get stderr: {}", e));
            }
        };
        let output = match self.jobs.get_mut(&id).unwrap().1.wait() {
            Err(e) => {
                // unset foreground job
                self.fg_job = None;
                // remove job
                self.jobs.remove(&id);
                self.update_terminal();
                return Err(format!("Couldn't wait for child to exit: {}", e));
            },
            Ok(v) => ProcessOutput {
                status: v,
                output: out,
                error: err
            }
        };
        // unset foreground job
        self.fg_job = None;
        // remove job
        self.jobs.remove(&id);
        // unhandle sigint
        self.unhandle_sigint();
        return Ok(output);
    }
}

<|endoftext|>


<commit_before>// Implements http://rosettacode.org/wiki/Arithmetic/Complex
#![cfg(not_tested)]

extern crate num;

use num::complex::Cmplx;

fn main() {
    let a = Cmplx::new(-4.0, 5.0);
    let b = Cmplx::new(1.0, 1.0);

    println!("a = {}", a);
    println!("b = {}", b);
    println!("a + b = {}", a + b);
    println!("a * b = {}", a * b);
    println!("1 / a = {}", Cmplx::new(1.0, 0.0) / a);
    println!("-a = {}", -a);
    println!("conj a = {}", a.conj());
}
<commit_msg>Fix Complex<commit_after>// Implements http://rosettacode.org/wiki/Arithmetic/Complex
#![cfg(not_tested)]

extern crate num;

use num::complex::Complex;

fn main() {
    let a = Complex::new(-4.0, 5.0);
    let b = Complex::new(1.0, 1.0);

    println!("a = {}", a);
    println!("b = {}", b);
    println!("a + b = {}", a + b);
    println!("a * b = {}", a * b);
    println!("1 / a = {}", Complex::new(1.0, 0.0) / a);
    println!("-a = {}", -a);
    println!("conj a = {}", a.conj());
}
<|endoftext|>


<commit_before>//! Functions for corner point detection, also known as interest points.

use image::{
    GenericImage,
    Luma
};

use std::cmp;

/// A location and score for a detected corner.
/// The scores need not be comparable between different
/// corner detectors.
pub struct Corner {
    pub x: u32,
    pub y: u32,
    pub score: Option<f32>
}

impl Corner {
    pub fn new(x: u32, y: u32, score: Option<f32>) -> Corner {
        Corner {x: x, y: y, score: score}
    }
}

/// Finds corners using FAST-12 features. Given a point P, define
/// C_P to be the 16-pixel Bresenham circle of radius 3 centred at P.
/// We define a point P of intensity I to be a corner if at least
/// 12 contiguous pixels in C_P have an intensity strictly
/// greater than I + threshold or strictly less than I - threshold.
/// https://en.wikipedia.org/wiki/Features_from_accelerated_segment_test
pub fn corners_fast12<I>(image: &I, threshold: u8) -> Vec<Corner>
    where I: GenericImage<Pixel=Luma<u8>> + 'static {

    let (width, height) = image.dimensions();
    let mut corners = vec![];

    for y in 0..height {
        for x in 0..width {
            if is_corner_fast12(image, threshold, x, y) {
                corners.push(Corner::new(x,y, None));
            }
        }
    }

    corners
}

/// Checks if the given pixel is a corner.
fn is_corner_fast12<I>(image: &I, threshold: u8, x: u32, y: u32)
        -> bool
    where I: GenericImage<Pixel=Luma<u8>> + 'static {

    let (width, height) = image.dimensions();

    if x < 3 || y < 3 || x >= width - 3 || y >= height - 3 {
        return false;
    }

    let c = image.get_pixel(x, y)[0];
    let low_thresh: i16  = c as i16 - threshold as i16;
    let high_thresh: i16 = c as i16 + threshold as i16;

    // Circle pixels are labelled clockwise, starting at 0 directly above
    // the center, and finishing at 15, three pixels above and one to
    // the left of the center

    let p0 = image.get_pixel(x, y - 3)[0] as i16;
    let p8 = image.get_pixel(x, y + 3)[0] as i16;

    let mut above = p0 > high_thresh && p8 > high_thresh;
    let mut below = p0 < low_thresh  && p8 < low_thresh;

    if !above && !below {
        return false;
    }

    let p4  = image.get_pixel(x + 3, y)[0] as i16;
    let p12 = image.get_pixel(x - 3, y)[0] as i16;

    above = above && ((p4 > high_thresh) || (p12 > high_thresh));
    below = below && ((p4 < low_thresh)  || (p12 < low_thresh));

    if !above && !below {
        return false;
    }

    // TODO: Generate a list of pixel offsets once per image,
    // TODO: and use those offsets directly when reading pixels.
    // TODO: This is a little tricky as we can't always do it - we'd
    // TODO: need to distinguish between GenericImages and ImageBuffers.
    // TODO: We can also reduce the number of checks we do below.

    let mut pixels = [0i16; 16];

    pixels[0]  = p0;
    pixels[1]  = image.get_pixel(x + 1, y - 3)[0] as i16;
    pixels[2]  = image.get_pixel(x + 2, y - 2)[0] as i16;
    pixels[3]  = image.get_pixel(x + 3, y - 1)[0] as i16;
    pixels[4]  = p4;
    pixels[5]  = image.get_pixel(x + 3, y + 1)[0] as i16;
    pixels[6]  = image.get_pixel(x + 2, y + 2)[0] as i16;
    pixels[7]  = image.get_pixel(x + 1, y + 3)[0] as i16;
    pixels[8]  = p8;
    pixels[9]  = image.get_pixel(x - 1, y + 3)[0] as i16;
    pixels[10] = image.get_pixel(x - 2, y + 2)[0] as i16;
    pixels[11] = image.get_pixel(x - 3, y + 1)[0] as i16;
    pixels[12] = p12;
    pixels[13] = image.get_pixel(x - 3, y - 1)[0] as i16;
    pixels[14] = image.get_pixel(x - 2, y - 2)[0] as i16;
    pixels[15] = image.get_pixel(x - 1, y - 3)[0] as i16;

    if above {
        for i in 0..15 {
            let mut is_corner = true;
            let end   = i + 12;
            let upper = cmp::min(end, 16);
            let wrap  = if end > 16 { end - 16 } else { 0 };

            for j in i..upper {
                if pixels[j] <= high_thresh {
                    is_corner = false;
                    break;
                }
            }
            if is_corner {
                for j in 0..wrap {
                    if pixels[j] <= high_thresh {
                        is_corner = false;
                        break;
                    }
                }
            }
            if is_corner {
                return true;
            }
        }
    }
    else {
        for i in 0..15 {
            let mut is_corner = true;
            let end   = i + 12;
            let upper = cmp::min(end, 16);
            let wrap  = if end > 16 { end - 16 } else { 0 };

            for j in i..upper {
                if pixels[j] >= low_thresh {
                    is_corner = false;
                    break;
                }
            }
            if is_corner {
                for j in 0..wrap {
                    if pixels[j] >= low_thresh {
                        is_corner = false;
                        break;
                    }
                }
            }
            if is_corner {
                return true;
            }
        }
    }

    false
}

#[cfg(test)]
mod test {

    use super::{
        is_corner_fast12
    };
    use image::{
        GrayImage,
        ImageBuffer
    };

    #[test]
    fn test_is_corner_fast12_12_contiguous_darker_pixels() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            10, 10, 00, 00, 00, 10, 10,
            10, 00, 10, 10, 10, 00, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            10, 00, 10, 10, 10, 10, 10,
            10, 10, 00, 00, 00, 10, 10]).unwrap();

        assert_eq!(is_corner_fast12(&image, 8, 3, 3), true);
    }

    #[test]
    fn test_is_corner_fast12_12_contiguous_darker_pixels_large_threshold() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            10, 10, 00, 00, 00, 10, 10,
            10, 00, 10, 10, 10, 00, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            10, 00, 10, 10, 10, 10, 10,
            10, 10, 00, 00, 00, 10, 10]).unwrap();

        assert_eq!(is_corner_fast12(&image, 15, 3, 3), false);
    }

    #[test]
    fn test_is_corner_fast12_12_contiguous_lighter_pixels() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            00, 00, 10, 10, 10, 00, 00,
            00, 10, 00, 00, 00, 10, 00,
            10, 00, 00, 00, 00, 00, 00,
            10, 00, 00, 00, 00, 00, 00,
            10, 00, 00, 00, 00, 00, 00,
            00, 10, 00, 00, 00, 00, 00,
            00, 00, 10, 10, 10, 00, 00]).unwrap();

        assert_eq!(is_corner_fast12(&image, 8, 3, 3), true);
    }

    #[test]
    fn test_is_corner_fast12_12_noncontiguous() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            10, 10, 00, 00, 00, 10, 10,
            10, 00, 10, 10, 10, 00, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            10, 10, 10, 10, 10, 10, 00,
            10, 00, 10, 10, 10, 10, 10,
            10, 10, 00, 00, 00, 10, 10]).unwrap();

        assert_eq!(is_corner_fast12(&image, 8, 3, 3), false);
    }

    #[test]
    fn test_is_corner_fast12_near_image_boundary() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            10, 10, 00, 00, 00, 10, 10,
            10, 00, 10, 10, 10, 00, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            10, 00, 10, 10, 10, 10, 10,
            10, 10, 00, 00, 00, 10, 10]).unwrap();

        assert_eq!(is_corner_fast12(&image, 8, 1, 1), false);
    }
}
<commit_msg>Add corner_score_fast12<commit_after>//! Functions for corner point detection, also known as interest points.

use image::{
    GenericImage,
    Luma
};

use std::cmp;

/// A location and score for a detected corner.
/// The scores need not be comparable between different
/// corner detectors.
pub struct Corner {
    pub x: u32,
    pub y: u32,
    pub score: Option<f32>
}

impl Corner {
    pub fn new(x: u32, y: u32, score: Option<f32>) -> Corner {
        Corner {x: x, y: y, score: score}
    }
}

/// Finds corners using FAST-12 features. Given a point P, define
/// C_P to be the 16-pixel Bresenham circle of radius 3 centred at P.
/// We define a point P of intensity I to be a corner if at least
/// 12 contiguous pixels in C_P have an intensity strictly
/// greater than I + threshold or strictly less than I - threshold.
/// https://en.wikipedia.org/wiki/Features_from_accelerated_segment_test
pub fn corners_fast12<I>(image: &I, threshold: u8) -> Vec<Corner>
    where I: GenericImage<Pixel=Luma<u8>> + 'static {

    let (width, height) = image.dimensions();
    let mut corners = vec![];

    for y in 0..height {
        for x in 0..width {
            if is_corner_fast12(image, threshold, x, y) {
                corners.push(Corner::new(x,y, None));
            }
        }
    }

    corners
}

/// The score of a corner detected using the FAST12
/// detector is the largest threshold for which this
/// pixel is still a corner. We input the threshold at which
/// the corner was detected as a lower bound on the search.
/// Note that the corner check uses a strict inequality, so if
/// the smallest intensity difference between the center pixel
/// and a corner pixel is n then the corner will have a score of n - 1.
pub fn corner_score_fast12<I>(image: &I, threshold: u8, x: u32, y: u32)
        -> u8
    where I: GenericImage<Pixel=Luma<u8>> + 'static {

    let mut max = 255u8;
    let mut min = threshold;

    loop {
        if max == min {
            return max;
        }

        let mean = ((max as u16 + min as u16) / 2u16) as u8;
        let probe = if max == min + 1 { max } else { mean };

        if is_corner_fast12(image, probe, x, y) {
            min = probe;
        }
        else {
            max = probe - 1;
        }
    }
}

/// Checks if the given pixel is a corner.
fn is_corner_fast12<I>(image: &I, threshold: u8, x: u32, y: u32)
        -> bool
    where I: GenericImage<Pixel=Luma<u8>> + 'static {

    let (width, height) = image.dimensions();

    if x < 3 || y < 3 || x >= width - 3 || y >= height - 3 {
        return false;
    }

    let c = image.get_pixel(x, y)[0];
    let low_thresh: i16  = c as i16 - threshold as i16;
    let high_thresh: i16 = c as i16 + threshold as i16;

    // Circle pixels are labelled clockwise, starting at 0 directly above
    // the center, and finishing at 15, three pixels above and one to
    // the left of the center

    let p0 = image.get_pixel(x, y - 3)[0] as i16;
    let p8 = image.get_pixel(x, y + 3)[0] as i16;

    let mut above = p0 > high_thresh && p8 > high_thresh;
    let mut below = p0 < low_thresh  && p8 < low_thresh;

    if !above && !below {
        return false;
    }

    let p4  = image.get_pixel(x + 3, y)[0] as i16;
    let p12 = image.get_pixel(x - 3, y)[0] as i16;

    above = above && ((p4 > high_thresh) || (p12 > high_thresh));
    below = below && ((p4 < low_thresh)  || (p12 < low_thresh));

    if !above && !below {
        return false;
    }

    // TODO: Generate a list of pixel offsets once per image,
    // TODO: and use those offsets directly when reading pixels.
    // TODO: This is a little tricky as we can't always do it - we'd
    // TODO: need to distinguish between GenericImages and ImageBuffers.
    // TODO: We can also reduce the number of checks we do below.

    let mut pixels = [0i16; 16];

    pixels[0]  = p0;
    pixels[1]  = image.get_pixel(x + 1, y - 3)[0] as i16;
    pixels[2]  = image.get_pixel(x + 2, y - 2)[0] as i16;
    pixels[3]  = image.get_pixel(x + 3, y - 1)[0] as i16;
    pixels[4]  = p4;
    pixels[5]  = image.get_pixel(x + 3, y + 1)[0] as i16;
    pixels[6]  = image.get_pixel(x + 2, y + 2)[0] as i16;
    pixels[7]  = image.get_pixel(x + 1, y + 3)[0] as i16;
    pixels[8]  = p8;
    pixels[9]  = image.get_pixel(x - 1, y + 3)[0] as i16;
    pixels[10] = image.get_pixel(x - 2, y + 2)[0] as i16;
    pixels[11] = image.get_pixel(x - 3, y + 1)[0] as i16;
    pixels[12] = p12;
    pixels[13] = image.get_pixel(x - 3, y - 1)[0] as i16;
    pixels[14] = image.get_pixel(x - 2, y - 2)[0] as i16;
    pixels[15] = image.get_pixel(x - 1, y - 3)[0] as i16;

    if above {
        for i in 0..15 {
            let mut is_corner = true;
            let end   = i + 12;
            let upper = cmp::min(end, 16);
            let wrap  = if end > 16 { end - 16 } else { 0 };

            for j in i..upper {
                if pixels[j] <= high_thresh {
                    is_corner = false;
                    break;
                }
            }
            if is_corner {
                for j in 0..wrap {
                    if pixels[j] <= high_thresh {
                        is_corner = false;
                        break;
                    }
                }
            }
            if is_corner {
                return true;
            }
        }
    }
    else {
        for i in 0..15 {
            let mut is_corner = true;
            let end   = i + 12;
            let upper = cmp::min(end, 16);
            let wrap  = if end > 16 { end - 16 } else { 0 };

            for j in i..upper {
                if pixels[j] >= low_thresh {
                    is_corner = false;
                    break;
                }
            }
            if is_corner {
                for j in 0..wrap {
                    if pixels[j] >= low_thresh {
                        is_corner = false;
                        break;
                    }
                }
            }
            if is_corner {
                return true;
            }
        }
    }

    false
}

#[cfg(test)]
mod test {

    use super::{
        corner_score_fast12,
        is_corner_fast12
    };
    use image::{
        GrayImage,
        ImageBuffer
    };

    #[test]
    fn test_is_corner_fast12_12_contiguous_darker_pixels() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            10, 10, 00, 00, 00, 10, 10,
            10, 00, 10, 10, 10, 00, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            10, 00, 10, 10, 10, 10, 10,
            10, 10, 00, 00, 00, 10, 10]).unwrap();

        assert_eq!(is_corner_fast12(&image, 8, 3, 3), true);
    }

    #[test]
    fn test_is_corner_fast12_12_contiguous_darker_pixels_large_threshold() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            10, 10, 00, 00, 00, 10, 10,
            10, 00, 10, 10, 10, 00, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            10, 00, 10, 10, 10, 10, 10,
            10, 10, 00, 00, 00, 10, 10]).unwrap();

        assert_eq!(is_corner_fast12(&image, 15, 3, 3), false);
    }

    #[test]
    fn test_is_corner_fast12_12_contiguous_lighter_pixels() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            00, 00, 10, 10, 10, 00, 00,
            00, 10, 00, 00, 00, 10, 00,
            10, 00, 00, 00, 00, 00, 00,
            10, 00, 00, 00, 00, 00, 00,
            10, 00, 00, 00, 00, 00, 00,
            00, 10, 00, 00, 00, 00, 00,
            00, 00, 10, 10, 10, 00, 00]).unwrap();

        assert_eq!(is_corner_fast12(&image, 8, 3, 3), true);
    }

    #[test]
    fn test_is_corner_fast12_12_noncontiguous() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            10, 10, 00, 00, 00, 10, 10,
            10, 00, 10, 10, 10, 00, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            10, 10, 10, 10, 10, 10, 00,
            10, 00, 10, 10, 10, 10, 10,
            10, 10, 00, 00, 00, 10, 10]).unwrap();

        assert_eq!(is_corner_fast12(&image, 8, 3, 3), false);
    }

    #[test]
    fn test_is_corner_fast12_near_image_boundary() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            10, 10, 00, 00, 00, 10, 10,
            10, 00, 10, 10, 10, 00, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            10, 00, 10, 10, 10, 10, 10,
            10, 10, 00, 00, 00, 10, 10]).unwrap();

        assert_eq!(is_corner_fast12(&image, 8, 1, 1), false);
    }

    #[test]
    fn test_corner_score_fast12() {
        let image: GrayImage = ImageBuffer::from_raw(7, 7, vec![
            10, 10, 00, 00, 00, 10, 10,
            10, 00, 10, 10, 10, 00, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            00, 10, 10, 10, 10, 10, 10,
            10, 00, 10, 10, 10, 10, 10,
            10, 10, 00, 00, 00, 10, 10]).unwrap();

        let score = corner_score_fast12(&image, 5, 3, 3);
        assert_eq!(score, 9);

        let score = corner_score_fast12(&image, 9, 3, 3);
        assert_eq!(score, 9);
    }
}
<|endoftext|>


<commit_before>use cron::Schedule;
use chrono::{FixedOffset, Local};

use std::str::FromStr;
use std::thread;
use std::time::Duration;

use command::Command;

/// The object to create and execute cronjobs for yout application.
pub struct CronJob {
    name: String,
    command: Box<dyn Command>,
    seconds: Option<String>,
    minutes: Option<String>,
    hours: Option<String>,
    day_of_month: Option<String>,
    month: Option<String>,
    day_of_week: Option<String>,
    year: Option<String>,
    offset: Option<FixedOffset>,
}

impl CronJob {
    /// Constructs new `CronJob` object.
    pub fn new<C: Command>(name: &str, command: C) -> Self {
        CronJob {
            name: name.to_string(),
            command: Box::new(command),
            seconds: None,
            minutes: None,
            hours: None,
            day_of_month: None,
            month: None,
            day_of_week: None,
            year: None,
            offset: None,
        }
    }

    pub fn seconds(&mut self, seconds: &str) -> &mut Self {
        self.seconds = Some(seconds.to_string());
        self
    }

    pub fn minutes(&mut self, minutes: &str) -> &mut Self {
        self.minutes = Some(minutes.to_string());
        self
    }

    pub fn hours(&mut self, hours: &str) -> &mut Self {
        self.hours = Some(hours.to_string());
        self
    }

    pub fn day_of_month(&mut self, day_of_month: &str) -> &mut Self {
        self.day_of_month = Some(day_of_month.to_string());
        self
    }

    pub fn month(&mut self, month: &str) -> &mut Self {
        self.month = Some(month.to_string());
        self
    }

    pub fn day_of_week(&mut self, day_of_week: &str) -> &mut Self {
        self.day_of_week = Some(day_of_week.to_string());
        self
    }

    pub fn year(&mut self, year: &str) -> &mut Self {
        self.year = Some(year.to_string());
        self
    }

    pub fn offset(&mut self, timezone_offset: i32) -> &mut Self {
        self.offset = Some(FixedOffset::east(timezone_offset));
        self
    }

    /// Returns the schedule for the cronjob, with this you are able to get the next occurences.
    pub fn get_schedule(&mut self) -> Schedule {
        let asterix = String::from("*");
        let cron = format!("{} {} {} {} {} {} {}",
                           self.seconds.as_ref().unwrap_or(&asterix),
                           self.minutes.as_ref().unwrap_or(&asterix),
                           self.hours.as_ref().unwrap_or(&asterix),
                           self.day_of_month.as_ref().unwrap_or(&asterix),
                           self.month.as_ref().unwrap_or(&asterix),
                           self.day_of_week.as_ref().unwrap_or(&asterix),
                           self.year.as_ref().unwrap_or(&asterix));
        Schedule::from_str(&cron).unwrap()
    }

    /// Starts the cronjob without threading.
    pub fn start_job(&mut self) {
        let schedule = self.get_schedule();
        let offset = self.offset.unwrap_or(FixedOffset::east(0));

        loop {
            let mut upcoming = schedule.upcoming(offset).take(1);
            thread::sleep(Duration::from_millis(500));
            let local = &Local::now();

            if let Some(datetime) = upcoming.next() {
                if datetime.timestamp() <= local.timestamp() {
                    self.command.execute(&self.name);
                }
            }
        }
    }

    /// Starts the cronjob with threading. Stops when application quits.
    pub fn start_job_threaded(mut cronjob: CronJob) {
        thread::Builder::new()
            .name(cronjob.name.clone())
            .spawn(move || { cronjob.start_job(); })
            .expect("There was an error in an cronjob");
    }
}
<commit_msg>fixed clippy warning<commit_after>use chrono::{FixedOffset, Local};
use cron::Schedule;

use std::str::FromStr;
use std::thread;
use std::time::Duration;

use command::Command;

/// The object to create and execute cronjobs for yout application.
pub struct CronJob {
    name: String,
    command: Box<dyn Command>,
    seconds: Option<String>,
    minutes: Option<String>,
    hours: Option<String>,
    day_of_month: Option<String>,
    month: Option<String>,
    day_of_week: Option<String>,
    year: Option<String>,
    offset: Option<FixedOffset>,
}

impl CronJob {
    /// Constructs new `CronJob` object.
    pub fn new<C: Command>(name: &str, command: C) -> Self {
        CronJob {
            name: name.to_string(),
            command: Box::new(command),
            seconds: None,
            minutes: None,
            hours: None,
            day_of_month: None,
            month: None,
            day_of_week: None,
            year: None,
            offset: None,
        }
    }

    pub fn seconds(&mut self, seconds: &str) -> &mut Self {
        self.seconds = Some(seconds.to_string());
        self
    }

    pub fn minutes(&mut self, minutes: &str) -> &mut Self {
        self.minutes = Some(minutes.to_string());
        self
    }

    pub fn hours(&mut self, hours: &str) -> &mut Self {
        self.hours = Some(hours.to_string());
        self
    }

    pub fn day_of_month(&mut self, day_of_month: &str) -> &mut Self {
        self.day_of_month = Some(day_of_month.to_string());
        self
    }

    pub fn month(&mut self, month: &str) -> &mut Self {
        self.month = Some(month.to_string());
        self
    }

    pub fn day_of_week(&mut self, day_of_week: &str) -> &mut Self {
        self.day_of_week = Some(day_of_week.to_string());
        self
    }

    pub fn year(&mut self, year: &str) -> &mut Self {
        self.year = Some(year.to_string());
        self
    }

    pub fn offset(&mut self, timezone_offset: i32) -> &mut Self {
        self.offset = Some(FixedOffset::east(timezone_offset));
        self
    }

    /// Returns the schedule for the cronjob, with this you are able to get the next occurences.
    pub fn get_schedule(&mut self) -> Schedule {
        let asterix = String::from("*");
        let cron = format!(
            "{} {} {} {} {} {} {}",
            self.seconds.as_ref().unwrap_or(&asterix),
            self.minutes.as_ref().unwrap_or(&asterix),
            self.hours.as_ref().unwrap_or(&asterix),
            self.day_of_month.as_ref().unwrap_or(&asterix),
            self.month.as_ref().unwrap_or(&asterix),
            self.day_of_week.as_ref().unwrap_or(&asterix),
            self.year.as_ref().unwrap_or(&asterix)
        );
        Schedule::from_str(&cron).unwrap()
    }

    /// Starts the cronjob without threading.
    pub fn start_job(&mut self) {
        let schedule = self.get_schedule();
        let offset = self.offset.unwrap_or_else(|| FixedOffset::east(0));

        loop {
            let mut upcoming = schedule.upcoming(offset).take(1);
            thread::sleep(Duration::from_millis(500));
            let local = &Local::now();

            if let Some(datetime) = upcoming.next() {
                if datetime.timestamp() <= local.timestamp() {
                    self.command.execute(&self.name);
                }
            }
        }
    }

    /// Starts the cronjob with threading. Stops when application quits.
    pub fn start_job_threaded(mut cronjob: CronJob) {
        thread::Builder::new()
            .name(cronjob.name.clone())
            .spawn(move || {
                cronjob.start_job();
            })
            .expect("There was an error in an cronjob");
    }
}
<|endoftext|>


<commit_before>//! Simple module to store files in database.
//!
//! cratesfyi is generating more than 5 million files, they are small and mostly html files.
//! They are using so many inodes and it is better to store them in database instead of
//! filesystem. This module is adding files into database and retrieving them.

use std::path::{PathBuf, Path};
use postgres::Connection;
use rustc_serialize::json::{Json, ToJson};
use std::cmp;
use std::fs;
use std::io::Read;
use error::Result;
use failure::err_msg;
use rusoto_s3::{S3, PutObjectRequest, GetObjectRequest, S3Client};
use rusoto_core::region::Region;
use rusoto_credential::DefaultCredentialsProvider;
use std::ffi::OsStr;

const MAX_CONCURRENT_UPLOADS: usize = 1000;

pub(super) static S3_BUCKET_NAME: &str = "rust-docs-rs";

fn get_file_list_from_dir<P: AsRef<Path>>(path: P,
                                          files: &mut Vec<PathBuf>)
                                          -> Result<()> {
    let path = path.as_ref();

    for file in path.read_dir()? {
        let file = file?;

        if file.file_type()?.is_file() {
            files.push(file.path());
        } else if file.file_type()?.is_dir() {
            get_file_list_from_dir(file.path(), files)?;
        }
    }

    Ok(())
}

fn get_file_list<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>> {
    let path = path.as_ref();
    let mut files = Vec::new();

    if !path.exists() {
        return Err(err_msg("File not found"));
    } else if path.is_file() {
        files.push(PathBuf::from(path.file_name().unwrap()));
    } else if path.is_dir() {
        get_file_list_from_dir(path, &mut files)?;
        for file_path in &mut files {
            // We want the paths in this list to not be {path}/bar.txt but just bar.txt
            *file_path = PathBuf::from(file_path.strip_prefix(path).unwrap());
        }
    }

    Ok(files)
}

pub struct Blob {
    pub path: String,
    pub mime: String,
    pub date_updated: time::Timespec,
    pub content: Vec<u8>,
}

pub fn get_path(conn: &Connection, path: &str) -> Option<Blob> {
    if let Some(client) = s3_client() {
        let res = client.get_object(GetObjectRequest {
            bucket: S3_BUCKET_NAME.into(),
            key: path.into(),
            ..Default::default()
        }).sync();

        let res = match res {
            Ok(r) => r,
            Err(_) => {
                return None;
            }
        };

        let mut b = res.body.unwrap().into_blocking_read();
        let mut content = Vec::new();
        b.read_to_end(&mut content).unwrap();

        let last_modified = res.last_modified.unwrap();
        let last_modified = time::strptime(&last_modified, "%a, %d %b %Y %H:%M:%S %Z")
            .unwrap_or_else(|e| panic!("failed to parse {:?} as timespec: {:?}", last_modified, e))
            .to_timespec();

        Some(Blob {
            path: path.into(),
            mime: res.content_type.unwrap(),
            date_updated: last_modified,
            content,
        })
    } else {
        let rows = conn.query("SELECT path, mime, date_updated, content
                            FROM files
                            WHERE path = $1", &[&path]).unwrap();

        if rows.len() == 0 {
            None
        } else {
            let row = rows.get(0);

            Some(Blob {
                path: row.get(0),
                mime: row.get(1),
                date_updated: row.get(2),
                content: row.get(3),
            })
        }
    }
}

pub(super) fn s3_client() -> Option<S3Client> {
    // If AWS keys aren't configured, then presume we should use the DB exclusively
    // for file storage.
    if std::env::var_os("AWS_ACCESS_KEY_ID").is_none() && std::env::var_os("FORCE_S3").is_none() {
        return None;
    }
    let creds = match DefaultCredentialsProvider::new() {
        Ok(creds) => creds,
        Err(err) => {
            warn!("failed to retrieve AWS credentials: {}", err);
            return None;
        }
    };
    Some(S3Client::new_with(
        rusoto_core::request::HttpClient::new().unwrap(),
        creds,
        std::env::var("S3_ENDPOINT").ok().map(|e| Region::Custom {
            name: std::env::var("S3_REGION")
                .unwrap_or_else(|_| "us-west-1".to_owned()),
            endpoint: e,
        }).unwrap_or(Region::UsWest1),
    ))
}

/// Store all files in a directory and return [[mimetype, filename]] as Json
///
/// If there is an S3 Client configured, store files into an S3 bucket;
/// otherwise, stores files into the 'files' table of the local database.
///
/// The mimetype is detected using `magic`.
///
/// Note that this function is used for uploading both sources
/// and files generated by rustdoc.
pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
                                              prefix: &str,
                                              path: P)
                                              -> Result<Json> {
    use std::collections::HashMap;
    use futures::future::Future;

    let trans = conn.transaction()?;
    let mut file_paths_and_mimes: HashMap<PathBuf, String> = HashMap::new();

    let mut rt = ::tokio::runtime::Runtime::new().unwrap();

    let mut to_upload = get_file_list(&path)?;
    let mut batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
    let mut currently_uploading: Vec<_> = to_upload.drain(..batch_size).collect();
    let mut attempts = 0;

    while !to_upload.is_empty() || !currently_uploading.is_empty() {

        let mut futures = Vec::new();
        let client = s3_client();

        for file_path in &currently_uploading {
            let path = Path::new(path.as_ref()).join(&file_path);
            // Some files have insufficient permissions (like .lock file created by cargo in
            // documentation directory). We are skipping this files.
            let mut file = match fs::File::open(path) {
                Ok(f) => f,
                Err(_) => continue,
            };
            let mut content: Vec<u8> = Vec::new();
            file.read_to_end(&mut content)?;

            let bucket_path = Path::new(prefix).join(&file_path);

            #[cfg(windows)] // On windows, we need to normalize \\ to / so the route logic works
            let bucket_path = path_slash::PathBufExt::to_slash(&bucket_path).unwrap();
            #[cfg(not(windows))]
            let bucket_path = bucket_path.into_os_string().into_string().unwrap();

            let mime = detect_mime(&file_path)?;

            if let Some(client) = &client {
                futures.push(client.put_object(PutObjectRequest {
                    bucket: S3_BUCKET_NAME.into(),
                    key: bucket_path.clone(),
                    body: Some(content.into()),
                    content_type: Some(mime.to_owned()),
                    ..Default::default()
                }).inspect(|_| {
                    crate::web::metrics::UPLOADED_FILES_TOTAL.inc_by(1);
                }));
            } else {
                // If AWS credentials are configured, don't insert/update the database
                // check if file already exists in database
                let rows = conn.query("SELECT COUNT(*) FROM files WHERE path = $1", &[&bucket_path])?;

                if rows.get(0).get::<usize, i64>(0) == 0 {
                    trans.query("INSERT INTO files (path, mime, content) VALUES ($1, $2, $3)",
                                    &[&bucket_path, &mime, &content])?;
                } else {
                    trans.query("UPDATE files SET mime = $2, content = $3, date_updated = NOW() \
                                    WHERE path = $1",
                                    &[&bucket_path, &mime, &content])?;
                }
            }

            file_paths_and_mimes.insert(file_path.clone(), mime.to_owned());
        }

        if !futures.is_empty() {
            attempts += 1;

            match rt.block_on(::futures::future::join_all(futures)) {
                Ok(_) => {
                    // this batch was successful, start another batch if there are still more files
                    batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
                    currently_uploading = to_upload.drain(..batch_size).collect();
                    attempts = 0;
                },
                Err(err) => {
                    error!("failed to upload to s3: {:?}", err);
                    // if any futures error, leave `currently_uploading` in tact so that we can retry the batch
                    if attempts > 2 {
                        panic!("failed to upload 3 times, exiting");
                    }
                }
            }
        } else {
            batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
            currently_uploading = to_upload.drain(..batch_size).collect();
        }
    }

    trans.commit()?;

    let file_list_with_mimes: Vec<(String, PathBuf)> = file_paths_and_mimes
        .into_iter()
        .map(|(file_path, mime)| (mime, file_path))
        .collect();
    file_list_to_json(file_list_with_mimes)
}

fn detect_mime(file_path: &Path) -> Result<&'static str> {
    let mime = mime_guess::from_path(file_path).first_raw().map(|m| m).unwrap_or("text/plain");
    Ok(match mime {
        "text/plain" | "text/troff" | "text/x-markdown" | "text/x-rust" | "text/x-toml" => {
            match file_path.extension().and_then(OsStr::to_str) {
                Some("md") => "text/markdown",
                Some("rs") => "text/rust",
                Some("markdown") => "text/markdown",
                Some("css") => "text/css",
                Some("toml") => "text/toml",
                Some("js") => "application/javascript",
                Some("json") => "application/json",
                _ => mime
            }
        },
        "image/svg" => "image/svg+xml",
        _ => mime
    })
}

fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result<Json> {

    let mut file_list_json: Vec<Json> = Vec::new();

    for file in file_list {
        let mut v: Vec<String> = Vec::new();
        v.push(file.0.clone());
        v.push(file.1.into_os_string().into_string().unwrap());
        file_list_json.push(v.to_json());
    }

    Ok(file_list_json.to_json())
}

pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize> {
    let trans = conn.transaction()?;
    let client = s3_client().expect("configured s3");

    let rows = trans.query(
            &format!("SELECT path, mime, content FROM files WHERE content != E'in-s3' LIMIT {}", n),
            &[])?;
    let count = rows.len();

    let mut rt = ::tokio::runtime::Runtime::new().unwrap();
    let mut futures = Vec::new();
    for row in &rows {
        let path: String = row.get(0);
        let mime: String = row.get(1);
        let content: Vec<u8> = row.get(2);
        let path_1 = path.clone();
        futures.push(client.put_object(PutObjectRequest {
            bucket: S3_BUCKET_NAME.into(),
            key: path.clone(),
            body: Some(content.into()),
            content_type: Some(mime),
            ..Default::default()
        }).map(move |_| {
            path_1
        }).map_err(move |e| {
            panic!("failed to upload to {}: {:?}", path, e)
        }));
    }

    use ::futures::future::Future;
    match rt.block_on(::futures::future::join_all(futures)) {
        Ok(paths) => {
            let statement = trans.prepare("DELETE FROM files WHERE path = $1").unwrap();
            for path in paths {
                statement.execute(&[&path]).unwrap();
            }
        }
        Err(e) => {
            panic!("results err: {:?}", e);
        }
    }

    trans.commit()?;

    Ok(count)
}

#[cfg(test)]
mod test {
    extern crate env_logger;
    use std::env;
    use super::*;

    #[test]
    fn test_get_file_list() {
        let _ = env_logger::try_init();

        let files = get_file_list(env::current_dir().unwrap());
        assert!(files.is_ok());
        assert!(files.unwrap().len() > 0);

        let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap();
        assert_eq!(files[0], std::path::Path::new("Cargo.toml"));
    }
    #[test]
    fn test_mime_types() {
        check_mime(".gitignore", "text/plain");
        check_mime("hello.toml","text/toml");
        check_mime("hello.css","text/css");
        check_mime("hello.js","application/javascript");
        check_mime("hello.html","text/html");
        check_mime("hello.hello.md","text/markdown");
        check_mime("hello.markdown","text/markdown");
        check_mime("hello.json","application/json");
        check_mime("hello.txt","text/plain");
        check_mime("file.rs", "text/rust");
        check_mime("important.svg", "image/svg+xml");
    }

    fn check_mime(path: &str, expected_mime: &str) {
        let detected_mime = detect_mime(Path::new(&path));
        let detected_mime = detected_mime.expect("no mime was given");
        assert_eq!(detected_mime, expected_mime);
    }
}

<commit_msg>Limit maximum uploads to a reasonable number<commit_after>//! Simple module to store files in database.
//!
//! cratesfyi is generating more than 5 million files, they are small and mostly html files.
//! They are using so many inodes and it is better to store them in database instead of
//! filesystem. This module is adding files into database and retrieving them.

use std::path::{PathBuf, Path};
use postgres::Connection;
use rustc_serialize::json::{Json, ToJson};
use std::cmp;
use std::fs;
use std::io::Read;
use error::Result;
use failure::err_msg;
use rusoto_s3::{S3, PutObjectRequest, GetObjectRequest, S3Client};
use rusoto_core::region::Region;
use rusoto_credential::DefaultCredentialsProvider;
use std::ffi::OsStr;

const MAX_CONCURRENT_UPLOADS: usize = 50;

pub(super) static S3_BUCKET_NAME: &str = "rust-docs-rs";

fn get_file_list_from_dir<P: AsRef<Path>>(path: P,
                                          files: &mut Vec<PathBuf>)
                                          -> Result<()> {
    let path = path.as_ref();

    for file in path.read_dir()? {
        let file = file?;

        if file.file_type()?.is_file() {
            files.push(file.path());
        } else if file.file_type()?.is_dir() {
            get_file_list_from_dir(file.path(), files)?;
        }
    }

    Ok(())
}

fn get_file_list<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>> {
    let path = path.as_ref();
    let mut files = Vec::new();

    if !path.exists() {
        return Err(err_msg("File not found"));
    } else if path.is_file() {
        files.push(PathBuf::from(path.file_name().unwrap()));
    } else if path.is_dir() {
        get_file_list_from_dir(path, &mut files)?;
        for file_path in &mut files {
            // We want the paths in this list to not be {path}/bar.txt but just bar.txt
            *file_path = PathBuf::from(file_path.strip_prefix(path).unwrap());
        }
    }

    Ok(files)
}

pub struct Blob {
    pub path: String,
    pub mime: String,
    pub date_updated: time::Timespec,
    pub content: Vec<u8>,
}

pub fn get_path(conn: &Connection, path: &str) -> Option<Blob> {
    if let Some(client) = s3_client() {
        let res = client.get_object(GetObjectRequest {
            bucket: S3_BUCKET_NAME.into(),
            key: path.into(),
            ..Default::default()
        }).sync();

        let res = match res {
            Ok(r) => r,
            Err(_) => {
                return None;
            }
        };

        let mut b = res.body.unwrap().into_blocking_read();
        let mut content = Vec::new();
        b.read_to_end(&mut content).unwrap();

        let last_modified = res.last_modified.unwrap();
        let last_modified = time::strptime(&last_modified, "%a, %d %b %Y %H:%M:%S %Z")
            .unwrap_or_else(|e| panic!("failed to parse {:?} as timespec: {:?}", last_modified, e))
            .to_timespec();

        Some(Blob {
            path: path.into(),
            mime: res.content_type.unwrap(),
            date_updated: last_modified,
            content,
        })
    } else {
        let rows = conn.query("SELECT path, mime, date_updated, content
                            FROM files
                            WHERE path = $1", &[&path]).unwrap();

        if rows.len() == 0 {
            None
        } else {
            let row = rows.get(0);

            Some(Blob {
                path: row.get(0),
                mime: row.get(1),
                date_updated: row.get(2),
                content: row.get(3),
            })
        }
    }
}

pub(super) fn s3_client() -> Option<S3Client> {
    // If AWS keys aren't configured, then presume we should use the DB exclusively
    // for file storage.
    if std::env::var_os("AWS_ACCESS_KEY_ID").is_none() && std::env::var_os("FORCE_S3").is_none() {
        return None;
    }
    let creds = match DefaultCredentialsProvider::new() {
        Ok(creds) => creds,
        Err(err) => {
            warn!("failed to retrieve AWS credentials: {}", err);
            return None;
        }
    };
    Some(S3Client::new_with(
        rusoto_core::request::HttpClient::new().unwrap(),
        creds,
        std::env::var("S3_ENDPOINT").ok().map(|e| Region::Custom {
            name: std::env::var("S3_REGION")
                .unwrap_or_else(|_| "us-west-1".to_owned()),
            endpoint: e,
        }).unwrap_or(Region::UsWest1),
    ))
}

/// Store all files in a directory and return [[mimetype, filename]] as Json
///
/// If there is an S3 Client configured, store files into an S3 bucket;
/// otherwise, stores files into the 'files' table of the local database.
///
/// The mimetype is detected using `magic`.
///
/// Note that this function is used for uploading both sources
/// and files generated by rustdoc.
pub fn add_path_into_database<P: AsRef<Path>>(conn: &Connection,
                                              prefix: &str,
                                              path: P)
                                              -> Result<Json> {
    use std::collections::HashMap;
    use futures::future::Future;

    let trans = conn.transaction()?;
    let mut file_paths_and_mimes: HashMap<PathBuf, String> = HashMap::new();

    let mut rt = ::tokio::runtime::Runtime::new().unwrap();

    let mut to_upload = get_file_list(&path)?;
    let mut batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
    let mut currently_uploading: Vec<_> = to_upload.drain(..batch_size).collect();
    let mut attempts = 0;

    while !to_upload.is_empty() || !currently_uploading.is_empty() {

        let mut futures = Vec::new();
        let client = s3_client();

        for file_path in &currently_uploading {
            let path = Path::new(path.as_ref()).join(&file_path);
            // Some files have insufficient permissions (like .lock file created by cargo in
            // documentation directory). We are skipping this files.
            let mut file = match fs::File::open(path) {
                Ok(f) => f,
                Err(_) => continue,
            };
            let mut content: Vec<u8> = Vec::new();
            file.read_to_end(&mut content)?;

            let bucket_path = Path::new(prefix).join(&file_path);

            #[cfg(windows)] // On windows, we need to normalize \\ to / so the route logic works
            let bucket_path = path_slash::PathBufExt::to_slash(&bucket_path).unwrap();
            #[cfg(not(windows))]
            let bucket_path = bucket_path.into_os_string().into_string().unwrap();

            let mime = detect_mime(&file_path)?;

            if let Some(client) = &client {
                futures.push(client.put_object(PutObjectRequest {
                    bucket: S3_BUCKET_NAME.into(),
                    key: bucket_path.clone(),
                    body: Some(content.into()),
                    content_type: Some(mime.to_owned()),
                    ..Default::default()
                }).inspect(|_| {
                    crate::web::metrics::UPLOADED_FILES_TOTAL.inc_by(1);
                }));
            } else {
                // If AWS credentials are configured, don't insert/update the database
                // check if file already exists in database
                let rows = conn.query("SELECT COUNT(*) FROM files WHERE path = $1", &[&bucket_path])?;

                if rows.get(0).get::<usize, i64>(0) == 0 {
                    trans.query("INSERT INTO files (path, mime, content) VALUES ($1, $2, $3)",
                                    &[&bucket_path, &mime, &content])?;
                } else {
                    trans.query("UPDATE files SET mime = $2, content = $3, date_updated = NOW() \
                                    WHERE path = $1",
                                    &[&bucket_path, &mime, &content])?;
                }
            }

            file_paths_and_mimes.insert(file_path.clone(), mime.to_owned());
        }

        if !futures.is_empty() {
            attempts += 1;

            match rt.block_on(::futures::future::join_all(futures)) {
                Ok(_) => {
                    // this batch was successful, start another batch if there are still more files
                    batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
                    currently_uploading = to_upload.drain(..batch_size).collect();
                    attempts = 0;
                },
                Err(err) => {
                    error!("failed to upload to s3: {:?}", err);
                    // if any futures error, leave `currently_uploading` in tact so that we can retry the batch
                    if attempts > 2 {
                        panic!("failed to upload 3 times, exiting");
                    }
                }
            }
        } else {
            batch_size = cmp::min(to_upload.len(), MAX_CONCURRENT_UPLOADS);
            currently_uploading = to_upload.drain(..batch_size).collect();
        }
    }

    trans.commit()?;

    let file_list_with_mimes: Vec<(String, PathBuf)> = file_paths_and_mimes
        .into_iter()
        .map(|(file_path, mime)| (mime, file_path))
        .collect();
    file_list_to_json(file_list_with_mimes)
}

fn detect_mime(file_path: &Path) -> Result<&'static str> {
    let mime = mime_guess::from_path(file_path).first_raw().map(|m| m).unwrap_or("text/plain");
    Ok(match mime {
        "text/plain" | "text/troff" | "text/x-markdown" | "text/x-rust" | "text/x-toml" => {
            match file_path.extension().and_then(OsStr::to_str) {
                Some("md") => "text/markdown",
                Some("rs") => "text/rust",
                Some("markdown") => "text/markdown",
                Some("css") => "text/css",
                Some("toml") => "text/toml",
                Some("js") => "application/javascript",
                Some("json") => "application/json",
                _ => mime
            }
        },
        "image/svg" => "image/svg+xml",
        _ => mime
    })
}

fn file_list_to_json(file_list: Vec<(String, PathBuf)>) -> Result<Json> {

    let mut file_list_json: Vec<Json> = Vec::new();

    for file in file_list {
        let mut v: Vec<String> = Vec::new();
        v.push(file.0.clone());
        v.push(file.1.into_os_string().into_string().unwrap());
        file_list_json.push(v.to_json());
    }

    Ok(file_list_json.to_json())
}

pub fn move_to_s3(conn: &Connection, n: usize) -> Result<usize> {
    let trans = conn.transaction()?;
    let client = s3_client().expect("configured s3");

    let rows = trans.query(
            &format!("SELECT path, mime, content FROM files WHERE content != E'in-s3' LIMIT {}", n),
            &[])?;
    let count = rows.len();

    let mut rt = ::tokio::runtime::Runtime::new().unwrap();
    let mut futures = Vec::new();
    for row in &rows {
        let path: String = row.get(0);
        let mime: String = row.get(1);
        let content: Vec<u8> = row.get(2);
        let path_1 = path.clone();
        futures.push(client.put_object(PutObjectRequest {
            bucket: S3_BUCKET_NAME.into(),
            key: path.clone(),
            body: Some(content.into()),
            content_type: Some(mime),
            ..Default::default()
        }).map(move |_| {
            path_1
        }).map_err(move |e| {
            panic!("failed to upload to {}: {:?}", path, e)
        }));
    }

    use ::futures::future::Future;
    match rt.block_on(::futures::future::join_all(futures)) {
        Ok(paths) => {
            let statement = trans.prepare("DELETE FROM files WHERE path = $1").unwrap();
            for path in paths {
                statement.execute(&[&path]).unwrap();
            }
        }
        Err(e) => {
            panic!("results err: {:?}", e);
        }
    }

    trans.commit()?;

    Ok(count)
}

#[cfg(test)]
mod test {
    extern crate env_logger;
    use std::env;
    use super::*;

    #[test]
    fn test_get_file_list() {
        let _ = env_logger::try_init();

        let files = get_file_list(env::current_dir().unwrap());
        assert!(files.is_ok());
        assert!(files.unwrap().len() > 0);

        let files = get_file_list(env::current_dir().unwrap().join("Cargo.toml")).unwrap();
        assert_eq!(files[0], std::path::Path::new("Cargo.toml"));
    }
    #[test]
    fn test_mime_types() {
        check_mime(".gitignore", "text/plain");
        check_mime("hello.toml","text/toml");
        check_mime("hello.css","text/css");
        check_mime("hello.js","application/javascript");
        check_mime("hello.html","text/html");
        check_mime("hello.hello.md","text/markdown");
        check_mime("hello.markdown","text/markdown");
        check_mime("hello.json","application/json");
        check_mime("hello.txt","text/plain");
        check_mime("file.rs", "text/rust");
        check_mime("important.svg", "image/svg+xml");
    }

    fn check_mime(path: &str, expected_mime: &str) {
        let detected_mime = detect_mime(Path::new(&path));
        let detected_mime = detected_mime.expect("no mime was given");
        assert_eq!(detected_mime, expected_mime);
    }
}

<|endoftext|>