<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! A priority queue implemented with a binary heap

use core::container::{Container, Mutable};
use core::cmp::Ord;
use core::prelude::*;
use core::ptr::addr_of;
use core::vec;

#[abi = "rust-intrinsic"]
extern "C" mod rusti {
    fn move_val_init<T>(dst: &mut T, -src: T);
}

pub struct PriorityQueue <T: Ord>{
    priv data: ~[T],
}

impl <T: Ord> PriorityQueue<T>: Container {
    /// Returns the length of the queue
    pure fn len(&self) -> uint { self.data.len() }

    /// Returns true if a queue contains no elements
    pure fn is_empty(&self) -> bool { self.data.is_empty() }
}

impl <T: Ord> PriorityQueue<T>: Mutable {
    /// Drop all items from the queue
    fn clear(&mut self) { self.data.truncate(0) }
}

impl <T: Ord> PriorityQueue<T> {
    /// Returns the greatest item in the queue - fails if empty
    pure fn top(&self) -> &self/T { &self.data[0] }

    /// Returns the greatest item in the queue - None if empty
    pure fn maybe_top(&self) -> Option<&self/T> {
        if self.is_empty() { None } else { Some(self.top()) }
    }

    /// Returns true if a queue contains some elements
    pure fn is_not_empty(&self) -> bool { self.data.is_not_empty() }

    /// Returns the number of elements the queue can hold without reallocating
    pure fn capacity(&self) -> uint { vec::capacity(&self.data) }

    fn reserve(&mut self, n: uint) { vec::reserve(&mut self.data, n) }

    fn reserve_at_least(&mut self, n: uint) {
        vec::reserve_at_least(&mut self.data, n)
    }

    /// Pop the greatest item from the queue - fails if empty
    fn pop(&mut self) -> T {
        let mut item = self.data.pop();
        if self.is_not_empty() { item <-> self.data[0]; self.siftdown(0); }
        item
    }

    /// Pop the greatest item from the queue - None if empty
    fn maybe_pop(&mut self) -> Option<T> {
        if self.is_empty() { None } else { Some(self.pop()) }
    }

    /// Push an item onto the queue
    fn push(&mut self, item: T) {
        self.data.push(item);
        self.siftup(0, self.len() - 1);
    }

    /// Optimized version of a push followed by a pop
    fn push_pop(&mut self, item: T) -> T {
        let mut item = item;
        if self.is_not_empty() && self.data[0] > item {
            item <-> self.data[0];
            self.siftdown(0);
        }
        item
    }

    /// Optimized version of a pop followed by a push - fails if empty
    fn replace(&mut self, item: T) -> T {
        let mut item = item;
        item <-> self.data[0];
        self.siftdown(0);
        item
    }

    /// Consume the PriorityQueue and return the underlying vector
    pure fn to_vec(self) -> ~[T] { let PriorityQueue{data: v} = self; v }

    /// Consume the PriorityQueue and return a vector in sorted
    /// (ascending) order
    pure fn to_sorted_vec(self) -> ~[T] {
        let mut q = self;
        let mut end = q.len();
        while end > 1 {
            end -= 1;
            q.data[end] <-> q.data[0];
            unsafe { q.siftdown_range(0, end) } // purity-checking workaround
        }
        q.to_vec()
    }

    /// Create an empty PriorityQueue
    static pure fn new() -> PriorityQueue<T> { PriorityQueue{data: ~[],} }

    /// Create a PriorityQueue from a vector (heapify)
    static pure fn from_vec(xs: ~[T]) -> PriorityQueue<T> {
        let mut q = PriorityQueue{data: xs,};
        let mut n = q.len() / 2;
        while n > 0 {
            n -= 1;
            unsafe { q.siftdown(n) }; // purity-checking workaround
        }
        q
    }

    // The implementations of siftup and siftdown use unsafe blocks in
    // order to move an element out of the vector (leaving behind a
    // junk element), shift along the others and move it back into the
    // vector over the junk element.  This reduces the constant factor
    // compared to using swaps, which involves twice as many moves.

    priv fn siftup(&mut self, start: uint, pos: uint) {
        unsafe {
            let mut pos = pos;
            let new = move *addr_of(&self.data[pos]);

            while pos > start {
                let parent = (pos - 1) >> 1;
                if new > self.data[parent] {
                    rusti::move_val_init(&mut self.data[pos],
                                         move *addr_of(&self.data[parent]));
                    pos = parent;
                    loop
                }
                break
            }
            rusti::move_val_init(&mut self.data[pos], move new);
        }
    }

    priv fn siftdown_range(&mut self, pos: uint, end: uint) {
        unsafe {
            let mut pos = pos;
            let start = pos;
            let new = move *addr_of(&self.data[pos]);

            let mut child = 2 * pos + 1;
            while child < end {
                let right = child + 1;
                if right < end && !(self.data[child] > self.data[right]) {
                    child = right;
                }
                rusti::move_val_init(&mut self.data[pos],
                                     move *addr_of(&self.data[child]));
                pos = child;
                child = 2 * pos + 1;
            }

            rusti::move_val_init(&mut self.data[pos], move new);
            self.siftup(start, pos);
        }
    }

    priv fn siftdown(&mut self, pos: uint) {
        self.siftdown_range(pos, self.len());
    }
}

#[cfg(test)]
mod tests {
    use sort::merge_sort;
    use core::cmp::le;
    use priority_queue::PriorityQueue::{from_vec, new};

    #[test]
    fn test_top_and_pop() {
        let data = ~[2, 4, 6, 2, 1, 8, 10, 3, 5, 7, 0, 9, 1];
        let mut sorted = merge_sort(data, le);
        let mut heap = from_vec(data);
        while heap.is_not_empty() {
            assert *heap.top() == sorted.last();
            assert heap.pop() == sorted.pop();
        }
    }

    #[test]
    fn test_push() {
        let mut heap = from_vec(~[2, 4, 9]);
        assert heap.len() == 3;
        assert *heap.top() == 9;
        heap.push(11);
        assert heap.len() == 4;
        assert *heap.top() == 11;
        heap.push(5);
        assert heap.len() == 5;
        assert *heap.top() == 11;
        heap.push(27);
        assert heap.len() == 6;
        assert *heap.top() == 27;
        heap.push(3);
        assert heap.len() == 7;
        assert *heap.top() == 27;
        heap.push(103);
        assert heap.len() == 8;
        assert *heap.top() == 103;
    }

    #[test]
    fn test_push_unique() {
        let mut heap = from_vec(~[~2, ~4, ~9]);
        assert heap.len() == 3;
        assert *heap.top() == ~9;
        heap.push(~11);
        assert heap.len() == 4;
        assert *heap.top() == ~11;
        heap.push(~5);
        assert heap.len() == 5;
        assert *heap.top() == ~11;
        heap.push(~27);
        assert heap.len() == 6;
        assert *heap.top() == ~27;
        heap.push(~3);
        assert heap.len() == 7;
        assert *heap.top() == ~27;
        heap.push(~103);
        assert heap.len() == 8;
        assert *heap.top() == ~103;
    }

    #[test]
    fn test_push_pop() {
        let mut heap = from_vec(~[5, 5, 2, 1, 3]);
        assert heap.len() == 5;
        assert heap.push_pop(6) == 6;
        assert heap.len() == 5;
        assert heap.push_pop(0) == 5;
        assert heap.len() == 5;
        assert heap.push_pop(4) == 5;
        assert heap.len() == 5;
        assert heap.push_pop(1) == 4;
        assert heap.len() == 5;
    }

    #[test]
    fn test_replace() {
        let mut heap = from_vec(~[5, 5, 2, 1, 3]);
        assert heap.len() == 5;
        assert heap.replace(6) == 5;
        assert heap.len() == 5;
        assert heap.replace(0) == 6;
        assert heap.len() == 5;
        assert heap.replace(4) == 5;
        assert heap.len() == 5;
        assert heap.replace(1) == 4;
        assert heap.len() == 5;
    }

    fn check_to_vec(data: ~[int]) {
        let heap = from_vec(data);
        assert merge_sort(heap.to_vec(), le) == merge_sort(data, le);
        assert heap.to_sorted_vec() == merge_sort(data, le);
    }

    #[test]
    fn test_to_vec() {
        check_to_vec(~[]);
        check_to_vec(~[5]);
        check_to_vec(~[3, 2]);
        check_to_vec(~[2, 3]);
        check_to_vec(~[5, 1, 2]);
        check_to_vec(~[1, 100, 2, 3]);
        check_to_vec(~[1, 3, 5, 7, 9, 2, 4, 6, 8, 0]);
        check_to_vec(~[2, 4, 6, 2, 1, 8, 10, 3, 5, 7, 0, 9, 1]);
        check_to_vec(~[9, 11, 9, 9, 9, 9, 11, 2, 3, 4, 11, 9, 0, 0, 0, 0]);
        check_to_vec(~[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
        check_to_vec(~[10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]);
        check_to_vec(~[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 1, 2]);
        check_to_vec(~[5, 4, 3, 2, 1, 5, 4, 3, 2, 1, 5, 4, 3, 2, 1]);
    }

    #[test]
    #[should_fail]
    #[ignore(cfg(windows))]
    fn test_empty_pop() { let mut heap = new::<int>(); heap.pop(); }

    #[test]
    fn test_empty_maybe_pop() {
        let mut heap = new::<int>();
        assert heap.maybe_pop().is_none();
    }

    #[test]
    #[should_fail]
    #[ignore(cfg(windows))]
    fn test_empty_top() { let empty = new::<int>(); empty.top(); }

    #[test]
    fn test_empty_maybe_top() {
        let empty = new::<int>();
        assert empty.maybe_top().is_none();
    }

    #[test]
    #[should_fail]
    #[ignore(cfg(windows))]
    fn test_empty_replace() { let mut heap = new(); heap.replace(5); }
}
<commit_msg>remove is_not_empty method from PriorityQueue<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! A priority queue implemented with a binary heap

use core::container::{Container, Mutable};
use core::cmp::Ord;
use core::prelude::*;
use core::ptr::addr_of;
use core::vec;

#[abi = "rust-intrinsic"]
extern "C" mod rusti {
    fn move_val_init<T>(dst: &mut T, -src: T);
}

pub struct PriorityQueue <T: Ord>{
    priv data: ~[T],
}

impl <T: Ord> PriorityQueue<T>: Container {
    /// Returns the length of the queue
    pure fn len(&self) -> uint { self.data.len() }

    /// Returns true if a queue contains no elements
    pure fn is_empty(&self) -> bool { self.data.is_empty() }
}

impl <T: Ord> PriorityQueue<T>: Mutable {
    /// Drop all items from the queue
    fn clear(&mut self) { self.data.truncate(0) }
}

impl <T: Ord> PriorityQueue<T> {
    /// Returns the greatest item in the queue - fails if empty
    pure fn top(&self) -> &self/T { &self.data[0] }

    /// Returns the greatest item in the queue - None if empty
    pure fn maybe_top(&self) -> Option<&self/T> {
        if self.is_empty() { None } else { Some(self.top()) }
    }

    /// Returns the number of elements the queue can hold without reallocating
    pure fn capacity(&self) -> uint { vec::capacity(&self.data) }

    fn reserve(&mut self, n: uint) { vec::reserve(&mut self.data, n) }

    fn reserve_at_least(&mut self, n: uint) {
        vec::reserve_at_least(&mut self.data, n)
    }

    /// Pop the greatest item from the queue - fails if empty
    fn pop(&mut self) -> T {
        let mut item = self.data.pop();
        if !self.is_empty() { item <-> self.data[0]; self.siftdown(0); }
        item
    }

    /// Pop the greatest item from the queue - None if empty
    fn maybe_pop(&mut self) -> Option<T> {
        if self.is_empty() { None } else { Some(self.pop()) }
    }

    /// Push an item onto the queue
    fn push(&mut self, item: T) {
        self.data.push(item);
        self.siftup(0, self.len() - 1);
    }

    /// Optimized version of a push followed by a pop
    fn push_pop(&mut self, item: T) -> T {
        let mut item = item;
        if !self.is_empty() && self.data[0] > item {
            item <-> self.data[0];
            self.siftdown(0);
        }
        item
    }

    /// Optimized version of a pop followed by a push - fails if empty
    fn replace(&mut self, item: T) -> T {
        let mut item = item;
        item <-> self.data[0];
        self.siftdown(0);
        item
    }

    /// Consume the PriorityQueue and return the underlying vector
    pure fn to_vec(self) -> ~[T] { let PriorityQueue{data: v} = self; v }

    /// Consume the PriorityQueue and return a vector in sorted
    /// (ascending) order
    pure fn to_sorted_vec(self) -> ~[T] {
        let mut q = self;
        let mut end = q.len();
        while end > 1 {
            end -= 1;
            q.data[end] <-> q.data[0];
            unsafe { q.siftdown_range(0, end) } // purity-checking workaround
        }
        q.to_vec()
    }

    /// Create an empty PriorityQueue
    static pure fn new() -> PriorityQueue<T> { PriorityQueue{data: ~[],} }

    /// Create a PriorityQueue from a vector (heapify)
    static pure fn from_vec(xs: ~[T]) -> PriorityQueue<T> {
        let mut q = PriorityQueue{data: xs,};
        let mut n = q.len() / 2;
        while n > 0 {
            n -= 1;
            unsafe { q.siftdown(n) }; // purity-checking workaround
        }
        q
    }

    // The implementations of siftup and siftdown use unsafe blocks in
    // order to move an element out of the vector (leaving behind a
    // junk element), shift along the others and move it back into the
    // vector over the junk element.  This reduces the constant factor
    // compared to using swaps, which involves twice as many moves.

    priv fn siftup(&mut self, start: uint, pos: uint) {
        unsafe {
            let mut pos = pos;
            let new = move *addr_of(&self.data[pos]);

            while pos > start {
                let parent = (pos - 1) >> 1;
                if new > self.data[parent] {
                    rusti::move_val_init(&mut self.data[pos],
                                         move *addr_of(&self.data[parent]));
                    pos = parent;
                    loop
                }
                break
            }
            rusti::move_val_init(&mut self.data[pos], move new);
        }
    }

    priv fn siftdown_range(&mut self, pos: uint, end: uint) {
        unsafe {
            let mut pos = pos;
            let start = pos;
            let new = move *addr_of(&self.data[pos]);

            let mut child = 2 * pos + 1;
            while child < end {
                let right = child + 1;
                if right < end && !(self.data[child] > self.data[right]) {
                    child = right;
                }
                rusti::move_val_init(&mut self.data[pos],
                                     move *addr_of(&self.data[child]));
                pos = child;
                child = 2 * pos + 1;
            }

            rusti::move_val_init(&mut self.data[pos], move new);
            self.siftup(start, pos);
        }
    }

    priv fn siftdown(&mut self, pos: uint) {
        self.siftdown_range(pos, self.len());
    }
}

#[cfg(test)]
mod tests {
    use sort::merge_sort;
    use core::cmp::le;
    use priority_queue::PriorityQueue::{from_vec, new};

    #[test]
    fn test_top_and_pop() {
        let data = ~[2, 4, 6, 2, 1, 8, 10, 3, 5, 7, 0, 9, 1];
        let mut sorted = merge_sort(data, le);
        let mut heap = from_vec(data);
        while !heap.is_empty() {
            assert *heap.top() == sorted.last();
            assert heap.pop() == sorted.pop();
        }
    }

    #[test]
    fn test_push() {
        let mut heap = from_vec(~[2, 4, 9]);
        assert heap.len() == 3;
        assert *heap.top() == 9;
        heap.push(11);
        assert heap.len() == 4;
        assert *heap.top() == 11;
        heap.push(5);
        assert heap.len() == 5;
        assert *heap.top() == 11;
        heap.push(27);
        assert heap.len() == 6;
        assert *heap.top() == 27;
        heap.push(3);
        assert heap.len() == 7;
        assert *heap.top() == 27;
        heap.push(103);
        assert heap.len() == 8;
        assert *heap.top() == 103;
    }

    #[test]
    fn test_push_unique() {
        let mut heap = from_vec(~[~2, ~4, ~9]);
        assert heap.len() == 3;
        assert *heap.top() == ~9;
        heap.push(~11);
        assert heap.len() == 4;
        assert *heap.top() == ~11;
        heap.push(~5);
        assert heap.len() == 5;
        assert *heap.top() == ~11;
        heap.push(~27);
        assert heap.len() == 6;
        assert *heap.top() == ~27;
        heap.push(~3);
        assert heap.len() == 7;
        assert *heap.top() == ~27;
        heap.push(~103);
        assert heap.len() == 8;
        assert *heap.top() == ~103;
    }

    #[test]
    fn test_push_pop() {
        let mut heap = from_vec(~[5, 5, 2, 1, 3]);
        assert heap.len() == 5;
        assert heap.push_pop(6) == 6;
        assert heap.len() == 5;
        assert heap.push_pop(0) == 5;
        assert heap.len() == 5;
        assert heap.push_pop(4) == 5;
        assert heap.len() == 5;
        assert heap.push_pop(1) == 4;
        assert heap.len() == 5;
    }

    #[test]
    fn test_replace() {
        let mut heap = from_vec(~[5, 5, 2, 1, 3]);
        assert heap.len() == 5;
        assert heap.replace(6) == 5;
        assert heap.len() == 5;
        assert heap.replace(0) == 6;
        assert heap.len() == 5;
        assert heap.replace(4) == 5;
        assert heap.len() == 5;
        assert heap.replace(1) == 4;
        assert heap.len() == 5;
    }

    fn check_to_vec(data: ~[int]) {
        let heap = from_vec(data);
        assert merge_sort(heap.to_vec(), le) == merge_sort(data, le);
        assert heap.to_sorted_vec() == merge_sort(data, le);
    }

    #[test]
    fn test_to_vec() {
        check_to_vec(~[]);
        check_to_vec(~[5]);
        check_to_vec(~[3, 2]);
        check_to_vec(~[2, 3]);
        check_to_vec(~[5, 1, 2]);
        check_to_vec(~[1, 100, 2, 3]);
        check_to_vec(~[1, 3, 5, 7, 9, 2, 4, 6, 8, 0]);
        check_to_vec(~[2, 4, 6, 2, 1, 8, 10, 3, 5, 7, 0, 9, 1]);
        check_to_vec(~[9, 11, 9, 9, 9, 9, 11, 2, 3, 4, 11, 9, 0, 0, 0, 0]);
        check_to_vec(~[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
        check_to_vec(~[10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]);
        check_to_vec(~[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 1, 2]);
        check_to_vec(~[5, 4, 3, 2, 1, 5, 4, 3, 2, 1, 5, 4, 3, 2, 1]);
    }

    #[test]
    #[should_fail]
    #[ignore(cfg(windows))]
    fn test_empty_pop() { let mut heap = new::<int>(); heap.pop(); }

    #[test]
    fn test_empty_maybe_pop() {
        let mut heap = new::<int>();
        assert heap.maybe_pop().is_none();
    }

    #[test]
    #[should_fail]
    #[ignore(cfg(windows))]
    fn test_empty_top() { let empty = new::<int>(); empty.top(); }

    #[test]
    fn test_empty_maybe_top() {
        let empty = new::<int>();
        assert empty.maybe_top().is_none();
    }

    #[test]
    #[should_fail]
    #[ignore(cfg(windows))]
    fn test_empty_replace() { let mut heap = new(); heap.replace(5); }
}
<|endoftext|>


<commit_before>//! Neural Network Layers

use linalg::{Matrix, MatrixSlice, BaseMatrix};

use learning::LearningResult;
use learning::error::{Error, ErrorKind};
use learning::toolkit::activ_fn::ActivationFunc;

use rand::thread_rng;
use rand::distributions::Sample;
use rand::distributions::normal::Normal;

use std::fmt::Debug;

/// Trait for neural net layers
pub trait NetLayer : Debug {
    /// The result of propogating data forward through this layer
    fn forward(&self, input: &Matrix<f64>, params: MatrixSlice<f64>) -> LearningResult<Matrix<f64>>;

    /// The gradient of the output of this layer with respect to its input
    fn back_input(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, output: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64>;

    /// The gradient of the output of this layer with respect to its parameters
    fn back_params(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, output: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64>;

    /// The default value of the parameters of this layer before training
    fn default_params(&self) -> Vec<f64>;

    /// The shape of the parameters used by this layer
    fn param_shape(&self) -> (usize, usize);

    /// The number of parameters used by this layer
    fn num_params(&self) -> usize {
        let shape = self.param_shape();
        shape.0 * shape.1
    }
}

/// Linear network layer
///
/// Represents a fully connected layer with optional bias term
///
/// The parameters are a matrix of weights of size I x O
/// where O is the dimensionality of the output and I the dimensionality of the input
#[derive(Debug, Clone, Copy)]
pub struct Linear {
    /// The number of dimensions of the input
    input_size: usize,
    /// The number of dimensions of the output
    output_size: usize,
    /// Whether or not to include a bias term
    has_bias: bool,
}

impl Linear {
    /// Construct a new Linear layer
    pub fn new(input_size: usize, output_size: usize) -> Linear {
        Linear {
            input_size: input_size + 1,
            output_size: output_size,
            has_bias: true
        }
    }

    /// Construct a Linear layer with a bias term
    pub fn without_bias(input_size: usize, output_size: usize) -> Linear {
        Linear {
            input_size: input_size,
            output_size: output_size,
            has_bias: false
        }
    }
}

impl NetLayer for Linear {
    /// Computes a matrix product
    ///
    /// input should have dimensions N x I
    /// where N is the number of samples and I is the dimensionality of the input
    fn forward(&self, input: &Matrix<f64>, params: MatrixSlice<f64>) -> LearningResult<Matrix<f64>> {
        if self.has_bias {
            if input.cols()+1 != params.rows() {
                Err(Error::new(ErrorKind::InvalidData, "The input had the wrong number of columns"))
            } else {
                Ok(&Matrix::ones(input.rows(), 1).hcat(input) * &params)
            }
        } else {
            if input.cols() != params.rows() {
                Err(Error::new(ErrorKind::InvalidData, "The input had the wrong number of columns"))
            } else {
                Ok(input * &params)
            }
        }
    }

    fn back_input(&self, out_grad: &Matrix<f64>, _: &Matrix<f64>, _: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64> {
        debug_assert_eq!(out_grad.cols(), params.cols());
        let gradient = out_grad * &params.transpose();
        if self.has_bias {
            let rows = gradient.rows();
            let cols = gradient.cols() - 1;
            gradient.sub_slice([0, 1], rows, cols).into()
        } else {
            gradient
        }
    }

    fn back_params(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, _: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        debug_assert_eq!(input.rows(), out_grad.rows());
        if self.has_bias {
            &Matrix::ones(input.rows(), 1).hcat(input).transpose() * out_grad
        } else {
            &input.transpose() * out_grad
        }
    }

    /// Initializes weights using Xavier initialization
    ///
    /// weights drawn from gaussian distribution with 0 mean and variance 2/(input_size+output_size)
    fn default_params(&self) -> Vec<f64> {
        let mut distro = Normal::new(0.0, (2.0/(self.input_size+self.output_size) as f64).sqrt());
        let mut rng = thread_rng();

        (0..self.input_size*self.output_size).map(|_| distro.sample(&mut rng))
                                             .collect()
    }

    fn param_shape(&self) -> (usize, usize) {
        (self.input_size, self.output_size)
    }
}

impl<T: ActivationFunc> NetLayer for T {
    /// Applies the activation function to each element of the input
    fn forward(&self, input: &Matrix<f64>, _: MatrixSlice<f64>) -> LearningResult<Matrix<f64>> {
        let mut output = Vec::with_capacity(input.rows()*input.cols());
        for val in input.data() {
            output.push(T::func(*val));
        }
        Ok(Matrix::new(input.rows(), input.cols(), output))
    }

    fn back_input(&self, out_grad: &Matrix<f64>, _: &Matrix<f64>, output: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        let mut in_grad = Vec::with_capacity(output.rows()*output.cols());
        for (y, g) in output.data().iter().zip(out_grad.data()) {
            in_grad.push(T::func_grad_from_output(*y) * g);
        }
        Matrix::new(output.rows(), output.cols(), in_grad)
    }

    fn back_params(&self, _: &Matrix<f64>, _: &Matrix<f64>, _: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        Matrix::new(0, 0, Vec::new())
    }

    fn default_params(&self) -> Vec<f64> {
        Vec::new()
    }

    fn param_shape(&self) -> (usize, usize) {
        (0, 0)
    }
}
<commit_msg>Fixed typo<commit_after>//! Neural Network Layers

use linalg::{Matrix, MatrixSlice, BaseMatrix};

use learning::LearningResult;
use learning::error::{Error, ErrorKind};
use learning::toolkit::activ_fn::ActivationFunc;

use rand::thread_rng;
use rand::distributions::Sample;
use rand::distributions::normal::Normal;

use std::fmt::Debug;

/// Trait for neural net layers
pub trait NetLayer : Debug {
    /// The result of propogating data forward through this layer
    fn forward(&self, input: &Matrix<f64>, params: MatrixSlice<f64>) -> LearningResult<Matrix<f64>>;

    /// The gradient of the output of this layer with respect to its input
    fn back_input(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, output: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64>;

    /// The gradient of the output of this layer with respect to its parameters
    fn back_params(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, output: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64>;

    /// The default value of the parameters of this layer before training
    fn default_params(&self) -> Vec<f64>;

    /// The shape of the parameters used by this layer
    fn param_shape(&self) -> (usize, usize);

    /// The number of parameters used by this layer
    fn num_params(&self) -> usize {
        let shape = self.param_shape();
        shape.0 * shape.1
    }
}

/// Linear network layer
///
/// Represents a fully connected layer with optional bias term
///
/// The parameters are a matrix of weights of size I x O
/// where O is the dimensionality of the output and I the dimensionality of the input
#[derive(Debug, Clone, Copy)]
pub struct Linear {
    /// The number of dimensions of the input
    input_size: usize,
    /// The number of dimensions of the output
    output_size: usize,
    /// Whether or not to include a bias term
    has_bias: bool,
}

impl Linear {
    /// Construct a new Linear layer
    pub fn new(input_size: usize, output_size: usize) -> Linear {
        Linear {
            input_size: input_size + 1,
            output_size: output_size,
            has_bias: true
        }
    }

    /// Construct a Linear layer without a bias term
    pub fn without_bias(input_size: usize, output_size: usize) -> Linear {
        Linear {
            input_size: input_size,
            output_size: output_size,
            has_bias: false
        }
    }
}

impl NetLayer for Linear {
    /// Computes a matrix product
    ///
    /// input should have dimensions N x I
    /// where N is the number of samples and I is the dimensionality of the input
    fn forward(&self, input: &Matrix<f64>, params: MatrixSlice<f64>) -> LearningResult<Matrix<f64>> {
        if self.has_bias {
            if input.cols()+1 != params.rows() {
                Err(Error::new(ErrorKind::InvalidData, "The input had the wrong number of columns"))
            } else {
                Ok(&Matrix::ones(input.rows(), 1).hcat(input) * &params)
            }
        } else {
            if input.cols() != params.rows() {
                Err(Error::new(ErrorKind::InvalidData, "The input had the wrong number of columns"))
            } else {
                Ok(input * &params)
            }
        }
    }

    fn back_input(&self, out_grad: &Matrix<f64>, _: &Matrix<f64>, _: &Matrix<f64>, params: MatrixSlice<f64>) -> Matrix<f64> {
        debug_assert_eq!(out_grad.cols(), params.cols());
        let gradient = out_grad * &params.transpose();
        if self.has_bias {
            let rows = gradient.rows();
            let cols = gradient.cols() - 1;
            gradient.sub_slice([0, 1], rows, cols).into()
        } else {
            gradient
        }
    }

    fn back_params(&self, out_grad: &Matrix<f64>, input: &Matrix<f64>, _: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        debug_assert_eq!(input.rows(), out_grad.rows());
        if self.has_bias {
            &Matrix::ones(input.rows(), 1).hcat(input).transpose() * out_grad
        } else {
            &input.transpose() * out_grad
        }
    }

    /// Initializes weights using Xavier initialization
    ///
    /// weights drawn from gaussian distribution with 0 mean and variance 2/(input_size+output_size)
    fn default_params(&self) -> Vec<f64> {
        let mut distro = Normal::new(0.0, (2.0/(self.input_size+self.output_size) as f64).sqrt());
        let mut rng = thread_rng();

        (0..self.input_size*self.output_size).map(|_| distro.sample(&mut rng))
                                             .collect()
    }

    fn param_shape(&self) -> (usize, usize) {
        (self.input_size, self.output_size)
    }
}

impl<T: ActivationFunc> NetLayer for T {
    /// Applies the activation function to each element of the input
    fn forward(&self, input: &Matrix<f64>, _: MatrixSlice<f64>) -> LearningResult<Matrix<f64>> {
        let mut output = Vec::with_capacity(input.rows()*input.cols());
        for val in input.data() {
            output.push(T::func(*val));
        }
        Ok(Matrix::new(input.rows(), input.cols(), output))
    }

    fn back_input(&self, out_grad: &Matrix<f64>, _: &Matrix<f64>, output: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        let mut in_grad = Vec::with_capacity(output.rows()*output.cols());
        for (y, g) in output.data().iter().zip(out_grad.data()) {
            in_grad.push(T::func_grad_from_output(*y) * g);
        }
        Matrix::new(output.rows(), output.cols(), in_grad)
    }

    fn back_params(&self, _: &Matrix<f64>, _: &Matrix<f64>, _: &Matrix<f64>, _: MatrixSlice<f64>) -> Matrix<f64> {
        Matrix::new(0, 0, Vec::new())
    }

    fn default_params(&self) -> Vec<f64> {
        Vec::new()
    }

    fn param_shape(&self) -> (usize, usize) {
        (0, 0)
    }
}
<|endoftext|>


<commit_before>#[feature(macro_rules)];

use std::io::{file_writer, Create, Truncate};
use std::os;

pub mod branchify;
pub mod status;
pub mod read_method;

fn main() {
    let args = os::args();
    match args.len() {
        0 => {
            println("usage: codegen [read_method|status].rs");
            os::set_exit_status(1);
        },
        3 => {
            let output_dir = GenericPath::from_str(args[2]);
            // TODO: maybe not 0777?
            os::make_dir(&output_dir, 0b111_111_111);

            match args[1] {
                ~"read_method.rs" => read_method::generate(&output_dir),
                ~"status.rs" => status::generate(&output_dir),
                s => {
                    println!("unknown thing-to-generate '{}'", s);
                    os::set_exit_status(1);
                }
            }
        },
        _ => {
            println!("usage: {} [read_method|status].rs", args[0]);
            os::set_exit_status(1);
        }
    }
}

pub fn get_writer(output_dir: &Path, filename: &str) -> @Writer {
    match file_writer(&output_dir.push(filename), [Create, Truncate]) {
        Ok(writer) => writer,
        Err(msg) => fail!("Unable to write file: %s", msg),
    }
}
<commit_msg>Changes for new Path API.<commit_after>#[feature(macro_rules)];

use std::io::{file_writer, Create, Truncate};
use std::os;

pub mod branchify;
pub mod status;
pub mod read_method;

fn main() {
    let args = os::args();
    match args.len() {
        0 => {
            println("usage: codegen [read_method|status].rs");
            os::set_exit_status(1);
        },
        3 => {
            let output_dir = from_str(args[2]).unwrap();
            // TODO: maybe not 0777?
            os::make_dir(&output_dir, 0b111_111_111);

            match args[1] {
                ~"read_method.rs" => read_method::generate(&output_dir),
                ~"status.rs" => status::generate(&output_dir),
                s => {
                    println!("unknown thing-to-generate '{}'", s);
                    os::set_exit_status(1);
                }
            }
        },
        _ => {
            println!("usage: {} [read_method|status].rs", args[0]);
            os::set_exit_status(1);
        }
    }
}

pub fn get_writer(output_dir: &Path, filename: &str) -> @Writer {
    let mut output_dir = output_dir.clone();
    output_dir.push(filename);
    match file_writer(&output_dir, [Create, Truncate]) {
        Ok(writer) => writer,
        Err(msg) => fail!("Unable to write file: %s", msg),
    }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use super::link;
use super::write;
use rustc::session::{self, config};
use llvm;
use llvm::archive_ro::ArchiveRO;
use llvm::{ModuleRef, TargetMachineRef, True, False};
use rustc::metadata::cstore;
use rustc::util::common::time;

use libc;
use flate;

use std::ffi::CString;
use std::iter;
use std::mem;
use std::num::Int;

pub fn run(sess: &session::Session, llmod: ModuleRef,
           tm: TargetMachineRef, reachable: &[String]) {
    if sess.opts.cg.prefer_dynamic {
        sess.err("cannot prefer dynamic linking when performing LTO");
        sess.note("only 'staticlib' and 'bin' outputs are supported with LTO");
        sess.abort_if_errors();
    }

    // Make sure we actually can run LTO
    for crate_type in sess.crate_types.borrow().iter() {
        match *crate_type {
            config::CrateTypeExecutable | config::CrateTypeStaticlib => {}
            _ => {
                sess.fatal("lto can only be run for executables and \
                            static library outputs");
            }
        }
    }

    // For each of our upstream dependencies, find the corresponding rlib and
    // load the bitcode from the archive. Then merge it into the current LLVM
    // module that we've got.
    let crates = sess.cstore.get_used_crates(cstore::RequireStatic);
    for (cnum, path) in crates.into_iter() {
        let name = sess.cstore.get_crate_data(cnum).name.clone();
        let path = match path {
            Some(p) => p,
            None => {
                sess.fatal(format!("could not find rlib for: `{}`",
                                   name).index(&FullRange));
            }
        };

        let archive = ArchiveRO::open(&path).expect("wanted an rlib");
        let file = path.filename_str().unwrap();
        let file = file.index(&(3..(file.len() - 5))); // chop off lib/.rlib
        debug!("reading {}", file);
        for i in iter::count(0u, 1) {
            let bc_encoded = time(sess.time_passes(),
                                  format!("check for {}.{}.bytecode.deflate", name, i).as_slice(),
                                  (),
                                  |_| {
                                      archive.read(format!("{}.{}.bytecode.deflate",
                                                           file, i).index(&FullRange))
                                  });
            let bc_encoded = match bc_encoded {
                Some(data) => data,
                None => {
                    if i == 0 {
                        // No bitcode was found at all.
                        sess.fatal(format!("missing compressed bytecode in {}",
                                           path.display()).index(&FullRange));
                    }
                    // No more bitcode files to read.
                    break;
                },
            };

            let bc_decoded = if is_versioned_bytecode_format(bc_encoded) {
                time(sess.time_passes(), format!("decode {}.{}.bc", file, i).as_slice(), (), |_| {
                    // Read the version
                    let version = extract_bytecode_format_version(bc_encoded);

                    if version == 1 {
                        // The only version existing so far
                        let data_size = extract_compressed_bytecode_size_v1(bc_encoded);
                        let compressed_data = bc_encoded.index(&(
                            link::RLIB_BYTECODE_OBJECT_V1_DATA_OFFSET..
                            (link::RLIB_BYTECODE_OBJECT_V1_DATA_OFFSET + data_size as uint)));

                        match flate::inflate_bytes(compressed_data) {
                            Some(inflated) => inflated,
                            None => {
                                sess.fatal(format!("failed to decompress bc of `{}`",
                                                   name).index(&FullRange))
                            }
                        }
                    } else {
                        sess.fatal(format!("Unsupported bytecode format version {}",
                                           version).index(&FullRange))
                    }
                })
            } else {
                time(sess.time_passes(), format!("decode {}.{}.bc", file, i).as_slice(), (), |_| {
                // the object must be in the old, pre-versioning format, so simply
                // inflate everything and let LLVM decide if it can make sense of it
                    match flate::inflate_bytes(bc_encoded) {
                        Some(bc) => bc,
                        None => {
                            sess.fatal(format!("failed to decompress bc of `{}`",
                                               name).index(&FullRange))
                        }
                    }
                })
            };

            let ptr = bc_decoded.as_slice().as_ptr();
            debug!("linking {}, part {}", name, i);
            time(sess.time_passes(),
                 format!("ll link {}.{}", name, i).index(&FullRange),
                 (),
                 |()| unsafe {
                if !llvm::LLVMRustLinkInExternalBitcode(llmod,
                                                        ptr as *const libc::c_char,
                                                        bc_decoded.len() as libc::size_t) {
                    write::llvm_err(sess.diagnostic().handler(),
                                    format!("failed to load bc of `{}`",
                                            name.index(&FullRange)));
                }
            });
        }
    }

    // Internalize everything but the reachable symbols of the current module
    let cstrs: Vec<CString> = reachable.iter().map(|s| {
        CString::from_slice(s.as_bytes())
    }).collect();
    let arr: Vec<*const i8> = cstrs.iter().map(|c| c.as_ptr()).collect();
    let ptr = arr.as_ptr();
    unsafe {
        llvm::LLVMRustRunRestrictionPass(llmod,
                                         ptr as *const *const libc::c_char,
                                         arr.len() as libc::size_t);
    }

    if sess.no_landing_pads() {
        unsafe {
            llvm::LLVMRustMarkAllFunctionsNounwind(llmod);
        }
    }

    // Now we have one massive module inside of llmod. Time to run the
    // LTO-specific optimization passes that LLVM provides.
    //
    // This code is based off the code found in llvm's LTO code generator:
    //      tools/lto/LTOCodeGenerator.cpp
    debug!("running the pass manager");
    unsafe {
        let pm = llvm::LLVMCreatePassManager();
        llvm::LLVMRustAddAnalysisPasses(tm, pm, llmod);
        llvm::LLVMRustAddPass(pm, "verify\0".as_ptr() as *const _);

        let builder = llvm::LLVMPassManagerBuilderCreate();
        llvm::LLVMPassManagerBuilderPopulateLTOPassManager(builder, pm,
            /* Internalize = */ False,
            /* RunInliner = */ True);
        llvm::LLVMPassManagerBuilderDispose(builder);

        llvm::LLVMRustAddPass(pm, "verify\0".as_ptr() as *const _);

        time(sess.time_passes(), "LTO passes", (), |()|
             llvm::LLVMRunPassManager(pm, llmod));

        llvm::LLVMDisposePassManager(pm);
    }
    debug!("lto done");
}

fn is_versioned_bytecode_format(bc: &[u8]) -> bool {
    let magic_id_byte_count = link::RLIB_BYTECODE_OBJECT_MAGIC.len();
    return bc.len() > magic_id_byte_count &&
           bc.index(&(0..magic_id_byte_count)) == link::RLIB_BYTECODE_OBJECT_MAGIC;
}

fn extract_bytecode_format_version(bc: &[u8]) -> u32 {
    return read_from_le_bytes::<u32>(bc, link::RLIB_BYTECODE_OBJECT_VERSION_OFFSET);
}

fn extract_compressed_bytecode_size_v1(bc: &[u8]) -> u64 {
    return read_from_le_bytes::<u64>(bc, link::RLIB_BYTECODE_OBJECT_V1_DATASIZE_OFFSET);
}

fn read_from_le_bytes<T: Int>(bytes: &[u8], position_in_bytes: uint) -> T {
    let byte_data = bytes.index(&(position_in_bytes..
                                  (position_in_bytes + mem::size_of::<T>())));
    let data = unsafe {
        *(byte_data.as_ptr() as *const T)
    };

    Int::from_le(data)
}

<commit_msg>Fix: `libc::c_char` is not always `i8`<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use super::link;
use super::write;
use rustc::session::{self, config};
use llvm;
use llvm::archive_ro::ArchiveRO;
use llvm::{ModuleRef, TargetMachineRef, True, False};
use rustc::metadata::cstore;
use rustc::util::common::time;

use libc;
use flate;

use std::ffi::CString;
use std::iter;
use std::mem;
use std::num::Int;

pub fn run(sess: &session::Session, llmod: ModuleRef,
           tm: TargetMachineRef, reachable: &[String]) {
    if sess.opts.cg.prefer_dynamic {
        sess.err("cannot prefer dynamic linking when performing LTO");
        sess.note("only 'staticlib' and 'bin' outputs are supported with LTO");
        sess.abort_if_errors();
    }

    // Make sure we actually can run LTO
    for crate_type in sess.crate_types.borrow().iter() {
        match *crate_type {
            config::CrateTypeExecutable | config::CrateTypeStaticlib => {}
            _ => {
                sess.fatal("lto can only be run for executables and \
                            static library outputs");
            }
        }
    }

    // For each of our upstream dependencies, find the corresponding rlib and
    // load the bitcode from the archive. Then merge it into the current LLVM
    // module that we've got.
    let crates = sess.cstore.get_used_crates(cstore::RequireStatic);
    for (cnum, path) in crates.into_iter() {
        let name = sess.cstore.get_crate_data(cnum).name.clone();
        let path = match path {
            Some(p) => p,
            None => {
                sess.fatal(format!("could not find rlib for: `{}`",
                                   name).index(&FullRange));
            }
        };

        let archive = ArchiveRO::open(&path).expect("wanted an rlib");
        let file = path.filename_str().unwrap();
        let file = file.index(&(3..(file.len() - 5))); // chop off lib/.rlib
        debug!("reading {}", file);
        for i in iter::count(0u, 1) {
            let bc_encoded = time(sess.time_passes(),
                                  format!("check for {}.{}.bytecode.deflate", name, i).as_slice(),
                                  (),
                                  |_| {
                                      archive.read(format!("{}.{}.bytecode.deflate",
                                                           file, i).index(&FullRange))
                                  });
            let bc_encoded = match bc_encoded {
                Some(data) => data,
                None => {
                    if i == 0 {
                        // No bitcode was found at all.
                        sess.fatal(format!("missing compressed bytecode in {}",
                                           path.display()).index(&FullRange));
                    }
                    // No more bitcode files to read.
                    break;
                },
            };

            let bc_decoded = if is_versioned_bytecode_format(bc_encoded) {
                time(sess.time_passes(), format!("decode {}.{}.bc", file, i).as_slice(), (), |_| {
                    // Read the version
                    let version = extract_bytecode_format_version(bc_encoded);

                    if version == 1 {
                        // The only version existing so far
                        let data_size = extract_compressed_bytecode_size_v1(bc_encoded);
                        let compressed_data = bc_encoded.index(&(
                            link::RLIB_BYTECODE_OBJECT_V1_DATA_OFFSET..
                            (link::RLIB_BYTECODE_OBJECT_V1_DATA_OFFSET + data_size as uint)));

                        match flate::inflate_bytes(compressed_data) {
                            Some(inflated) => inflated,
                            None => {
                                sess.fatal(format!("failed to decompress bc of `{}`",
                                                   name).index(&FullRange))
                            }
                        }
                    } else {
                        sess.fatal(format!("Unsupported bytecode format version {}",
                                           version).index(&FullRange))
                    }
                })
            } else {
                time(sess.time_passes(), format!("decode {}.{}.bc", file, i).as_slice(), (), |_| {
                // the object must be in the old, pre-versioning format, so simply
                // inflate everything and let LLVM decide if it can make sense of it
                    match flate::inflate_bytes(bc_encoded) {
                        Some(bc) => bc,
                        None => {
                            sess.fatal(format!("failed to decompress bc of `{}`",
                                               name).index(&FullRange))
                        }
                    }
                })
            };

            let ptr = bc_decoded.as_slice().as_ptr();
            debug!("linking {}, part {}", name, i);
            time(sess.time_passes(),
                 format!("ll link {}.{}", name, i).index(&FullRange),
                 (),
                 |()| unsafe {
                if !llvm::LLVMRustLinkInExternalBitcode(llmod,
                                                        ptr as *const libc::c_char,
                                                        bc_decoded.len() as libc::size_t) {
                    write::llvm_err(sess.diagnostic().handler(),
                                    format!("failed to load bc of `{}`",
                                            name.index(&FullRange)));
                }
            });
        }
    }

    // Internalize everything but the reachable symbols of the current module
    let cstrs: Vec<CString> = reachable.iter().map(|s| {
        CString::from_slice(s.as_bytes())
    }).collect();
    let arr: Vec<*const libc::c_char> = cstrs.iter().map(|c| c.as_ptr()).collect();
    let ptr = arr.as_ptr();
    unsafe {
        llvm::LLVMRustRunRestrictionPass(llmod,
                                         ptr as *const *const libc::c_char,
                                         arr.len() as libc::size_t);
    }

    if sess.no_landing_pads() {
        unsafe {
            llvm::LLVMRustMarkAllFunctionsNounwind(llmod);
        }
    }

    // Now we have one massive module inside of llmod. Time to run the
    // LTO-specific optimization passes that LLVM provides.
    //
    // This code is based off the code found in llvm's LTO code generator:
    //      tools/lto/LTOCodeGenerator.cpp
    debug!("running the pass manager");
    unsafe {
        let pm = llvm::LLVMCreatePassManager();
        llvm::LLVMRustAddAnalysisPasses(tm, pm, llmod);
        llvm::LLVMRustAddPass(pm, "verify\0".as_ptr() as *const _);

        let builder = llvm::LLVMPassManagerBuilderCreate();
        llvm::LLVMPassManagerBuilderPopulateLTOPassManager(builder, pm,
            /* Internalize = */ False,
            /* RunInliner = */ True);
        llvm::LLVMPassManagerBuilderDispose(builder);

        llvm::LLVMRustAddPass(pm, "verify\0".as_ptr() as *const _);

        time(sess.time_passes(), "LTO passes", (), |()|
             llvm::LLVMRunPassManager(pm, llmod));

        llvm::LLVMDisposePassManager(pm);
    }
    debug!("lto done");
}

fn is_versioned_bytecode_format(bc: &[u8]) -> bool {
    let magic_id_byte_count = link::RLIB_BYTECODE_OBJECT_MAGIC.len();
    return bc.len() > magic_id_byte_count &&
           bc.index(&(0..magic_id_byte_count)) == link::RLIB_BYTECODE_OBJECT_MAGIC;
}

fn extract_bytecode_format_version(bc: &[u8]) -> u32 {
    return read_from_le_bytes::<u32>(bc, link::RLIB_BYTECODE_OBJECT_VERSION_OFFSET);
}

fn extract_compressed_bytecode_size_v1(bc: &[u8]) -> u64 {
    return read_from_le_bytes::<u64>(bc, link::RLIB_BYTECODE_OBJECT_V1_DATASIZE_OFFSET);
}

fn read_from_le_bytes<T: Int>(bytes: &[u8], position_in_bytes: uint) -> T {
    let byte_data = bytes.index(&(position_in_bytes..
                                  (position_in_bytes + mem::size_of::<T>())));
    let data = unsafe {
        *(byte_data.as_ptr() as *const T)
    };

    Int::from_le(data)
}

<|endoftext|>


<commit_before>#![no_std]
#![allow(incomplete_features)]
#![feature(repr_simd, platform_intrinsics, simd_ffi, const_generics)]
#![feature(extended_key_value_attributes)]
#![warn(missing_docs)]
//! Portable SIMD module.

#[macro_use]
mod first;
#[macro_use]
mod permute;
#[macro_use]
mod transmute;

mod comparisons;
mod fmt;
mod intrinsics;
mod ops;
mod round;

mod math;

mod lanes_at_most_64;
pub use lanes_at_most_64::LanesAtMost32;

mod masks;
pub use masks::*;

mod vector;
pub use vector::*;
<commit_msg>Add floating-point classification functions<commit_after>#![no_std]
#![allow(incomplete_features)]
#![feature(repr_simd, platform_intrinsics, simd_ffi, const_generics)]
#![feature(extended_key_value_attributes)]
#![warn(missing_docs)]
//! Portable SIMD module.

#[macro_use]
mod first;
#[macro_use]
mod permute;
#[macro_use]
mod transmute;

mod comparisons;
mod fmt;
mod intrinsics;
mod ops;
mod round;
mod comparisons;

mod math;

mod lanes_at_most_64;
pub use lanes_at_most_64::LanesAtMost32;

mod masks;
pub use masks::*;

mod vector;
pub use vector::*;
<|endoftext|>


<commit_before>extern crate env_logger;

use super::{MaxMindDBError, Reader};

use std::net::IpAddr;
use std::str::FromStr;

#[test]
fn test_decoder() {
    let _ = env_logger::try_init();

    #[allow(non_snake_case)]
    #[derive(Deserialize, Debug, Eq, PartialEq)]
    struct MapXType {
        arrayX: Vec<u32>,
        utf8_stringX: String,
    };

    #[allow(non_snake_case)]
    #[derive(Deserialize, Debug, Eq, PartialEq)]
    struct MapType {
        mapX: MapXType,
    };

    #[derive(Deserialize, Debug)]
    struct TestType {
        array: Vec<u32>,
        boolean: bool,
        bytes: Vec<u8>,
        double: f64,
        float: f32,
        int32: i32,
        map: MapType,
        uint16: u16,
        uint32: u32,
        uint64: u64,
        uint128: Vec<u8>,
        utf8_string: String,
    }

    let r = Reader::open("test-data/test-data/MaxMind-DB-test-decoder.mmdb");
    if let Err(err) = r {
        panic!(format!("error opening mmdb: {:?}", err));
    }
    let r = r.unwrap();
    let ip: IpAddr = FromStr::from_str("1.1.1.0").unwrap();
    let result: TestType = r.lookup(ip).unwrap();

    assert_eq!(result.array, vec![1u32, 2u32, 3u32]);
    assert_eq!(result.boolean, true);
    assert_eq!(result.bytes, vec![0u8, 0u8, 0u8, 42u8]);
    assert_eq!(result.double, 42.123456);
    assert_eq!(result.float, 1.1);
    assert_eq!(result.int32, -268435456);

    assert_eq!(
        result.map,
        MapType {
            mapX: MapXType {
                arrayX: vec![7, 8, 9],
                utf8_stringX: "hello".to_string(),
            },
        }
    );

    assert_eq!(result.uint16, 100);
    assert_eq!(result.uint32, 268435456);
    assert_eq!(result.uint64, 1152921504606846976);
    assert_eq!(
        result.uint128,
        vec![1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
    );

    assert_eq!(result.utf8_string, "unicode! ☯ - ♫".to_string());
}

#[test]
fn test_broken_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("test-data/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb")
        .ok()
        .unwrap();
    let ip: IpAddr = FromStr::from_str("2001:220::").unwrap();

    #[derive(Deserialize, Debug)]
    struct TestType;
    match r.lookup::<TestType>(ip) {
        Err(e) => assert_eq!(
            e,
            MaxMindDBError::InvalidDatabaseError("double of size 2".to_string())
        ),
        Ok(_) => panic!("Error expected"),
    }
}

#[test]
fn test_missing_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("file-does-not-exist.mmdb");
    match r {
        Ok(_) => panic!("Received Reader when opening non-existent file"),
        Err(MaxMindDBError::IoError(_)) => assert!(true),
        Err(_) => assert!(false),
    }
}

#[test]
fn test_non_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("README.md");
    match r {
        Ok(_) => panic!("Received Reader when opening a non-MMDB file"),
        Err(e) => assert_eq!(
            e,
            MaxMindDBError::InvalidDatabaseError(
                "Could not find MaxMind DB metadata \
                 in file."
                    .to_string(),
            )
        ),
    }
}

#[test]
fn test_reader() {
    let _ = env_logger::try_init();

    let sizes = [24usize, 28, 32];
    for record_size in sizes.iter() {
        let versions = [4usize, 6];
        for ip_version in versions.iter() {
            let filename = format!(
                "test-data/test-data/MaxMind-DB-test-ipv{}-{}.mmdb",
                ip_version, record_size
            );
            let reader = Reader::open(filename.as_ref()).ok().unwrap();

            check_metadata(&reader, *ip_version, *record_size);
            check_ip(&reader, *ip_version);
        }
    }
}

#[test]
fn test_lookup_city() {
    use super::geoip2::City;
    let _ = env_logger::try_init();

    let filename = "test-data/test-data/GeoIP2-City-Test.mmdb";

    let reader = Reader::open(filename).unwrap();

    let ip: IpAddr = FromStr::from_str("89.160.20.112").unwrap();
    let city: City = reader.lookup(ip).unwrap();

    let iso_code = city.country.and_then(|cy| cy.iso_code);

    assert_eq!(iso_code, Some("SE".to_owned()));
}

fn check_metadata(reader: &Reader, ip_version: usize, record_size: usize) {
    let metadata = &reader.metadata;

    assert_eq!(metadata.binary_format_major_version, 2u16);

    assert_eq!(metadata.binary_format_minor_version, 0u16);
    assert!(metadata.build_epoch >= 1397457605);
    assert_eq!(metadata.database_type, "Test".to_string());

    assert_eq!(
        *metadata.description.get(&"en".to_string()).unwrap(),
        "Test Database".to_string()
    );
    assert_eq!(
        *metadata.description.get(&"zh".to_string()).unwrap(),
        "Test Database Chinese".to_string()
    );

    assert_eq!(metadata.ip_version, ip_version as u16);
    assert_eq!(metadata.languages, vec!["en".to_string(), "zh".to_string()]);

    if ip_version == 4 {
        assert_eq!(metadata.node_count, 164)
    } else {
        assert_eq!(metadata.node_count, 416)
    }

    assert_eq!(metadata.record_size, record_size as u16)
}

fn check_ip(reader: &Reader, ip_version: usize) {
    let subnets = match ip_version {
        6 => [
            "::1:ffff:ffff",
            "::2:0:0",
            "::2:0:0",
            "::2:0:0",
            "::2:0:0",
            "::2:0:40",
            "::2:0:40",
            "::2:0:40",
            "::2:0:50",
            "::2:0:50",
            "::2:0:50",
            "::2:0:58",
            "::2:0:58",
        ],
        _ => [
            "1.1.1.1", "1.1.1.2", "1.1.1.2", "1.1.1.4", "1.1.1.4", "1.1.1.4", "1.1.1.4", "1.1.1.8",
            "1.1.1.8", "1.1.1.8", "1.1.1.16", "1.1.1.16", "1.1.1.16",
        ],
    };

    #[derive(Deserialize, Debug)]
    struct IpType {
        ip: String,
    }

    for subnet in subnets.iter() {
        let ip: IpAddr = FromStr::from_str(&subnet).unwrap();
        let value: IpType = reader.lookup(ip).unwrap();

        assert_eq!(value.ip, subnet.to_string());
    }

    let no_record = ["1.1.1.33", "255.254.253.123", "89fa::"];

    for &address in no_record.iter() {
        let ip: IpAddr = FromStr::from_str(address).unwrap();
        match reader.lookup::<IpType>(ip) {
            Ok(v) => panic!("received an unexpected value: {:?}", v),
            Err(e) => assert_eq!(
                e,
                MaxMindDBError::AddressNotFoundError("Address not found in database".to_string())
            ),
        }
    }
}
<commit_msg>removed obsolete .as_ref() and added test specifically exercising the open_readfile() code path when open() defaults to the mmap implementation<commit_after>extern crate env_logger;

use super::{MaxMindDBError, Reader};

use std::net::IpAddr;
use std::str::FromStr;

#[test]
fn test_decoder() {
    let _ = env_logger::try_init();

    #[allow(non_snake_case)]
    #[derive(Deserialize, Debug, Eq, PartialEq)]
    struct MapXType {
        arrayX: Vec<u32>,
        utf8_stringX: String,
    };

    #[allow(non_snake_case)]
    #[derive(Deserialize, Debug, Eq, PartialEq)]
    struct MapType {
        mapX: MapXType,
    };

    #[derive(Deserialize, Debug)]
    struct TestType {
        array: Vec<u32>,
        boolean: bool,
        bytes: Vec<u8>,
        double: f64,
        float: f32,
        int32: i32,
        map: MapType,
        uint16: u16,
        uint32: u32,
        uint64: u64,
        uint128: Vec<u8>,
        utf8_string: String,
    }

    let r = Reader::open("test-data/test-data/MaxMind-DB-test-decoder.mmdb");
    if let Err(err) = r {
        panic!(format!("error opening mmdb: {:?}", err));
    }
    let r = r.unwrap();
    let ip: IpAddr = FromStr::from_str("1.1.1.0").unwrap();
    let result: TestType = r.lookup(ip).unwrap();

    assert_eq!(result.array, vec![1u32, 2u32, 3u32]);
    assert_eq!(result.boolean, true);
    assert_eq!(result.bytes, vec![0u8, 0u8, 0u8, 42u8]);
    assert_eq!(result.double, 42.123456);
    assert_eq!(result.float, 1.1);
    assert_eq!(result.int32, -268435456);

    assert_eq!(
        result.map,
        MapType {
            mapX: MapXType {
                arrayX: vec![7, 8, 9],
                utf8_stringX: "hello".to_string(),
            },
        }
    );

    assert_eq!(result.uint16, 100);
    assert_eq!(result.uint32, 268435456);
    assert_eq!(result.uint64, 1152921504606846976);
    assert_eq!(
        result.uint128,
        vec![1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
    );

    assert_eq!(result.utf8_string, "unicode! ☯ - ♫".to_string());
}

#[test]
fn test_broken_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("test-data/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb")
        .ok()
        .unwrap();
    let ip: IpAddr = FromStr::from_str("2001:220::").unwrap();

    #[derive(Deserialize, Debug)]
    struct TestType;
    match r.lookup::<TestType>(ip) {
        Err(e) => assert_eq!(
            e,
            MaxMindDBError::InvalidDatabaseError("double of size 2".to_string())
        ),
        Ok(_) => panic!("Error expected"),
    }
}

#[test]
fn test_missing_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("file-does-not-exist.mmdb");
    match r {
        Ok(_) => panic!("Received Reader when opening non-existent file"),
        Err(MaxMindDBError::IoError(_)) => assert!(true),
        Err(_) => assert!(false),
    }
}

#[test]
fn test_non_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("README.md");
    match r {
        Ok(_) => panic!("Received Reader when opening a non-MMDB file"),
        Err(e) => assert_eq!(
            e,
            MaxMindDBError::InvalidDatabaseError(
                "Could not find MaxMind DB metadata \
                 in file."
                    .to_string(),
            )
        ),
    }
}

#[test]
fn test_reader() {
    let _ = env_logger::try_init();

    let sizes = [24usize, 28, 32];
    for record_size in sizes.iter() {
        let versions = [4usize, 6];
        for ip_version in versions.iter() {
            let filename = format!(
                "test-data/test-data/MaxMind-DB-test-ipv{}-{}.mmdb",
                ip_version, record_size
            );
            let reader = Reader::open(filename).ok().unwrap();

            check_metadata(&reader, *ip_version, *record_size);
            check_ip(&reader, *ip_version);
        }
    }
}

/// Create Reader by explicitly reading the entire file into a buffer.
#[test]
#[cfg(feature = "mmap")]
fn test_reader_readfile() {
    let _ = env_logger::try_init();

    let sizes = [24usize, 28, 32];
    for record_size in sizes.iter() {
        let versions = [4usize, 6];
        for ip_version in versions.iter() {
            let filename = format!(
                "test-data/test-data/MaxMind-DB-test-ipv{}-{}.mmdb",
                ip_version, record_size
            );
            let reader = Reader::open_readfile(filename).ok().unwrap();

            check_metadata(&reader, *ip_version, *record_size);
            check_ip(&reader, *ip_version);
        }
    }
}

#[test]
fn test_lookup_city() {
    use super::geoip2::City;
    let _ = env_logger::try_init();

    let filename = "test-data/test-data/GeoIP2-City-Test.mmdb";

    let reader = Reader::open(filename).unwrap();

    let ip: IpAddr = FromStr::from_str("89.160.20.112").unwrap();
    let city: City = reader.lookup(ip).unwrap();

    let iso_code = city.country.and_then(|cy| cy.iso_code);

    assert_eq!(iso_code, Some("SE".to_owned()));
}

fn check_metadata(reader: &Reader, ip_version: usize, record_size: usize) {
    let metadata = &reader.metadata;

    assert_eq!(metadata.binary_format_major_version, 2u16);

    assert_eq!(metadata.binary_format_minor_version, 0u16);
    assert!(metadata.build_epoch >= 1397457605);
    assert_eq!(metadata.database_type, "Test".to_string());

    assert_eq!(
        *metadata.description.get(&"en".to_string()).unwrap(),
        "Test Database".to_string()
    );
    assert_eq!(
        *metadata.description.get(&"zh".to_string()).unwrap(),
        "Test Database Chinese".to_string()
    );

    assert_eq!(metadata.ip_version, ip_version as u16);
    assert_eq!(metadata.languages, vec!["en".to_string(), "zh".to_string()]);

    if ip_version == 4 {
        assert_eq!(metadata.node_count, 164)
    } else {
        assert_eq!(metadata.node_count, 416)
    }

    assert_eq!(metadata.record_size, record_size as u16)
}

fn check_ip(reader: &Reader, ip_version: usize) {
    let subnets = match ip_version {
        6 => [
            "::1:ffff:ffff",
            "::2:0:0",
            "::2:0:0",
            "::2:0:0",
            "::2:0:0",
            "::2:0:40",
            "::2:0:40",
            "::2:0:40",
            "::2:0:50",
            "::2:0:50",
            "::2:0:50",
            "::2:0:58",
            "::2:0:58",
        ],
        _ => [
            "1.1.1.1", "1.1.1.2", "1.1.1.2", "1.1.1.4", "1.1.1.4", "1.1.1.4", "1.1.1.4", "1.1.1.8",
            "1.1.1.8", "1.1.1.8", "1.1.1.16", "1.1.1.16", "1.1.1.16",
        ],
    };

    #[derive(Deserialize, Debug)]
    struct IpType {
        ip: String,
    }

    for subnet in subnets.iter() {
        let ip: IpAddr = FromStr::from_str(&subnet).unwrap();
        let value: IpType = reader.lookup(ip).unwrap();

        assert_eq!(value.ip, subnet.to_string());
    }

    let no_record = ["1.1.1.33", "255.254.253.123", "89fa::"];

    for &address in no_record.iter() {
        let ip: IpAddr = FromStr::from_str(address).unwrap();
        match reader.lookup::<IpType>(ip) {
            Ok(v) => panic!("received an unexpected value: {:?}", v),
            Err(e) => assert_eq!(
                e,
                MaxMindDBError::AddressNotFoundError("Address not found in database".to_string())
            ),
        }
    }
}
<|endoftext|>


<commit_before>use std;
use build_env_info;

fn dirty() -> bool{
    match *build_env_info::BUILD_ENV_INFO.get("GIT_STATUS").unwrap(){
        Some(v) => v.contains("modified"),
        None => true //because we don't know
    }
}

fn dirty_star() -> &'static str{
    if dirty(){
        "*"
    }else{ ""}
}

fn describe_always_dirty() -> String{
    if dirty(){
        format!("{}*",build_env_info::GIT_DESCRIBE_ALWAYS)
    }else{
        format!("{}",build_env_info::GIT_DESCRIBE_ALWAYS)
    }
}
fn commit9_and_date() -> String{
    format!("{}{} {}", &build_env_info::GIT_COMMIT[0..9], dirty_star(), build_env_info::GENERATED_DATE_UTC)
}

fn one_line_suffix() -> String{
    let c9d = commit9_and_date();
    format!("({}) for {}", c9d, build_env_info::TARGET)
}

pub fn one_line_version() -> String{
    let branch = build_env_info::BUILD_ENV_INFO.get("GIT_OPTIONAL_BRANCH").unwrap();
    match build_env_info::BUILT_ON_CI{
        false => {
            format!("unofficial build of {} {}", describe_always_dirty(), one_line_suffix() )
        }
        true => {
            match build_env_info::BUILD_ENV_INFO.get("GET_OPTIONAL_TAG").unwrap(){
                &Some(ref tag) if tag.starts_with("v") => {
                    format!("release {} {}", &tag[1..], one_line_suffix() )
                }
                _  => {
                    if let Some(ref branch_name) = *branch {
                        format!("nightly build {} from branch {} {}", describe_always_dirty(), branch_name, one_line_suffix())
                    } else {
                        format!("unknown build {} {} ", describe_always_dirty(), one_line_suffix())
                    }
                }
            }
        }
    }
}

pub fn all_build_info_pairs() -> String{
    //channel matters
    //tagged status matters

    let mut pairs: Vec<(&&'static str, &Option<&'static str>)> = build_env_info::BUILD_ENV_INFO.iter().collect();
    pairs.sort_by(|a, b| {
        let a_lines = match *a.1{
            Some(text) => text.lines().count(),
            None => 0
        };
        let b_lines = match *b.1 {
            Some(text) => text.lines().count(),
            None => 0
        };
        if a_lines > 1 || b_lines > 1{
            match a_lines.cmp(&b_lines){
                std::cmp::Ordering::Equal => a.0.cmp(b.0),
                other => other
            }
        }else {
            a.0.cmp(b.0)
        }
    });

    let mut s = String::new();

    for (k, v) in pairs{
        let line = match *v {
            Some(ref s) => format!("{}={}\n", k, s),
            None => format!("{} (None)\n", k)
        };
        s += &line;
    }
    s
}<commit_msg>Fix typo GET->GIT; use module alias<commit_after>use std;
use build_env_info as benv;

fn dirty() -> bool{
    match *benv::BUILD_ENV_INFO.get("GIT_STATUS").unwrap(){
        Some(v) => v.contains("modified"),
        None => true //because we don't know
    }
}

fn dirty_star() -> &'static str{
    if dirty(){
        "*"
    }else{ ""}
}

fn describe_always_dirty() -> String{
    if dirty(){
        format!("{}*",benv::GIT_DESCRIBE_ALWAYS)
    }else{
        format!("{}",benv::GIT_DESCRIBE_ALWAYS)
    }
}
fn commit9_and_date() -> String{
    format!("{}{} {}", &benv::GIT_COMMIT[0..9], dirty_star(), benv::GENERATED_DATE_UTC)
}

fn one_line_suffix() -> String{
    let c9d = commit9_and_date();
    format!("({}) for {}", c9d, benv::TARGET)
}

pub fn one_line_version() -> String{
    let branch = benv::BUILD_ENV_INFO.get("GIT_OPTIONAL_BRANCH").unwrap();
    match benv::BUILT_ON_CI{
        false => {
            format!("unofficial build of {} {}", describe_always_dirty(), one_line_suffix() )
        }
        true => {
            match benv::BUILD_ENV_INFO.get("GIT_OPTIONAL_TAG").unwrap(){
                &Some(ref tag) if tag.starts_with("v") => {
                    format!("release {} {}", &tag[1..], one_line_suffix() )
                }
                _  => {
                    if let Some(ref branch_name) = *branch {
                        format!("nightly build {} from branch {} {}", describe_always_dirty(), branch_name, one_line_suffix())
                    } else {
                        format!("unknown build {} {} ", describe_always_dirty(), one_line_suffix())
                    }
                }
            }
        }
    }
}

pub fn all_build_info_pairs() -> String{
    //channel matters
    //tagged status matters

    let mut pairs: Vec<(&&'static str, &Option<&'static str>)> = benv::BUILD_ENV_INFO.iter().collect();
    pairs.sort_by(|a, b| {
        let a_lines = match *a.1{
            Some(text) => text.lines().count(),
            None => 0
        };
        let b_lines = match *b.1 {
            Some(text) => text.lines().count(),
            None => 0
        };
        if a_lines > 1 || b_lines > 1{
            match a_lines.cmp(&b_lines){
                std::cmp::Ordering::Equal => a.0.cmp(b.0),
                other => other
            }
        }else {
            a.0.cmp(b.0)
        }
    });

    let mut s = String::new();

    for (k, v) in pairs{
        let line = match *v {
            Some(ref s) => format!("{}={}\n", k, s),
            None => format!("{} (None)\n", k)
        };
        s += &line;
    }
    s
}<|endoftext|>


<commit_before>//! Routing entity for handling endpoints
//!
//! This module is intended for matching and converting a passed URLs by a client
//! in request into certain queue/topic names.
//!

pub mod endpoint;

pub use self::endpoint::{Endpoint, extract_endpoints};

use std::collections::{HashMap};
use std::clone::{Clone};

use super::super::error::{Result, PathfinderError};

/// A struct which is stores a mapping of resources that can be
/// represented in requests and transformed into certain API endpoints of
/// microservices.
///
/// # Examples
///
/// Attempt to get an endpoint with correct URL will return all expected data:
///
/// ```
/// use config::{get_config};
/// use engine::router::{Router, Endpoint, extract_endpoints};
///
/// let url = "/api/matchmaking/search";
/// let config = get_config(&"../../../tests/files/config_with_valid_endpoints.yaml");
/// let endpoints = extract_endpoints(config);
/// let router = Box::new(Router::new(endpoints))
///
/// let endpoint = route.match_url(url).unwrap();
/// assert_eq!(endpoint.get_url(), "/api/matchmaking/search");
/// assert_eq!(endpoint.get_microservice(), "microservice.search");
/// ```
///
/// For not matched URL will be returned an error:
///
/// ```
/// use config::{get_config};
/// use engine::router::{Router, Endpoint, extract_endpoints};
///
/// let url = "/api/matchmaking/search";
/// let config = get_config(&"../../../tests/files/config_with_invalid_endpoints.yaml");
/// let endpoints = extract_endpoints(config);
/// let router = Box::new(Router::new(endpoints))
///
/// assert_eq!(route.match_url(url).is_err(), true);
/// ```
///
pub struct Router {
    endpoints: HashMap<String, Box<Endpoint>>
}

impl Router {
    /// Returns a new instance of `Router` that contains a mapping for resources.
    pub fn new(endpoints: HashMap<String, Box<Endpoint>>) -> Router {
        Router {
            endpoints: endpoints
        }
    }

    /// Returns an endpoint that was found for a passed URL.
    pub fn match_url(&self, url: &str) -> Result<Box<Endpoint>> {
        match self.endpoints.contains_key(url) {
            true => Ok(self.endpoints[url].clone()),
            false => Err(PathfinderError::EndpointNotFound(url.to_string()))
        }
    }
}

#[cfg(test)]
mod tests {
    use config::{get_config};
    use engine::router::{Router, Endpoint, extract_endpoints};
    use error::{Result};

    fn get_route(file_path: &str, url: &str) -> Result<Box<Endpoint>> {
        let config = get_config(file_path);
        let endpoints = extract_endpoints(config);
        let router = Box::new(Router::new(endpoints));
        router.match_url(url)
    }

    #[test]
    fn test_router_returns_endpoint_for_a_match() {
        let route = get_route(
            &"./tests/files/config_with_valid_endpoints.yaml",
            "/api/matchmaking/search"
        );

        assert_eq!(route.is_ok(), true);
        let endpoint = route.unwrap();
        assert_eq!(endpoint.get_url(), "/api/matchmaking/search");
        assert_eq!(endpoint.get_microservice(), "microservice.search");
    }

    #[test]
    fn test_router_returns_an_error_for_non_existing_endpoint() {
        let route = get_route(
            &"./tests/files/config_with_invalid_endpoints.yaml",
            "/api/matchmaking/search"
        );

        assert_eq!(route.is_err(), true);
    }
}
<commit_msg>Implemented match_url_or_default method<commit_after>//! Routing entity for handling endpoints
//!
//! This module is intended for matching and converting a passed URLs by a client
//! in request into certain queue/topic names.
//!

pub mod endpoint;

pub use self::endpoint::{Endpoint, extract_endpoints};

use std::collections::{HashMap};
use std::clone::{Clone};

use super::super::error::{Result, PathfinderError};

/// A struct which is stores a mapping of resources that can be
/// represented in requests and transformed into certain API endpoints of
/// microservices.
///
/// # Examples
///
/// Attempt to get an endpoint with correct URL will return all expected data:
///
/// ```
/// use config::{get_config};
/// use engine::router::{Router, Endpoint, extract_endpoints};
///
/// let url = "/api/matchmaking/search";
/// let config = get_config(&"../../../tests/files/config_with_valid_endpoints.yaml");
/// let endpoints = extract_endpoints(config);
/// let router = Box::new(Router::new(endpoints))
///
/// let endpoint = route.match_url(url).unwrap();
/// assert_eq!(endpoint.get_url(), "/api/matchmaking/search");
/// assert_eq!(endpoint.get_microservice(), "microservice.search");
/// ```
///
/// For not matched URL will be returned an error:
///
/// ```
/// use config::{get_config};
/// use engine::router::{Router, Endpoint, extract_endpoints};
///
/// let url = "/api/matchmaking/search";
/// let config = get_config(&"../../../tests/files/config_with_invalid_endpoints.yaml");
/// let endpoints = extract_endpoints(config);
/// let router = Box::new(Router::new(endpoints))
///
/// assert_eq!(route.match_url(url).is_err(), true);
/// ```
///
pub struct Router {
    endpoints: HashMap<String, Box<Endpoint>>
}

impl Router {
    /// Returns a new instance of `Router` that contains a mapping for resources.
    pub fn new(endpoints: HashMap<String, Box<Endpoint>>) -> Router {
        Router {
            endpoints: endpoints
        }
    }

    /// Returns an endpoint that was found for a passed URL.
    pub fn match_url(&self, url: &str) -> Result<Box<Endpoint>> {
        match self.endpoints.contains_key(url) {
            true => Ok(self.endpoints[url].clone()),
            false => Err(PathfinderError::EndpointNotFound(url.to_string()))
        }
    }

    /// Returns an endpoint for the matched URL. If wasn't found returns a processed
    /// URL as endpoint like in normal cases.
    pub fn match_url_or_default(&self, url: &str) -> Box<Endpoint> {
        match self.match_url(url) {
            Ok(endpoint) => endpoint,
            Err(_) => Box::new(Endpoint {
                url: url.to_string(),
                microservice: self.convert_url_into_microservice(url)
            })
        }
    }

    /// Converts a URL to the certain microservice name, so that it will be used as a
    /// queue/topic name further.
    fn convert_url_into_microservice(&self, url: &str) -> String {
        let mut external_url = url.clone();
        external_url = external_url.trim_left_matches("/");
        external_url = external_url.trim_right_matches("/");
        external_url.replace("/", ".")
    }
}

#[cfg(test)]
mod tests {
    use config::{get_config};
    use engine::router::{Router, Endpoint, extract_endpoints};
    use error::{Result};

    fn get_route(file_path: &str, url: &str) -> Result<Box<Endpoint>> {
        let config = get_config(file_path);
        let endpoints = extract_endpoints(config);
        let router = Box::new(Router::new(endpoints));
        router.match_url(url)
    }

    #[test]
    fn test_router_returns_endpoint_for_a_match() {
        let route = get_route(
            &"./tests/files/config_with_valid_endpoints.yaml",
            "/api/matchmaking/search"
        );

        assert_eq!(route.is_ok(), true);
        let endpoint = route.unwrap();
        assert_eq!(endpoint.get_url(), "/api/matchmaking/search");
        assert_eq!(endpoint.get_microservice(), "microservice.search");
    }

    #[test]
    fn test_router_returns_an_error_for_non_existing_endpoint() {
        let route = get_route(
            &"./tests/files/config_with_invalid_endpoints.yaml",
            "/api/matchmaking/search"
        );

        assert_eq!(route.is_err(), true);
    }
}
<|endoftext|>


<commit_before>pub use x11::keysym::*;
pub use x11::xcursor::*;
pub use x11::xf86vmode::*;
pub use x11::xlib::*;
pub use x11::xlib::xkb::*;

pub use self::glx::types::GLXContext;

/// GLX bindings
pub mod glx {
    include!(concat!(env!("OUT_DIR"), "/glx_bindings.rs"));
}

/// Functions that are not necessarly always available
pub mod glx_extra {
    include!(concat!(env!("OUT_DIR"), "/glx_extra_bindings.rs"));
}
<commit_msg>Removed import of x11::xlib::xkb, which has been absorbed by x11::xlib.<commit_after>pub use x11::keysym::*;
pub use x11::xcursor::*;
pub use x11::xf86vmode::*;
pub use x11::xlib::*;

pub use self::glx::types::GLXContext;

/// GLX bindings
pub mod glx {
    include!(concat!(env!("OUT_DIR"), "/glx_bindings.rs"));
}

/// Functions that are not necessarly always available
pub mod glx_extra {
    include!(concat!(env!("OUT_DIR"), "/glx_extra_bindings.rs"));
}
<|endoftext|>


<commit_before>use std::fmt;
use super::{Array, Dimension};

/// HACK: fmt::rt::FlagAlternate has been hidden away
const FlagAlternate: usize = 2;

fn format_array<A, D: Dimension, F>(view: &Array<A, D>, f: &mut fmt::Formatter,
                                    mut format: F) -> fmt::Result where
    F: FnMut(&mut fmt::Formatter, &A) -> fmt::Result,
{
    let sz = view.dim.slice().len();
    /* private nowadays
    if sz > 0 && f.width.is_none() {
        f.width = Some(4)
    }
    */
    // None will be an empty iter.
    let mut last_index = match view.dim.first_index() {
        None => view.dim.clone(),
        Some(ix) => ix,
    };
    for _ in (0..sz) {
        try!(write!(f, "["));
    }
    let mut first = true;
    // Simply use the indexed iterator, and take the index wraparounds
    // as cues for when to add []'s and how many to add.
    for (index, elt) in view.indexed_iter() {
        let mut update_index = false;
        for (i, (a, b)) in index.slice().iter().take(sz-1)
                        .zip(last_index.slice().iter())
                        .enumerate()
        {
            if a != b {
                // New row.
                // # of ['s needed
                let n = sz - i - 1;
                for _ in (0..n) {
                    try!(write!(f, "]"));
                }
                try!(write!(f, ","));
                if f.flags() & (1 << FlagAlternate) == 0 {
                    try!(write!(f, "\n"));
                }
                for _ in (0..sz - n) {
                    try!(write!(f, " "));
                }
                for _ in (0..n) {
                    try!(write!(f, "["));
                }
                first = true;
                update_index = true;
                break;
            }
        }
        if !first {
            try!(write!(f, ", "));
        }
        first = false;
        try!(format(f, elt));

        if update_index {
            last_index = index;
        }
    }
    for _ in (0..sz) {
        try!(write!(f, "]"));
    }
    Ok(())
}

// NOTE: We can impl other fmt traits here
impl<'a, A: fmt::Display, D: Dimension> fmt::Display for Array<A, D>
{
    /// Format the array using `Display` and apply the formatting parameters used
    /// to each element.
    ///
    /// The array is shown in multiline style, unless the alternate form
    /// is used -- i.e. `{:#}`.
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        format_array(self, f, |f, elt| elt.fmt(f))
    }
}

impl<'a, A: fmt::Debug, D: Dimension> fmt::Debug for Array<A, D>
{
    /// Format the array using `Debug` and apply the formatting parameters used
    /// to each element.
    ///
    /// The array is shown in multiline style, unless the alternate form
    /// is used -- i.e. `{:#}`.
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        format_array(self, f, |f, elt| elt.fmt(f))
    }
}

impl<'a, A: fmt::LowerExp, D: Dimension> fmt::LowerExp for Array<A, D>
{
    /// Format the array using `LowerExp` and apply the formatting parameters used
    /// to each element.
    ///
    /// The array is shown in multiline style, unless the alternate form
    /// is used -- i.e. `{:#e}`.
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        format_array(self, f, |f, elt| elt.fmt(f))
    }
}

impl<'a, A: fmt::UpperExp, D: Dimension> fmt::UpperExp for Array<A, D>
{
    /// Format the array using `UpperExp` and apply the formatting parameters used
    /// to each element.
    ///
    /// The array is shown in multiline style, unless the alternate form
    /// is used -- i.e. `{:#E}`.
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        format_array(self, f, |f, elt| elt.fmt(f))
    }
}

<commit_msg>Fix arrayformat for 0 - 1 case in unsigned arith<commit_after>use std::fmt;
use super::{Array, Dimension};

/// HACK: fmt::rt::FlagAlternate has been hidden away
const FlagAlternate: usize = 2;

fn format_array<A, D: Dimension, F>(view: &Array<A, D>, f: &mut fmt::Formatter,
                                    mut format: F) -> fmt::Result where
    F: FnMut(&mut fmt::Formatter, &A) -> fmt::Result,
{
    let ndim = view.dim.slice().len();
    /* private nowadays
    if ndim > 0 && f.width.is_none() {
        f.width = Some(4)
    }
    */
    // None will be an empty iter.
    let mut last_index = match view.dim.first_index() {
        None => view.dim.clone(),
        Some(ix) => ix,
    };
    for _ in (0..ndim) {
        try!(write!(f, "["));
    }
    let mut first = true;
    // Simply use the indexed iterator, and take the index wraparounds
    // as cues for when to add []'s and how many to add.
    for (index, elt) in view.indexed_iter() {
        let take_n = if ndim == 0 { 1 } else { ndim - 1 };
        let mut update_index = false;
        for (i, (a, b)) in index.slice().iter().take(take_n)
                        .zip(last_index.slice().iter())
                        .enumerate()
        {
            if a != b {
                // New row.
                // # of ['s needed
                let n = ndim - i - 1;
                for _ in (0..n) {
                    try!(write!(f, "]"));
                }
                try!(write!(f, ","));
                if f.flags() & (1 << FlagAlternate) == 0 {
                    try!(write!(f, "\n"));
                }
                for _ in (0..ndim - n) {
                    try!(write!(f, " "));
                }
                for _ in (0..n) {
                    try!(write!(f, "["));
                }
                first = true;
                update_index = true;
                break;
            }
        }
        if !first {
            try!(write!(f, ", "));
        }
        first = false;
        try!(format(f, elt));

        if update_index {
            last_index = index;
        }
    }
    for _ in (0..ndim) {
        try!(write!(f, "]"));
    }
    Ok(())
}

// NOTE: We can impl other fmt traits here
impl<'a, A: fmt::Display, D: Dimension> fmt::Display for Array<A, D>
{
    /// Format the array using `Display` and apply the formatting parameters used
    /// to each element.
    ///
    /// The array is shown in multiline style, unless the alternate form
    /// is used -- i.e. `{:#}`.
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        format_array(self, f, |f, elt| elt.fmt(f))
    }
}

impl<'a, A: fmt::Debug, D: Dimension> fmt::Debug for Array<A, D>
{
    /// Format the array using `Debug` and apply the formatting parameters used
    /// to each element.
    ///
    /// The array is shown in multiline style, unless the alternate form
    /// is used -- i.e. `{:#}`.
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        format_array(self, f, |f, elt| elt.fmt(f))
    }
}

impl<'a, A: fmt::LowerExp, D: Dimension> fmt::LowerExp for Array<A, D>
{
    /// Format the array using `LowerExp` and apply the formatting parameters used
    /// to each element.
    ///
    /// The array is shown in multiline style, unless the alternate form
    /// is used -- i.e. `{:#e}`.
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        format_array(self, f, |f, elt| elt.fmt(f))
    }
}

impl<'a, A: fmt::UpperExp, D: Dimension> fmt::UpperExp for Array<A, D>
{
    /// Format the array using `UpperExp` and apply the formatting parameters used
    /// to each element.
    ///
    /// The array is shown in multiline style, unless the alternate form
    /// is used -- i.e. `{:#E}`.
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        format_array(self, f, |f, elt| elt.fmt(f))
    }
}

<|endoftext|>