<commit_before>#[macro_use]
extern crate clap;
extern crate bytes;
extern crate futures;
extern crate tokio_core;
extern crate net2;

use std::net::SocketAddr;
use net2::UdpBuilder;
use std::thread;
use std::sync::atomic::{ATOMIC_USIZE_INIT, AtomicUsize, Ordering};

use tokio_core::reactor::{Core, Interval};
use tokio_core::net::{UdpSocket, UdpCodec};

use bytes::{Bytes, IntoBuf, Buf};
use futures::{Future, Stream, Sink};
use futures::sync::mpsc::channel;

use clap::{App, Arg};

pub static CHUNK_COUNTER: AtomicUsize = ATOMIC_USIZE_INIT;

struct DgramCodec {
    addr: SocketAddr,
}

impl DgramCodec {
    pub fn new(addr: SocketAddr) -> Self {
        DgramCodec { addr: addr }
    }
}

impl UdpCodec for DgramCodec {
    type In = Bytes;
    type Out = Bytes;

    fn decode(&mut self,
              _src: &SocketAddr,
              buf: &[u8])
              -> ::std::result::Result<Self::In, ::std::io::Error> {
        Ok(buf.into())
    }

    fn encode(&mut self, msg: Self::Out, buf: &mut Vec<u8>) -> SocketAddr {
        use std::io::Read;
        msg.into_buf().reader().read_to_end(buf).unwrap();
        self.addr
    }
}

fn main() {
    let matches = App::new("MEGA UDP DUPLICATOR")
        .version("0.0.0.0.0.0.1alpha")
        .author("Sergey Noskov <snoskov@avito.ru>")
        .about("UDP multiplexor: copies packets from listened address to specified backends")
        .arg(Arg::with_name("listen")
             .short("l")
             .long("listen")
             .value_name("IP:PORT")
             .help("ip and port to listen to")
             .takes_value(true))
        .arg(Arg::with_name("queue_size")
             .short("q")
             .long("queue")
             .takes_value(true)
             .default_value("1048576")
            )
        .arg(Arg::with_name("backend")
             .short("b")
             .long("backend")
             .value_name("IP:PORT")
             .help("IP and port of a backend to forward data to. Can be specified multiple times.")
             .multiple(true)
             .takes_value(true)
            )
         .arg(Arg::with_name("nthreads")
             .short("n")
             .long("nthreads")
             .takes_value(true)
             .default_value("0")
            )
        .get_matches();

    let listen = value_t!(matches.value_of("listen"), SocketAddr).unwrap_or_else(|e| e.exit());
    let backends = values_t!(matches.values_of("backend"), SocketAddr).unwrap_or_else(|e| e.exit());
    let size = value_t!(matches.value_of("queue_size"), usize).unwrap_or_else(|e| e.exit());
    let nthreads = value_t!(matches.value_of("nthreads"), usize).unwrap_or_else(|e| e.exit());

    // Init chunk counter
    // In the main thread start a reporting timer
    let mut core = Core::new().unwrap();
    let handle = core.handle();

    let timer = Interval::new(::std::time::Duration::from_secs(1), &handle).unwrap();
    let timer = timer.for_each(|()|{
                println!("chunks: {:?}", CHUNK_COUNTER.load(Ordering::Relaxed));
                CHUNK_COUNTER.store(0, Ordering::Relaxed);
                //println!("total drops: {:?}", METRIC_DROPS.load(Ordering::Relaxed));
                Ok(())
    });

    let socket = UdpBuilder::new_v4().unwrap();
    let socket = socket.bind(&listen).unwrap();

    // Create sockets from backend addresses
    let senders = backends
        .iter()
        .map(|_| {
                 let socket = UdpBuilder::new_v4().unwrap();
                 let bind: SocketAddr = "0.0.0.0:0".parse().unwrap();
                 socket.bind(&bind).unwrap()
             })
        .collect::<Vec<_>>();

    for _ in 1..8 {
        // create clone of each sender socket for thread
        let senders = senders
            .iter()
            .map(|socket| socket.try_clone().unwrap())
            .collect::<Vec<_>>();

        let socket = socket.try_clone().unwrap();
        let mut backends = backends.clone();
        thread::spawn(move || {
            // each thread runs it's own core
            let mut core = Core::new().unwrap();
            let handle = core.handle();

            // make senders into framed UDP
            let mut senders = senders
                .into_iter()
                .map(|socket| {
                         let sender = UdpSocket::from_socket(socket, &handle).unwrap();
                         // socket order becomes reversed but we don't care about this
                         // because the order is same everywhere
                         let sender = sender.framed(DgramCodec::new(backends.pop().unwrap()));
                         sender
                     })
                .collect::<Vec<_>>();

            // create server now
            let socket = UdpSocket::from_socket(socket, &handle).unwrap();
            let server = socket.framed(DgramCodec::new(listen));

            let server = server.for_each(move |buf| {
                CHUNK_COUNTER.fetch_add(1, Ordering::Relaxed);
                senders
                    .iter_mut()
                    .map(|sender| {
                             sender.start_send(buf.clone()).unwrap();
                             sender.poll_complete().unwrap();
                         })
                    .last();
                Ok(())
            });

            core.run(server).unwrap();
        });
    }
    core.run(timer).unwrap();
}
<commit_msg>Fixed warnings and unused vars<commit_after>#[macro_use]
extern crate clap;
extern crate bytes;
extern crate futures;
extern crate tokio_core;
extern crate net2;

use std::net::SocketAddr;
use net2::UdpBuilder;
use std::thread;
use std::sync::atomic::{ATOMIC_USIZE_INIT, AtomicUsize, Ordering};

use tokio_core::reactor::{Core, Interval};
use tokio_core::net::{UdpSocket, UdpCodec};

use bytes::{Bytes, IntoBuf, Buf};
use futures::{Stream, Sink};

use clap::{App, Arg};

pub static CHUNK_COUNTER: AtomicUsize = ATOMIC_USIZE_INIT;

struct DgramCodec {
    addr: SocketAddr,
}

impl DgramCodec {
    pub fn new(addr: SocketAddr) -> Self {
        DgramCodec { addr: addr }
    }
}

impl UdpCodec for DgramCodec {
    type In = Bytes;
    type Out = Bytes;

    fn decode(&mut self,
              _src: &SocketAddr,
              buf: &[u8])
              -> ::std::result::Result<Self::In, ::std::io::Error> {
        Ok(buf.into())
    }

    fn encode(&mut self, msg: Self::Out, buf: &mut Vec<u8>) -> SocketAddr {
        use std::io::Read;
        msg.into_buf().reader().read_to_end(buf).unwrap();
        self.addr
    }
}

fn main() {
    let matches = App::new("MEGA UDP DUPLICATOR")
        .version("0.0.0.0.0.0.1alpha")
        .author("Sergey Noskov <snoskov@avito.ru>")
        .about("UDP multiplexor: copies packets from listened address to specified backends")
        .arg(Arg::with_name("listen")
             .short("l")
             .long("listen")
             .value_name("IP:PORT")
             .help("ip and port to listen to")
             .takes_value(true))
        .arg(Arg::with_name("backend")
             .short("b")
             .long("backend")
             .value_name("IP:PORT")
             .help("IP and port of a backend to forward data to. Can be specified multiple times.")
             .multiple(true)
             .takes_value(true)
            )
         .arg(Arg::with_name("nthreads")
             .short("n")
             .long("nthreads")
             .takes_value(true)
             .default_value("0")
            )
        .get_matches();

    let listen = value_t!(matches.value_of("listen"), SocketAddr).unwrap_or_else(|e| e.exit());
    let backends = values_t!(matches.values_of("backend"), SocketAddr).unwrap_or_else(|e| e.exit());
    let nthreads = value_t!(matches.value_of("nthreads"), usize).unwrap_or_else(|e| e.exit());

    // Init chunk counter
    // In the main thread start a reporting timer
    let mut core = Core::new().unwrap();
    let handle = core.handle();

    let timer = Interval::new(::std::time::Duration::from_secs(1), &handle).unwrap();
    let timer = timer.for_each(|()|{
                println!("chunks: {:?}", CHUNK_COUNTER.load(Ordering::Relaxed));
                CHUNK_COUNTER.store(0, Ordering::Relaxed);
                //println!("total drops: {:?}", METRIC_DROPS.load(Ordering::Relaxed));
                Ok(())
    });

    let socket = UdpBuilder::new_v4().unwrap();
    let socket = socket.bind(&listen).unwrap();

    // Create sockets from backend addresses
    let senders = backends
        .iter()
        .map(|_| {
                 let socket = UdpBuilder::new_v4().unwrap();
                 let bind: SocketAddr = "0.0.0.0:0".parse().unwrap();
                 socket.bind(&bind).unwrap()
             })
        .collect::<Vec<_>>();

    for _ in 0..nthreads {
        // create clone of each sender socket for thread
        let senders = senders
            .iter()
            .map(|socket| socket.try_clone().unwrap())
            .collect::<Vec<_>>();

        let socket = socket.try_clone().unwrap();
        let mut backends = backends.clone();
        thread::spawn(move || {
            // each thread runs it's own core
            let mut core = Core::new().unwrap();
            let handle = core.handle();

            // make senders into framed UDP
            let mut senders = senders
                .into_iter()
                .map(|socket| {
                         let sender = UdpSocket::from_socket(socket, &handle).unwrap();
                         // socket order becomes reversed but we don't care about this
                         // because the order is same everywhere
                         let sender = sender.framed(DgramCodec::new(backends.pop().unwrap()));
                         sender
                     })
                .collect::<Vec<_>>();

            // create server now
            let socket = UdpSocket::from_socket(socket, &handle).unwrap();
            let server = socket.framed(DgramCodec::new(listen));

            let server = server.for_each(move |buf| {
                CHUNK_COUNTER.fetch_add(1, Ordering::Relaxed);
                senders
                    .iter_mut()
                    .map(|sender| {
                             sender.start_send(buf.clone()).unwrap();
                             sender.poll_complete().unwrap();
                         })
                    .last();
                Ok(())
            });

            core.run(server).unwrap();
        });
    }
    core.run(timer).unwrap();
}
<|endoftext|>


<commit_before>extern crate "rustc-serialize" as serialize;
extern crate handlebars;

use std::old_io as io;
use serialize::json::Json;
use handlebars::Handlebars;

fn main() {
   let input: String = if io::stdio::stdin_raw().isatty() {
     io::stdin().read_line().ok().expect("json expected")
   } else {
     "{}".to_string()
   };
   let json: Json = Json::from_str(&input).ok().expect("malformed json");
   let mut handlebars = Handlebars::new();
   handlebars.register_template_string("t","hello {{foo}}".to_string()).ok().expect("expected template");
   println!("{}", handlebars.render("t", &json).ok().unwrap());
}
<commit_msg>annot not needed<commit_after>extern crate "rustc-serialize" as serialize;
extern crate handlebars;

use std::old_io as io;
use serialize::json::Json;
use handlebars::Handlebars;

fn main() {
   let input = if io::stdio::stdin_raw().isatty() {
     io::stdin().read_line().ok().expect("json expected")
   } else {
     "{}".to_string()
   };
   let json = Json::from_str(&input).ok().expect("malformed json");
   let mut handlebars = Handlebars::new();
   handlebars.register_template_string("t","hello {{foo}}".to_string()).ok().expect("expected template");
   println!("{}", handlebars.render("t", &json).ok().unwrap());
}
<|endoftext|>


<commit_before>#![feature(phase)]
#![feature(unboxed_closures)]

#[phase(plugin)]
extern crate glium_macros;

extern crate glutin;
extern crate glium;

use glium::Surface;

mod support;

#[test]
fn attribute_types_matching() {
    let display = support::build_display();

    #[vertex_format]
    #[deriving(Copy)]
    struct Vertex {
        field1: [f32, ..2],
        field2: [i32, ..3],
    }

    let vertex_buffer = glium::VertexBuffer::new(&display, vec![
            Vertex { field1: [0.0, 0.0], field2: [0, 0, 0] }
        ]);
    let index_buffer = glium::IndexBuffer::new(&display,
                            glium::index_buffer::PointsList(vec![0u16]));

    let program = glium::Program::new(&display,
        // vertex shader
        "
            #version 110

            attribute vec2 field1;
            attribute ivec3 field2;

            void main() {
                float z = 0.0;
                gl_Position = vec4(field1, z, 1.0);
            }
        ",
        "
            #version 110
            void main() {
                gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0);
            }
        ",

        // geometry shader
        None)
        .unwrap();

    // drawing a frame
    let mut target = display.draw();
    target.draw(&vertex_buffer, &index_buffer, &program, &glium::uniforms::EmptyUniforms,
                &std::default::Default::default());
    target.finish();

    display.assert_no_error();
}

#[test]
#[should_fail(expected = "The program attribute `field1` does not match the vertex format")]
fn attribute_types_mismatch() {
    let display = support::build_display();

    #[vertex_format]
    #[deriving(Copy)]
    struct Vertex {
        field1: [f32, ..4],
    }

    let vertex_buffer = glium::VertexBuffer::new(&display, Vec::<Vertex>::new());
    let index_buffer = glium::IndexBuffer::new(&display,
                            glium::index_buffer::PointsList(Vec::<u16>::new()));

    let program = glium::Program::new(&display,
        // vertex shader
        "
            #version 110

            attribute vec2 field1;

            void main() {
                gl_Position = vec4(field1, 0.0, 1.0);
            }
        ",
        "
            #version 110
            void main() {
                gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0);
            }
        ",

        // geometry shader
        None)
        .unwrap();

    // drawing a frame
    let mut target = display.draw();
    target.draw(&vertex_buffer, &index_buffer, &program, &glium::uniforms::EmptyUniforms,
                &std::default::Default::default());
    target.finish();

    display.assert_no_error();
}

#[test]
#[should_fail(expected = "The program attribute `field2` is missing in the vertex bindings")]
fn missing_attribute() {
    let display = support::build_display();

    #[vertex_format]
    #[deriving(Copy)]
    struct Vertex {
        field1: [f32, ..4],
    }

    let vertex_buffer = glium::VertexBuffer::new(&display, Vec::<Vertex>::new());
    let index_buffer = glium::IndexBuffer::new(&display,
                            glium::index_buffer::PointsList(Vec::<u16>::new()));

    let program = glium::Program::new(&display,
        // vertex shader
        "
            #version 110

            attribute vec2 field2;

            void main() {
                gl_Position = vec4(field2, 0.0, 1.0);
            }
        ",
        "
            #version 110
            void main() {
                gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0);
            }
        ",

        // geometry shader
        None)
        .unwrap();

    // drawing a frame
    let mut target = display.draw();
    target.draw(&vertex_buffer, &index_buffer, &program, &glium::uniforms::EmptyUniforms,
                &std::default::Default::default());
    target.finish();

    display.assert_no_error();
}
<commit_msg>Fix tests<commit_after>#![feature(phase)]
#![feature(unboxed_closures)]

#[phase(plugin)]
extern crate glium_macros;

extern crate glutin;
extern crate glium;

use glium::Surface;

mod support;

#[test]
fn attribute_types_matching() {
    let display = support::build_display();

    #[vertex_format]
    #[deriving(Copy)]
    struct Vertex {
        field1: [f32, ..2],
    }

    let vertex_buffer = glium::VertexBuffer::new(&display, vec![
            Vertex { field1: [0.0, 0.0] }
        ]);
    let index_buffer = glium::IndexBuffer::new(&display,
                            glium::index_buffer::PointsList(vec![0u16]));

    let program = glium::Program::new(&display,
        // vertex shader
        "
            #version 110

            attribute vec2 field1;

            void main() {
                gl_Position = vec4(field1, 0.0, 1.0);
            }
        ",
        "
            #version 110
            void main() {
                gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0);
            }
        ",

        // geometry shader
        None)
        .unwrap();

    // drawing a frame
    let mut target = display.draw();
    target.draw(&vertex_buffer, &index_buffer, &program, &glium::uniforms::EmptyUniforms,
                &std::default::Default::default());
    target.finish();

    display.assert_no_error();
}

#[test]
#[should_fail(expected = "The program attribute `field1` does not match the vertex format")]
fn attribute_types_mismatch() {
    let display = support::build_display();

    #[vertex_format]
    #[deriving(Copy)]
    struct Vertex {
        field1: [f32, ..4],
    }

    let vertex_buffer = glium::VertexBuffer::new(&display, Vec::<Vertex>::new());
    let index_buffer = glium::IndexBuffer::new(&display,
                            glium::index_buffer::PointsList(Vec::<u16>::new()));

    let program = glium::Program::new(&display,
        // vertex shader
        "
            #version 110

            attribute vec2 field1;

            void main() {
                gl_Position = vec4(field1, 0.0, 1.0);
            }
        ",
        "
            #version 110
            void main() {
                gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0);
            }
        ",

        // geometry shader
        None)
        .unwrap();

    // drawing a frame
    let mut target = display.draw();
    target.draw(&vertex_buffer, &index_buffer, &program, &glium::uniforms::EmptyUniforms,
                &std::default::Default::default());
    target.finish();

    display.assert_no_error();
}

#[test]
#[should_fail(expected = "The program attribute `field2` is missing in the vertex bindings")]
fn missing_attribute() {
    let display = support::build_display();

    #[vertex_format]
    #[deriving(Copy)]
    struct Vertex {
        field1: [f32, ..4],
    }

    let vertex_buffer = glium::VertexBuffer::new(&display, Vec::<Vertex>::new());
    let index_buffer = glium::IndexBuffer::new(&display,
                            glium::index_buffer::PointsList(Vec::<u16>::new()));

    let program = glium::Program::new(&display,
        // vertex shader
        "
            #version 110

            attribute vec2 field2;

            void main() {
                gl_Position = vec4(field2, 0.0, 1.0);
            }
        ",
        "
            #version 110
            void main() {
                gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0);
            }
        ",

        // geometry shader
        None)
        .unwrap();

    // drawing a frame
    let mut target = display.draw();
    target.draw(&vertex_buffer, &index_buffer, &program, &glium::uniforms::EmptyUniforms,
                &std::default::Default::default());
    target.finish();

    display.assert_no_error();
}
<|endoftext|>


<commit_before>#![allow(non_camel_case_types)]

/// Implements inherent methods for a float vector `$name` containing multiple
/// `$lanes` of float `$type`, which uses `$bits_ty` as its binary
/// representation. Called from `define_float_vector!`.
macro_rules! impl_float_vector {
    { $name:ident, $type:ident, $bits_ty:ident, $mask_ty:ident, $mask_impl_ty:ident } => {
        impl_vector! { $name, $type }
        impl_float_reductions! { $name, $type }

        impl<const LANES: usize> $name<LANES>
        where
            Self: crate::LanesAtMost32,
            crate::$bits_ty<LANES>: crate::LanesAtMost32,
        {
            /// Raw transmutation to an unsigned integer vector type with the
            /// same size and number of lanes.
            #[inline]
            pub fn to_bits(self) -> crate::$bits_ty<LANES> {
                assert_eq!(core::mem::size_of::<Self>(), core::mem::size_of::<crate::$bits_ty<LANES>>());
                unsafe { core::mem::transmute_copy(&self) }
            }

            /// Raw transmutation from an unsigned integer vector type with the
            /// same size and number of lanes.
            #[inline]
            pub fn from_bits(bits: crate::$bits_ty<LANES>) -> Self {
                assert_eq!(core::mem::size_of::<Self>(), core::mem::size_of::<crate::$bits_ty<LANES>>());
                unsafe { core::mem::transmute_copy(&bits) }
            }

            /// Produces a vector where every lane has the absolute value of the
            /// equivalently-indexed lane in `self`.
            #[inline]
            pub fn abs(self) -> Self {
                unsafe { crate::intrinsics::simd_fabs(self) }
            }

            /// Fused multiply-add.  Computes `(self * a) + b` with only one rounding error,
            /// yielding a more accurate result than an unfused multiply-add.
            ///
            /// Using `mul_add` *may* be more performant than an unfused multiply-add if the target
            /// architecture has a dedicated `fma` CPU instruction.  However, this is not always
            /// true, and will be heavily dependent on designing algorithms with specific target
            /// hardware in mind.
            #[inline]
            pub fn mul_add(self, a: Self, b: Self) -> Self {
                unsafe { crate::intrinsics::simd_fma(self, a, b) }
            }

            /// Produces a vector where every lane has the square root value
            /// of the equivalently-indexed lane in `self`
            #[inline]
            #[cfg(feature = "std")]
            pub fn sqrt(self) -> Self {
                unsafe { crate::intrinsics::simd_fsqrt(self) }
            }

            /// Takes the reciprocal (inverse) of each lane, `1/x`.
            #[inline]
            pub fn recip(self) -> Self {
                Self::splat(1.0) / self
            }

            /// Converts each lane from radians to degrees.
            #[inline]
            pub fn to_degrees(self) -> Self {
                // to_degrees uses a special constant for better precision, so extract that constant
                self * Self::splat($type::to_degrees(1.))
            }

            /// Converts each lane from degrees to radians.
            #[inline]
            pub fn to_radians(self) -> Self {
                self * Self::splat(core::$type::consts::PI / 180.)
            }
        }

        impl<const LANES: usize> $name<LANES>
        where
            Self: crate::LanesAtMost32,
            crate::$bits_ty<LANES>: crate::LanesAtMost32,
            crate::$mask_impl_ty<LANES>: crate::LanesAtMost32,
            crate::$mask_ty<LANES>: crate::Mask,
        {
            /// Returns true for each lane if it has a positive sign, including
            /// `+0.0`, `NaN`s with positive sign bit and positive infinity.
            #[inline]
            pub fn is_sign_positive(self) -> crate::$mask_ty<LANES> {
                !self.is_sign_negative()
            }

            /// Returns true for each lane if it has a negative sign, including
            /// `-0.0`, `NaN`s with negative sign bit and negative infinity.
            #[inline]
            pub fn is_sign_negative(self) -> crate::$mask_ty<LANES> {
                let sign_bits = self.to_bits() & crate::$bits_ty::splat((!0 >> 1) + 1);
                sign_bits.lanes_gt(crate::$bits_ty::splat(0))
            }

            /// Returns true for each lane if its value is `NaN`.
            #[inline]
            pub fn is_nan(self) -> crate::$mask_ty<LANES> {
                self.lanes_ne(self)
            }

            /// Returns true for each lane if its value is positive infinity or negative infinity.
            #[inline]
            pub fn is_infinite(self) -> crate::$mask_ty<LANES> {
                self.abs().lanes_eq(Self::splat(<$type>::INFINITY))
            }

            /// Returns true for each lane if its value is neither infinite nor `NaN`.
            #[inline]
            pub fn is_finite(self) -> crate::$mask_ty<LANES> {
                self.abs().lanes_lt(Self::splat(<$type>::INFINITY))
            }

            /// Returns true for each lane if its value is subnormal.
            #[inline]
            pub fn is_subnormal(self) -> crate::$mask_ty<LANES> {
                self.abs().lanes_ne(Self::splat(0.0)) & (self.to_bits() & Self::splat(<$type>::INFINITY).to_bits()).lanes_eq(crate::$bits_ty::splat(0))
            }

            /// Returns true for each lane if its value is neither neither zero, infinite,
            /// subnormal, or `NaN`.
            #[inline]
            pub fn is_normal(self) -> crate::$mask_ty<LANES> {
                !(self.abs().lanes_eq(Self::splat(0.0)) | self.is_nan() | self.is_subnormal() | self.is_infinite())
            }

            /// Replaces each lane with a number that represents its sign.
            ///
            /// * `1.0` if the number is positive, `+0.0`, or `INFINITY`
            /// * `-1.0` if the number is negative, `-0.0`, or `NEG_INFINITY`
            /// * `NAN` if the number is `NAN`
            #[inline]
            pub fn signum(self) -> Self {
                self.is_nan().select(Self::splat($type::NAN), Self::splat(1.0).copysign(self))
            }

            /// Returns each lane with the magnitude of `self` and the sign of `sign`.
            ///
            /// If any lane is a `NAN`, then a `NAN` with the sign of `sign` is returned.
            #[inline]
            pub fn copysign(self, sign: Self) -> Self {
                let sign_bit = sign.to_bits() & Self::splat(-0.).to_bits();
                let magnitude = self.to_bits() & !Self::splat(-0.).to_bits();
                Self::from_bits(sign_bit | magnitude)
            }

            /// Returns the minimum of each lane.
            ///
            /// If one of the values is `NAN`, then the other value is returned.
            #[inline]
            pub fn min(self, other: Self) -> Self {
                // TODO consider using an intrinsic
                self.is_nan().select(
                    other,
                    self.lanes_ge(other).select(other, self)
                )
            }

            /// Returns the maximum of each lane.
            ///
            /// If one of the values is `NAN`, then the other value is returned.
            #[inline]
            pub fn max(self, other: Self) -> Self {
                // TODO consider using an intrinsic
                self.is_nan().select(
                    other,
                    self.lanes_le(other).select(other, self)
                )
            }

            /// Restrict each lane to a certain interval unless it is NaN.
            ///
            /// For each lane in `self`, returns the corresponding lane in `max` if the lane is
            /// greater than `max`, and the corresponding lane in `min` if the lane is less
            /// than `min`.  Otherwise returns the lane in `self`.
            #[inline]
            pub fn clamp(self, min: Self, max: Self) -> Self {
                assert!(
                    min.lanes_le(max).all(),
                    "each lane in `min` must be less than or equal to the corresponding lane in `max`",
                );
                let mut x = self;
                x = x.lanes_lt(min).select(min, x);
                x = x.lanes_gt(max).select(max, x);
                x
            }
        }
    };
}

/// A SIMD vector of containing `LANES` `f32` values.
#[repr(simd)]
pub struct SimdF32<const LANES: usize>([f32; LANES])
where
    Self: crate::LanesAtMost32;

impl_float_vector! { SimdF32, f32, SimdU32, Mask32, SimdI32 }

from_transmute_x86! { unsafe f32x4 => __m128 }
from_transmute_x86! { unsafe f32x8 => __m256 }
//from_transmute_x86! { unsafe f32x16 => __m512 }

/// A SIMD vector of containing `LANES` `f64` values.
#[repr(simd)]
pub struct SimdF64<const LANES: usize>([f64; LANES])
where
    Self: crate::LanesAtMost32;

impl_float_vector! { SimdF64, f64, SimdU64, Mask64, SimdI64 }

from_transmute_x86! { unsafe f64x2 => __m128d }
from_transmute_x86! { unsafe f64x4 => __m256d }
//from_transmute_x86! { unsafe f64x8 => __m512d }

/// Vector of two `f32` values
pub type f32x2 = SimdF32<2>;

/// Vector of four `f32` values
pub type f32x4 = SimdF32<4>;

/// Vector of eight `f32` values
pub type f32x8 = SimdF32<8>;

/// Vector of 16 `f32` values
pub type f32x16 = SimdF32<16>;

/// Vector of two `f64` values
pub type f64x2 = SimdF64<2>;

/// Vector of four `f64` values
pub type f64x4 = SimdF64<4>;

/// Vector of eight `f64` values
pub type f64x8 = SimdF64<8>;
<commit_msg>Extract constant from scalar to_radians as well<commit_after>#![allow(non_camel_case_types)]

/// Implements inherent methods for a float vector `$name` containing multiple
/// `$lanes` of float `$type`, which uses `$bits_ty` as its binary
/// representation. Called from `define_float_vector!`.
macro_rules! impl_float_vector {
    { $name:ident, $type:ident, $bits_ty:ident, $mask_ty:ident, $mask_impl_ty:ident } => {
        impl_vector! { $name, $type }
        impl_float_reductions! { $name, $type }

        impl<const LANES: usize> $name<LANES>
        where
            Self: crate::LanesAtMost32,
            crate::$bits_ty<LANES>: crate::LanesAtMost32,
        {
            /// Raw transmutation to an unsigned integer vector type with the
            /// same size and number of lanes.
            #[inline]
            pub fn to_bits(self) -> crate::$bits_ty<LANES> {
                assert_eq!(core::mem::size_of::<Self>(), core::mem::size_of::<crate::$bits_ty<LANES>>());
                unsafe { core::mem::transmute_copy(&self) }
            }

            /// Raw transmutation from an unsigned integer vector type with the
            /// same size and number of lanes.
            #[inline]
            pub fn from_bits(bits: crate::$bits_ty<LANES>) -> Self {
                assert_eq!(core::mem::size_of::<Self>(), core::mem::size_of::<crate::$bits_ty<LANES>>());
                unsafe { core::mem::transmute_copy(&bits) }
            }

            /// Produces a vector where every lane has the absolute value of the
            /// equivalently-indexed lane in `self`.
            #[inline]
            pub fn abs(self) -> Self {
                unsafe { crate::intrinsics::simd_fabs(self) }
            }

            /// Fused multiply-add.  Computes `(self * a) + b` with only one rounding error,
            /// yielding a more accurate result than an unfused multiply-add.
            ///
            /// Using `mul_add` *may* be more performant than an unfused multiply-add if the target
            /// architecture has a dedicated `fma` CPU instruction.  However, this is not always
            /// true, and will be heavily dependent on designing algorithms with specific target
            /// hardware in mind.
            #[inline]
            pub fn mul_add(self, a: Self, b: Self) -> Self {
                unsafe { crate::intrinsics::simd_fma(self, a, b) }
            }

            /// Produces a vector where every lane has the square root value
            /// of the equivalently-indexed lane in `self`
            #[inline]
            #[cfg(feature = "std")]
            pub fn sqrt(self) -> Self {
                unsafe { crate::intrinsics::simd_fsqrt(self) }
            }

            /// Takes the reciprocal (inverse) of each lane, `1/x`.
            #[inline]
            pub fn recip(self) -> Self {
                Self::splat(1.0) / self
            }

            /// Converts each lane from radians to degrees.
            #[inline]
            pub fn to_degrees(self) -> Self {
                // to_degrees uses a special constant for better precision, so extract that constant
                self * Self::splat($type::to_degrees(1.))
            }

            /// Converts each lane from degrees to radians.
            #[inline]
            pub fn to_radians(self) -> Self {
                self * Self::splat($type::to_radians(1.))
            }
        }

        impl<const LANES: usize> $name<LANES>
        where
            Self: crate::LanesAtMost32,
            crate::$bits_ty<LANES>: crate::LanesAtMost32,
            crate::$mask_impl_ty<LANES>: crate::LanesAtMost32,
            crate::$mask_ty<LANES>: crate::Mask,
        {
            /// Returns true for each lane if it has a positive sign, including
            /// `+0.0`, `NaN`s with positive sign bit and positive infinity.
            #[inline]
            pub fn is_sign_positive(self) -> crate::$mask_ty<LANES> {
                !self.is_sign_negative()
            }

            /// Returns true for each lane if it has a negative sign, including
            /// `-0.0`, `NaN`s with negative sign bit and negative infinity.
            #[inline]
            pub fn is_sign_negative(self) -> crate::$mask_ty<LANES> {
                let sign_bits = self.to_bits() & crate::$bits_ty::splat((!0 >> 1) + 1);
                sign_bits.lanes_gt(crate::$bits_ty::splat(0))
            }

            /// Returns true for each lane if its value is `NaN`.
            #[inline]
            pub fn is_nan(self) -> crate::$mask_ty<LANES> {
                self.lanes_ne(self)
            }

            /// Returns true for each lane if its value is positive infinity or negative infinity.
            #[inline]
            pub fn is_infinite(self) -> crate::$mask_ty<LANES> {
                self.abs().lanes_eq(Self::splat(<$type>::INFINITY))
            }

            /// Returns true for each lane if its value is neither infinite nor `NaN`.
            #[inline]
            pub fn is_finite(self) -> crate::$mask_ty<LANES> {
                self.abs().lanes_lt(Self::splat(<$type>::INFINITY))
            }

            /// Returns true for each lane if its value is subnormal.
            #[inline]
            pub fn is_subnormal(self) -> crate::$mask_ty<LANES> {
                self.abs().lanes_ne(Self::splat(0.0)) & (self.to_bits() & Self::splat(<$type>::INFINITY).to_bits()).lanes_eq(crate::$bits_ty::splat(0))
            }

            /// Returns true for each lane if its value is neither neither zero, infinite,
            /// subnormal, or `NaN`.
            #[inline]
            pub fn is_normal(self) -> crate::$mask_ty<LANES> {
                !(self.abs().lanes_eq(Self::splat(0.0)) | self.is_nan() | self.is_subnormal() | self.is_infinite())
            }

            /// Replaces each lane with a number that represents its sign.
            ///
            /// * `1.0` if the number is positive, `+0.0`, or `INFINITY`
            /// * `-1.0` if the number is negative, `-0.0`, or `NEG_INFINITY`
            /// * `NAN` if the number is `NAN`
            #[inline]
            pub fn signum(self) -> Self {
                self.is_nan().select(Self::splat($type::NAN), Self::splat(1.0).copysign(self))
            }

            /// Returns each lane with the magnitude of `self` and the sign of `sign`.
            ///
            /// If any lane is a `NAN`, then a `NAN` with the sign of `sign` is returned.
            #[inline]
            pub fn copysign(self, sign: Self) -> Self {
                let sign_bit = sign.to_bits() & Self::splat(-0.).to_bits();
                let magnitude = self.to_bits() & !Self::splat(-0.).to_bits();
                Self::from_bits(sign_bit | magnitude)
            }

            /// Returns the minimum of each lane.
            ///
            /// If one of the values is `NAN`, then the other value is returned.
            #[inline]
            pub fn min(self, other: Self) -> Self {
                // TODO consider using an intrinsic
                self.is_nan().select(
                    other,
                    self.lanes_ge(other).select(other, self)
                )
            }

            /// Returns the maximum of each lane.
            ///
            /// If one of the values is `NAN`, then the other value is returned.
            #[inline]
            pub fn max(self, other: Self) -> Self {
                // TODO consider using an intrinsic
                self.is_nan().select(
                    other,
                    self.lanes_le(other).select(other, self)
                )
            }

            /// Restrict each lane to a certain interval unless it is NaN.
            ///
            /// For each lane in `self`, returns the corresponding lane in `max` if the lane is
            /// greater than `max`, and the corresponding lane in `min` if the lane is less
            /// than `min`.  Otherwise returns the lane in `self`.
            #[inline]
            pub fn clamp(self, min: Self, max: Self) -> Self {
                assert!(
                    min.lanes_le(max).all(),
                    "each lane in `min` must be less than or equal to the corresponding lane in `max`",
                );
                let mut x = self;
                x = x.lanes_lt(min).select(min, x);
                x = x.lanes_gt(max).select(max, x);
                x
            }
        }
    };
}

/// A SIMD vector of containing `LANES` `f32` values.
#[repr(simd)]
pub struct SimdF32<const LANES: usize>([f32; LANES])
where
    Self: crate::LanesAtMost32;

impl_float_vector! { SimdF32, f32, SimdU32, Mask32, SimdI32 }

from_transmute_x86! { unsafe f32x4 => __m128 }
from_transmute_x86! { unsafe f32x8 => __m256 }
//from_transmute_x86! { unsafe f32x16 => __m512 }

/// A SIMD vector of containing `LANES` `f64` values.
#[repr(simd)]
pub struct SimdF64<const LANES: usize>([f64; LANES])
where
    Self: crate::LanesAtMost32;

impl_float_vector! { SimdF64, f64, SimdU64, Mask64, SimdI64 }

from_transmute_x86! { unsafe f64x2 => __m128d }
from_transmute_x86! { unsafe f64x4 => __m256d }
//from_transmute_x86! { unsafe f64x8 => __m512d }

/// Vector of two `f32` values
pub type f32x2 = SimdF32<2>;

/// Vector of four `f32` values
pub type f32x4 = SimdF32<4>;

/// Vector of eight `f32` values
pub type f32x8 = SimdF32<8>;

/// Vector of 16 `f32` values
pub type f32x16 = SimdF32<16>;

/// Vector of two `f64` values
pub type f64x2 = SimdF64<2>;

/// Vector of four `f64` values
pub type f64x4 = SimdF64<4>;

/// Vector of eight `f64` values
pub type f64x8 = SimdF64<8>;
<|endoftext|>


<commit_before>#[cfg(feature = "zstd")]
use zstd::block::{compress, decompress};

use crate::*;

use super::{
    arr_to_u32, gc_blobs, pwrite_all, raw_segment_iter_from, u32_to_arr,
    u64_to_arr, DiskPtr, LogIter, LogKind, LogOffset, Lsn, MessageKind,
};

/// A snapshot of the state required to quickly restart
/// the `PageCache` and `SegmentAccountant`.
#[derive(Debug, Default)]
#[cfg_attr(test, derive(Clone, PartialEq))]
pub struct Snapshot {
    /// The last read message lsn
    pub last_lsn: Lsn,
    /// The last read message lid
    pub last_lid: LogOffset,
    /// the mapping from pages to (lsn, lid)
    pub pt: Vec<PageState>,
}

#[derive(Clone, Debug, PartialEq)]
pub enum PageState {
    Present { base: (Lsn, DiskPtr, u64), frags: Vec<(Lsn, DiskPtr, u64)> },
    Free(Lsn, DiskPtr),
    Uninitialized,
}

impl PageState {
    fn push(&mut self, item: (Lsn, DiskPtr, u64)) {
        match *self {
            PageState::Present { base, ref mut frags } => {
                if frags.last().map_or(base.0, |f| f.0) < item.0 {
                    frags.push(item)
                } else {
                    debug!(
                        "skipping merging item {:?} into \
                        existing PageState::Present({:?})",
                        item, frags
                    );
                }
            }
            _ => panic!("pushed frags to {:?}", self),
        }
    }

    pub(crate) fn is_free(&self) -> bool {
        match *self {
            PageState::Free(_, _) => true,
            _ => false,
        }
    }
}

impl Snapshot {
    fn apply(
        &mut self,
        log_kind: LogKind,
        pid: PageId,
        lsn: Lsn,
        disk_ptr: DiskPtr,
        sz: u64,
    ) {
        // unwrapping this because it's already passed the crc check
        // in the log iterator
        trace!("trying to deserialize buf for ptr {} lsn {}", disk_ptr, lsn);
        let _measure = Measure::new(&M.snapshot_apply);

        if self.pt.len() <= usize::try_from(pid).unwrap() {
            self.pt.resize(
                usize::try_from(pid + 1).unwrap(),
                PageState::Uninitialized,
            );
        }

        match log_kind {
            LogKind::Replace => {
                trace!(
                    "compact of pid {} at ptr {} lsn {}",
                    pid,
                    disk_ptr,
                    lsn,
                );

                self.pt[usize::try_from(pid).unwrap()] = PageState::Present {
                    base: (lsn, disk_ptr, sz),
                    frags: vec![],
                };
            }
            LogKind::Link => {
                // Because we rewrite pages over time, we may have relocated
                // a page's initial Compact to a later segment. We should skip
                // over pages here unless we've encountered a Compact for them.
                if let Some(lids @ PageState::Present { .. }) =
                    self.pt.get_mut(usize::try_from(pid).unwrap())
                {
                    trace!(
                        "append of pid {} at lid {} lsn {}",
                        pid,
                        disk_ptr,
                        lsn,
                    );

                    if lids.is_free() {
                        // this can happen if the allocate or replace
                        // has been moved to a later segment.

                        trace!(
                            "we have not yet encountered an \
                             allocation of this page, skipping push"
                        );

                        return;
                    }

                    lids.push((lsn, disk_ptr, sz));
                } else {
                    trace!(
                        "skipping dangling append of pid {} at lid {} lsn {}",
                        pid,
                        disk_ptr,
                        lsn,
                    );
                }
            }
            LogKind::Free => {
                trace!("free of pid {} at ptr {} lsn {}", pid, disk_ptr, lsn);
                self.pt[usize::try_from(pid).unwrap()] =
                    PageState::Free(lsn, disk_ptr);
            }
            LogKind::Corrupted | LogKind::Skip => panic!(
                "unexppected messagekind in snapshot application: {:?}",
                log_kind
            ),
        }
    }
}

fn advance_snapshot(
    mut iter: LogIter,
    mut snapshot: Snapshot,
    config: &RunningConfig,
) -> Result<Snapshot> {
    let _measure = Measure::new(&M.advance_snapshot);

    trace!("building on top of old snapshot: {:?}", snapshot);

    let old_last_lsn = snapshot.last_lsn;

    while let Some((log_kind, pid, lsn, ptr, sz)) = iter.next() {
        trace!(
            "in advance_snapshot looking at item with pid {} lsn {} ptr {}",
            pid,
            lsn,
            ptr
        );

        if lsn < snapshot.last_lsn {
            // don't process already-processed Lsn's. last_lsn is for the last
            // item ALREADY INCLUDED lsn in the snapshot.
            trace!(
                "continuing in advance_snapshot, lsn {} ptr {} last_lsn {}",
                lsn,
                ptr,
                snapshot.last_lsn
            );
            continue;
        }

        assert!(lsn >= snapshot.last_lsn);
        snapshot.last_lsn = lsn + Lsn::try_from(sz).unwrap();
        snapshot.last_lid = ptr.lid() + sz;

        snapshot.apply(log_kind, pid, lsn, ptr, sz);
    }

    trace!("generated snapshot: {:#?}", snapshot);

    if snapshot.last_lsn != old_last_lsn {
        write_snapshot(config, &snapshot)?;
    }

    let to_zero = iter.segments;
    for lid in to_zero.values() {
        debug!("zeroing torn segment at lid {}", lid);

        // NB we intentionally corrupt this header to prevent any segment
        // from being allocated which would duplicate its LSN, messing
        // up recovery in the future.
        io_fail!(config, "segment initial free zero");
        pwrite_all(
            &config.file,
            &*vec![MessageKind::Corrupted.into(); SEG_HEADER_LEN],
            *lid,
        )?;
        if !config.temporary {
            config.file.sync_all()?;
        }
    }

    // remove all blob files larger than our stable offset
    gc_blobs(config, snapshot.last_lsn)?;

    // zero the tail of the segment after the recovered tip
    let segment_size = config.segment_size;
    let last_segment_base = config.normalize(snapshot.last_lid);
    let next_segment_base = last_segment_base + segment_size as LogOffset;
    let shred_len =
        usize::try_from(next_segment_base - snapshot.last_lid).unwrap();
    let shred_zone = vec![MessageKind::Corrupted.into(); shred_len];
    pwrite_all(&config.file, &shred_zone, snapshot.last_lid)?;
    config.file.sync_all()?;

    #[cfg(feature = "event_log")]
    config.event_log.recovered_lsn(snapshot.last_lsn);

    Ok(snapshot)
}

/// Read a `Snapshot` or generate a default, then advance it to
/// the tip of the data file, if present.
pub fn read_snapshot_or_default(config: &RunningConfig) -> Result<Snapshot> {
    let last_snap = read_snapshot(config)?.unwrap_or_else(Snapshot::default);

    let log_iter = raw_segment_iter_from(last_snap.last_lsn, config)?;

    let res = advance_snapshot(log_iter, last_snap, config)?;

    Ok(res)
}

/// Read a `Snapshot` from disk.
fn read_snapshot(config: &RunningConfig) -> std::io::Result<Option<Snapshot>> {
    let mut f = loop {
        let mut candidates = config.get_snapshot_files()?;
        if candidates.is_empty() {
            debug!("no previous snapshot found");
            return Ok(None);
        }

        candidates.sort();

        let path = candidates.pop().unwrap();

        match std::fs::OpenOptions::new().read(true).open(&path) {
            Ok(f) => break f,
            Err(ref e) if e.kind() == std::io::ErrorKind::NotFound => {
                // this can happen if there's a race
                continue;
            }
            Err(other) => return Err(other),
        }
    };

    if f.metadata()?.len() <= 12 {
        warn!("empty/corrupt snapshot file found");
        return Ok(None);
    }

    let mut buf = vec![];
    let _read = f.read_to_end(&mut buf)?;
    let len = buf.len();
    let mut len_expected_bytes = [0; 8];
    len_expected_bytes.copy_from_slice(&buf[len - 12..len - 4]);

    let mut crc_expected_bytes = [0; 4];
    crc_expected_bytes.copy_from_slice(&buf[len - 4..]);

    let _ = buf.split_off(len - 12);
    let crc_expected: u32 = arr_to_u32(&crc_expected_bytes);

    let crc_actual = crc32(&buf);

    if crc_expected != crc_actual {
        return Ok(None);
    }

    #[cfg(feature = "zstd")]
    let bytes = if config.use_compression {
        use std::convert::TryInto;

        let len_expected: u64 =
            u64::from_le_bytes(len_expected_bytes.as_ref().try_into().unwrap());

        decompress(&*buf, usize::try_from(len_expected).unwrap()).unwrap()
    } else {
        buf
    };

    #[cfg(not(feature = "zstd"))]
    let bytes = buf;

    Ok(Snapshot::deserialize(&mut bytes.as_slice()).ok())
}

fn write_snapshot(config: &RunningConfig, snapshot: &Snapshot) -> Result<()> {
    trace!("writing snapshot {:?}", snapshot);

    let raw_bytes = snapshot.serialize();
    let decompressed_len = raw_bytes.len();

    #[cfg(feature = "zstd")]
    let bytes = if config.use_compression {
        compress(&*raw_bytes, config.compression_factor).unwrap()
    } else {
        raw_bytes
    };

    #[cfg(not(feature = "zstd"))]
    let bytes = raw_bytes;

    let crc32: [u8; 4] = u32_to_arr(crc32(&bytes));
    let len_bytes: [u8; 8] = u64_to_arr(decompressed_len as u64);

    let path_1_suffix = format!("snap.{:016X}.generating", snapshot.last_lsn);

    let mut path_1 = config.get_path();
    path_1.push(path_1_suffix);

    let path_2_suffix = format!("snap.{:016X}", snapshot.last_lsn);

    let mut path_2 = config.get_path();
    path_2.push(path_2_suffix);

    let parent = path_1.parent().unwrap();
    std::fs::create_dir_all(parent)?;
    let mut f =
        std::fs::OpenOptions::new().write(true).create(true).open(&path_1)?;

    // write the snapshot bytes, followed by a crc64 checksum at the end
    io_fail!(config, "snap write");
    f.write_all(&*bytes)?;
    io_fail!(config, "snap write len");
    f.write_all(&len_bytes)?;
    io_fail!(config, "snap write crc");
    f.write_all(&crc32)?;
    io_fail!(config, "snap write post");
    f.sync_all()?;

    trace!("wrote snapshot to {}", path_1.to_string_lossy());

    io_fail!(config, "snap write mv");
    std::fs::rename(&path_1, &path_2)?;
    io_fail!(config, "snap write mv post");

    trace!("renamed snapshot to {}", path_2.to_string_lossy());

    // clean up any old snapshots
    let candidates = config.get_snapshot_files()?;
    for path in candidates {
        let path_str = path.file_name().unwrap().to_str().unwrap();
        if !path_2.to_string_lossy().ends_with(&*path_str) {
            debug!("removing old snapshot file {:?}", path);

            io_fail!(config, "snap write rm old");

            if let Err(e) = std::fs::remove_file(&path) {
                // TODO should this just be a try return?
                warn!(
                    "failed to remove old snapshot file, maybe snapshot race? {}",
                    e
                );
            }
        }
    }
    Ok(())
}
<commit_msg>Avoid call to fs metadata<commit_after>#[cfg(feature = "zstd")]
use zstd::block::{compress, decompress};

use crate::*;

use super::{
    arr_to_u32, gc_blobs, pwrite_all, raw_segment_iter_from, u32_to_arr,
    u64_to_arr, DiskPtr, LogIter, LogKind, LogOffset, Lsn, MessageKind,
};

/// A snapshot of the state required to quickly restart
/// the `PageCache` and `SegmentAccountant`.
#[derive(Debug, Default)]
#[cfg_attr(test, derive(Clone, PartialEq))]
pub struct Snapshot {
    /// The last read message lsn
    pub last_lsn: Lsn,
    /// The last read message lid
    pub last_lid: LogOffset,
    /// the mapping from pages to (lsn, lid)
    pub pt: Vec<PageState>,
}

#[derive(Clone, Debug, PartialEq)]
pub enum PageState {
    Present { base: (Lsn, DiskPtr, u64), frags: Vec<(Lsn, DiskPtr, u64)> },
    Free(Lsn, DiskPtr),
    Uninitialized,
}

impl PageState {
    fn push(&mut self, item: (Lsn, DiskPtr, u64)) {
        match *self {
            PageState::Present { base, ref mut frags } => {
                if frags.last().map_or(base.0, |f| f.0) < item.0 {
                    frags.push(item)
                } else {
                    debug!(
                        "skipping merging item {:?} into \
                        existing PageState::Present({:?})",
                        item, frags
                    );
                }
            }
            _ => panic!("pushed frags to {:?}", self),
        }
    }

    pub(crate) fn is_free(&self) -> bool {
        match *self {
            PageState::Free(_, _) => true,
            _ => false,
        }
    }
}

impl Snapshot {
    fn apply(
        &mut self,
        log_kind: LogKind,
        pid: PageId,
        lsn: Lsn,
        disk_ptr: DiskPtr,
        sz: u64,
    ) {
        // unwrapping this because it's already passed the crc check
        // in the log iterator
        trace!("trying to deserialize buf for ptr {} lsn {}", disk_ptr, lsn);
        let _measure = Measure::new(&M.snapshot_apply);

        if self.pt.len() <= usize::try_from(pid).unwrap() {
            self.pt.resize(
                usize::try_from(pid + 1).unwrap(),
                PageState::Uninitialized,
            );
        }

        match log_kind {
            LogKind::Replace => {
                trace!(
                    "compact of pid {} at ptr {} lsn {}",
                    pid,
                    disk_ptr,
                    lsn,
                );

                self.pt[usize::try_from(pid).unwrap()] = PageState::Present {
                    base: (lsn, disk_ptr, sz),
                    frags: vec![],
                };
            }
            LogKind::Link => {
                // Because we rewrite pages over time, we may have relocated
                // a page's initial Compact to a later segment. We should skip
                // over pages here unless we've encountered a Compact for them.
                if let Some(lids @ PageState::Present { .. }) =
                    self.pt.get_mut(usize::try_from(pid).unwrap())
                {
                    trace!(
                        "append of pid {} at lid {} lsn {}",
                        pid,
                        disk_ptr,
                        lsn,
                    );

                    if lids.is_free() {
                        // this can happen if the allocate or replace
                        // has been moved to a later segment.

                        trace!(
                            "we have not yet encountered an \
                             allocation of this page, skipping push"
                        );

                        return;
                    }

                    lids.push((lsn, disk_ptr, sz));
                } else {
                    trace!(
                        "skipping dangling append of pid {} at lid {} lsn {}",
                        pid,
                        disk_ptr,
                        lsn,
                    );
                }
            }
            LogKind::Free => {
                trace!("free of pid {} at ptr {} lsn {}", pid, disk_ptr, lsn);
                self.pt[usize::try_from(pid).unwrap()] =
                    PageState::Free(lsn, disk_ptr);
            }
            LogKind::Corrupted | LogKind::Skip => panic!(
                "unexppected messagekind in snapshot application: {:?}",
                log_kind
            ),
        }
    }
}

fn advance_snapshot(
    mut iter: LogIter,
    mut snapshot: Snapshot,
    config: &RunningConfig,
) -> Result<Snapshot> {
    let _measure = Measure::new(&M.advance_snapshot);

    trace!("building on top of old snapshot: {:?}", snapshot);

    let old_last_lsn = snapshot.last_lsn;

    while let Some((log_kind, pid, lsn, ptr, sz)) = iter.next() {
        trace!(
            "in advance_snapshot looking at item with pid {} lsn {} ptr {}",
            pid,
            lsn,
            ptr
        );

        if lsn < snapshot.last_lsn {
            // don't process already-processed Lsn's. last_lsn is for the last
            // item ALREADY INCLUDED lsn in the snapshot.
            trace!(
                "continuing in advance_snapshot, lsn {} ptr {} last_lsn {}",
                lsn,
                ptr,
                snapshot.last_lsn
            );
            continue;
        }

        assert!(lsn >= snapshot.last_lsn);
        snapshot.last_lsn = lsn + Lsn::try_from(sz).unwrap();
        snapshot.last_lid = ptr.lid() + sz;

        snapshot.apply(log_kind, pid, lsn, ptr, sz);
    }

    trace!("generated snapshot: {:#?}", snapshot);

    if snapshot.last_lsn != old_last_lsn {
        write_snapshot(config, &snapshot)?;
    }

    let to_zero = iter.segments;
    for lid in to_zero.values() {
        debug!("zeroing torn segment at lid {}", lid);

        // NB we intentionally corrupt this header to prevent any segment
        // from being allocated which would duplicate its LSN, messing
        // up recovery in the future.
        io_fail!(config, "segment initial free zero");
        pwrite_all(
            &config.file,
            &*vec![MessageKind::Corrupted.into(); SEG_HEADER_LEN],
            *lid,
        )?;
        if !config.temporary {
            config.file.sync_all()?;
        }
    }

    // remove all blob files larger than our stable offset
    gc_blobs(config, snapshot.last_lsn)?;

    // zero the tail of the segment after the recovered tip
    let segment_size = config.segment_size;
    let last_segment_base = config.normalize(snapshot.last_lid);
    let next_segment_base = last_segment_base + segment_size as LogOffset;
    let shred_len =
        usize::try_from(next_segment_base - snapshot.last_lid).unwrap();
    let shred_zone = vec![MessageKind::Corrupted.into(); shred_len];
    pwrite_all(&config.file, &shred_zone, snapshot.last_lid)?;
    config.file.sync_all()?;

    #[cfg(feature = "event_log")]
    config.event_log.recovered_lsn(snapshot.last_lsn);

    Ok(snapshot)
}

/// Read a `Snapshot` or generate a default, then advance it to
/// the tip of the data file, if present.
pub fn read_snapshot_or_default(config: &RunningConfig) -> Result<Snapshot> {
    let last_snap = read_snapshot(config)?.unwrap_or_else(Snapshot::default);

    let log_iter = raw_segment_iter_from(last_snap.last_lsn, config)?;

    let res = advance_snapshot(log_iter, last_snap, config)?;

    Ok(res)
}

/// Read a `Snapshot` from disk.
fn read_snapshot(config: &RunningConfig) -> std::io::Result<Option<Snapshot>> {
    let mut f = loop {
        let mut candidates = config.get_snapshot_files()?;
        if candidates.is_empty() {
            debug!("no previous snapshot found");
            return Ok(None);
        }

        candidates.sort();

        let path = candidates.pop().unwrap();

        match std::fs::OpenOptions::new().read(true).open(&path) {
            Ok(f) => break f,
            Err(ref e) if e.kind() == std::io::ErrorKind::NotFound => {
                // this can happen if there's a race
                continue;
            }
            Err(other) => return Err(other),
        }
    };

    let mut buf = vec![];
    let _read = f.read_to_end(&mut buf)?;
    let len = buf.len();
    if len <= 12 {
        warn!("empty/corrupt snapshot file found");
        return Ok(None);
    }

    let mut len_expected_bytes = [0; 8];
    len_expected_bytes.copy_from_slice(&buf[len - 12..len - 4]);

    let mut crc_expected_bytes = [0; 4];
    crc_expected_bytes.copy_from_slice(&buf[len - 4..]);

    let _ = buf.split_off(len - 12);
    let crc_expected: u32 = arr_to_u32(&crc_expected_bytes);

    let crc_actual = crc32(&buf);

    if crc_expected != crc_actual {
        return Ok(None);
    }

    #[cfg(feature = "zstd")]
    let bytes = if config.use_compression {
        use std::convert::TryInto;

        let len_expected: u64 =
            u64::from_le_bytes(len_expected_bytes.as_ref().try_into().unwrap());

        decompress(&*buf, usize::try_from(len_expected).unwrap()).unwrap()
    } else {
        buf
    };

    #[cfg(not(feature = "zstd"))]
    let bytes = buf;

    Ok(Snapshot::deserialize(&mut bytes.as_slice()).ok())
}

fn write_snapshot(config: &RunningConfig, snapshot: &Snapshot) -> Result<()> {
    trace!("writing snapshot {:?}", snapshot);

    let raw_bytes = snapshot.serialize();
    let decompressed_len = raw_bytes.len();

    #[cfg(feature = "zstd")]
    let bytes = if config.use_compression {
        compress(&*raw_bytes, config.compression_factor).unwrap()
    } else {
        raw_bytes
    };

    #[cfg(not(feature = "zstd"))]
    let bytes = raw_bytes;

    let crc32: [u8; 4] = u32_to_arr(crc32(&bytes));
    let len_bytes: [u8; 8] = u64_to_arr(decompressed_len as u64);

    let path_1_suffix = format!("snap.{:016X}.generating", snapshot.last_lsn);

    let mut path_1 = config.get_path();
    path_1.push(path_1_suffix);

    let path_2_suffix = format!("snap.{:016X}", snapshot.last_lsn);

    let mut path_2 = config.get_path();
    path_2.push(path_2_suffix);

    let parent = path_1.parent().unwrap();
    std::fs::create_dir_all(parent)?;
    let mut f =
        std::fs::OpenOptions::new().write(true).create(true).open(&path_1)?;

    // write the snapshot bytes, followed by a crc64 checksum at the end
    io_fail!(config, "snap write");
    f.write_all(&*bytes)?;
    io_fail!(config, "snap write len");
    f.write_all(&len_bytes)?;
    io_fail!(config, "snap write crc");
    f.write_all(&crc32)?;
    io_fail!(config, "snap write post");
    f.sync_all()?;

    trace!("wrote snapshot to {}", path_1.to_string_lossy());

    io_fail!(config, "snap write mv");
    std::fs::rename(&path_1, &path_2)?;
    io_fail!(config, "snap write mv post");

    trace!("renamed snapshot to {}", path_2.to_string_lossy());

    // clean up any old snapshots
    let candidates = config.get_snapshot_files()?;
    for path in candidates {
        let path_str = path.file_name().unwrap().to_str().unwrap();
        if !path_2.to_string_lossy().ends_with(&*path_str) {
            debug!("removing old snapshot file {:?}", path);

            io_fail!(config, "snap write rm old");

            if let Err(e) = std::fs::remove_file(&path) {
                // TODO should this just be a try return?
                warn!(
                    "failed to remove old snapshot file, maybe snapshot race? {}",
                    e
                );
            }
        }
    }
    Ok(())
}
<|endoftext|>


<commit_before>#[macro_use] extern crate clap;
extern crate secstr;
extern crate colorhash256;
extern crate interactor;
extern crate rusterpassword;
extern crate sodiumoxide;
extern crate ansi_term;
extern crate freepass_core;

use std::{fs,env,io};
use std::io::prelude::*;
use ansi_term::Colour::Fixed;
use ansi_term::ANSIStrings;
use secstr::*;
use interactor::*;
use rusterpassword::*;
use freepass_core::data::*;

fn opt_or_env(matches: &clap::ArgMatches, opt_name: &str, env_name: &str) -> String {
    match matches.value_of(opt_name).map(|x| x.to_owned()).or(env::var_os(env_name).and_then(|s| s.into_string().ok())) {
        Some(s) => s,
        None => panic!("Option {} or environment variable {} not found", opt_name, env_name)
    }
}

fn main() {
    let matches = clap_app!(freepass =>
        (version: env!("CARGO_PKG_VERSION"))
        (author: "Greg V <greg@unrelenting.technology>")
        (about: "The free password manager for power users")
        (@arg FILE: -f --file +takes_value "Sets the vault file to use, by default: $FREEPASS_FILE")
        (@arg NAME: -n --name +takes_value "Sets the user name to use (must be always the same for a vault file!), by default: $FREEPASS_NAME")
        (@subcommand interact =>
            (about: "Launches interactive mode")
        )
    ).get_matches();

    let file_path = opt_or_env(&matches, "FILE", "FREEPASS_FILE");
    let user_name = opt_or_env(&matches, "NAME", "FREEPASS_NAME");

    sodiumoxide::init();

    // Do this early because we don't want to ask for the password when we get permission denied or something.
    // Ensure we can write! Maybe someone somewhere would want to open the vault in read-only mode...
    // But the frustration of trying to save the vault while only having read permissions would be worse.
    let file = match fs::OpenOptions::new().read(true).write(true).open(&file_path) {
        Ok(file) => Some(file),
        Err(ref err) if err.kind() == io::ErrorKind::NotFound => None,
        Err(ref err) => panic!("Could not open file {}: {}", &file_path, err),
    };

    let master_key = {
        let read_result = read_from_tty(|buf, _, tty| {
            let color_string = if buf.len() < 8 {
                // Make it a bit harder to recover the password by e.g. someone filming how you're entering your password
                // Although if you're entering your password on camera, you're kinda screwed anyway
                b"\rPassword: ~~~~~~".to_vec()
            } else {
                let colors = colorhash256::hash_as_ansi(buf);
                format!("\rPassword: {}",
                    ANSIStrings(&[
                        Fixed(colors[0] as u8).paint("~~"),
                        Fixed(colors[1] as u8).paint("~~"),
                        Fixed(colors[2] as u8).paint("~~"),
                    ])).into_bytes()
            };
            tty.write(&color_string).unwrap();
        }, true).unwrap();
        gen_master_key(SecStr::new(read_result), &user_name).unwrap()
    };
    let outer_key = gen_outer_key(&master_key);
    let entries_key = gen_entries_key(&master_key);

    let vault = match file {
        Some(f) => Vault::open(&outer_key, f).unwrap(),
        None => Vault::new(),
    };
}
<commit_msg>add interaction macro<commit_after>#[macro_use] extern crate clap;
extern crate secstr;
extern crate colorhash256;
extern crate interactor;
extern crate rusterpassword;
extern crate sodiumoxide;
extern crate ansi_term;
extern crate freepass_core;

use std::{fs,env,io};
use std::io::prelude::*;
use ansi_term::Colour::Fixed;
use ansi_term::ANSIStrings;
use secstr::*;
use interactor::*;
use rusterpassword::*;
use freepass_core::data::*;

fn opt_or_env(matches: &clap::ArgMatches, opt_name: &str, env_name: &str) -> String {
    match matches.value_of(opt_name).map(|x| x.to_owned()).or(env::var_os(env_name).and_then(|s| s.into_string().ok())) {
        Some(s) => s,
        None => panic!("Option {} or environment variable {} not found", opt_name, env_name)
    }
}

fn main() {
    let matches = clap_app!(freepass =>
        (version: env!("CARGO_PKG_VERSION"))
        (author: "Greg V <greg@unrelenting.technology>")
        (about: "The free password manager for power users")
        (@arg FILE: -f --file +takes_value "Sets the vault file to use, by default: $FREEPASS_FILE")
        (@arg NAME: -n --name +takes_value "Sets the user name to use (must be always the same for a vault file!), by default: $FREEPASS_NAME")
        (@subcommand interact =>
            (about: "Launches interactive mode")
        )
    ).get_matches();

    let file_path = opt_or_env(&matches, "FILE", "FREEPASS_FILE");
    let user_name = opt_or_env(&matches, "NAME", "FREEPASS_NAME");

    sodiumoxide::init();

    // Do this early because we don't want to ask for the password when we get permission denied or something.
    // Ensure we can write! Maybe someone somewhere would want to open the vault in read-only mode...
    // But the frustration of trying to save the vault while only having read permissions would be worse.
    let file = match fs::OpenOptions::new().read(true).write(true).open(&file_path) {
        Ok(file) => Some(file),
        Err(ref err) if err.kind() == io::ErrorKind::NotFound => None,
        Err(ref err) => panic!("Could not open file {}: {}", &file_path, err),
    };

    let master_key = {
        let read_result = read_from_tty(|buf, _, tty| {
            let color_string = if buf.len() < 8 {
                // Make it a bit harder to recover the password by e.g. someone filming how you're entering your password
                // Although if you're entering your password on camera, you're kinda screwed anyway
                b"\rPassword: ~~~~~~".to_vec()
            } else {
                let colors = colorhash256::hash_as_ansi(buf);
                format!("\rPassword: {}",
                    ANSIStrings(&[
                        Fixed(colors[0] as u8).paint("~~"),
                        Fixed(colors[1] as u8).paint("~~"),
                        Fixed(colors[2] as u8).paint("~~"),
                    ])).into_bytes()
            };
            tty.write(&color_string).unwrap();
        }, true).unwrap();
        gen_master_key(SecStr::new(read_result), &user_name).unwrap()
    };
    let outer_key = gen_outer_key(&master_key);

    let mut vault = match file {
        Some(f) => Vault::open(&outer_key, f).unwrap(),
        None => Vault::new(),
    };

    {// XXX: TEST ENTRY
        let entries_key = gen_entries_key(&master_key);
        let mut twitter = Entry::new();
            twitter.fields.insert("password".to_owned(), Field::Derived {
                counter: 4, site_name: Some("twitter.com".to_owned()), usage: DerivedUsage::Password(PasswordTemplate::Maximum)
            });
            twitter.fields.insert("old_password".to_owned(), Field::Stored {
                data: SecStr::from("h0rse"), usage: StoredUsage::Password
            });
        let mut metadata = EntryMetadata::new();
        vault.put_entry(&entries_key, "twitter", &twitter, &mut metadata).unwrap();
    }

    interact_entries(&mut vault, &master_key);
}

macro_rules! interaction {
    ( { $($action_name:expr => $action_fn:expr),+ }, $data:expr, $data_fn:expr ) => {
        {
            let mut items = vec![$(">> ".to_string() + $action_name),+];
            let data_items : Vec<String> = $data.clone().map(|x| " | ".to_string() + x).collect();
            items.extend(data_items.iter().cloned());
            match pick_from_list(default_menu_cmd().as_mut(), &items[..], "Selection: ").unwrap() {
                $(ref x if *x == ">> ".to_string() + $action_name => $action_fn),+
                ref x if data_items.contains(x) => ($data_fn)(&x[3..]),
                ref x => panic!("Unknown selection: {}", x),
            }
        }
    }
}

fn interact_entries(vault: &mut Vault, master_key: &SecStr) {
    let entries_key = gen_entries_key(&master_key);
    interaction!({
        "Add new entry" => {
            println!("YO");
        }
    }, vault.entry_names(), { |x|
        println!("pick {}", x)
    })
}
<|endoftext|>


<commit_before>fn process_response_case(phrase: &str, expected_response: &str) {
    assert_eq!(bob::reply(phrase), expected_response);
}

#[test]
#[ignore]
/// ending with whitespace
fn test_ending_with_whitespace() {
    process_response_case("Okay if like my  spacebar  quite a bit?   ", "Sure.");
}

#[test]
#[ignore]
/// shouting numbers
fn test_shouting_numbers() {
    process_response_case("1, 2, 3 GO!", "Whoa, chill out!");
}

#[test]
#[ignore]
/// other whitespace
fn test_other_whitespace() {
    process_response_case("\r\r 	", "Fine. Be that way!");
}

#[test]
#[ignore]
/// shouting with special characters
fn test_shouting_with_special_characters() {
    process_response_case("ZOMG THE %^*@#$(*^ ZOMBIES ARE COMING!!11!!1!", "Whoa, chill out!");
}

#[test]
#[ignore]
/// talking forcefully
fn test_talking_forcefully() {
    process_response_case("Hi there!", "Whatever.");
}

#[test]
#[ignore]
/// prattling on
fn test_prattling_on() {
    process_response_case("Wait! Hang on. Are you going to be OK?", "Sure.");
}

#[test]
#[ignore]
/// forceful question
fn test_forceful_question() {
   process_response_case("WHAT'S GOING ON?", "Calm down, I know what I'm doing!");
}

#[test]
#[ignore]
/// shouting with no exclamation mark
fn test_shouting_with_no_exclamation_mark() {
    process_response_case("I HATE THE DENTIST", "Whoa, chill out!");
}

#[test]
#[ignore]
/// asking gibberish
fn test_asking_gibberish() {
    process_response_case("fffbbcbeab?", "Sure.");
}

#[test]
#[ignore]
/// question with no letters
fn test_question_with_no_letters() {
    process_response_case("4?", "Sure.");
}

#[test]
#[ignore]
/// no letters
fn test_no_letters() {
    process_response_case("1, 2, 3", "Whatever.");
}

#[test]
#[ignore]
/// statement containing question mark
fn test_statement_containing_question_mark() {
    process_response_case("Ending with ? means a question.", "Whatever.");
}

//NEW
#[test]
#[ignore]
/// multiple line question
fn test_multiple_line_question() {
    process_response_case("\rDoes this cryogenic chamber make me look fat?\rNo.", "Whatever.");
}

#[test]
#[ignore]
/// non-question ending with whitespace
fn test_nonquestion_ending_with_whitespace() {
    process_response_case("This is a statement ending with whitespace      ", "Whatever.");
}

#[test]
#[ignore]
/// shouting
fn test_shouting() {
    process_response_case("WATCH OUT!", "Whoa, chill out!");
}

#[test]
#[ignore]
/// non-letters with question
fn test_nonletters_with_question() {
    process_response_case(":) ?", "Sure.");
}

#[test]
#[ignore]
/// stating something
fn test_stating_something() {
    process_response_case("Tom-ay-to, tom-aaaah-to.", "Whatever.");
}

#[test]
#[ignore]
/// shouting gibberish
fn test_shouting_gibberish() {
    process_response_case("FCECDFCAAB", "Whoa, chill out!");
}

#[test]
#[ignore]
/// asking a question
fn test_asking_a_question() {
    process_response_case("Does this cryogenic chamber make me look fat?", "Sure.");
}

#[test]
#[ignore]
/// asking a numeric question
fn test_asking_a_numeric_question() {
    process_response_case("You are, what, like 15?", "Sure.");
}

#[test]
#[ignore]
/// silence
fn test_silence() {
    process_response_case("", "Fine. Be that way!");
}

#[test]
#[ignore]
/// starting with whitespace
fn test_starting_with_whitespace() {
    process_response_case("         hmmmmmmm...", "Whatever.");
}

#[test]
#[ignore]
/// using acronyms in regular speech
fn test_using_acronyms_in_regular_speech() {
    process_response_case("It's OK if you don't want to go work for NASA.", "Whatever.");
}

#[test]
#[ignore]
/// alternate silence
fn test_alternate_silence() {
    process_response_case("										", "Fine. Be that way!");
}

#[test]
#[ignore]
/// prolonged silence
fn test_prolonged_silence() {
    process_response_case("          ", "Fine. Be that way!");
}

<commit_msg>Remove ignore for first test-case<commit_after>fn process_response_case(phrase: &str, expected_response: &str) {
    assert_eq!(bob::reply(phrase), expected_response);
}

#[test]
/// stating something
fn test_stating_something() {
    process_response_case("Tom-ay-to, tom-aaaah-to.", "Whatever.");
}

#[test]
#[ignore]
/// ending with whitespace
fn test_ending_with_whitespace() {
    process_response_case("Okay if like my  spacebar  quite a bit?   ", "Sure.");
}

#[test]
#[ignore]
/// shouting numbers
fn test_shouting_numbers() {
    process_response_case("1, 2, 3 GO!", "Whoa, chill out!");
}

#[test]
#[ignore]
/// other whitespace
fn test_other_whitespace() {
    process_response_case("\r\r 	", "Fine. Be that way!");
}

#[test]
#[ignore]
/// shouting with special characters
fn test_shouting_with_special_characters() {
    process_response_case("ZOMG THE %^*@#$(*^ ZOMBIES ARE COMING!!11!!1!", "Whoa, chill out!");
}

#[test]
#[ignore]
/// talking forcefully
fn test_talking_forcefully() {
    process_response_case("Hi there!", "Whatever.");
}

#[test]
#[ignore]
/// prattling on
fn test_prattling_on() {
    process_response_case("Wait! Hang on. Are you going to be OK?", "Sure.");
}

#[test]
#[ignore]
/// forceful question
fn test_forceful_question() {
   process_response_case("WHAT'S GOING ON?", "Calm down, I know what I'm doing!");
}

#[test]
#[ignore]
/// shouting with no exclamation mark
fn test_shouting_with_no_exclamation_mark() {
    process_response_case("I HATE THE DENTIST", "Whoa, chill out!");
}

#[test]
#[ignore]
/// asking gibberish
fn test_asking_gibberish() {
    process_response_case("fffbbcbeab?", "Sure.");
}

#[test]
#[ignore]
/// question with no letters
fn test_question_with_no_letters() {
    process_response_case("4?", "Sure.");
}

#[test]
#[ignore]
/// no letters
fn test_no_letters() {
    process_response_case("1, 2, 3", "Whatever.");
}

#[test]
#[ignore]
/// statement containing question mark
fn test_statement_containing_question_mark() {
    process_response_case("Ending with ? means a question.", "Whatever.");
}

//NEW
#[test]
#[ignore]
/// multiple line question
fn test_multiple_line_question() {
    process_response_case("\rDoes this cryogenic chamber make me look fat?\rNo.", "Whatever.");
}

#[test]
#[ignore]
/// non-question ending with whitespace
fn test_nonquestion_ending_with_whitespace() {
    process_response_case("This is a statement ending with whitespace      ", "Whatever.");
}

#[test]
#[ignore]
/// shouting
fn test_shouting() {
    process_response_case("WATCH OUT!", "Whoa, chill out!");
}

#[test]
#[ignore]
/// non-letters with question
fn test_nonletters_with_question() {
    process_response_case(":) ?", "Sure.");
}

#[test]
#[ignore]
/// shouting gibberish
fn test_shouting_gibberish() {
    process_response_case("FCECDFCAAB", "Whoa, chill out!");
}

#[test]
#[ignore]
/// asking a question
fn test_asking_a_question() {
    process_response_case("Does this cryogenic chamber make me look fat?", "Sure.");
}

#[test]
#[ignore]
/// asking a numeric question
fn test_asking_a_numeric_question() {
    process_response_case("You are, what, like 15?", "Sure.");
}

#[test]
#[ignore]
/// silence
fn test_silence() {
    process_response_case("", "Fine. Be that way!");
}

#[test]
#[ignore]
/// starting with whitespace
fn test_starting_with_whitespace() {
    process_response_case("         hmmmmmmm...", "Whatever.");
}

#[test]
#[ignore]
/// using acronyms in regular speech
fn test_using_acronyms_in_regular_speech() {
    process_response_case("It's OK if you don't want to go work for NASA.", "Whatever.");
}

#[test]
#[ignore]
/// alternate silence
fn test_alternate_silence() {
    process_response_case("										", "Fine. Be that way!");
}

#[test]
#[ignore]
/// prolonged silence
fn test_prolonged_silence() {
    process_response_case("          ", "Fine. Be that way!");
}

<|endoftext|>


<commit_before>
extern crate ndarray;
#[macro_use]
extern crate ndarray_linalg;

use ndarray::*;
use ndarray_linalg::*;

fn test1d<A, Sa, Sb, Tol>(uplo: UPLO, a: ArrayBase<Sa, Ix2>, b: ArrayBase<Sb, Ix1>, tol: Tol)
where
    A: Scalar + Absolute<Output = Tol>,
    Sa: Data<Elem = A>,
    Sb: DataMut<Elem = A> + DataOwned,
    Tol: RealScalar,
{
    println!("a = {:?}", &a);
    println!("b = {:?}", &b);
    let x = a.solve_triangular(uplo, Diag::NonUnit, &b).unwrap();
    println!("x = {:?}", &x);
    let b_ = a.dot(&x);
    println!("Ax = {:?}", &b_);
    assert_close_l2!(&b_, &b, tol);
}

fn test2d<A, Sa, Sb, Tol>(uplo: UPLO, a: ArrayBase<Sa, Ix2>, b: ArrayBase<Sb, Ix2>, tol: Tol)
where
    A: Scalar + Absolute<Output = Tol>,
    Sa: Data<Elem = A>,
    Sb: DataMut<Elem = A> + DataOwned + DataClone,
    Tol: RealScalar,
{
    println!("a = {:?}", &a);
    println!("b = {:?}", &b);
    let ans = b.clone();
    let x = a.solve_triangular(uplo, Diag::NonUnit, b).unwrap();
    println!("x = {:?}", &x);
    let b_ = a.dot(&x);
    println!("Ax = {:?}", &b_);
    assert_close_l2!(&b_, &ans, tol);
}

#[test]
fn triangular_1d_upper() {
    let n = 3;
    let b: Array1<f64> = random(n);
    let a: Array2<f64> = random((n, n)).into_triangular(UPLO::Upper);
    test1d(UPLO::Upper, a, b, 1e-7);
}

#[test]
fn triangular_1d_lower() {
    let n = 3;
    let b: Array1<f64> = random(n);
    let a: Array2<f64> = random((n, n)).into_triangular(UPLO::Lower);
    test1d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_1d_upper_t() {
    let n = 3;
    let b: Array1<f64> = random(n);
    let a: Array2<f64> = random((n, n).f()).into_triangular(UPLO::Upper);
    test1d(UPLO::Upper, a, b, 1e-7);
}

#[test]
fn triangular_1d_lower_t() {
    let n = 3;
    let b: Array1<f64> = random(n);
    let a: Array2<f64> = random((n, n).f()).into_triangular(UPLO::Lower);
    test1d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_2d_upper() {
    let b: Array2<f64> = random((3, 4));
    let a: Array2<f64> = random((3, 3)).into_triangular(UPLO::Upper);
    test2d(UPLO::Upper, a, b, 1e-7);
}

#[test]
fn triangular_2d_lower() {
    let b: Array2<f64> = random((3, 4));
    let a: Array2<f64> = random((3, 3)).into_triangular(UPLO::Lower);
    test2d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_2d_lower_t() {
    let b: Array2<f64> = random((3, 4));
    let a: Array2<f64> = random((3, 3).f()).into_triangular(UPLO::Lower);
    test2d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_2d_upper_t() {
    let b: Array2<f64> = random((3, 4));
    let a: Array2<f64> = random((3, 3).f()).into_triangular(UPLO::Upper);
    test2d(UPLO::Upper, a, b, 1e-7);
}

#[test]
fn triangular_2d_upper_bt() {
    let b: Array2<f64> = random((3, 4).f());
    let a: Array2<f64> = random((3, 3)).into_triangular(UPLO::Upper);
    test2d(UPLO::Upper, a, b, 1e-7);
}

#[test]
fn triangular_2d_lower_bt() {
    let b: Array2<f64> = random((3, 4).f());
    let a: Array2<f64> = random((3, 3)).into_triangular(UPLO::Lower);
    test2d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_2d_lower_t_bt() {
    let b: Array2<f64> = random((3, 4).f());
    let a: Array2<f64> = random((3, 3).f()).into_triangular(UPLO::Lower);
    test2d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_2d_upper_t_bt() {
    let b: Array2<f64> = random((3, 4).f());
    let a: Array2<f64> = random((3, 3).f()).into_triangular(UPLO::Upper);
    test2d(UPLO::Upper, a, b, 1e-7);
}
<commit_msg>Remove Tol type also in test<commit_after>
extern crate ndarray;
#[macro_use]
extern crate ndarray_linalg;

use ndarray::*;
use ndarray_linalg::*;

fn test1d<A, Sa, Sb>(uplo: UPLO, a: ArrayBase<Sa, Ix2>, b: ArrayBase<Sb, Ix1>, tol: A::Real)
where
    A: Scalar,
    Sa: Data<Elem = A>,
    Sb: DataMut<Elem = A> + DataOwned,
{
    println!("a = {:?}", &a);
    println!("b = {:?}", &b);
    let x = a.solve_triangular(uplo, Diag::NonUnit, &b).unwrap();
    println!("x = {:?}", &x);
    let b_ = a.dot(&x);
    println!("Ax = {:?}", &b_);
    assert_close_l2!(&b_, &b, tol);
}

fn test2d<A, Sa, Sb>(uplo: UPLO, a: ArrayBase<Sa, Ix2>, b: ArrayBase<Sb, Ix2>, tol: A::Real)
where
    A: Scalar,
    Sa: Data<Elem = A>,
    Sb: DataMut<Elem = A> + DataOwned + DataClone,
{
    println!("a = {:?}", &a);
    println!("b = {:?}", &b);
    let ans = b.clone();
    let x = a.solve_triangular(uplo, Diag::NonUnit, b).unwrap();
    println!("x = {:?}", &x);
    let b_ = a.dot(&x);
    println!("Ax = {:?}", &b_);
    assert_close_l2!(&b_, &ans, tol);
}

#[test]
fn triangular_1d_upper() {
    let n = 3;
    let b: Array1<f64> = random(n);
    let a: Array2<f64> = random((n, n)).into_triangular(UPLO::Upper);
    test1d(UPLO::Upper, a, b, 1e-7);
}

#[test]
fn triangular_1d_lower() {
    let n = 3;
    let b: Array1<f64> = random(n);
    let a: Array2<f64> = random((n, n)).into_triangular(UPLO::Lower);
    test1d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_1d_upper_t() {
    let n = 3;
    let b: Array1<f64> = random(n);
    let a: Array2<f64> = random((n, n).f()).into_triangular(UPLO::Upper);
    test1d(UPLO::Upper, a, b, 1e-7);
}

#[test]
fn triangular_1d_lower_t() {
    let n = 3;
    let b: Array1<f64> = random(n);
    let a: Array2<f64> = random((n, n).f()).into_triangular(UPLO::Lower);
    test1d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_2d_upper() {
    let b: Array2<f64> = random((3, 4));
    let a: Array2<f64> = random((3, 3)).into_triangular(UPLO::Upper);
    test2d(UPLO::Upper, a, b, 1e-7);
}

#[test]
fn triangular_2d_lower() {
    let b: Array2<f64> = random((3, 4));
    let a: Array2<f64> = random((3, 3)).into_triangular(UPLO::Lower);
    test2d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_2d_lower_t() {
    let b: Array2<f64> = random((3, 4));
    let a: Array2<f64> = random((3, 3).f()).into_triangular(UPLO::Lower);
    test2d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_2d_upper_t() {
    let b: Array2<f64> = random((3, 4));
    let a: Array2<f64> = random((3, 3).f()).into_triangular(UPLO::Upper);
    test2d(UPLO::Upper, a, b, 1e-7);
}

#[test]
fn triangular_2d_upper_bt() {
    let b: Array2<f64> = random((3, 4).f());
    let a: Array2<f64> = random((3, 3)).into_triangular(UPLO::Upper);
    test2d(UPLO::Upper, a, b, 1e-7);
}

#[test]
fn triangular_2d_lower_bt() {
    let b: Array2<f64> = random((3, 4).f());
    let a: Array2<f64> = random((3, 3)).into_triangular(UPLO::Lower);
    test2d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_2d_lower_t_bt() {
    let b: Array2<f64> = random((3, 4).f());
    let a: Array2<f64> = random((3, 3).f()).into_triangular(UPLO::Lower);
    test2d(UPLO::Lower, a, b, 1e-7);
}

#[test]
fn triangular_2d_upper_t_bt() {
    let b: Array2<f64> = random((3, 4).f());
    let a: Array2<f64> = random((3, 3).f()).into_triangular(UPLO::Upper);
    test2d(UPLO::Upper, a, b, 1e-7);
}
<|endoftext|>


<commit_before>use crate::cell::UnsafeCell;
use crate::sys::condvar::Condvar;
use crate::sys::mutex::Mutex;

pub struct RWLock {
    lock: Mutex,
    cond: Condvar,
    state: UnsafeCell<State>,
}

pub type MovableRWLock = Box<RWLock>;

enum State {
    Unlocked,
    Reading(usize),
    Writing,
}

unsafe impl Send for RWLock {}
unsafe impl Sync for RWLock {}

// This rwlock implementation is a relatively simple implementation which has a
// condition variable for readers/writers as well as a mutex protecting the
// internal state of the lock. A current downside of the implementation is that
// unlocking the lock will notify *all* waiters rather than just readers or just
// writers. This can cause lots of "thundering stampede" problems. While
// hopefully correct this implementation is very likely to want to be changed in
// the future.

impl RWLock {
    pub const fn new() -> RWLock {
        RWLock { lock: Mutex::new(), cond: Condvar::new(), state: UnsafeCell::new(State::Unlocked) }
    }

    #[inline]
    pub unsafe fn read(&self) {
        self.lock.lock();
        while !(*self.state.get()).inc_readers() {
            self.cond.wait(&self.lock);
        }
        self.lock.unlock();
    }

    #[inline]
    pub unsafe fn try_read(&self) -> bool {
        self.lock.lock();
        let ok = (*self.state.get()).inc_readers();
        self.lock.unlock();
        return ok;
    }

    #[inline]
    pub unsafe fn write(&self) {
        self.lock.lock();
        while !(*self.state.get()).inc_writers() {
            self.cond.wait(&self.lock);
        }
        self.lock.unlock();
    }

    #[inline]
    pub unsafe fn try_write(&self) -> bool {
        self.lock.lock();
        let ok = (*self.state.get()).inc_writers();
        self.lock.unlock();
        return ok;
    }

    #[inline]
    pub unsafe fn read_unlock(&self) {
        self.lock.lock();
        let notify = (*self.state.get()).dec_readers();
        self.lock.unlock();
        if notify {
            // FIXME: should only wake up one of these some of the time
            self.cond.notify_all();
        }
    }

    #[inline]
    pub unsafe fn write_unlock(&self) {
        self.lock.lock();
        (*self.state.get()).dec_writers();
        self.lock.unlock();
        // FIXME: should only wake up one of these some of the time
        self.cond.notify_all();
    }

    #[inline]
    pub unsafe fn destroy(&self) {
        self.lock.destroy();
        self.cond.destroy();
    }
}

impl State {
    fn inc_readers(&mut self) -> bool {
        match *self {
            State::Unlocked => {
                *self = State::Reading(1);
                true
            }
            State::Reading(ref mut cnt) => {
                *cnt += 1;
                true
            }
            State::Writing => false,
        }
    }

    fn inc_writers(&mut self) -> bool {
        match *self {
            State::Unlocked => {
                *self = State::Writing;
                true
            }
            State::Reading(_) | State::Writing => false,
        }
    }

    fn dec_readers(&mut self) -> bool {
        let zero = match *self {
            State::Reading(ref mut cnt) => {
                *cnt -= 1;
                *cnt == 0
            }
            State::Unlocked | State::Writing => invalid(),
        };
        if zero {
            *self = State::Unlocked;
        }
        zero
    }

    fn dec_writers(&mut self) {
        match *self {
            State::Writing => {}
            State::Unlocked | State::Reading(_) => invalid(),
        }
        *self = State::Unlocked;
    }
}

fn invalid() -> ! {
    panic!("inconsistent rwlock");
}
<commit_msg>Don't put hermit rwlocks in a box.<commit_after>use crate::cell::UnsafeCell;
use crate::sys::condvar::Condvar;
use crate::sys::mutex::Mutex;

pub struct RWLock {
    lock: Mutex,
    cond: Condvar,
    state: UnsafeCell<State>,
}

pub type MovableRWLock = RWLock;

enum State {
    Unlocked,
    Reading(usize),
    Writing,
}

unsafe impl Send for RWLock {}
unsafe impl Sync for RWLock {}

// This rwlock implementation is a relatively simple implementation which has a
// condition variable for readers/writers as well as a mutex protecting the
// internal state of the lock. A current downside of the implementation is that
// unlocking the lock will notify *all* waiters rather than just readers or just
// writers. This can cause lots of "thundering stampede" problems. While
// hopefully correct this implementation is very likely to want to be changed in
// the future.

impl RWLock {
    pub const fn new() -> RWLock {
        RWLock { lock: Mutex::new(), cond: Condvar::new(), state: UnsafeCell::new(State::Unlocked) }
    }

    #[inline]
    pub unsafe fn read(&self) {
        self.lock.lock();
        while !(*self.state.get()).inc_readers() {
            self.cond.wait(&self.lock);
        }
        self.lock.unlock();
    }

    #[inline]
    pub unsafe fn try_read(&self) -> bool {
        self.lock.lock();
        let ok = (*self.state.get()).inc_readers();
        self.lock.unlock();
        return ok;
    }

    #[inline]
    pub unsafe fn write(&self) {
        self.lock.lock();
        while !(*self.state.get()).inc_writers() {
            self.cond.wait(&self.lock);
        }
        self.lock.unlock();
    }

    #[inline]
    pub unsafe fn try_write(&self) -> bool {
        self.lock.lock();
        let ok = (*self.state.get()).inc_writers();
        self.lock.unlock();
        return ok;
    }

    #[inline]
    pub unsafe fn read_unlock(&self) {
        self.lock.lock();
        let notify = (*self.state.get()).dec_readers();
        self.lock.unlock();
        if notify {
            // FIXME: should only wake up one of these some of the time
            self.cond.notify_all();
        }
    }

    #[inline]
    pub unsafe fn write_unlock(&self) {
        self.lock.lock();
        (*self.state.get()).dec_writers();
        self.lock.unlock();
        // FIXME: should only wake up one of these some of the time
        self.cond.notify_all();
    }

    #[inline]
    pub unsafe fn destroy(&self) {
        self.lock.destroy();
        self.cond.destroy();
    }
}

impl State {
    fn inc_readers(&mut self) -> bool {
        match *self {
            State::Unlocked => {
                *self = State::Reading(1);
                true
            }
            State::Reading(ref mut cnt) => {
                *cnt += 1;
                true
            }
            State::Writing => false,
        }
    }

    fn inc_writers(&mut self) -> bool {
        match *self {
            State::Unlocked => {
                *self = State::Writing;
                true
            }
            State::Reading(_) | State::Writing => false,
        }
    }

    fn dec_readers(&mut self) -> bool {
        let zero = match *self {
            State::Reading(ref mut cnt) => {
                *cnt -= 1;
                *cnt == 0
            }
            State::Unlocked | State::Writing => invalid(),
        };
        if zero {
            *self = State::Unlocked;
        }
        zero
    }

    fn dec_writers(&mut self) {
        match *self {
            State::Writing => {}
            State::Unlocked | State::Reading(_) => invalid(),
        }
        *self = State::Unlocked;
    }
}

fn invalid() -> ! {
    panic!("inconsistent rwlock");
}
<|endoftext|>