<commit_before>use std::io;
use std::ops::{Deref, DerefMut};
use std::sync::Arc;

use futures::{Async, Poll, Stream, Sink, StartSend, AsyncSink};
use futures::sync::BiLock;

use io::Io;

/// A reference counted buffer of bytes.
///
/// An `EasyBuf` is a representation of a byte buffer where sub-slices of it can
/// be handed out efficiently, each with a `'static` lifetime which keeps the
/// data alive. The buffer also supports mutation but may require bytes to be
/// copied to complete the operation.
pub struct EasyBuf {
    buf: Arc<Vec<u8>>,
    start: usize,
    end: usize,
}

/// An RAII object returned from `get_mut` which provides mutable access to the
/// underlying `Vec<u8>`.
pub struct EasyBufMut<'a> {
    buf: &'a mut Vec<u8>,
    end: &'a mut usize,
}

impl EasyBuf {
    /// Creates a new EasyBuf with no data and the default capacity.
    pub fn new() -> EasyBuf {
        EasyBuf::with_capacity(8 * 1024)
    }

    /// Creates a new EasyBuf with `cap` capacity.
    pub fn with_capacity(cap: usize) -> EasyBuf {
        EasyBuf {
            buf: Arc::new(Vec::with_capacity(cap)),
            start: 0,
            end: 0,
        }
    }

    /// Changes the starting index of this window to the index specified.
    ///
    /// Returns the windows back to chain multiple calls to this method.
    ///
    /// # Panics
    ///
    /// This method will panic if `start` is out of bounds for the underlying
    /// slice or if it comes after the `end` configured in this window.
    fn set_start(&mut self, start: usize) -> &mut EasyBuf {
        assert!(start <= self.buf.as_ref().len());
        assert!(start <= self.end);
        self.start = start;
        self
    }

    /// Changes the end index of this window to the index specified.
    ///
    /// Returns the windows back to chain multiple calls to this method.
    ///
    /// # Panics
    ///
    /// This method will panic if `end` is out of bounds for the underlying
    /// slice or if it comes after the `end` configured in this window.
    fn set_end(&mut self, end: usize) -> &mut EasyBuf {
        assert!(end <= self.buf.len());
        assert!(self.start <= end);
        self.end = end;
        self
    }

    /// Returns the number of bytes contained in this `EasyBuf`.
    pub fn len(&self) -> usize {
        self.end - self.start
    }

    /// Returns the inner contents of this `EasyBuf` as a slice.
    pub fn as_slice(&self) -> &[u8] {
        self.as_ref()
    }

    /// Splits the buffer into two at the given index.
    ///
    /// Afterwards `self` contains elements `[0, at)`, and the returned `EasyBuf`
    /// contains elements `[at, len)`.
    ///
    /// This is an O(1) operation that just increases the reference count and
    /// sets a few indexes.
    ///
    /// # Panics
    ///
    /// Panics if `at > len`
    pub fn split_off(&mut self, at: usize) -> EasyBuf {
        let mut other = EasyBuf { buf: self.buf.clone(), ..*self };
        let idx = self.start + at;
        other.set_start(idx);
        self.set_end(idx);
        return other
    }

    /// Splits the buffer into two at the given index.
    ///
    /// Afterwards `self` contains elements `[at, len)`, and the returned `EasyBuf`
    /// contains elements `[0, at)`.
    ///
    /// This is an O(1) operation that just increases the reference count and
    /// sets a few indexes.
    ///
    /// # Panics
    ///
    /// Panics if `at > len`
    pub fn drain_to(&mut self, at: usize) -> EasyBuf {
        let mut other = EasyBuf { buf: self.buf.clone(), ..*self };
        let idx = self.start + at;
        other.set_end(idx);
        self.set_start(idx);
        return other
    }

    /// Returns a mutable reference to the underlying growable buffer of bytes.
    ///
    /// If this `EasyBuf` is the only instance pointing at the underlying buffer
    /// of bytes, a direct mutable reference will be returned. Otherwise the
    /// contents of this `EasyBuf` will be reallocated in a fresh `Vec<u8>`
    /// allocation with the same capacity as this allocation, and that
    /// allocation will be returned.
    ///
    /// This operation **is not O(1)** as it may clone the entire contents of
    /// this buffer.
    ///
    /// The returned `EasyBufMut` type implement `Deref` and `DerefMut` to
    /// `Vec<u8>` can the byte buffer can be manipulated using the standard
    /// `Vec<u8>` methods.
    pub fn get_mut(&mut self) -> EasyBufMut {
        // Fast path if we can get mutable access to our own current
        // buffer.
        //
        // TODO: this should be a match or an if-let
        if Arc::get_mut(&mut self.buf).is_some() {
            let buf = Arc::get_mut(&mut self.buf).unwrap();
            buf.drain(..self.start);
            self.start = 0;
            return EasyBufMut { buf: buf, end: &mut self.end }
        }

        // If we couldn't get access above then we give ourself a new buffer
        // here.
        let mut v = Vec::with_capacity(self.buf.capacity());
        v.extend_from_slice(self.as_ref());
        self.start = 0;
        self.buf = Arc::new(v);
        EasyBufMut {
            buf: Arc::get_mut(&mut self.buf).unwrap(),
            end: &mut self.end,
        }
    }
}

impl AsRef<[u8]> for EasyBuf {
    fn as_ref(&self) -> &[u8] {
        &self.buf[self.start..self.end]
    }
}

impl<'a> Deref for EasyBufMut<'a> {
    type Target = Vec<u8>;

    fn deref(&self) -> &Vec<u8> {
        self.buf
    }
}

impl<'a> DerefMut for EasyBufMut<'a> {
    fn deref_mut(&mut self) -> &mut Vec<u8> {
        self.buf
    }
}

impl<'a> Drop for EasyBufMut<'a> {
    fn drop(&mut self) {
        *self.end = self.buf.len();
    }
}

/// Encoding and decoding of frames via buffers.
///
/// This trait is used when constructing an instance of `Framed`. It provides
/// two types: `In`, for decoded input frames, and `Out`, for outgoing frames
/// that need to be encoded. It also provides methods to actually perform the
/// encoding and decoding, which work with corresponding buffer types.
///
/// The trait itself is implemented on a type that can track state for decoding
/// or encoding, which is particularly useful for streaming parsers. In many
/// cases, though, this type will simply be a unit struct (e.g. `struct
/// HttpCodec`).
pub trait Codec {
    /// The type of decoded frames.
    type In;

    /// The type of frames to be encoded.
    type Out;

    /// Attempts to decode a frame from the provided buffer of bytes.
    ///
    /// This method is called by `Framed` whenever bytes are ready to be parsed.
    /// The provided buffer of bytes is what's been read so far, and this
    /// instance of `Decode` can determine whether an entire frame is in the
    /// buffer and is ready to be returned.
    ///
    /// If an entire frame is available, then this instance will remove those
    /// bytes from the buffer provided and return them as a decoded
    /// frame. Note that removing bytes from the provided buffer doesn't always
    /// necessarily copy the bytes, so this should be an efficient operation in
    /// most circumstances.
    ///
    /// If the bytes look valid, but a frame isn't fully available yet, then
    /// `Ok(None)` is returned. This indicates to the `Framed` instance that
    /// it needs to read some more bytes before calling this method again.
    ///
    /// Finally, if the bytes in the buffer are malformed then an error is
    /// returned indicating why. This informs `Framed` that the stream is now
    /// corrupt and should be terminated.
    fn decode(&mut self, buf: &mut EasyBuf) -> Result<Option<Self::In>, io::Error>;

    /// A default method available to be called when there are no more bytes
    /// available to be read from the underlying I/O.
    ///
    /// This method defaults to calling `decode` and returns an error if
    /// `Ok(None)` is returned. Typically this doesn't need to be implemented
    /// unless the framing protocol differs near the end of the stream.
    fn decode_eof(&mut self, buf: &mut EasyBuf) -> io::Result<Self::In> {
        match try!(self.decode(buf)) {
            Some(frame) => Ok(frame),
            None => Err(io::Error::new(io::ErrorKind::Other,
                                       "bytes remaining on stream")),
        }
    }

    /// Encodes a frame into the buffer provided.
    ///
    /// This method will encode `msg` into the byte buffer provided by `buf`.
    /// The `buf` provided is an internal buffer of the `Framed` instance and
    /// will be written out when possible.
    fn encode(&mut self, msg: Self::Out, buf: &mut Vec<u8>);
}

/// A `Stream` interface to an underlying `Io` object, using the `Decode` trait
/// to decode frames.
pub struct FramedRead<T, C> {
    framed: BiLock<Framed<T, C>>,
}

impl<T: Io, C: Codec> Stream for FramedRead<T, C> {
    type Item = C::In;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Option<C::In>, io::Error> {
        if let Async::Ready(mut guard) = self.framed.poll_lock() {
            guard.poll()
        } else {
            Ok(Async::NotReady)
        }
    }
}

/// A `Sink` interface to an underlying `Io` object, using the `Encode` trait
/// to encode frames.
pub struct FramedWrite<T, C> {
    framed: BiLock<Framed<T, C>>,
}

impl<T: Io, C: Codec> Sink for FramedWrite<T, C> {
    type SinkItem = C::Out;
    type SinkError = io::Error;

    fn start_send(&mut self, item: C::Out) -> StartSend<C::Out, io::Error> {
        if let Async::Ready(mut guard) = self.framed.poll_lock() {
            guard.start_send(item)
        } else {
            Ok(AsyncSink::NotReady(item))
        }
    }

    fn poll_complete(&mut self) -> Poll<(), io::Error> {
        if let Async::Ready(mut guard) = self.framed.poll_lock() {
            guard.poll_complete()
        } else {
            Ok(Async::NotReady)
        }
    }
}

/// A unified `Stream` and `Sink` interface to an underlying `Io` object, using
/// the `Encode` and `Decode` traits to encode and decode frames.
///
/// You can acquire a `Framed` instance by using the `Io::framed` adapter.
pub struct Framed<T, C> {
    upstream: T,
    codec: C,
    eof: bool,
    is_readable: bool,
    rd: EasyBuf,
    wr: Vec<u8>,
}

impl<T: Io, C: Codec> Stream for Framed<T, C> {
    type Item = C::In;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Option<C::In>, io::Error> {
        loop {
            // If the read buffer has any pending data, then it could be
            // possible that `decode` will return a new frame. We leave it to
            // the decoder to optimize detecting that more data is required.
            if self.is_readable {
                if self.eof {
                    if self.rd.len() == 0 {
                        return Ok(None.into())
                    } else {
                        let frame = try!(self.codec.decode_eof(&mut self.rd));
                        return Ok(Async::Ready(Some(frame)))
                    }
                }
                trace!("attempting to decode a frame");
                if let Some(frame) = try!(self.codec.decode(&mut self.rd)) {
                    trace!("frame decoded from buffer");
                    return Ok(Async::Ready(Some(frame)));
                }
                self.is_readable = false;
            }

            assert!(!self.eof);

            // Otherwise, try to read more data and try again
            //
            // TODO: shouldn't read_to_end, that may read a lot
            let before = self.rd.len();
            let ret = self.upstream.read_to_end(&mut self.rd.get_mut());
            match ret {
                Ok(_n) => self.eof = true,
                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
                    if self.rd.len() == before {
                        return Ok(Async::NotReady)
                    }
                }
                Err(e) => return Err(e),
            }
            self.is_readable = true;
        }
    }
}

impl<T: Io, C: Codec> Sink for Framed<T, C> {
    type SinkItem = C::Out;
    type SinkError = io::Error;

    fn start_send(&mut self, item: C::Out) -> StartSend<C::Out, io::Error> {
        // If the buffer is already over 8KiB, then attempt to flush it. If after flushing it's
        // *still* over 8KiB, then apply backpressure (reject the send).
        if self.wr.len() > 8 * 1024 {
            try!(self.poll_complete());
            if self.wr.len() > 8 * 1024 {
                return Ok(AsyncSink::NotReady(item));
            }
        }

        self.codec.encode(item, &mut self.wr);
        Ok(AsyncSink::Ready)
    }

    fn poll_complete(&mut self) -> Poll<(), io::Error> {
        trace!("flushing framed transport");

        while !self.wr.is_empty() {
            trace!("writing; remaining={}", self.wr.len());
            let n = try_nb!(self.upstream.write(&self.wr));
            if n == 0 {
                return Err(io::Error::new(io::ErrorKind::WriteZero,
                                          "failed to write frame to transport"));
            }
            self.wr.drain(..n);
        }

        // Try flushing the underlying IO
        try_nb!(self.upstream.flush());

        trace!("framed transport flushed");
        return Ok(Async::Ready(()));
    }
}

pub fn framed<T, C>(io: T, codec: C) -> Framed<T, C> {
    Framed {
        upstream: io,
        codec: codec,
        eof: false,
        is_readable: false,
        rd: EasyBuf::new(),
        wr: Vec::with_capacity(8 * 1024),
    }
}

impl<T, C> Framed<T, C> {
    /// Splits this `Stream + Sink` object into separate `Stream` and `Sink`
    /// objects, which can be useful when you want to split ownership between
    /// tasks, or allow direct interaction between the two objects (e.g. via
    /// `Sink::send_all`).
    pub fn split(self) -> (FramedRead<T, C>, FramedWrite<T, C>) {
        let (a, b) = BiLock::new(self);
        let read = FramedRead { framed: a };
        let write = FramedWrite { framed: b };
        (read, write)
    }

    /// Returns a reference to the underlying I/O stream wrapped by `Framed`.
    ///
    /// Note that care should be taken to not tamper with the underlying stream
    /// of data coming in as it may corrupt the stream of frames otherwise being
    /// worked with.
    pub fn get_ref(&self) -> &T {
        &self.upstream
    }

    /// Returns a mutable reference to the underlying I/O stream wrapped by
    /// `Framed`.
    ///
    /// Note that care should be taken to not tamper with the underlying stream
    /// of data coming in as it may corrupt the stream of frames otherwise being
    /// worked with.
    pub fn get_mut(&mut self) -> &mut T {
        &mut self.upstream
    }

    /// Consumes the `Framed`, returning its underlying I/O stream.
    ///
    /// Note that care should be taken to not tamper with the underlying stream
    /// of data coming in as it may corrupt the stream of frames otherwise being
    /// worked with.
    pub fn into_inner(self) -> T {
        self.upstream
    }
}
<commit_msg>Updated 'encode' return value to Result, removed parameter passing buffer<commit_after>use std::io;
use std::ops::{Deref, DerefMut};
use std::sync::Arc;

use futures::{Async, Poll, Stream, Sink, StartSend, AsyncSink};
use futures::sync::BiLock;

use io::Io;

/// A reference counted buffer of bytes.
///
/// An `EasyBuf` is a representation of a byte buffer where sub-slices of it can
/// be handed out efficiently, each with a `'static` lifetime which keeps the
/// data alive. The buffer also supports mutation but may require bytes to be
/// copied to complete the operation.
pub struct EasyBuf {
    buf: Arc<Vec<u8>>,
    start: usize,
    end: usize,
}

/// An RAII object returned from `get_mut` which provides mutable access to the
/// underlying `Vec<u8>`.
pub struct EasyBufMut<'a> {
    buf: &'a mut Vec<u8>,
    end: &'a mut usize,
}

impl EasyBuf {
    /// Creates a new EasyBuf with no data and the default capacity.
    pub fn new() -> EasyBuf {
        EasyBuf::with_capacity(8 * 1024)
    }

    /// Creates a new EasyBuf with `cap` capacity.
    pub fn with_capacity(cap: usize) -> EasyBuf {
        EasyBuf {
            buf: Arc::new(Vec::with_capacity(cap)),
            start: 0,
            end: 0,
        }
    }

    /// Changes the starting index of this window to the index specified.
    ///
    /// Returns the windows back to chain multiple calls to this method.
    ///
    /// # Panics
    ///
    /// This method will panic if `start` is out of bounds for the underlying
    /// slice or if it comes after the `end` configured in this window.
    fn set_start(&mut self, start: usize) -> &mut EasyBuf {
        assert!(start <= self.buf.as_ref().len());
        assert!(start <= self.end);
        self.start = start;
        self
    }

    /// Changes the end index of this window to the index specified.
    ///
    /// Returns the windows back to chain multiple calls to this method.
    ///
    /// # Panics
    ///
    /// This method will panic if `end` is out of bounds for the underlying
    /// slice or if it comes after the `end` configured in this window.
    fn set_end(&mut self, end: usize) -> &mut EasyBuf {
        assert!(end <= self.buf.len());
        assert!(self.start <= end);
        self.end = end;
        self
    }

    /// Returns the number of bytes contained in this `EasyBuf`.
    pub fn len(&self) -> usize {
        self.end - self.start
    }

    /// Returns the inner contents of this `EasyBuf` as a slice.
    pub fn as_slice(&self) -> &[u8] {
        self.as_ref()
    }

    /// Splits the buffer into two at the given index.
    ///
    /// Afterwards `self` contains elements `[0, at)`, and the returned `EasyBuf`
    /// contains elements `[at, len)`.
    ///
    /// This is an O(1) operation that just increases the reference count and
    /// sets a few indexes.
    ///
    /// # Panics
    ///
    /// Panics if `at > len`
    pub fn split_off(&mut self, at: usize) -> EasyBuf {
        let mut other = EasyBuf { buf: self.buf.clone(), ..*self };
        let idx = self.start + at;
        other.set_start(idx);
        self.set_end(idx);
        return other
    }

    /// Splits the buffer into two at the given index.
    ///
    /// Afterwards `self` contains elements `[at, len)`, and the returned `EasyBuf`
    /// contains elements `[0, at)`.
    ///
    /// This is an O(1) operation that just increases the reference count and
    /// sets a few indexes.
    ///
    /// # Panics
    ///
    /// Panics if `at > len`
    pub fn drain_to(&mut self, at: usize) -> EasyBuf {
        let mut other = EasyBuf { buf: self.buf.clone(), ..*self };
        let idx = self.start + at;
        other.set_end(idx);
        self.set_start(idx);
        return other
    }

    /// Returns a mutable reference to the underlying growable buffer of bytes.
    ///
    /// If this `EasyBuf` is the only instance pointing at the underlying buffer
    /// of bytes, a direct mutable reference will be returned. Otherwise the
    /// contents of this `EasyBuf` will be reallocated in a fresh `Vec<u8>`
    /// allocation with the same capacity as this allocation, and that
    /// allocation will be returned.
    ///
    /// This operation **is not O(1)** as it may clone the entire contents of
    /// this buffer.
    ///
    /// The returned `EasyBufMut` type implement `Deref` and `DerefMut` to
    /// `Vec<u8>` can the byte buffer can be manipulated using the standard
    /// `Vec<u8>` methods.
    pub fn get_mut(&mut self) -> EasyBufMut {
        // Fast path if we can get mutable access to our own current
        // buffer.
        //
        // TODO: this should be a match or an if-let
        if Arc::get_mut(&mut self.buf).is_some() {
            let buf = Arc::get_mut(&mut self.buf).unwrap();
            buf.drain(..self.start);
            self.start = 0;
            return EasyBufMut { buf: buf, end: &mut self.end }
        }

        // If we couldn't get access above then we give ourself a new buffer
        // here.
        let mut v = Vec::with_capacity(self.buf.capacity());
        v.extend_from_slice(self.as_ref());
        self.start = 0;
        self.buf = Arc::new(v);
        EasyBufMut {
            buf: Arc::get_mut(&mut self.buf).unwrap(),
            end: &mut self.end,
        }
    }
}

impl AsRef<[u8]> for EasyBuf {
    fn as_ref(&self) -> &[u8] {
        &self.buf[self.start..self.end]
    }
}

impl<'a> Deref for EasyBufMut<'a> {
    type Target = Vec<u8>;

    fn deref(&self) -> &Vec<u8> {
        self.buf
    }
}

impl<'a> DerefMut for EasyBufMut<'a> {
    fn deref_mut(&mut self) -> &mut Vec<u8> {
        self.buf
    }
}

impl<'a> Drop for EasyBufMut<'a> {
    fn drop(&mut self) {
        *self.end = self.buf.len();
    }
}

/// Encoding and decoding of frames via buffers.
///
/// This trait is used when constructing an instance of `Framed`. It provides
/// two types: `In`, for decoded input frames, and `Out`, for outgoing frames
/// that need to be encoded. It also provides methods to actually perform the
/// encoding and decoding, which work with corresponding buffer types.
///
/// The trait itself is implemented on a type that can track state for decoding
/// or encoding, which is particularly useful for streaming parsers. In many
/// cases, though, this type will simply be a unit struct (e.g. `struct
/// HttpCodec`).
pub trait Codec {
    /// The type of decoded frames.
    type In;

    /// The type of frames to be encoded.
    type Out;

    /// Attempts to decode a frame from the provided buffer of bytes.
    ///
    /// This method is called by `Framed` whenever bytes are ready to be parsed.
    /// The provided buffer of bytes is what's been read so far, and this
    /// instance of `Decode` can determine whether an entire frame is in the
    /// buffer and is ready to be returned.
    ///
    /// If an entire frame is available, then this instance will remove those
    /// bytes from the buffer provided and return them as a decoded
    /// frame. Note that removing bytes from the provided buffer doesn't always
    /// necessarily copy the bytes, so this should be an efficient operation in
    /// most circumstances.
    ///
    /// If the bytes look valid, but a frame isn't fully available yet, then
    /// `Ok(None)` is returned. This indicates to the `Framed` instance that
    /// it needs to read some more bytes before calling this method again.
    ///
    /// Finally, if the bytes in the buffer are malformed then an error is
    /// returned indicating why. This informs `Framed` that the stream is now
    /// corrupt and should be terminated.
    fn decode(&mut self, buf: &mut EasyBuf) -> Result<Option<Self::In>, io::Error>;

    /// A default method available to be called when there are no more bytes
    /// available to be read from the underlying I/O.
    ///
    /// This method defaults to calling `decode` and returns an error if
    /// `Ok(None)` is returned. Typically this doesn't need to be implemented
    /// unless the framing protocol differs near the end of the stream.
    fn decode_eof(&mut self, buf: &mut EasyBuf) -> io::Result<Self::In> {
        match try!(self.decode(buf)) {
            Some(frame) => Ok(frame),
            None => Err(io::Error::new(io::ErrorKind::Other,
                                       "bytes remaining on stream")),
        }
    }

    /// Encodes a frame into the buffer provided.
    ///
    /// This method will encode `msg` into the byte buffer provided by `buf`.
    /// The `buf` provided is an internal buffer of the `Framed` instance and
    /// will be written out when possible.
    fn encode(&mut self, msg: Self::Out) -> Result<Vec<u8>, io::Error>;
}

/// A `Stream` interface to an underlying `Io` object, using the `Decode` trait
/// to decode frames.
pub struct FramedRead<T, C> {
    framed: BiLock<Framed<T, C>>,
}

impl<T: Io, C: Codec> Stream for FramedRead<T, C> {
    type Item = C::In;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Option<C::In>, io::Error> {
        if let Async::Ready(mut guard) = self.framed.poll_lock() {
            guard.poll()
        } else {
            Ok(Async::NotReady)
        }
    }
}

/// A `Sink` interface to an underlying `Io` object, using the `Encode` trait
/// to encode frames.
pub struct FramedWrite<T, C> {
    framed: BiLock<Framed<T, C>>,
}

impl<T: Io, C: Codec> Sink for FramedWrite<T, C> {
    type SinkItem = C::Out;
    type SinkError = io::Error;

    fn start_send(&mut self, item: C::Out) -> StartSend<C::Out, io::Error> {
        if let Async::Ready(mut guard) = self.framed.poll_lock() {
            guard.start_send(item)
        } else {
            Ok(AsyncSink::NotReady(item))
        }
    }

    fn poll_complete(&mut self) -> Poll<(), io::Error> {
        if let Async::Ready(mut guard) = self.framed.poll_lock() {
            guard.poll_complete()
        } else {
            Ok(Async::NotReady)
        }
    }
}

/// A unified `Stream` and `Sink` interface to an underlying `Io` object, using
/// the `Encode` and `Decode` traits to encode and decode frames.
///
/// You can acquire a `Framed` instance by using the `Io::framed` adapter.
pub struct Framed<T, C> {
    upstream: T,
    codec: C,
    eof: bool,
    is_readable: bool,
    rd: EasyBuf,
    wr: Vec<u8>,
}

impl<T: Io, C: Codec> Stream for Framed<T, C> {
    type Item = C::In;
    type Error = io::Error;

    fn poll(&mut self) -> Poll<Option<C::In>, io::Error> {
        loop {
            // If the read buffer has any pending data, then it could be
            // possible that `decode` will return a new frame. We leave it to
            // the decoder to optimize detecting that more data is required.
            if self.is_readable {
                if self.eof {
                    if self.rd.len() == 0 {
                        return Ok(None.into())
                    } else {
                        let frame = try!(self.codec.decode_eof(&mut self.rd));
                        return Ok(Async::Ready(Some(frame)))
                    }
                }
                trace!("attempting to decode a frame");
                if let Some(frame) = try!(self.codec.decode(&mut self.rd)) {
                    trace!("frame decoded from buffer");
                    return Ok(Async::Ready(Some(frame)));
                }
                self.is_readable = false;
            }

            assert!(!self.eof);

            // Otherwise, try to read more data and try again
            //
            // TODO: shouldn't read_to_end, that may read a lot
            let before = self.rd.len();
            let ret = self.upstream.read_to_end(&mut self.rd.get_mut());
            match ret {
                Ok(_n) => self.eof = true,
                Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
                    if self.rd.len() == before {
                        return Ok(Async::NotReady)
                    }
                }
                Err(e) => return Err(e),
            }
            self.is_readable = true;
        }
    }
}

impl<T: Io, C: Codec> Sink for Framed<T, C> {
    type SinkItem = C::Out;
    type SinkError = io::Error;

    fn start_send(&mut self, item: C::Out) -> StartSend<C::Out, io::Error> {
        // If the buffer is already over 8KiB, then attempt to flush it. If after flushing it's
        // *still* over 8KiB, then apply backpressure (reject the send).
        if self.wr.len() > 8 * 1024 {
            try!(self.poll_complete());
            if self.wr.len() > 8 * 1024 {
                return Ok(AsyncSink::NotReady(item));
            }
        }

        self.wr = self.codec.encode(item).unwrap();
        Ok(AsyncSink::Ready)
    }

    fn poll_complete(&mut self) -> Poll<(), io::Error> {
        trace!("flushing framed transport");

        while !self.wr.is_empty() {
            trace!("writing; remaining={}", self.wr.len());
            let n = try_nb!(self.upstream.write(&self.wr));
            if n == 0 {
                return Err(io::Error::new(io::ErrorKind::WriteZero,
                                          "failed to write frame to transport"));
            }
            self.wr.drain(..n);
        }

        // Try flushing the underlying IO
        try_nb!(self.upstream.flush());

        trace!("framed transport flushed");
        return Ok(Async::Ready(()));
    }
}

pub fn framed<T, C>(io: T, codec: C) -> Framed<T, C> {
    Framed {
        upstream: io,
        codec: codec,
        eof: false,
        is_readable: false,
        rd: EasyBuf::new(),
        wr: Vec::with_capacity(8 * 1024),
    }
}

impl<T, C> Framed<T, C> {
    /// Splits this `Stream + Sink` object into separate `Stream` and `Sink`
    /// objects, which can be useful when you want to split ownership between
    /// tasks, or allow direct interaction between the two objects (e.g. via
    /// `Sink::send_all`).
    pub fn split(self) -> (FramedRead<T, C>, FramedWrite<T, C>) {
        let (a, b) = BiLock::new(self);
        let read = FramedRead { framed: a };
        let write = FramedWrite { framed: b };
        (read, write)
    }

    /// Returns a reference to the underlying I/O stream wrapped by `Framed`.
    ///
    /// Note that care should be taken to not tamper with the underlying stream
    /// of data coming in as it may corrupt the stream of frames otherwise being
    /// worked with.
    pub fn get_ref(&self) -> &T {
        &self.upstream
    }

    /// Returns a mutable reference to the underlying I/O stream wrapped by
    /// `Framed`.
    ///
    /// Note that care should be taken to not tamper with the underlying stream
    /// of data coming in as it may corrupt the stream of frames otherwise being
    /// worked with.
    pub fn get_mut(&mut self) -> &mut T {
        &mut self.upstream
    }

    /// Consumes the `Framed`, returning its underlying I/O stream.
    ///
    /// Note that care should be taken to not tamper with the underlying stream
    /// of data coming in as it may corrupt the stream of frames otherwise being
    /// worked with.
    pub fn into_inner(self) -> T {
        self.upstream
    }
}
<|endoftext|>


<commit_before>//! Page management.
//!
//! Pages are virtual data units of size 4088 bytes. They're represented on disk somewhat
//! non-obviously, since clusters can hold more than one page at once (compression). Every cluster
//! will maximize the number of pages held and when it's filled up, a new cluster will be fetched.

struct Metacluster {
    free: Vec<cluster::Pointer>,
}

enum Error {
    OutOfClusters,
    Disk(disk::Error),
}

/// A state of a page manager.
struct State {
    /// The state block.
    ///
    /// The state block stores the state of the file system including allocation state,
    /// configuration, and more.
    state_block: state_block::StateBlock,
    /// The head of the freelist.
    ///
    /// This list is used as the allocation primitive of TFS. It is a simple freelist-based extent
    /// allocation system, but there is one twist: To optimize the data locality, the list is
    /// unrolled.
    freelist_head: Metacluster,
}

/// The page manager.
///
/// This is the center point of the I/O stack, providing allocation, deallocation, compression,
/// etc. It manages the clusters (with the page abstraction) and caches the disks.
struct Manager<D> {
    /// The cached disk header.
    ///
    /// The disk header contains various very basic information about the disk and how to interact
    /// with it.
    ///
    /// In reality, we could fetch this from the `disk` field as-we-go, but that hurts performance,
    /// so we cache it in memory.
    header: header::DiskHeader,
    /// The inner disk.
    disk: Cache<D>,
    /// The state of the manager.
    state: State,
    /// The state of the manager on the time of last cache commit.
    ///
    /// This contains the state of the page manager upon the last cache commit (pipeline flush). It
    /// is used to roll back the page manager when an error occurs.
    committed_state: State,
}

impl<D: Disk> Manager<D> {
    fn commit(&mut self) {
        // Update the stored committed state to the current state, which we will commit.
        self.committed_state = self.state.clone();
        // Commit the cache pipeline.
        self.disk.commit();
    }

    fn revert(&mut self) {
        // Revert the state to when it was committed last time.
        self.state = self.committed_state.clone();
        // Revert the cache pipeline.
        self.disk.revert();
    }

    fn queue_freelist_pop(&mut self) -> Result<cluster::Pointer, Error> {
        // Pop from the metacluster.
        if let Some(cluster) = self.freelist_head.free.pop() {
            if self.freelist.head.free.is_empty() {
                // The head metacluster is exhausted, so we load the next metacluster (specified to be
                // the last pointer in the metacluster), i.e. `cluster`. The old metacluster is then
                // used as the popped cluster.
                mem::swap(&mut self.state.state_block.cluster, &mut cluster);
                self.state.freelist_head.update(self.disk.read(self.state.freelist_head.cluster)?);

                // We've updated the state block, so we queue a flush to the disk.
                self.queue_state_block_flush();
            } else {
                // Since the freelist head was changed after the pop, we queue a flush.
                self.queue_freelist_head_flush();
            }

            Ok(cluster)
        } else {
            // We ran out of clusters :(.
            Err(Error::OutOfClusters)
        }
    }

    fn queue_freelist_push(&mut self, cluster: cluster::Pointer) -> Result<(), Error> {
        if self.state.freelist_head.free.len() == cluster::SIZE / cluster::POINTER_SIZE {
            // The freelist head is full, and therefore we use following algorithm:
            //
            // 1. Create a new metacluster at `cluster`.
            // 2. Link said metacluster to the old metacluster.
            // 3. Queue a flush.

            // Clear the in-memory freelist head mirror.
            self.state.freelist_head.free.clear();
            // Put the link to the old freelist head into the new metacluster.
            self.state.freelist_head.push(state_block.freelist_head);

            // Update the freelist head pointer to point to the new metacluster.
            self.state.state_block.freelist_head = cluster;
            // Queue a flush of the new freelist head. This won't leave the system in an
            // inconsistent state as it merely creates a new metacluster, which is first linked
            // later. If the state block flush fails, the metacluster will merely be an orphan
            // cluster, and therefore simply leaked space.
            self.queue_freelist_head_flush();
            // Queue a flush of the state block (or, in particular, the freelist head pointer).
            // This is completely consistent as the freelist head must flush before, thus rendering
            // the pointed cluster a valid metacluster.
            self.queue_state_block_flush();
        } else {
            // There is space for more clusters in the head metacluster.

            // Push the cluster pointer to the freelist head.
            self.state.freelist_head.free.push(cluster);
            // Queue a flush of the new freelist head.
            self.queue_freelist_head_flush();
        }
    }
}
<commit_msg>Add documentation to the page management.<commit_after>//! Page management.
//!
//! Pages are virtual data units of size 4088 bytes. They're represented on disk somewhat
//! non-obviously, since clusters can hold more than one page at once (compression). Every cluster
//! will maximize the number of pages held and when it's filled up, a new cluster will be fetched.

/// A metacluster.
///
/// Metaclusters holds some number of links to other clusters, forming a link in an unrolled
/// freelist.
struct Metacluster {
    /// The free clusters in this metacluster.
    ///
    /// If any, the bottom pointer points to another metacluster.
    free: Vec<cluster::Pointer>,
}

/// A page management error.
enum Error {
    /// No clusters left in the freelist.
    ///
    /// This is the equivalent to OOM, but with disk space.
    OutOfClusters,
    /// A disk error.
    Disk(disk::Error),
}

/// A state of a page manager.
struct State {
    /// The state block.
    ///
    /// The state block stores the state of the file system including allocation state,
    /// configuration, and more.
    state_block: state_block::StateBlock,
    /// The head of the freelist.
    ///
    /// This list is used as the allocation primitive of TFS. It is a simple freelist-based extent
    /// allocation system, but there is one twist: To optimize the data locality, the list is
    /// unrolled.
    freelist_head: Metacluster,
}

/// The page manager.
///
/// This is the center point of the I/O stack, providing allocation, deallocation, compression,
/// etc. It manages the clusters (with the page abstraction) and caches the disks.
struct Manager<D> {
    /// The cached disk header.
    ///
    /// The disk header contains various very basic information about the disk and how to interact
    /// with it.
    ///
    /// In reality, we could fetch this from the `disk` field as-we-go, but that hurts performance,
    /// so we cache it in memory.
    header: header::DiskHeader,
    /// The inner disk.
    disk: Cache<D>,
    /// The state of the manager.
    state: State,
    /// The state of the manager on the time of last cache commit.
    ///
    /// This contains the state of the page manager upon the last cache commit (pipeline flush). It
    /// is used to roll back the page manager when an error occurs.
    committed_state: State,
}

impl<D: Disk> Manager<D> {
    /// Commit the transactions in the pipeline to the cache.
    ///
    /// This runs over the transactions in the pipeline and applies them to the cache. In a sense,
    /// it can be seen as a form of checkpoint as you can revert to the last commit through
    /// `.revert()`, as it stores the old state.
    fn commit(&mut self) {
        // Update the stored committed state to the current state, which we will commit.
        self.committed_state = self.state.clone();
        // Commit the cache pipeline.
        self.disk.commit();
    }

    /// Revert to the last commit.
    ///
    /// This will reset the state to after the previous cache commit.
    fn revert(&mut self) {
        // Revert the state to when it was committed last time.
        self.state = self.committed_state.clone();
        // Revert the cache pipeline.
        self.disk.revert();
    }

    /// Queue a pop from the freelist.
    ///
    /// This adds a new transaction to the cache pipeline, which will pop from the top of the
    /// freelist and return the result.
    fn queue_freelist_pop(&mut self) -> Result<cluster::Pointer, Error> {
        // Pop from the metacluster.
        if let Some(cluster) = self.freelist_head.free.pop() {
            if self.freelist.head.free.is_empty() {
                // The head metacluster is exhausted, so we load the next metacluster (specified to be
                // the last pointer in the metacluster), i.e. `cluster`. The old metacluster is then
                // used as the popped cluster.
                mem::swap(&mut self.state.state_block.cluster, &mut cluster);
                self.state.freelist_head.update(self.disk.read(self.state.freelist_head.cluster)?);

                // We've updated the state block, so we queue a flush to the disk.
                self.queue_state_block_flush();
            } else {
                // Since the freelist head was changed after the pop, we queue a flush.
                self.queue_freelist_head_flush();
            }

            Ok(cluster)
        } else {
            // We ran out of clusters :(.
            Err(Error::OutOfClusters)
        }
    }

    /// Queue a push to the freelist.
    ///
    /// This adds a new transaction to the cache pipeline, which will push some free cluster to the
    /// top of the freelist.
    fn queue_freelist_push(&mut self, cluster: cluster::Pointer) -> Result<(), Error> {
        if self.state.freelist_head.free.len() == cluster::SIZE / cluster::POINTER_SIZE {
            // The freelist head is full, and therefore we use following algorithm:
            //
            // 1. Create a new metacluster at `cluster`.
            // 2. Link said metacluster to the old metacluster.
            // 3. Queue a flush.

            // Clear the in-memory freelist head mirror.
            self.state.freelist_head.free.clear();
            // Put the link to the old freelist head into the new metacluster.
            self.state.freelist_head.push(state_block.freelist_head);

            // Update the freelist head pointer to point to the new metacluster.
            self.state.state_block.freelist_head = cluster;
            // Queue a flush of the new freelist head. This won't leave the system in an
            // inconsistent state as it merely creates a new metacluster, which is first linked
            // later. If the state block flush fails, the metacluster will merely be an orphan
            // cluster, and therefore simply leaked space.
            self.queue_freelist_head_flush();
            // Queue a flush of the state block (or, in particular, the freelist head pointer).
            // This is completely consistent as the freelist head must flush before, thus rendering
            // the pointed cluster a valid metacluster.
            self.queue_state_block_flush();
        } else {
            // There is space for more clusters in the head metacluster.

            // Push the cluster pointer to the freelist head.
            self.state.freelist_head.free.push(cluster);
            // Queue a flush of the new freelist head.
            self.queue_freelist_head_flush();

            // lulz @ these comments. like shit, ticki, they add basically nothing you fuking dumb
            // monkey. seriously stop it
        }
    }
}
<|endoftext|>


<commit_before>/// A value::Value literal.
///
/// # Example
///
/// ```rust
/// # use liquid_value::ValueView;
/// #
/// # fn main() {
/// liquid_value::value!(5)
///     .as_scalar().unwrap()
///     .to_integer().unwrap();
/// liquid_value::value!("foo")
///     .as_scalar().unwrap()
///     .to_kstr();
/// liquid_value::value!([1, 2, 3])
///     .as_array().unwrap();
/// liquid_value::value!({"foo": 5})
///     .as_object().unwrap();
/// # }
/// ```
#[macro_export(local_inner_macros)]
macro_rules! value {
    ($($value:tt)+) => {
        value_internal!($($value)+)
    };
}

#[macro_export(local_inner_macros)]
#[doc(hidden)]
macro_rules! value_internal {
    // Done with trailing comma.
    (@array [$($elems:expr,)*]) => {
        value_internal_vec![$($elems,)*]
    };

    // Done without trailing comma.
    (@array [$($elems:expr),*]) => {
        value_internal_vec![$($elems),*]
    };

    // Next element is `nil`.
    (@array [$($elems:expr,)*] nil $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!(nil)] $($rest)*)
    };

    // Next element is `true`.
    (@array [$($elems:expr,)*] true $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!(true)] $($rest)*)
    };

    // Next element is `false`.
    (@array [$($elems:expr,)*] false $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!(false)] $($rest)*)
    };

    // Next element is an array.
    (@array [$($elems:expr,)*] [$($array:tt)*] $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!([$($array)*])] $($rest)*)
    };

    // Next element is a map.
    (@array [$($elems:expr,)*] {$($map:tt)*} $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!({$($map)*})] $($rest)*)
    };

    // Next element is an expression followed by comma.
    (@array [$($elems:expr,)*] $next:expr, $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!($next),] $($rest)*)
    };

    // Last element is an expression with no trailing comma.
    (@array [$($elems:expr,)*] $last:expr) => {
        value_internal!(@array [$($elems,)* value_internal!($last)])
    };

    // Comma after the most recent element.
    (@array [$($elems:expr),*] , $($rest:tt)*) => {
        value_internal!(@array [$($elems,)*] $($rest)*)
    };

    // Unexpected token after most recent element.
    (@array [$($elems:expr),*] $unexpected:tt $($rest:tt)*) => {
        value_unexpected!($unexpected)
    };

    //////////////////////////////////////////////////////////////////////////
    // TT muncher for parsing the inside of an object {...}. Each entry is
    // inserted into the given map variable.
    //
    // Must be invoked as: value_internal!(@object $map () ($($tt)*) ($($tt)*))
    //
    // We require two copies of the input tokens so that we can match on one
    // copy and trigger errors on the other copy.
    //////////////////////////////////////////////////////////////////////////

    // Done.
    (@object $object:ident () () ()) => {};

    // Insert the current entry followed by trailing comma.
    (@object $object:ident [$($key:tt)+] ($value:expr) , $($rest:tt)*) => {
        let _ = $object.insert(($($key)+).into(), $value);
        value_internal!(@object $object () ($($rest)*) ($($rest)*));
    };

    // Current entry followed by unexpected token.
    (@object $object:ident [$($key:tt)+] ($value:expr) $unexpected:tt $($rest:tt)*) => {
        value_unexpected!($unexpected);
    };

    // Insert the last entry without trailing comma.
    (@object $object:ident [$($key:tt)+] ($value:expr)) => {
        let _ = $object.insert(($($key)+).into(), $value);
    };

    // Next value is `nil`.
    (@object $object:ident ($($key:tt)+) (: nil $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!(nil)) $($rest)*);
    };

    // Next value is `true`.
    (@object $object:ident ($($key:tt)+) (: true $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!(true)) $($rest)*);
    };

    // Next value is `false`.
    (@object $object:ident ($($key:tt)+) (: false $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!(false)) $($rest)*);
    };

    // Next value is an array.
    (@object $object:ident ($($key:tt)+) (: [$($array:tt)*] $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!([$($array)*])) $($rest)*);
    };

    // Next value is a map.
    (@object $object:ident ($($key:tt)+) (: {$($map:tt)*} $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!({$($map)*})) $($rest)*);
    };

    // Next value is an expression followed by comma.
    (@object $object:ident ($($key:tt)+) (: $value:expr , $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!($value)) , $($rest)*);
    };

    // Last value is an expression with no trailing comma.
    (@object $object:ident ($($key:tt)+) (: $value:expr) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!($value)));
    };

    // Missing value for last entry. Trigger a reasonable error message.
    (@object $object:ident ($($key:tt)+) (:) $copy:tt) => {
        // "unexpected end of macro invocation"
        value_internal!();
    };

    // Missing colon and value for last entry. Trigger a reasonable error
    // message.
    (@object $object:ident ($($key:tt)+) () $copy:tt) => {
        // "unexpected end of macro invocation"
        value_internal!();
    };

    // Misplaced colon. Trigger a reasonable error message.
    (@object $object:ident () (: $($rest:tt)*) ($colon:tt $($copy:tt)*)) => {
        // Takes no arguments so "no rules expected the token `:`".
        value_unexpected!($colon);
    };

    // Found a comma inside a key. Trigger a reasonable error message.
    (@object $object:ident ($($key:tt)*) (, $($rest:tt)*) ($comma:tt $($copy:tt)*)) => {
        // Takes no arguments so "no rules expected the token `,`".
        value_unexpected!($comma);
    };

    // Key is fully parenthesized. This avoids clippy double_parens false
    // positives because the parenthesization may be necessary here.
    (@object $object:ident () (($key:expr) : $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object ($key) (: $($rest)*) (: $($rest)*));
    };

    // Munch a token into the current key.
    (@object $object:ident ($($key:tt)*) ($tt:tt $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object ($($key)* $tt) ($($rest)*) ($($rest)*));
    };

    //////////////////////////////////////////////////////////////////////////
    // The main implementation.
    //
    // Must be invoked as: value_internal!($($value)+)
    //////////////////////////////////////////////////////////////////////////

    (nil) => {
        $crate::Value::Nil
    };

    (true) => {
        $crate::Value::scalar(true)
    };

    (false) => {
        $crate::Value::scalar(false)
    };

    ([]) => {
        $crate::Value::Array(value_internal_vec![])
    };

    ([ $($tt:tt)+ ]) => {
        $crate::Value::Array(value_internal!(@array [] $($tt)+))
    };

    ({}) => {
        $crate::Value::Object(Default::default())
    };

    ({ $($tt:tt)+ }) => {
        $crate::Value::Object({
            let mut object = $crate::Object::new();
            value_internal!(@object object () ($($tt)+) ($($tt)+));
            object
        })
    };

    ($other:ident) => {
        $other
    };

    // Any Serialize type: numbers, strings, struct literals, variables etc.
    // Must be below every other rule.
    ($other:expr) => {
        $crate::to_value($other).unwrap()
    };
}

#[macro_export]
#[doc(hidden)]
macro_rules! value_internal_vec {
    ($($content:tt)*) => {
        vec![$($content)*]
    };
}
<commit_msg>feat(value): Add scalar macro<commit_after>/// A value::Value literal.
///
/// # Example
///
/// ```rust
/// # use liquid_value::ValueView;
/// #
/// # fn main() {
/// liquid_value::value!(5)
///     .as_scalar().unwrap()
///     .to_integer().unwrap();
/// liquid_value::value!("foo")
///     .as_scalar().unwrap()
///     .to_kstr();
/// liquid_value::value!([1, 2, 3])
///     .as_array().unwrap();
/// liquid_value::value!({"foo": 5})
///     .as_object().unwrap();
/// # }
/// ```
#[macro_export(local_inner_macros)]
macro_rules! value {
    ($($value:tt)+) => {
        value_internal!($($value)+)
    };
}

/// A value::Scalar literal.
///
/// # Example
///
/// ```rust
/// # use liquid_value::ValueView;
/// #
/// # fn main() {
/// liquid_value::scalar!(5)
///     .to_integer().unwrap();
/// liquid_value::scalar!("foo")
///     .to_kstr();
/// # }
/// ```
#[macro_export(local_inner_macros)]
macro_rules! scalar {
    ($value:literal) => {
        $crate::Scalar::new($value)
    };

    ($other:ident) => {
        $other
    };

    // Any Serialize type: numbers, strings, struct literals, variables etc.
    // Must be below every other rule.
    ($other:expr) => {
        $crate::to_scalar($other).unwrap()
    };
}

#[macro_export(local_inner_macros)]
#[doc(hidden)]
macro_rules! value_internal {
    // Done with trailing comma.
    (@array [$($elems:expr,)*]) => {
        value_internal_vec![$($elems,)*]
    };

    // Done without trailing comma.
    (@array [$($elems:expr),*]) => {
        value_internal_vec![$($elems),*]
    };

    // Next element is `nil`.
    (@array [$($elems:expr,)*] nil $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!(nil)] $($rest)*)
    };

    // Next element is `true`.
    (@array [$($elems:expr,)*] true $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!(true)] $($rest)*)
    };

    // Next element is `false`.
    (@array [$($elems:expr,)*] false $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!(false)] $($rest)*)
    };

    // Next element is an array.
    (@array [$($elems:expr,)*] [$($array:tt)*] $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!([$($array)*])] $($rest)*)
    };

    // Next element is a map.
    (@array [$($elems:expr,)*] {$($map:tt)*} $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!({$($map)*})] $($rest)*)
    };

    // Next element is an expression followed by comma.
    (@array [$($elems:expr,)*] $next:expr, $($rest:tt)*) => {
        value_internal!(@array [$($elems,)* value_internal!($next),] $($rest)*)
    };

    // Last element is an expression with no trailing comma.
    (@array [$($elems:expr,)*] $last:expr) => {
        value_internal!(@array [$($elems,)* value_internal!($last)])
    };

    // Comma after the most recent element.
    (@array [$($elems:expr),*] , $($rest:tt)*) => {
        value_internal!(@array [$($elems,)*] $($rest)*)
    };

    // Unexpected token after most recent element.
    (@array [$($elems:expr),*] $unexpected:tt $($rest:tt)*) => {
        value_unexpected!($unexpected)
    };

    //////////////////////////////////////////////////////////////////////////
    // TT muncher for parsing the inside of an object {...}. Each entry is
    // inserted into the given map variable.
    //
    // Must be invoked as: value_internal!(@object $map () ($($tt)*) ($($tt)*))
    //
    // We require two copies of the input tokens so that we can match on one
    // copy and trigger errors on the other copy.
    //////////////////////////////////////////////////////////////////////////

    // Done.
    (@object $object:ident () () ()) => {};

    // Insert the current entry followed by trailing comma.
    (@object $object:ident [$($key:tt)+] ($value:expr) , $($rest:tt)*) => {
        let _ = $object.insert(($($key)+).into(), $value);
        value_internal!(@object $object () ($($rest)*) ($($rest)*));
    };

    // Current entry followed by unexpected token.
    (@object $object:ident [$($key:tt)+] ($value:expr) $unexpected:tt $($rest:tt)*) => {
        value_unexpected!($unexpected);
    };

    // Insert the last entry without trailing comma.
    (@object $object:ident [$($key:tt)+] ($value:expr)) => {
        let _ = $object.insert(($($key)+).into(), $value);
    };

    // Next value is `nil`.
    (@object $object:ident ($($key:tt)+) (: nil $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!(nil)) $($rest)*);
    };

    // Next value is `true`.
    (@object $object:ident ($($key:tt)+) (: true $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!(true)) $($rest)*);
    };

    // Next value is `false`.
    (@object $object:ident ($($key:tt)+) (: false $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!(false)) $($rest)*);
    };

    // Next value is an array.
    (@object $object:ident ($($key:tt)+) (: [$($array:tt)*] $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!([$($array)*])) $($rest)*);
    };

    // Next value is a map.
    (@object $object:ident ($($key:tt)+) (: {$($map:tt)*} $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!({$($map)*})) $($rest)*);
    };

    // Next value is an expression followed by comma.
    (@object $object:ident ($($key:tt)+) (: $value:expr , $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!($value)) , $($rest)*);
    };

    // Last value is an expression with no trailing comma.
    (@object $object:ident ($($key:tt)+) (: $value:expr) $copy:tt) => {
        value_internal!(@object $object [$($key)+] (value_internal!($value)));
    };

    // Missing value for last entry. Trigger a reasonable error message.
    (@object $object:ident ($($key:tt)+) (:) $copy:tt) => {
        // "unexpected end of macro invocation"
        value_internal!();
    };

    // Missing colon and value for last entry. Trigger a reasonable error
    // message.
    (@object $object:ident ($($key:tt)+) () $copy:tt) => {
        // "unexpected end of macro invocation"
        value_internal!();
    };

    // Misplaced colon. Trigger a reasonable error message.
    (@object $object:ident () (: $($rest:tt)*) ($colon:tt $($copy:tt)*)) => {
        // Takes no arguments so "no rules expected the token `:`".
        value_unexpected!($colon);
    };

    // Found a comma inside a key. Trigger a reasonable error message.
    (@object $object:ident ($($key:tt)*) (, $($rest:tt)*) ($comma:tt $($copy:tt)*)) => {
        // Takes no arguments so "no rules expected the token `,`".
        value_unexpected!($comma);
    };

    // Key is fully parenthesized. This avoids clippy double_parens false
    // positives because the parenthesization may be necessary here.
    (@object $object:ident () (($key:expr) : $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object ($key) (: $($rest)*) (: $($rest)*));
    };

    // Munch a token into the current key.
    (@object $object:ident ($($key:tt)*) ($tt:tt $($rest:tt)*) $copy:tt) => {
        value_internal!(@object $object ($($key)* $tt) ($($rest)*) ($($rest)*));
    };

    //////////////////////////////////////////////////////////////////////////
    // The main implementation.
    //
    // Must be invoked as: value_internal!($($value)+)
    //////////////////////////////////////////////////////////////////////////

    (nil) => {
        $crate::Value::Nil
    };

    (true) => {
        $crate::Value::scalar(true)
    };

    (false) => {
        $crate::Value::scalar(false)
    };

    ([]) => {
        $crate::Value::Array(value_internal_vec![])
    };

    ([ $($tt:tt)+ ]) => {
        $crate::Value::Array(value_internal!(@array [] $($tt)+))
    };

    ({}) => {
        $crate::Value::Object(Default::default())
    };

    ({ $($tt:tt)+ }) => {
        $crate::Value::Object({
            let mut object = $crate::Object::new();
            value_internal!(@object object () ($($tt)+) ($($tt)+));
            object
        })
    };

    ($other:ident) => {
        $other
    };

    // Any Serialize type: numbers, strings, struct literals, variables etc.
    // Must be below every other rule.
    ($other:expr) => {
        $crate::to_value($other).unwrap()
    };
}

#[macro_export]
#[doc(hidden)]
macro_rules! value_internal_vec {
    ($($content:tt)*) => {
        vec![$($content)*]
    };
}
<|endoftext|>


<commit_before>// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.

use collections::HashMap;
use grpcio::{ChannelBuilder, Environment};
use kvproto::{kvrpcpb::*, metapb, tikvpb::TikvClient};
use test_raftstore::*;
use tikv_util::HandyRwLock;

use std::sync::Arc;

#[test]
fn test_physical_scan_lock() {
    let (_cluster, client, ctx) = must_new_cluster_and_kv_client();

    // Generate kvs like k10, v10, ts=10; k11, v11, ts=11; ...
    let kv: Vec<_> = (10..20)
        .map(|i| (i, vec![b'k', i as u8], vec![b'v', i as u8]))
        .collect();

    for (ts, k, v) in &kv {
        let mut mutation = Mutation::default();
        mutation.set_op(Op::Put);
        mutation.set_key(k.clone());
        mutation.set_value(v.clone());
        must_kv_prewrite(&client, ctx.clone(), vec![mutation], k.clone(), *ts);
    }

    let all_locks: Vec<_> = kv
        .into_iter()
        .map(|(ts, k, _)| {
            // Create a LockInfo that matches the prewrite request in `must_kv_prewrite`.
            let mut lock_info = LockInfo::default();
            lock_info.set_primary_lock(k.clone());
            lock_info.set_lock_version(ts);
            lock_info.set_key(k);
            lock_info.set_lock_ttl(3000);
            lock_info.set_lock_type(Op::Put);
            lock_info.set_min_commit_ts(ts + 1);
            lock_info
        })
        .collect();

    let check_result = |got_locks: &[_], expected_locks: &[_]| {
        for i in 0..std::cmp::max(got_locks.len(), expected_locks.len()) {
            assert_eq!(got_locks[i], expected_locks[i], "lock {} mismatch", i);
        }
    };

    check_result(
        &must_physical_scan_lock(&client, ctx.clone(), 30, b"", 100),
        &all_locks,
    );
    check_result(
        &must_physical_scan_lock(&client, ctx.clone(), 15, b"", 100),
        &all_locks[0..=5],
    );
    check_result(
        &must_physical_scan_lock(&client, ctx.clone(), 10, b"", 100),
        &all_locks[0..1],
    );
    check_result(
        &must_physical_scan_lock(&client, ctx.clone(), 9, b"", 100),
        &[],
    );
    check_result(
        &must_physical_scan_lock(&client, ctx, 30, &[b'k', 13], 5),
        &all_locks[3..8],
    );
}

#[test]
fn test_applied_lock_collector() {
    let mut cluster = new_server_cluster(0, 3);
    cluster.pd_client.disable_default_operator();
    cluster.run();

    // Create all stores' clients.
    let env = Arc::new(Environment::new(1));
    let mut clients = HashMap::default();
    for node_id in cluster.get_node_ids() {
        let channel =
            ChannelBuilder::new(Arc::clone(&env)).connect(&cluster.sim.rl().get_addr(node_id));
        let client = TikvClient::new(channel);
        clients.insert(node_id, client);
    }

    // Create the ctx of the first region.
    let region = cluster.get_region(b"");
    let region_id = region.get_id();
    let leader_peer = cluster.leader_of_region(region_id).unwrap();
    let leader_store_id = leader_peer.get_store_id();
    let leader_client = clients.get(&leader_store_id).unwrap();
    let mut ctx = Context::default();
    ctx.set_region_id(region_id);
    ctx.set_peer(leader_peer);
    ctx.set_region_epoch(cluster.get_region_epoch(region_id));

    // It's used to make sure all stores applies all logs.
    let wait_for_apply = |cluster: &mut Cluster<_>, region: &metapb::Region| {
        let cluster = &mut *cluster;
        region.get_peers().iter().for_each(|p| {
            let resp = async_read_on_peer(cluster, p.clone(), region.clone(), b"key", true, true)
                .recv()
                .unwrap();
            assert!(!resp.get_header().has_error(), "{:?}", resp);
        });
    };

    let check_lock = |lock: &LockInfo, k: &[u8], pk: &[u8], ts| {
        assert_eq!(lock.get_key(), k);
        assert_eq!(lock.get_primary_lock(), pk);
        assert_eq!(lock.get_lock_version(), ts);
    };

    // Register lock observer at safe point 10000.
    let mut safe_point = 10000;
    clients.iter().for_each(|(_, c)| {
        // Should report error when checking non-existent observer.
        assert!(!check_lock_observer(c, safe_point).get_error().is_empty());
        must_register_lock_observer(c, safe_point);
        assert!(must_check_lock_observer(c, safe_point, true).is_empty());
    });

    // Lock observer should only collect values in lock CF.
    let key = b"key0";
    must_kv_prewrite(
        &leader_client,
        ctx.clone(),
        vec![new_mutation(Op::Put, key, &b"v".repeat(1024))],
        key.to_vec(),
        1,
    );
    must_kv_commit(&leader_client, ctx.clone(), vec![key.to_vec()], 1, 2, 2);
    wait_for_apply(&mut cluster, &region);
    clients.iter().for_each(|(_, c)| {
        let locks = must_check_lock_observer(c, safe_point, true);
        assert_eq!(locks.len(), 1);
        check_lock(&locks[0], key, key, 1);
    });

    // Lock observer shouldn't collect locks after the safe point.
    must_kv_prewrite(
        &leader_client,
        ctx.clone(),
        vec![new_mutation(Op::Put, key, b"v")],
        key.to_vec(),
        safe_point + 1,
    );
    wait_for_apply(&mut cluster, &region);
    clients.iter().for_each(|(_, c)| {
        let locks = must_check_lock_observer(c, safe_point, true);
        assert_eq!(locks.len(), 1);
        check_lock(&locks[0], key, key, 1);
    });

    // Write 999 locks whose timestamp is less than the safe point.
    let mutations = (1..1000)
        .map(|i| new_mutation(Op::Put, format!("key{}", i).as_bytes(), b"v"))
        .collect();
    must_kv_prewrite(&leader_client, ctx.clone(), mutations, b"key1".to_vec(), 10);
    wait_for_apply(&mut cluster, &region);
    clients.iter().for_each(|(_, c)| {
        let locks = must_check_lock_observer(c, safe_point, true);
        // Plus the first lock.
        assert_eq!(locks.len(), 1000);
    });

    // Add a new store and register lock observer.
    let store_id = cluster.add_new_engine();
    let channel =
        ChannelBuilder::new(Arc::clone(&env)).connect(&cluster.sim.rl().get_addr(store_id));
    let client = TikvClient::new(channel);
    must_register_lock_observer(&client, safe_point);

    // Add a new peer. Lock observer should collect all locks from snapshot.
    let peer = new_peer(store_id, store_id);
    cluster.pd_client.must_add_peer(region_id, peer.clone());
    cluster.pd_client.must_none_pending_peer(peer);
    wait_for_apply(&mut cluster, &region);
    let locks = must_check_lock_observer(&client, safe_point, true);
    assert_eq!(locks.len(), 999);

    // Should be dirty when collects too many locks.
    let mutations = (1000..1100)
        .map(|i| new_mutation(Op::Put, format!("key{}", i).as_bytes(), b"v"))
        .collect();
    must_kv_prewrite(
        &leader_client,
        ctx.clone(),
        mutations,
        b"key1000".to_vec(),
        100,
    );
    wait_for_apply(&mut cluster, &region);
    clients.insert(store_id, client);
    clients.iter().for_each(|(_, c)| {
        let resp = check_lock_observer(c, safe_point);
        assert!(resp.get_error().is_empty(), "{:?}", resp.get_error());
        assert!(!resp.get_is_clean());
        // MAX_COLLECT_SIZE is 1024.
        assert_eq!(resp.get_locks().len(), 1024);
    });

    // Reregister and check. It shouldn't clean up state.
    clients.iter().for_each(|(_, c)| {
        must_register_lock_observer(c, safe_point);
        let resp = check_lock_observer(c, safe_point);
        assert!(resp.get_error().is_empty(), "{:?}", resp.get_error());
        assert!(!resp.get_is_clean());
        // MAX_COLLECT_SIZE is 1024.
        assert_eq!(resp.get_locks().len(), 1024);
    });

    // Register lock observer at a later safe point. Lock observer should reset its state.
    safe_point += 1;
    clients.iter().for_each(|(_, c)| {
        must_register_lock_observer(c, safe_point);
        assert!(must_check_lock_observer(c, safe_point, true).is_empty());
        // Can't register observer with smaller max_ts.
        assert!(!register_lock_observer(&c, safe_point - 1)
            .get_error()
            .is_empty());
        assert!(must_check_lock_observer(c, safe_point, true).is_empty());
    });
    let leader_client = clients.get(&leader_store_id).unwrap();
    must_kv_prewrite(
        &leader_client,
        ctx,
        vec![new_mutation(Op::Put, b"key1100", b"v")],
        b"key1100".to_vec(),
        safe_point,
    );
    wait_for_apply(&mut cluster, &region);
    clients.iter().for_each(|(_, c)| {
        // Should collect locks according to the new max ts.
        let locks = must_check_lock_observer(c, safe_point, true);
        assert_eq!(locks.len(), 1, "{:?}", locks);
        // Shouldn't remove it with a wrong max ts.
        assert!(!remove_lock_observer(c, safe_point - 1)
            .get_error()
            .is_empty());
        let locks = must_check_lock_observer(c, safe_point, true);
        assert_eq!(locks.len(), 1, "{:?}", locks);
        // Remove lock observers.
        must_remove_lock_observer(c, safe_point);
        assert!(!check_lock_observer(c, safe_point).get_error().is_empty());
    });
}
<commit_msg>test: stablize test_applied_lock_collector (#9503)<commit_after>// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.

use collections::HashMap;
use grpcio::{ChannelBuilder, Environment};
use kvproto::{kvrpcpb::*, metapb, tikvpb::TikvClient};
use test_raftstore::*;
use tikv_util::HandyRwLock;

use std::sync::Arc;

#[test]
fn test_physical_scan_lock() {
    let (_cluster, client, ctx) = must_new_cluster_and_kv_client();

    // Generate kvs like k10, v10, ts=10; k11, v11, ts=11; ...
    let kv: Vec<_> = (10..20)
        .map(|i| (i, vec![b'k', i as u8], vec![b'v', i as u8]))
        .collect();

    for (ts, k, v) in &kv {
        let mut mutation = Mutation::default();
        mutation.set_op(Op::Put);
        mutation.set_key(k.clone());
        mutation.set_value(v.clone());
        must_kv_prewrite(&client, ctx.clone(), vec![mutation], k.clone(), *ts);
    }

    let all_locks: Vec<_> = kv
        .into_iter()
        .map(|(ts, k, _)| {
            // Create a LockInfo that matches the prewrite request in `must_kv_prewrite`.
            let mut lock_info = LockInfo::default();
            lock_info.set_primary_lock(k.clone());
            lock_info.set_lock_version(ts);
            lock_info.set_key(k);
            lock_info.set_lock_ttl(3000);
            lock_info.set_lock_type(Op::Put);
            lock_info.set_min_commit_ts(ts + 1);
            lock_info
        })
        .collect();

    let check_result = |got_locks: &[_], expected_locks: &[_]| {
        for i in 0..std::cmp::max(got_locks.len(), expected_locks.len()) {
            assert_eq!(got_locks[i], expected_locks[i], "lock {} mismatch", i);
        }
    };

    check_result(
        &must_physical_scan_lock(&client, ctx.clone(), 30, b"", 100),
        &all_locks,
    );
    check_result(
        &must_physical_scan_lock(&client, ctx.clone(), 15, b"", 100),
        &all_locks[0..=5],
    );
    check_result(
        &must_physical_scan_lock(&client, ctx.clone(), 10, b"", 100),
        &all_locks[0..1],
    );
    check_result(
        &must_physical_scan_lock(&client, ctx.clone(), 9, b"", 100),
        &[],
    );
    check_result(
        &must_physical_scan_lock(&client, ctx, 30, &[b'k', 13], 5),
        &all_locks[3..8],
    );
}

#[test]
fn test_applied_lock_collector() {
    let mut cluster = new_server_cluster(0, 3);
    cluster.pd_client.disable_default_operator();
    cluster.run();

    // Create all stores' clients.
    let env = Arc::new(Environment::new(1));
    let mut clients = HashMap::default();
    for node_id in cluster.get_node_ids() {
        let channel =
            ChannelBuilder::new(Arc::clone(&env)).connect(&cluster.sim.rl().get_addr(node_id));
        let client = TikvClient::new(channel);
        clients.insert(node_id, client);
    }

    // Create the ctx of the first region.
    let region = cluster.get_region(b"");
    let region_id = region.get_id();
    let leader_peer = cluster.leader_of_region(region_id).unwrap();
    let leader_store_id = leader_peer.get_store_id();
    let leader_client = clients.get(&leader_store_id).unwrap();
    let mut ctx = Context::default();
    ctx.set_region_id(region_id);
    ctx.set_peer(leader_peer);
    ctx.set_region_epoch(cluster.get_region_epoch(region_id));

    // It's used to make sure all stores applies all logs.
    let wait_for_apply = |cluster: &mut Cluster<_>, region: &metapb::Region| {
        let cluster = &mut *cluster;
        region.get_peers().iter().for_each(|p| {
            let mut retry_times = 1;
            loop {
                let resp =
                    async_read_on_peer(cluster, p.clone(), region.clone(), b"key", true, true)
                        .recv()
                        .unwrap();
                if !resp.get_header().has_error() {
                    return;
                }
                if retry_times >= 50 {
                    panic!("failed to read on {:?}: {:?}", p, resp);
                }
                retry_times += 1;
                sleep_ms(20);
            }
        });
    };

    let check_lock = |lock: &LockInfo, k: &[u8], pk: &[u8], ts| {
        assert_eq!(lock.get_key(), k);
        assert_eq!(lock.get_primary_lock(), pk);
        assert_eq!(lock.get_lock_version(), ts);
    };

    // Register lock observer at safe point 10000.
    let mut safe_point = 10000;
    clients.iter().for_each(|(_, c)| {
        // Should report error when checking non-existent observer.
        assert!(!check_lock_observer(c, safe_point).get_error().is_empty());
        must_register_lock_observer(c, safe_point);
        assert!(must_check_lock_observer(c, safe_point, true).is_empty());
    });

    // Lock observer should only collect values in lock CF.
    let key = b"key0";
    must_kv_prewrite(
        &leader_client,
        ctx.clone(),
        vec![new_mutation(Op::Put, key, &b"v".repeat(1024))],
        key.to_vec(),
        1,
    );
    must_kv_commit(&leader_client, ctx.clone(), vec![key.to_vec()], 1, 2, 2);
    wait_for_apply(&mut cluster, &region);
    clients.iter().for_each(|(_, c)| {
        let locks = must_check_lock_observer(c, safe_point, true);
        assert_eq!(locks.len(), 1);
        check_lock(&locks[0], key, key, 1);
    });

    // Lock observer shouldn't collect locks after the safe point.
    must_kv_prewrite(
        &leader_client,
        ctx.clone(),
        vec![new_mutation(Op::Put, key, b"v")],
        key.to_vec(),
        safe_point + 1,
    );
    wait_for_apply(&mut cluster, &region);
    clients.iter().for_each(|(_, c)| {
        let locks = must_check_lock_observer(c, safe_point, true);
        assert_eq!(locks.len(), 1);
        check_lock(&locks[0], key, key, 1);
    });

    // Write 999 locks whose timestamp is less than the safe point.
    let mutations = (1..1000)
        .map(|i| new_mutation(Op::Put, format!("key{}", i).as_bytes(), b"v"))
        .collect();
    must_kv_prewrite(&leader_client, ctx.clone(), mutations, b"key1".to_vec(), 10);
    wait_for_apply(&mut cluster, &region);
    clients.iter().for_each(|(_, c)| {
        let locks = must_check_lock_observer(c, safe_point, true);
        // Plus the first lock.
        assert_eq!(locks.len(), 1000);
    });

    // Add a new store and register lock observer.
    let store_id = cluster.add_new_engine();
    let channel =
        ChannelBuilder::new(Arc::clone(&env)).connect(&cluster.sim.rl().get_addr(store_id));
    let client = TikvClient::new(channel);
    must_register_lock_observer(&client, safe_point);

    // Add a new peer. Lock observer should collect all locks from snapshot.
    let peer = new_peer(store_id, store_id);
    cluster.pd_client.must_add_peer(region_id, peer.clone());
    cluster.pd_client.must_none_pending_peer(peer);
    wait_for_apply(&mut cluster, &region);
    let locks = must_check_lock_observer(&client, safe_point, true);
    assert_eq!(locks.len(), 999);

    // Should be dirty when collects too many locks.
    let mutations = (1000..1100)
        .map(|i| new_mutation(Op::Put, format!("key{}", i).as_bytes(), b"v"))
        .collect();
    must_kv_prewrite(
        &leader_client,
        ctx.clone(),
        mutations,
        b"key1000".to_vec(),
        100,
    );
    wait_for_apply(&mut cluster, &region);
    clients.insert(store_id, client);
    clients.iter().for_each(|(_, c)| {
        let resp = check_lock_observer(c, safe_point);
        assert!(resp.get_error().is_empty(), "{:?}", resp.get_error());
        assert!(!resp.get_is_clean());
        // MAX_COLLECT_SIZE is 1024.
        assert_eq!(resp.get_locks().len(), 1024);
    });

    // Reregister and check. It shouldn't clean up state.
    clients.iter().for_each(|(_, c)| {
        must_register_lock_observer(c, safe_point);
        let resp = check_lock_observer(c, safe_point);
        assert!(resp.get_error().is_empty(), "{:?}", resp.get_error());
        assert!(!resp.get_is_clean());
        // MAX_COLLECT_SIZE is 1024.
        assert_eq!(resp.get_locks().len(), 1024);
    });

    // Register lock observer at a later safe point. Lock observer should reset its state.
    safe_point += 1;
    clients.iter().for_each(|(_, c)| {
        must_register_lock_observer(c, safe_point);
        assert!(must_check_lock_observer(c, safe_point, true).is_empty());
        // Can't register observer with smaller max_ts.
        assert!(!register_lock_observer(&c, safe_point - 1)
            .get_error()
            .is_empty());
        assert!(must_check_lock_observer(c, safe_point, true).is_empty());
    });
    let leader_client = clients.get(&leader_store_id).unwrap();
    must_kv_prewrite(
        &leader_client,
        ctx,
        vec![new_mutation(Op::Put, b"key1100", b"v")],
        b"key1100".to_vec(),
        safe_point,
    );
    wait_for_apply(&mut cluster, &region);
    clients.iter().for_each(|(_, c)| {
        // Should collect locks according to the new max ts.
        let locks = must_check_lock_observer(c, safe_point, true);
        assert_eq!(locks.len(), 1, "{:?}", locks);
        // Shouldn't remove it with a wrong max ts.
        assert!(!remove_lock_observer(c, safe_point - 1)
            .get_error()
            .is_empty());
        let locks = must_check_lock_observer(c, safe_point, true);
        assert_eq!(locks.len(), 1, "{:?}", locks);
        // Remove lock observers.
        must_remove_lock_observer(c, safe_point);
        assert!(!check_lock_observer(c, safe_point).get_error().is_empty());
    });
}
<|endoftext|>


<commit_before>use std::io;
use std::io::{Read};

fn main() {
    let mut string = String::new();
    match io::stdin().read_to_string(&mut string) {
        Err(_) => println!("Problem reading from stdin"),
        Ok(_) => {
            let mut matrix: Vec<Vec<u64>> = Vec::new();
            for row_str in string.split('\n') {
                matrix.push(Vec::new());
                for col_str in row_str.split_whitespace() {
                    match (matrix.last_mut(), col_str.parse::<u64>().ok()) {
                        (Some(row), Some(val)) => row.push(val),
                        _ => ()
                    }
                }
            }
        },
    }
}
<commit_msg>Find max four elements in matrix, solving problem 11<commit_after>use std::io;
use std::io::{Read};

fn max_horiz(matrix: &Vec<Vec<u64>>, size: usize) -> u64 {
    let mut max = 0u64;
    for row in matrix {
        for i in 0..(row.len() - size + 1) {
            let product = (0..size).fold(1, |prod, next| prod * row[i + next]);
            if product > max {
                max = product;
            }
        }
    }
    max
}

fn max_vert(matrix: &Vec<Vec<u64>>, size: usize) -> u64 {
    let mut max = 0u64;
    for (row_idx, row) in matrix.iter().enumerate() {
        if row_idx < matrix.len() - size {
            for col_idx in 0..row.len() {
                let product = (0..size).fold(
                    1, |prod, next| prod * matrix[next + row_idx][col_idx]);
                if product > max {
                    max = product;
                }
            }
        }
    }
    max
}

fn max_desc(matrix: &Vec<Vec<u64>>, size: usize) -> u64 {
    let mut max = 0u64;
    for (row_idx, row) in matrix.iter().enumerate() {
        for (col_idx, col) in row.iter().enumerate() {
            if row_idx < matrix.len() - size
                && col_idx < row.len() - size {
                let product = (0..size).fold(
                    1, |prod, next| prod * matrix[next + row_idx][next + col_idx]);
                if product > max {
                    max = product;
                }
            }
        }
    }
    max
}

fn max_asc(matrix: &Vec<Vec<u64>>, size: usize) -> u64 {
    let mut max = 0u64;
    for (row_idx, row) in matrix.iter().enumerate() {
        for (col_idx, col) in row.iter().enumerate() {
            if row_idx >= size - 1 && col_idx < row.len() - size {
                let product = (0..size).fold(
                    1, |prod, next| prod * matrix[row_idx - next][next + col_idx]);
                if product > max {
                    max = product;
                }
            }
        }
    }
    max
}

fn main() {
    let mut string = String::new();
    match io::stdin().read_to_string(&mut string) {
        Err(_) => println!("Problem reading from stdin"),
        Ok(_) => {
            let problem = string.trim();
            let mut matrix: Vec<Vec<u64>> = Vec::new();
            for row_str in problem.split('\n') {
                matrix.push(Vec::new());
                for col_str in row_str.split_whitespace() {
                    match (matrix.last_mut(), col_str.parse::<u64>().ok()) {
                        (Some(row), Some(val)) => row.push(val),
                        _ => ()
                    }
                }
            };
            println!("{}", max_horiz(&matrix, 4));
            println!("{}", max_vert(&matrix, 4));
            println!("{}", max_asc(&matrix, 4));
            println!("{}", max_desc(&matrix, 4));
        },
    }
}
<|endoftext|>


<commit_before>// Copyright 2018 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use std::fs::File;
use std::io::{Error, Result};
use std::os::unix::io::AsRawFd;

use libc::{lseek64, ENXIO, SEEK_DATA, SEEK_HOLE};

/// A trait for seeking to the next hole or non-hole position in a file.
pub trait SeekHole {
    /// Seek to the first hole in a file at a position greater than or equal to `offset`.
    /// If no holes exist after `offset`, the seek position will be set to the end of the file.
    /// If `offset` is at or after the end of the file, the seek position is unchanged, and None is returned.
    /// Returns the current seek position after the seek or an error.
    fn seek_hole(&mut self, offset: u64) -> Result<Option<u64>>;

    /// Seek to the first data in a file at a position greater than or equal to `offset`.
    /// If no data exists after `offset`, the seek position is unchanged, and None is returned.
    /// Returns the current offset after the seek or an error.
    fn seek_data(&mut self, offset: u64) -> Result<Option<u64>>;
}

/// Safe wrapper for `libc::lseek64()`
fn lseek(file: &mut File, offset: i64, whence: i32) -> Result<Option<u64>> {
    // This is safe because we pass a known-good file descriptor.
    let res = unsafe { lseek64(file.as_raw_fd(), offset, whence) };

    if res < 0 {
        // Convert ENXIO into None; pass any other error as-is.
        let err = Error::last_os_error();
        if let Some(errno) = Error::raw_os_error(&err) {
            if errno == ENXIO {
                return Ok(None);
            }
        }
        Err(err)
    } else {
        Ok(Some(res as u64))
    }
}

impl SeekHole for File {
    fn seek_hole(&mut self, offset: u64) -> Result<Option<u64>> {
        lseek(self, offset as i64, SEEK_HOLE)
    }

    fn seek_data(&mut self, offset: u64) -> Result<Option<u64>> {
        lseek(self, offset as i64, SEEK_DATA)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs::File;
    use std::io::{Seek, SeekFrom, Write};
    use tempfile::tempfile;

    fn seek_cur(file: &mut File) -> u64 {
        file.seek(SeekFrom::Current(0)).unwrap()
    }

    #[test]
    fn seek_data() {
        let mut file = tempfile().unwrap();

        // Empty file
        assert_eq!(file.seek_data(0).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // File with non-zero length consisting entirely of a hole
        file.set_len(0x10000).unwrap();
        assert_eq!(file.seek_data(0).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // seek_data at or after the end of the file should return None
        assert_eq!(file.seek_data(0x10000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);
        assert_eq!(file.seek_data(0x10001).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // Write some data to [0x10000, 0x20000)
        let b = [0x55u8; 0x10000];
        file.seek(SeekFrom::Start(0x10000)).unwrap();
        file.write_all(&b).unwrap();
        assert_eq!(file.seek_data(0).unwrap(), Some(0x10000));
        assert_eq!(seek_cur(&mut file), 0x10000);

        // seek_data within data should return the same offset
        assert_eq!(file.seek_data(0x10000).unwrap(), Some(0x10000));
        assert_eq!(seek_cur(&mut file), 0x10000);
        assert_eq!(file.seek_data(0x10001).unwrap(), Some(0x10001));
        assert_eq!(seek_cur(&mut file), 0x10001);
        assert_eq!(file.seek_data(0x1FFFF).unwrap(), Some(0x1FFFF));
        assert_eq!(seek_cur(&mut file), 0x1FFFF);

        // Extend the file to add another hole after the data
        file.set_len(0x30000).unwrap();
        assert_eq!(file.seek_data(0).unwrap(), Some(0x10000));
        assert_eq!(seek_cur(&mut file), 0x10000);
        assert_eq!(file.seek_data(0x1FFFF).unwrap(), Some(0x1FFFF));
        assert_eq!(seek_cur(&mut file), 0x1FFFF);
        assert_eq!(file.seek_data(0x20000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0x1FFFF);
    }

    #[test]
    fn seek_hole() {
        let mut file = tempfile().unwrap();

        // Empty file
        assert_eq!(file.seek_hole(0).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // File with non-zero length consisting entirely of a hole
        file.set_len(0x10000).unwrap();
        assert_eq!(file.seek_hole(0).unwrap(), Some(0));
        assert_eq!(seek_cur(&mut file), 0);
        assert_eq!(file.seek_hole(0xFFFF).unwrap(), Some(0xFFFF));
        assert_eq!(seek_cur(&mut file), 0xFFFF);

        // seek_hole at or after the end of the file should return None
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x10000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);
        assert_eq!(file.seek_hole(0x10001).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // Write some data to [0x10000, 0x20000)
        let b = [0x55u8; 0x10000];
        file.seek(SeekFrom::Start(0x10000)).unwrap();
        file.write_all(&b).unwrap();

        // seek_hole within a hole should return the same offset
        assert_eq!(file.seek_hole(0).unwrap(), Some(0));
        assert_eq!(seek_cur(&mut file), 0);
        assert_eq!(file.seek_hole(0xFFFF).unwrap(), Some(0xFFFF));
        assert_eq!(seek_cur(&mut file), 0xFFFF);

        // seek_hole within data should return the next hole (EOF)
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x10000).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x10001).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x1FFFF).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);

        // seek_hole at EOF after data should return None
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x20000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // Extend the file to add another hole after the data
        file.set_len(0x30000).unwrap();
        assert_eq!(file.seek_hole(0).unwrap(), Some(0));
        assert_eq!(seek_cur(&mut file), 0);
        assert_eq!(file.seek_hole(0xFFFF).unwrap(), Some(0xFFFF));
        assert_eq!(seek_cur(&mut file), 0xFFFF);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x10000).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x1FFFF).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x20000).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x20001).unwrap(), Some(0x20001));
        assert_eq!(seek_cur(&mut file), 0x20001);

        // seek_hole at EOF after a hole should return None
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x30000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // Write some data to [0x20000, 0x30000)
        file.seek(SeekFrom::Start(0x20000)).unwrap();
        file.write_all(&b).unwrap();

        // seek_hole within [0x20000, 0x30000) should now find the hole at EOF
        assert_eq!(file.seek_hole(0x20000).unwrap(), Some(0x30000));
        assert_eq!(seek_cur(&mut file), 0x30000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x20001).unwrap(), Some(0x30000));
        assert_eq!(seek_cur(&mut file), 0x30000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x30000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);
    }
}
<commit_msg>sys_util: ignore seek_hole tests for kokoro uprev<commit_after>// Copyright 2018 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use std::fs::File;
use std::io::{Error, Result};
use std::os::unix::io::AsRawFd;

use libc::{lseek64, ENXIO, SEEK_DATA, SEEK_HOLE};

/// A trait for seeking to the next hole or non-hole position in a file.
pub trait SeekHole {
    /// Seek to the first hole in a file at a position greater than or equal to `offset`.
    /// If no holes exist after `offset`, the seek position will be set to the end of the file.
    /// If `offset` is at or after the end of the file, the seek position is unchanged, and None is returned.
    /// Returns the current seek position after the seek or an error.
    fn seek_hole(&mut self, offset: u64) -> Result<Option<u64>>;

    /// Seek to the first data in a file at a position greater than or equal to `offset`.
    /// If no data exists after `offset`, the seek position is unchanged, and None is returned.
    /// Returns the current offset after the seek or an error.
    fn seek_data(&mut self, offset: u64) -> Result<Option<u64>>;
}

/// Safe wrapper for `libc::lseek64()`
fn lseek(file: &mut File, offset: i64, whence: i32) -> Result<Option<u64>> {
    // This is safe because we pass a known-good file descriptor.
    let res = unsafe { lseek64(file.as_raw_fd(), offset, whence) };

    if res < 0 {
        // Convert ENXIO into None; pass any other error as-is.
        let err = Error::last_os_error();
        if let Some(errno) = Error::raw_os_error(&err) {
            if errno == ENXIO {
                return Ok(None);
            }
        }
        Err(err)
    } else {
        Ok(Some(res as u64))
    }
}

impl SeekHole for File {
    fn seek_hole(&mut self, offset: u64) -> Result<Option<u64>> {
        lseek(self, offset as i64, SEEK_HOLE)
    }

    fn seek_data(&mut self, offset: u64) -> Result<Option<u64>> {
        lseek(self, offset as i64, SEEK_DATA)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs::File;
    use std::io::{Seek, SeekFrom, Write};
    use tempfile::tempfile;

    fn seek_cur(file: &mut File) -> u64 {
        file.seek(SeekFrom::Current(0)).unwrap()
    }

    #[test]
    #[ignore] // b/208901617
    fn seek_data() {
        let mut file = tempfile().unwrap();

        // Empty file
        assert_eq!(file.seek_data(0).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // File with non-zero length consisting entirely of a hole
        file.set_len(0x10000).unwrap();
        assert_eq!(file.seek_data(0).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // seek_data at or after the end of the file should return None
        assert_eq!(file.seek_data(0x10000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);
        assert_eq!(file.seek_data(0x10001).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // Write some data to [0x10000, 0x20000)
        let b = [0x55u8; 0x10000];
        file.seek(SeekFrom::Start(0x10000)).unwrap();
        file.write_all(&b).unwrap();
        assert_eq!(file.seek_data(0).unwrap(), Some(0x10000));
        assert_eq!(seek_cur(&mut file), 0x10000);

        // seek_data within data should return the same offset
        assert_eq!(file.seek_data(0x10000).unwrap(), Some(0x10000));
        assert_eq!(seek_cur(&mut file), 0x10000);
        assert_eq!(file.seek_data(0x10001).unwrap(), Some(0x10001));
        assert_eq!(seek_cur(&mut file), 0x10001);
        assert_eq!(file.seek_data(0x1FFFF).unwrap(), Some(0x1FFFF));
        assert_eq!(seek_cur(&mut file), 0x1FFFF);

        // Extend the file to add another hole after the data
        file.set_len(0x30000).unwrap();
        assert_eq!(file.seek_data(0).unwrap(), Some(0x10000));
        assert_eq!(seek_cur(&mut file), 0x10000);
        assert_eq!(file.seek_data(0x1FFFF).unwrap(), Some(0x1FFFF));
        assert_eq!(seek_cur(&mut file), 0x1FFFF);
        assert_eq!(file.seek_data(0x20000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0x1FFFF);
    }

    #[test]
    #[ignore] // b/208901617
    fn seek_hole() {
        let mut file = tempfile().unwrap();

        // Empty file
        assert_eq!(file.seek_hole(0).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // File with non-zero length consisting entirely of a hole
        file.set_len(0x10000).unwrap();
        assert_eq!(file.seek_hole(0).unwrap(), Some(0));
        assert_eq!(seek_cur(&mut file), 0);
        assert_eq!(file.seek_hole(0xFFFF).unwrap(), Some(0xFFFF));
        assert_eq!(seek_cur(&mut file), 0xFFFF);

        // seek_hole at or after the end of the file should return None
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x10000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);
        assert_eq!(file.seek_hole(0x10001).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // Write some data to [0x10000, 0x20000)
        let b = [0x55u8; 0x10000];
        file.seek(SeekFrom::Start(0x10000)).unwrap();
        file.write_all(&b).unwrap();

        // seek_hole within a hole should return the same offset
        assert_eq!(file.seek_hole(0).unwrap(), Some(0));
        assert_eq!(seek_cur(&mut file), 0);
        assert_eq!(file.seek_hole(0xFFFF).unwrap(), Some(0xFFFF));
        assert_eq!(seek_cur(&mut file), 0xFFFF);

        // seek_hole within data should return the next hole (EOF)
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x10000).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x10001).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x1FFFF).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);

        // seek_hole at EOF after data should return None
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x20000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // Extend the file to add another hole after the data
        file.set_len(0x30000).unwrap();
        assert_eq!(file.seek_hole(0).unwrap(), Some(0));
        assert_eq!(seek_cur(&mut file), 0);
        assert_eq!(file.seek_hole(0xFFFF).unwrap(), Some(0xFFFF));
        assert_eq!(seek_cur(&mut file), 0xFFFF);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x10000).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x1FFFF).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x20000).unwrap(), Some(0x20000));
        assert_eq!(seek_cur(&mut file), 0x20000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x20001).unwrap(), Some(0x20001));
        assert_eq!(seek_cur(&mut file), 0x20001);

        // seek_hole at EOF after a hole should return None
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x30000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);

        // Write some data to [0x20000, 0x30000)
        file.seek(SeekFrom::Start(0x20000)).unwrap();
        file.write_all(&b).unwrap();

        // seek_hole within [0x20000, 0x30000) should now find the hole at EOF
        assert_eq!(file.seek_hole(0x20000).unwrap(), Some(0x30000));
        assert_eq!(seek_cur(&mut file), 0x30000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x20001).unwrap(), Some(0x30000));
        assert_eq!(seek_cur(&mut file), 0x30000);
        file.seek(SeekFrom::Start(0)).unwrap();
        assert_eq!(file.seek_hole(0x30000).unwrap(), None);
        assert_eq!(seek_cur(&mut file), 0);
    }
}
<|endoftext|>