<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Buffering wrappers for I/O traits
//!
//! It can be excessively inefficient to work directly with a `Reader` or
//! `Writer`. Every call to `read` or `write` on `TcpStream` results in a
//! system call, for example. This module provides structures that wrap
//! `Readers`, `Writers`, and `Streams` and buffer input and output to them.
//!
//! # Examples
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let reader = BufferedReader::new(tcp_stream);
//!
//! let mut buf: ~[u8] = vec::from_elem(100, 0u8);
//! match reader.read(buf.as_slice()) {
//!     Some(nread) => println!("Read {} bytes", nread),
//!     None => println!("At the end of the stream!")
//! }
//! ~~~
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let writer = BufferedWriter::new(tcp_stream);
//!
//! writer.write("hello, world".as_bytes());
//! writer.flush();
//! ~~~
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let stream = BufferedStream::new(tcp_stream);
//!
//! stream.write("hello, world".as_bytes());
//! stream.flush();
//!
//! let mut buf = vec::from_elem(100, 0u8);
//! match stream.read(buf.as_slice()) {
//!     Some(nread) => println!("Read {} bytes", nread),
//!     None => println!("At the end of the stream!")
//! }
//! ~~~
//!

use prelude::*;

use num;
use vec;
use super::{Reader, Writer, Stream, Decorator};

// libuv recommends 64k buffers to maximize throughput
// https://groups.google.com/forum/#!topic/libuv/oQO1HJAIDdA
static DEFAULT_CAPACITY: uint = 64 * 1024;

/// Wraps a Reader and buffers input from it
pub struct BufferedReader<R> {
    priv inner: R,
    priv buf: ~[u8],
    priv pos: uint,
    priv cap: uint
}

impl<R: Reader> BufferedReader<R> {
    /// Creates a new `BufferedReader` with with the specified buffer capacity
    pub fn with_capacity(cap: uint, inner: R) -> BufferedReader<R> {
        BufferedReader {
            inner: inner,
            buf: vec::from_elem(cap, 0u8),
            pos: 0,
            cap: 0
        }
    }

    /// Creates a new `BufferedReader` with a default buffer capacity
    pub fn new(inner: R) -> BufferedReader<R> {
        BufferedReader::with_capacity(DEFAULT_CAPACITY, inner)
    }
}

impl<R: Reader> Reader for BufferedReader<R> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        if self.pos == self.cap {
            match self.inner.read(self.buf) {
                Some(cap) => {
                    self.pos = 0;
                    self.cap = cap;
                }
                None => return None
            }
        }

        let src = self.buf.slice(self.pos, self.cap);
        let nread = num::min(src.len(), buf.len());
        vec::bytes::copy_memory(buf, src, nread);
        self.pos += nread;
        Some(nread)
    }

    fn eof(&mut self) -> bool {
        self.pos == self.cap && self.inner.eof()
    }
}

impl<R: Reader> Decorator<R> for BufferedReader<R> {
    fn inner(self) -> R {
        self.inner
    }

    fn inner_ref<'a>(&'a self) -> &'a R {
        &self.inner
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut R {
        &mut self.inner
    }
}

/// Wraps a Writer and buffers output to it
///
/// NOTE: `BufferedWriter` will NOT flush its buffer when dropped.
pub struct BufferedWriter<W> {
    priv inner: W,
    priv buf: ~[u8],
    priv pos: uint
}

impl<W: Writer> BufferedWriter<W> {
    /// Creates a new `BufferedWriter` with with the specified buffer capacity
    pub fn with_capacity(cap: uint, inner: W) -> BufferedWriter<W> {
        BufferedWriter {
            inner: inner,
            buf: vec::from_elem(cap, 0u8),
            pos: 0
        }
    }

    /// Creates a new `BufferedWriter` with a default buffer capacity
    pub fn new(inner: W) -> BufferedWriter<W> {
        BufferedWriter::with_capacity(DEFAULT_CAPACITY, inner)
    }
}

impl<W: Writer> Writer for BufferedWriter<W> {
    fn write(&mut self, buf: &[u8]) {
        if self.pos + buf.len() > self.buf.len() {
            self.flush();
        }

        if buf.len() > self.buf.len() {
            self.inner.write(buf);
        } else {
            let dst = self.buf.mut_slice_from(self.pos);
            vec::bytes::copy_memory(dst, buf, buf.len());
            self.pos += buf.len();
        }
    }

    fn flush(&mut self) {
        if self.pos != 0 {
            self.inner.write(self.buf.slice_to(self.pos));
            self.pos = 0;
        }
        self.inner.flush();
    }
}

impl<W: Writer> Decorator<W> for BufferedWriter<W> {
    fn inner(self) -> W {
        self.inner
    }

    fn inner_ref<'a>(&'a self) -> &'a W {
        &self.inner
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut W {
        &mut self.inner
    }
}

struct InternalBufferedWriter<W>(BufferedWriter<W>);

impl<W: Reader> Reader for InternalBufferedWriter<W> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        self.inner.read(buf)
    }

    fn eof(&mut self) -> bool {
        self.inner.eof()
    }
}

/// Wraps a Stream and buffers input and output to and from it
///
/// NOTE: `BufferedStream` will NOT flush its output buffer when dropped.
pub struct BufferedStream<S>(BufferedReader<InternalBufferedWriter<S>>);

impl<S: Stream> BufferedStream<S> {
    pub fn with_capacities(reader_cap: uint, writer_cap: uint, inner: S)
                           -> BufferedStream<S> {
        let writer = BufferedWriter::with_capacity(writer_cap, inner);
        let internal_writer = InternalBufferedWriter(writer);
        let reader = BufferedReader::with_capacity(reader_cap,
                                                   internal_writer);
        BufferedStream(reader)
    }

    pub fn new(inner: S) -> BufferedStream<S> {
        BufferedStream::with_capacities(DEFAULT_CAPACITY, DEFAULT_CAPACITY,
                                        inner)
    }
}

impl<S: Stream> Reader for BufferedStream<S> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        (**self).read(buf)
    }

    fn eof(&mut self) -> bool {
        (**self).eof()
    }
}

impl<S: Stream> Writer for BufferedStream<S> {
    fn write(&mut self, buf: &[u8]) {
        self.inner.write(buf)
    }

    fn flush(&mut self) {
        self.inner.flush()
    }
}

impl<S: Stream> Decorator<S> for BufferedStream<S> {
    fn inner(self) -> S {
        self.inner.inner()
    }

    fn inner_ref<'a>(&'a self) -> &'a S {
        self.inner.inner_ref()
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut S {
        self.inner.inner_mut_ref()
    }
}

#[cfg(test)]
mod test {
    use prelude::*;
    use super::*;
    use super::super::mem::{MemReader, MemWriter};

    #[test]
    fn test_buffered_reader() {
        let inner = MemReader::new(~[0, 1, 2, 3, 4]);
        let mut reader = BufferedReader::with_capacity(2, inner);

        let mut buf = [0, 0, 0];
        let nread = reader.read(buf);
        assert_eq!(Some(2), nread);
        assert_eq!([0, 1, 0], buf);
        assert!(!reader.eof());

        let mut buf = [0];
        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([2], buf);
        assert!(!reader.eof());

        let mut buf = [0, 0, 0];
        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([3, 0, 0], buf);
        assert!(!reader.eof());

        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([4, 0, 0], buf);
        assert!(reader.eof());

        assert_eq!(None, reader.read(buf));
    }

    #[test]
    fn test_buffered_writer() {
        let inner = MemWriter::new();
        let mut writer = BufferedWriter::with_capacity(2, inner);

        writer.write([0, 1]);
        assert_eq!([], writer.inner_ref().inner_ref().as_slice());

        writer.write([2]);
        assert_eq!([0, 1], writer.inner_ref().inner_ref().as_slice());

        writer.write([3]);
        assert_eq!([0, 1], writer.inner_ref().inner_ref().as_slice());

        writer.flush();
        assert_eq!([0, 1, 2, 3], writer.inner_ref().inner_ref().as_slice());

        writer.write([4]);
        writer.write([5]);
        assert_eq!([0, 1, 2, 3], writer.inner_ref().inner_ref().as_slice());

        writer.write([6]);
        assert_eq!([0, 1, 2, 3, 4, 5],
                   writer.inner_ref().inner_ref().as_slice());

        writer.write([7, 8]);
        assert_eq!([0, 1, 2, 3, 4, 5, 6],
                   writer.inner_ref().inner_ref().as_slice());

        writer.write([9, 10, 11]);
        assert_eq!([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
                   writer.inner_ref().inner_ref().as_slice());

        writer.flush();
        assert_eq!([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
                   writer.inner_ref().inner_ref().as_slice());
    }

    // This is just here to make sure that we don't infinite loop in the
    // newtype struct autoderef weirdness
    #[test]
    fn test_buffered_stream() {
        struct S;

        impl Writer for S {
            fn write(&mut self, _: &[u8]) {}
            fn flush(&mut self) {}
        }

        impl Reader for S {
            fn read(&mut self, _: &mut [u8]) -> Option<uint> { None }
            fn eof(&mut self) -> bool { true }
        }

        let mut stream = BufferedStream::new(S);
        let mut buf = [];
        stream.read(buf);
        stream.eof();
        stream.write(buf);
        stream.flush();
    }
}
<commit_msg>auto merge of #9156 : sfackler/rust/buffered-fix, r=huonw<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Buffering wrappers for I/O traits
//!
//! It can be excessively inefficient to work directly with a `Reader` or
//! `Writer`. Every call to `read` or `write` on `TcpStream` results in a
//! system call, for example. This module provides structures that wrap
//! `Readers`, `Writers`, and `Streams` and buffer input and output to them.
//!
//! # Examples
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let reader = BufferedReader::new(tcp_stream);
//!
//! let mut buf: ~[u8] = vec::from_elem(100, 0u8);
//! match reader.read(buf.as_slice()) {
//!     Some(nread) => println!("Read {} bytes", nread),
//!     None => println!("At the end of the stream!")
//! }
//! ~~~
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let writer = BufferedWriter::new(tcp_stream);
//!
//! writer.write("hello, world".as_bytes());
//! writer.flush();
//! ~~~
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let stream = BufferedStream::new(tcp_stream);
//!
//! stream.write("hello, world".as_bytes());
//! stream.flush();
//!
//! let mut buf = vec::from_elem(100, 0u8);
//! match stream.read(buf.as_slice()) {
//!     Some(nread) => println!("Read {} bytes", nread),
//!     None => println!("At the end of the stream!")
//! }
//! ~~~
//!

use prelude::*;

use num;
use vec;
use super::{Reader, Writer, Stream, Decorator};

// libuv recommends 64k buffers to maximize throughput
// https://groups.google.com/forum/#!topic/libuv/oQO1HJAIDdA
static DEFAULT_CAPACITY: uint = 64 * 1024;

/// Wraps a Reader and buffers input from it
pub struct BufferedReader<R> {
    priv inner: R,
    priv buf: ~[u8],
    priv pos: uint,
    priv cap: uint
}

impl<R: Reader> BufferedReader<R> {
    /// Creates a new `BufferedReader` with with the specified buffer capacity
    pub fn with_capacity(cap: uint, inner: R) -> BufferedReader<R> {
        BufferedReader {
            inner: inner,
            buf: vec::from_elem(cap, 0u8),
            pos: 0,
            cap: 0
        }
    }

    /// Creates a new `BufferedReader` with a default buffer capacity
    pub fn new(inner: R) -> BufferedReader<R> {
        BufferedReader::with_capacity(DEFAULT_CAPACITY, inner)
    }
}

impl<R: Reader> Reader for BufferedReader<R> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        if self.pos == self.cap {
            match self.inner.read(self.buf) {
                Some(cap) => {
                    self.pos = 0;
                    self.cap = cap;
                }
                None => return None
            }
        }

        let src = self.buf.slice(self.pos, self.cap);
        let nread = num::min(src.len(), buf.len());
        vec::bytes::copy_memory(buf, src, nread);
        self.pos += nread;
        Some(nread)
    }

    fn eof(&mut self) -> bool {
        self.pos == self.cap && self.inner.eof()
    }
}

impl<R: Reader> Decorator<R> for BufferedReader<R> {
    fn inner(self) -> R {
        self.inner
    }

    fn inner_ref<'a>(&'a self) -> &'a R {
        &self.inner
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut R {
        &mut self.inner
    }
}

/// Wraps a Writer and buffers output to it
///
/// NOTE: `BufferedWriter` will NOT flush its buffer when dropped.
pub struct BufferedWriter<W> {
    priv inner: W,
    priv buf: ~[u8],
    priv pos: uint
}

impl<W: Writer> BufferedWriter<W> {
    /// Creates a new `BufferedWriter` with with the specified buffer capacity
    pub fn with_capacity(cap: uint, inner: W) -> BufferedWriter<W> {
        BufferedWriter {
            inner: inner,
            buf: vec::from_elem(cap, 0u8),
            pos: 0
        }
    }

    /// Creates a new `BufferedWriter` with a default buffer capacity
    pub fn new(inner: W) -> BufferedWriter<W> {
        BufferedWriter::with_capacity(DEFAULT_CAPACITY, inner)
    }
}

impl<W: Writer> Writer for BufferedWriter<W> {
    fn write(&mut self, buf: &[u8]) {
        if self.pos + buf.len() > self.buf.len() {
            self.flush();
        }

        if buf.len() > self.buf.len() {
            self.inner.write(buf);
        } else {
            let dst = self.buf.mut_slice_from(self.pos);
            vec::bytes::copy_memory(dst, buf, buf.len());
            self.pos += buf.len();
        }
    }

    fn flush(&mut self) {
        if self.pos != 0 {
            self.inner.write(self.buf.slice_to(self.pos));
            self.pos = 0;
        }
        self.inner.flush();
    }
}

impl<W: Writer> Decorator<W> for BufferedWriter<W> {
    fn inner(self) -> W {
        self.inner
    }

    fn inner_ref<'a>(&'a self) -> &'a W {
        &self.inner
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut W {
        &mut self.inner
    }
}

// FIXME #9155 this should be a newtype struct
struct InternalBufferedWriter<W> {
    inner: BufferedWriter<W>
}

impl<W: Reader> Reader for InternalBufferedWriter<W> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        self.inner.inner.read(buf)
    }

    fn eof(&mut self) -> bool {
        self.inner.inner.eof()
    }
}

/// Wraps a Stream and buffers input and output to and from it
///
/// NOTE: `BufferedStream` will NOT flush its output buffer when dropped.
// FIXME #9155 this should be a newtype struct
pub struct BufferedStream<S> {
    priv inner: BufferedReader<InternalBufferedWriter<S>>
}

impl<S: Stream> BufferedStream<S> {
    pub fn with_capacities(reader_cap: uint, writer_cap: uint, inner: S)
                           -> BufferedStream<S> {
        let writer = BufferedWriter::with_capacity(writer_cap, inner);
        let internal_writer = InternalBufferedWriter { inner: writer };
        let reader = BufferedReader::with_capacity(reader_cap,
                                                   internal_writer);
        BufferedStream { inner: reader }
    }

    pub fn new(inner: S) -> BufferedStream<S> {
        BufferedStream::with_capacities(DEFAULT_CAPACITY, DEFAULT_CAPACITY,
                                        inner)
    }
}

impl<S: Stream> Reader for BufferedStream<S> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        self.inner.read(buf)
    }

    fn eof(&mut self) -> bool {
        self.inner.eof()
    }
}

impl<S: Stream> Writer for BufferedStream<S> {
    fn write(&mut self, buf: &[u8]) {
        self.inner.inner.inner.write(buf)
    }

    fn flush(&mut self) {
        self.inner.inner.inner.flush()
    }
}

impl<S: Stream> Decorator<S> for BufferedStream<S> {
    fn inner(self) -> S {
        self.inner.inner.inner.inner()
    }

    fn inner_ref<'a>(&'a self) -> &'a S {
        self.inner.inner.inner.inner_ref()
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut S {
        self.inner.inner.inner.inner_mut_ref()
    }
}

#[cfg(test)]
mod test {
    use prelude::*;
    use super::*;
    use super::super::mem::{MemReader, MemWriter};

    #[test]
    fn test_buffered_reader() {
        let inner = MemReader::new(~[0, 1, 2, 3, 4]);
        let mut reader = BufferedReader::with_capacity(2, inner);

        let mut buf = [0, 0, 0];
        let nread = reader.read(buf);
        assert_eq!(Some(2), nread);
        assert_eq!([0, 1, 0], buf);
        assert!(!reader.eof());

        let mut buf = [0];
        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([2], buf);
        assert!(!reader.eof());

        let mut buf = [0, 0, 0];
        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([3, 0, 0], buf);
        assert!(!reader.eof());

        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([4, 0, 0], buf);
        assert!(reader.eof());

        assert_eq!(None, reader.read(buf));
    }

    #[test]
    fn test_buffered_writer() {
        let inner = MemWriter::new();
        let mut writer = BufferedWriter::with_capacity(2, inner);

        writer.write([0, 1]);
        assert_eq!([], writer.inner_ref().inner_ref().as_slice());

        writer.write([2]);
        assert_eq!([0, 1], writer.inner_ref().inner_ref().as_slice());

        writer.write([3]);
        assert_eq!([0, 1], writer.inner_ref().inner_ref().as_slice());

        writer.flush();
        assert_eq!([0, 1, 2, 3], writer.inner_ref().inner_ref().as_slice());

        writer.write([4]);
        writer.write([5]);
        assert_eq!([0, 1, 2, 3], writer.inner_ref().inner_ref().as_slice());

        writer.write([6]);
        assert_eq!([0, 1, 2, 3, 4, 5],
                   writer.inner_ref().inner_ref().as_slice());

        writer.write([7, 8]);
        assert_eq!([0, 1, 2, 3, 4, 5, 6],
                   writer.inner_ref().inner_ref().as_slice());

        writer.write([9, 10, 11]);
        assert_eq!([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
                   writer.inner_ref().inner_ref().as_slice());

        writer.flush();
        assert_eq!([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
                   writer.inner_ref().inner_ref().as_slice());
    }

    // This is just here to make sure that we don't infinite loop in the
    // newtype struct autoderef weirdness
    #[test]
    fn test_buffered_stream() {
        struct S;

        impl Writer for S {
            fn write(&mut self, _: &[u8]) {}
            fn flush(&mut self) {}
        }

        impl Reader for S {
            fn read(&mut self, _: &mut [u8]) -> Option<uint> { None }
            fn eof(&mut self) -> bool { true }
        }

        let mut stream = BufferedStream::new(S);
        let mut buf = [];
        stream.read(buf);
        stream.eof();
        stream.write(buf);
        stream.flush();
    }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Buffering wrappers for I/O traits
//!
//! It can be excessively inefficient to work directly with a `Reader` or
//! `Writer`. Every call to `read` or `write` on `TcpStream` results in a
//! system call, for example. This module provides structures that wrap
//! `Readers`, `Writers`, and `Streams` and buffer input and output to them.
//!
//! # Examples
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let reader = BufferedReader::new(tcp_stream);
//!
//! let mut buf: ~[u8] = vec::from_elem(100, 0u8);
//! match reader.read(buf.as_slice()) {
//!     Some(nread) => println!("Read {} bytes", nread),
//!     None => println!("At the end of the stream!")
//! }
//! ~~~
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let writer = BufferedWriter::new(tcp_stream);
//!
//! writer.write("hello, world".as_bytes());
//! writer.flush();
//! ~~~
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let stream = BufferedStream::new(tcp_stream);
//!
//! stream.write("hello, world".as_bytes());
//! stream.flush();
//!
//! let mut buf = vec::from_elem(100, 0u8);
//! match stream.read(buf.as_slice()) {
//!     Some(nread) => println!("Read {} bytes", nread),
//!     None => println!("At the end of the stream!")
//! }
//! ~~~
//!

use prelude::*;

use num;
use vec;
use super::{Reader, Writer, Stream, Decorator};

// libuv recommends 64k buffers to maximize throughput
// https://groups.google.com/forum/#!topic/libuv/oQO1HJAIDdA
static DEFAULT_CAPACITY: uint = 64 * 1024;

/// Wraps a Reader and buffers input from it
pub struct BufferedReader<R> {
    priv inner: R,
    priv buf: ~[u8],
    priv pos: uint,
    priv cap: uint
}

impl<R: Reader> BufferedReader<R> {
    /// Creates a new `BufferedReader` with with the specified buffer capacity
    pub fn with_capacity(cap: uint, inner: R) -> BufferedReader<R> {
        BufferedReader {
            inner: inner,
            buf: vec::from_elem(cap, 0u8),
            pos: 0,
            cap: 0
        }
    }

    /// Creates a new `BufferedReader` with a default buffer capacity
    pub fn new(inner: R) -> BufferedReader<R> {
        BufferedReader::with_capacity(DEFAULT_CAPACITY, inner)
    }
}

impl<R: Reader> Reader for BufferedReader<R> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        if self.pos == self.cap {
            match self.inner.read(self.buf) {
                Some(cap) => {
                    self.pos = 0;
                    self.cap = cap;
                }
                None => return None
            }
        }

        let src = self.buf.slice(self.pos, self.cap);
        let nread = num::min(src.len(), buf.len());
        vec::bytes::copy_memory(buf, src, nread);
        self.pos += nread;
        Some(nread)
    }

    fn eof(&mut self) -> bool {
        self.pos == self.cap && self.inner.eof()
    }
}

impl<R: Reader> Decorator<R> for BufferedReader<R> {
    fn inner(self) -> R {
        self.inner
    }

    fn inner_ref<'a>(&'a self) -> &'a R {
        &self.inner
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut R {
        &mut self.inner
    }
}

/// Wraps a Writer and buffers output to it
///
/// NOTE: `BufferedWriter` will NOT flush its buffer when dropped.
pub struct BufferedWriter<W> {
    priv inner: W,
    priv buf: ~[u8],
    priv pos: uint
}

impl<W: Writer> BufferedWriter<W> {
    /// Creates a new `BufferedWriter` with with the specified buffer capacity
    pub fn with_capacity(cap: uint, inner: W) -> BufferedWriter<W> {
        BufferedWriter {
            inner: inner,
            buf: vec::from_elem(cap, 0u8),
            pos: 0
        }
    }

    /// Creates a new `BufferedWriter` with a default buffer capacity
    pub fn new(inner: W) -> BufferedWriter<W> {
        BufferedWriter::with_capacity(DEFAULT_CAPACITY, inner)
    }
}

impl<W: Writer> Writer for BufferedWriter<W> {
    fn write(&mut self, buf: &[u8]) {
        if self.pos + buf.len() > self.buf.len() {
            self.flush();
        }

        if buf.len() > self.buf.len() {
            self.inner.write(buf);
        } else {
            let dst = self.buf.mut_slice_from(self.pos);
            vec::bytes::copy_memory(dst, buf, buf.len());
            self.pos += buf.len();
        }
    }

    fn flush(&mut self) {
        if self.pos != 0 {
            self.inner.write(self.buf.slice_to(self.pos));
            self.pos = 0;
        }
        self.inner.flush();
    }
}

impl<W: Writer> Decorator<W> for BufferedWriter<W> {
    fn inner(self) -> W {
        self.inner
    }

    fn inner_ref<'a>(&'a self) -> &'a W {
        &self.inner
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut W {
        &mut self.inner
    }
}

struct InternalBufferedWriter<W>(BufferedWriter<W>);

impl<W: Reader> Reader for InternalBufferedWriter<W> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        self.inner.read(buf)
    }

    fn eof(&mut self) -> bool {
        self.inner.eof()
    }
}

/// Wraps a Stream and buffers input and output to and from it
///
/// NOTE: `BufferedStream` will NOT flush its output buffer when dropped.
pub struct BufferedStream<S>(BufferedReader<InternalBufferedWriter<S>>);

impl<S: Stream> BufferedStream<S> {
    pub fn with_capacities(reader_cap: uint, writer_cap: uint, inner: S)
                           -> BufferedStream<S> {
        let writer = BufferedWriter::with_capacity(writer_cap, inner);
        let internal_writer = InternalBufferedWriter(writer);
        let reader = BufferedReader::with_capacity(reader_cap,
                                                   internal_writer);
        BufferedStream(reader)
    }

    pub fn new(inner: S) -> BufferedStream<S> {
        BufferedStream::with_capacities(DEFAULT_CAPACITY, DEFAULT_CAPACITY,
                                        inner)
    }
}

impl<S: Stream> Reader for BufferedStream<S> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        (**self).read(buf)
    }

    fn eof(&mut self) -> bool {
        (**self).eof()
    }
}

impl<S: Stream> Writer for BufferedStream<S> {
    fn write(&mut self, buf: &[u8]) {
        self.inner.write(buf)
    }

    fn flush(&mut self) {
        self.inner.flush()
    }
}

impl<S: Stream> Decorator<S> for BufferedStream<S> {
    fn inner(self) -> S {
        self.inner.inner()
    }

    fn inner_ref<'a>(&'a self) -> &'a S {
        self.inner.inner_ref()
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut S {
        self.inner.inner_mut_ref()
    }
}

#[cfg(test)]
mod test {
    use prelude::*;
    use super::*;
    use super::super::mem::{MemReader, MemWriter};

    #[test]
    fn test_buffered_reader() {
        let inner = MemReader::new(~[0, 1, 2, 3, 4]);
        let mut reader = BufferedReader::with_capacity(2, inner);

        let mut buf = [0, 0, 0];
        let nread = reader.read(buf);
        assert_eq!(Some(2), nread);
        assert_eq!([0, 1, 0], buf);
        assert!(!reader.eof());

        let mut buf = [0];
        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([2], buf);
        assert!(!reader.eof());

        let mut buf = [0, 0, 0];
        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([3, 0, 0], buf);
        assert!(!reader.eof());

        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([4, 0, 0], buf);
        assert!(reader.eof());

        assert_eq!(None, reader.read(buf));
    }

    #[test]
    fn test_buffered_writer() {
        let inner = MemWriter::new();
        let mut writer = BufferedWriter::with_capacity(2, inner);

        writer.write([0, 1]);
        assert_eq!([], writer.inner_ref().inner_ref().as_slice());

        writer.write([2]);
        assert_eq!([0, 1], writer.inner_ref().inner_ref().as_slice());

        writer.write([3]);
        assert_eq!([0, 1], writer.inner_ref().inner_ref().as_slice());

        writer.flush();
        assert_eq!([0, 1, 2, 3], writer.inner_ref().inner_ref().as_slice());

        writer.write([4]);
        writer.write([5]);
        assert_eq!([0, 1, 2, 3], writer.inner_ref().inner_ref().as_slice());

        writer.write([6]);
        assert_eq!([0, 1, 2, 3, 4, 5],
                   writer.inner_ref().inner_ref().as_slice());

        writer.write([7, 8]);
        assert_eq!([0, 1, 2, 3, 4, 5, 6],
                   writer.inner_ref().inner_ref().as_slice());

        writer.write([9, 10, 11]);
        assert_eq!([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
                   writer.inner_ref().inner_ref().as_slice());

        writer.flush();
        assert_eq!([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
                   writer.inner_ref().inner_ref().as_slice());
    }

    // This is just here to make sure that we don't infinite loop in the
    // newtype struct autoderef weirdness
    #[test]
    fn test_buffered_stream() {
        struct S;

        impl Writer for S {
            fn write(&mut self, _: &[u8]) {}
            fn flush(&mut self) {}
        }

        impl Reader for S {
            fn read(&mut self, _: &mut [u8]) -> Option<uint> { None }
            fn eof(&mut self) -> bool { true }
        }

        let mut stream = BufferedStream::new(S);
        let mut buf = [];
        stream.read(buf);
        stream.eof();
        stream.write(buf);
        stream.flush();
    }
}
<commit_msg>Stop using newtypes in rt::io::buffered<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Buffering wrappers for I/O traits
//!
//! It can be excessively inefficient to work directly with a `Reader` or
//! `Writer`. Every call to `read` or `write` on `TcpStream` results in a
//! system call, for example. This module provides structures that wrap
//! `Readers`, `Writers`, and `Streams` and buffer input and output to them.
//!
//! # Examples
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let reader = BufferedReader::new(tcp_stream);
//!
//! let mut buf: ~[u8] = vec::from_elem(100, 0u8);
//! match reader.read(buf.as_slice()) {
//!     Some(nread) => println!("Read {} bytes", nread),
//!     None => println!("At the end of the stream!")
//! }
//! ~~~
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let writer = BufferedWriter::new(tcp_stream);
//!
//! writer.write("hello, world".as_bytes());
//! writer.flush();
//! ~~~
//!
//! ~~~
//! let tcp_stream = TcpStream::connect(addr);
//! let stream = BufferedStream::new(tcp_stream);
//!
//! stream.write("hello, world".as_bytes());
//! stream.flush();
//!
//! let mut buf = vec::from_elem(100, 0u8);
//! match stream.read(buf.as_slice()) {
//!     Some(nread) => println!("Read {} bytes", nread),
//!     None => println!("At the end of the stream!")
//! }
//! ~~~
//!

use prelude::*;

use num;
use vec;
use super::{Reader, Writer, Stream, Decorator};

// libuv recommends 64k buffers to maximize throughput
// https://groups.google.com/forum/#!topic/libuv/oQO1HJAIDdA
static DEFAULT_CAPACITY: uint = 64 * 1024;

/// Wraps a Reader and buffers input from it
pub struct BufferedReader<R> {
    priv inner: R,
    priv buf: ~[u8],
    priv pos: uint,
    priv cap: uint
}

impl<R: Reader> BufferedReader<R> {
    /// Creates a new `BufferedReader` with with the specified buffer capacity
    pub fn with_capacity(cap: uint, inner: R) -> BufferedReader<R> {
        BufferedReader {
            inner: inner,
            buf: vec::from_elem(cap, 0u8),
            pos: 0,
            cap: 0
        }
    }

    /// Creates a new `BufferedReader` with a default buffer capacity
    pub fn new(inner: R) -> BufferedReader<R> {
        BufferedReader::with_capacity(DEFAULT_CAPACITY, inner)
    }
}

impl<R: Reader> Reader for BufferedReader<R> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        if self.pos == self.cap {
            match self.inner.read(self.buf) {
                Some(cap) => {
                    self.pos = 0;
                    self.cap = cap;
                }
                None => return None
            }
        }

        let src = self.buf.slice(self.pos, self.cap);
        let nread = num::min(src.len(), buf.len());
        vec::bytes::copy_memory(buf, src, nread);
        self.pos += nread;
        Some(nread)
    }

    fn eof(&mut self) -> bool {
        self.pos == self.cap && self.inner.eof()
    }
}

impl<R: Reader> Decorator<R> for BufferedReader<R> {
    fn inner(self) -> R {
        self.inner
    }

    fn inner_ref<'a>(&'a self) -> &'a R {
        &self.inner
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut R {
        &mut self.inner
    }
}

/// Wraps a Writer and buffers output to it
///
/// NOTE: `BufferedWriter` will NOT flush its buffer when dropped.
pub struct BufferedWriter<W> {
    priv inner: W,
    priv buf: ~[u8],
    priv pos: uint
}

impl<W: Writer> BufferedWriter<W> {
    /// Creates a new `BufferedWriter` with with the specified buffer capacity
    pub fn with_capacity(cap: uint, inner: W) -> BufferedWriter<W> {
        BufferedWriter {
            inner: inner,
            buf: vec::from_elem(cap, 0u8),
            pos: 0
        }
    }

    /// Creates a new `BufferedWriter` with a default buffer capacity
    pub fn new(inner: W) -> BufferedWriter<W> {
        BufferedWriter::with_capacity(DEFAULT_CAPACITY, inner)
    }
}

impl<W: Writer> Writer for BufferedWriter<W> {
    fn write(&mut self, buf: &[u8]) {
        if self.pos + buf.len() > self.buf.len() {
            self.flush();
        }

        if buf.len() > self.buf.len() {
            self.inner.write(buf);
        } else {
            let dst = self.buf.mut_slice_from(self.pos);
            vec::bytes::copy_memory(dst, buf, buf.len());
            self.pos += buf.len();
        }
    }

    fn flush(&mut self) {
        if self.pos != 0 {
            self.inner.write(self.buf.slice_to(self.pos));
            self.pos = 0;
        }
        self.inner.flush();
    }
}

impl<W: Writer> Decorator<W> for BufferedWriter<W> {
    fn inner(self) -> W {
        self.inner
    }

    fn inner_ref<'a>(&'a self) -> &'a W {
        &self.inner
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut W {
        &mut self.inner
    }
}

// FIXME #9155 this should be a newtype struct
struct InternalBufferedWriter<W> {
    inner: BufferedWriter<W>
}

impl<W: Reader> Reader for InternalBufferedWriter<W> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        self.inner.inner.read(buf)
    }

    fn eof(&mut self) -> bool {
        self.inner.inner.eof()
    }
}

/// Wraps a Stream and buffers input and output to and from it
///
/// NOTE: `BufferedStream` will NOT flush its output buffer when dropped.
// FIXME #9155 this should be a newtype struct
pub struct BufferedStream<S> {
    priv inner: BufferedReader<InternalBufferedWriter<S>>
}

impl<S: Stream> BufferedStream<S> {
    pub fn with_capacities(reader_cap: uint, writer_cap: uint, inner: S)
                           -> BufferedStream<S> {
        let writer = BufferedWriter::with_capacity(writer_cap, inner);
        let internal_writer = InternalBufferedWriter { inner: writer };
        let reader = BufferedReader::with_capacity(reader_cap,
                                                   internal_writer);
        BufferedStream { inner: reader }
    }

    pub fn new(inner: S) -> BufferedStream<S> {
        BufferedStream::with_capacities(DEFAULT_CAPACITY, DEFAULT_CAPACITY,
                                        inner)
    }
}

impl<S: Stream> Reader for BufferedStream<S> {
    fn read(&mut self, buf: &mut [u8]) -> Option<uint> {
        self.inner.read(buf)
    }

    fn eof(&mut self) -> bool {
        self.inner.eof()
    }
}

impl<S: Stream> Writer for BufferedStream<S> {
    fn write(&mut self, buf: &[u8]) {
        self.inner.inner.inner.write(buf)
    }

    fn flush(&mut self) {
        self.inner.inner.inner.flush()
    }
}

impl<S: Stream> Decorator<S> for BufferedStream<S> {
    fn inner(self) -> S {
        self.inner.inner.inner.inner()
    }

    fn inner_ref<'a>(&'a self) -> &'a S {
        self.inner.inner.inner.inner_ref()
    }

    fn inner_mut_ref<'a>(&'a mut self) -> &'a mut S {
        self.inner.inner.inner.inner_mut_ref()
    }
}

#[cfg(test)]
mod test {
    use prelude::*;
    use super::*;
    use super::super::mem::{MemReader, MemWriter};

    #[test]
    fn test_buffered_reader() {
        let inner = MemReader::new(~[0, 1, 2, 3, 4]);
        let mut reader = BufferedReader::with_capacity(2, inner);

        let mut buf = [0, 0, 0];
        let nread = reader.read(buf);
        assert_eq!(Some(2), nread);
        assert_eq!([0, 1, 0], buf);
        assert!(!reader.eof());

        let mut buf = [0];
        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([2], buf);
        assert!(!reader.eof());

        let mut buf = [0, 0, 0];
        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([3, 0, 0], buf);
        assert!(!reader.eof());

        let nread = reader.read(buf);
        assert_eq!(Some(1), nread);
        assert_eq!([4, 0, 0], buf);
        assert!(reader.eof());

        assert_eq!(None, reader.read(buf));
    }

    #[test]
    fn test_buffered_writer() {
        let inner = MemWriter::new();
        let mut writer = BufferedWriter::with_capacity(2, inner);

        writer.write([0, 1]);
        assert_eq!([], writer.inner_ref().inner_ref().as_slice());

        writer.write([2]);
        assert_eq!([0, 1], writer.inner_ref().inner_ref().as_slice());

        writer.write([3]);
        assert_eq!([0, 1], writer.inner_ref().inner_ref().as_slice());

        writer.flush();
        assert_eq!([0, 1, 2, 3], writer.inner_ref().inner_ref().as_slice());

        writer.write([4]);
        writer.write([5]);
        assert_eq!([0, 1, 2, 3], writer.inner_ref().inner_ref().as_slice());

        writer.write([6]);
        assert_eq!([0, 1, 2, 3, 4, 5],
                   writer.inner_ref().inner_ref().as_slice());

        writer.write([7, 8]);
        assert_eq!([0, 1, 2, 3, 4, 5, 6],
                   writer.inner_ref().inner_ref().as_slice());

        writer.write([9, 10, 11]);
        assert_eq!([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
                   writer.inner_ref().inner_ref().as_slice());

        writer.flush();
        assert_eq!([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
                   writer.inner_ref().inner_ref().as_slice());
    }

    // This is just here to make sure that we don't infinite loop in the
    // newtype struct autoderef weirdness
    #[test]
    fn test_buffered_stream() {
        struct S;

        impl Writer for S {
            fn write(&mut self, _: &[u8]) {}
            fn flush(&mut self) {}
        }

        impl Reader for S {
            fn read(&mut self, _: &mut [u8]) -> Option<uint> { None }
            fn eof(&mut self) -> bool { true }
        }

        let mut stream = BufferedStream::new(S);
        let mut buf = [];
        stream.read(buf);
        stream.eof();
        stream.write(buf);
        stream.flush();
    }
}
<|endoftext|>


<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Abstraction of a thread pool for basic parallelism.

#![unstable(feature = "std_misc",
            reason = "the semantics of a failing task and whether a thread is \
                      re-attached to a thread pool are somewhat unclear, and the \
                      utility of this type in `std::sync` is questionable with \
                      respect to the jobs of other primitives")]

use core::prelude::*;

use sync::{Arc, Mutex};
use sync::mpsc::{channel, Sender, Receiver};
use thread::Thread;
use thunk::Thunk;

struct Sentinel<'a> {
    jobs: &'a Arc<Mutex<Receiver<Thunk>>>,
    active: bool
}

impl<'a> Sentinel<'a> {
    fn new(jobs: &Arc<Mutex<Receiver<Thunk>>>) -> Sentinel {
        Sentinel {
            jobs: jobs,
            active: true
        }
    }

    // Cancel and destroy this sentinel.
    fn cancel(mut self) {
        self.active = false;
    }
}

#[unsafe_destructor]
impl<'a> Drop for Sentinel<'a> {
    fn drop(&mut self) {
        if self.active {
            spawn_in_pool(self.jobs.clone())
        }
    }
}

/// A thread pool used to execute functions in parallel.
///
/// Spawns `n` worker threads and replenishes the pool if any worker threads
/// panic.
///
/// # Example
///
/// ```rust
/// use std::sync::TaskPool;
/// use std::iter::AdditiveIterator;
/// use std::sync::mpsc::channel;
///
/// let pool = TaskPool::new(4u);
///
/// let (tx, rx) = channel();
/// for _ in 0..8u {
///     let tx = tx.clone();
///     pool.execute(move|| {
///         tx.send(1u).unwrap();
///     });
/// }
///
/// assert_eq!(rx.iter().take(8u).sum(), 8u);
/// ```
pub struct TaskPool {
    // How the threadpool communicates with subthreads.
    //
    // This is the only such Sender, so when it is dropped all subthreads will
    // quit.
    jobs: Sender<Thunk>
}

impl TaskPool {
    /// Spawns a new thread pool with `threads` threads.
    ///
    /// # Panics
    ///
    /// This function will panic if `threads` is 0.
    pub fn new(threads: uint) -> TaskPool {
        assert!(threads >= 1);

        let (tx, rx) = channel::<Thunk>();
        let rx = Arc::new(Mutex::new(rx));

        // Threadpool threads
        for _ in 0..threads {
            spawn_in_pool(rx.clone());
        }

        TaskPool { jobs: tx }
    }

    /// Executes the function `job` on a thread in the pool.
    pub fn execute<F>(&self, job: F)
        where F : FnOnce(), F : Send
    {
        self.jobs.send(Thunk::new(job)).unwrap();
    }
}

fn spawn_in_pool(jobs: Arc<Mutex<Receiver<Thunk>>>) {
    Thread::spawn(move || {
        // Will spawn a new thread on panic unless it is cancelled.
        let sentinel = Sentinel::new(&jobs);

        loop {
            let message = {
                // Only lock jobs for the time it takes
                // to get a job, not run it.
                let lock = jobs.lock().unwrap();
                lock.recv()
            };

            match message {
                Ok(job) => job.invoke(()),

                // The Taskpool was dropped.
                Err(..) => break
            }
        }

        sentinel.cancel();
    });
}

#[cfg(test)]
mod test {
    use prelude::v1::*;
    use super::*;
    use sync::mpsc::channel;

    const TEST_TASKS: uint = 4u;

    #[test]
    fn test_works() {
        use iter::AdditiveIterator;

        let pool = TaskPool::new(TEST_TASKS);

        let (tx, rx) = channel();
        for _ in 0..TEST_TASKS {
            let tx = tx.clone();
            pool.execute(move|| {
                tx.send(1u).unwrap();
            });
        }

        assert_eq!(rx.iter().take(TEST_TASKS).sum(), TEST_TASKS);
    }

    #[test]
    #[should_fail]
    fn test_zero_tasks_panic() {
        TaskPool::new(0);
    }

    #[test]
    fn test_recovery_from_subtask_panic() {
        use iter::AdditiveIterator;

        let pool = TaskPool::new(TEST_TASKS);

        // Panic all the existing threads.
        for _ in 0..TEST_TASKS {
            pool.execute(move|| -> () { panic!() });
        }

        // Ensure new threads were spawned to compensate.
        let (tx, rx) = channel();
        for _ in 0..TEST_TASKS {
            let tx = tx.clone();
            pool.execute(move|| {
                tx.send(1u).unwrap();
            });
        }

        assert_eq!(rx.iter().take(TEST_TASKS).sum(), TEST_TASKS);
    }

    #[test]
    fn test_should_not_panic_on_drop_if_subtasks_panic_after_drop() {
        use sync::{Arc, Barrier};

        let pool = TaskPool::new(TEST_TASKS);
        let waiter = Arc::new(Barrier::new(TEST_TASKS + 1));

        // Panic all the existing threads in a bit.
        for _ in 0..TEST_TASKS {
            let waiter = waiter.clone();
            pool.execute(move|| {
                waiter.wait();
                panic!();
            });
        }

        drop(pool);

        // Kick off the failure.
        waiter.wait();
    }
}
<commit_msg>Deprecate std::sync::TaskPool<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Abstraction of a thread pool for basic parallelism.

#![deprecated(since = "1.0.0",
              reason = "This kind of API needs some time to bake in \
                        crates.io. Consider trying \
                        https://github.com/carllerche/syncbox")]
#![unstable(feature = "std_misc")]

use core::prelude::*;

use sync::{Arc, Mutex};
use sync::mpsc::{channel, Sender, Receiver};
use thread::Thread;
use thunk::Thunk;

struct Sentinel<'a> {
    jobs: &'a Arc<Mutex<Receiver<Thunk>>>,
    active: bool
}

impl<'a> Sentinel<'a> {
    fn new(jobs: &Arc<Mutex<Receiver<Thunk>>>) -> Sentinel {
        Sentinel {
            jobs: jobs,
            active: true
        }
    }

    // Cancel and destroy this sentinel.
    fn cancel(mut self) {
        self.active = false;
    }
}

#[unsafe_destructor]
impl<'a> Drop for Sentinel<'a> {
    fn drop(&mut self) {
        if self.active {
            spawn_in_pool(self.jobs.clone())
        }
    }
}

/// A thread pool used to execute functions in parallel.
///
/// Spawns `n` worker threads and replenishes the pool if any worker threads
/// panic.
///
/// # Example
///
/// ```rust
/// use std::sync::TaskPool;
/// use std::iter::AdditiveIterator;
/// use std::sync::mpsc::channel;
///
/// let pool = TaskPool::new(4u);
///
/// let (tx, rx) = channel();
/// for _ in 0..8u {
///     let tx = tx.clone();
///     pool.execute(move|| {
///         tx.send(1u).unwrap();
///     });
/// }
///
/// assert_eq!(rx.iter().take(8u).sum(), 8u);
/// ```
pub struct TaskPool {
    // How the threadpool communicates with subthreads.
    //
    // This is the only such Sender, so when it is dropped all subthreads will
    // quit.
    jobs: Sender<Thunk>
}

impl TaskPool {
    /// Spawns a new thread pool with `threads` threads.
    ///
    /// # Panics
    ///
    /// This function will panic if `threads` is 0.
    pub fn new(threads: uint) -> TaskPool {
        assert!(threads >= 1);

        let (tx, rx) = channel::<Thunk>();
        let rx = Arc::new(Mutex::new(rx));

        // Threadpool threads
        for _ in 0..threads {
            spawn_in_pool(rx.clone());
        }

        TaskPool { jobs: tx }
    }

    /// Executes the function `job` on a thread in the pool.
    pub fn execute<F>(&self, job: F)
        where F : FnOnce(), F : Send
    {
        self.jobs.send(Thunk::new(job)).unwrap();
    }
}

fn spawn_in_pool(jobs: Arc<Mutex<Receiver<Thunk>>>) {
    Thread::spawn(move || {
        // Will spawn a new thread on panic unless it is cancelled.
        let sentinel = Sentinel::new(&jobs);

        loop {
            let message = {
                // Only lock jobs for the time it takes
                // to get a job, not run it.
                let lock = jobs.lock().unwrap();
                lock.recv()
            };

            match message {
                Ok(job) => job.invoke(()),

                // The Taskpool was dropped.
                Err(..) => break
            }
        }

        sentinel.cancel();
    });
}

#[cfg(test)]
mod test {
    use prelude::v1::*;
    use super::*;
    use sync::mpsc::channel;

    const TEST_TASKS: uint = 4u;

    #[test]
    fn test_works() {
        use iter::AdditiveIterator;

        let pool = TaskPool::new(TEST_TASKS);

        let (tx, rx) = channel();
        for _ in 0..TEST_TASKS {
            let tx = tx.clone();
            pool.execute(move|| {
                tx.send(1u).unwrap();
            });
        }

        assert_eq!(rx.iter().take(TEST_TASKS).sum(), TEST_TASKS);
    }

    #[test]
    #[should_fail]
    fn test_zero_tasks_panic() {
        TaskPool::new(0);
    }

    #[test]
    fn test_recovery_from_subtask_panic() {
        use iter::AdditiveIterator;

        let pool = TaskPool::new(TEST_TASKS);

        // Panic all the existing threads.
        for _ in 0..TEST_TASKS {
            pool.execute(move|| -> () { panic!() });
        }

        // Ensure new threads were spawned to compensate.
        let (tx, rx) = channel();
        for _ in 0..TEST_TASKS {
            let tx = tx.clone();
            pool.execute(move|| {
                tx.send(1u).unwrap();
            });
        }

        assert_eq!(rx.iter().take(TEST_TASKS).sum(), TEST_TASKS);
    }

    #[test]
    fn test_should_not_panic_on_drop_if_subtasks_panic_after_drop() {
        use sync::{Arc, Barrier};

        let pool = TaskPool::new(TEST_TASKS);
        let waiter = Arc::new(Barrier::new(TEST_TASKS + 1));

        // Panic all the existing threads in a bit.
        for _ in 0..TEST_TASKS {
            let waiter = waiter.clone();
            pool.execute(move|| {
                waiter.wait();
                panic!();
            });
        }

        drop(pool);

        // Kick off the failure.
        waiter.wait();
    }
}
<|endoftext|>


<commit_before>//! Tests for the -Zrustdoc-map feature.

use cargo_test_support::registry::Package;
use cargo_test_support::{is_nightly, project, Project};

fn basic_project() -> Project {
    Package::new("bar", "1.0.0")
        .file("src/lib.rs", "pub struct Straw;")
        .publish();

    project()
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "foo"
                version = "0.1.0"
                edition = "2018"

                [dependencies]
                bar = "1.0"
            "#,
        )
        .file(
            "src/lib.rs",
            r#"
                pub fn myfun() -> Option<bar::Straw> {
                    None
                }
            "#,
        )
        .build()
}

fn docs_rs(p: &Project) {
    p.change_file(
        ".cargo/config",
        r#"
            [doc.extern-map.registries]
            crates-io = "https://docs.rs/"
        "#,
    );
}

#[cargo_test]
fn ignores_on_stable() {
    // Requires -Zrustdoc-map to use.
    let p = basic_project();
    docs_rs(&p);
    p.cargo("doc -v --no-deps")
        .with_stderr_does_not_contain("[..]--extern-html-root-url[..]")
        .run();
}

#[cargo_test]
fn simple() {
    // Basic test that it works with crates.io.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    let p = basic_project();
    docs_rs(&p);
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo [..]bar=https://docs.rs/bar/1.0.0/[..]",
        )
        .run();
    let myfun = p.read_file("target/doc/foo/fn.myfun.html");
    assert!(myfun.contains(r#"href="https://docs.rs/bar/1.0.0/bar/struct.Straw.html""#));
}

#[cargo_test]
fn std_docs() {
    // Mapping std docs somewhere else.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    let p = basic_project();
    p.change_file(
        ".cargo/config",
        r#"
            [doc.extern-map]
            std = "local"
        "#,
    );
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains("[RUNNING] `rustdoc [..]--crate-name foo [..]std=file://[..]")
        .run();
    let myfun = p.read_file("target/doc/foo/fn.myfun.html");
    assert!(myfun.contains(r#"share/doc/rust/html/core/option/enum.Option.html""#));

    p.change_file(
        ".cargo/config",
        r#"
            [doc.extern-map]
            std = "https://example.com/rust/"
        "#,
    );
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo [..]std=https://example.com/rust/[..]",
        )
        .run();
    let myfun = p.read_file("target/doc/foo/fn.myfun.html");
    assert!(myfun.contains(r#"href="https://example.com/rust/core/option/enum.Option.html""#));
}

#[cargo_test]
fn renamed_dep() {
    // Handles renamed dependencies.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    Package::new("bar", "1.0.0")
        .file("src/lib.rs", "pub struct Straw;")
        .publish();

    let p = project()
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "foo"
                version = "0.1.0"
                edition = "2018"

                [dependencies]
                groovy = { version = "1.0", package = "bar" }
            "#,
        )
        .file(
            "src/lib.rs",
            r#"
                pub fn myfun() -> Option<groovy::Straw> {
                    None
                }
            "#,
        )
        .build();
    docs_rs(&p);
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo [..]bar=https://docs.rs/bar/1.0.0/[..]",
        )
        .run();
    let myfun = p.read_file("target/doc/foo/fn.myfun.html");
    assert!(myfun.contains(r#"href="https://docs.rs/bar/1.0.0/bar/struct.Straw.html""#));
}

#[cargo_test]
fn lib_name() {
    // Handles lib name != package name.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    Package::new("bar", "1.0.0")
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "bar"
                version = "1.0.0"

                [lib]
                name = "rumpelstiltskin"
            "#,
        )
        .file("src/lib.rs", "pub struct Straw;")
        .publish();

    let p = project()
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "foo"
                version = "0.1.0"

                [dependencies]
                bar = "1.0"
            "#,
        )
        .file(
            "src/lib.rs",
            r#"
                pub fn myfun() -> Option<rumpelstiltskin::Straw> {
                    None
                }
            "#,
        )
        .build();
    docs_rs(&p);
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo [..]rumpelstiltskin=https://docs.rs/bar/1.0.0/[..]",
        )
        .run();
    let myfun = p.read_file("target/doc/foo/fn.myfun.html");
    assert!(myfun.contains(r#"href="https://docs.rs/bar/1.0.0/rumpelstiltskin/struct.Straw.html""#));
}

#[cargo_test]
fn alt_registry() {
    // Supports other registry names.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    Package::new("bar", "1.0.0")
        .alternative(true)
        .file(
            "src/lib.rs",
            r#"
                extern crate baz;
                pub struct Queen;
                pub use baz::King;
            "#,
        )
        .registry_dep("baz", "1.0")
        .publish();
    Package::new("baz", "1.0.0")
        .alternative(true)
        .file("src/lib.rs", "pub struct King;")
        .publish();
    Package::new("grimm", "1.0.0")
        .file("src/lib.rs", "pub struct Gold;")
        .publish();

    let p = project()
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "foo"
                version = "0.1.0"
                edition = "2018"

                [dependencies]
                bar = { version = "1.0", registry="alternative" }
                grimm = "1.0"
            "#,
        )
        .file(
            "src/lib.rs",
            r#"
                pub fn queen() -> bar::Queen { bar::Queen }
                pub fn king() -> bar::King { bar::King }
                pub fn gold() -> grimm::Gold { grimm::Gold }
            "#,
        )
        .file(
            ".cargo/config",
            r#"
                [doc.extern-map.registries]
                alternative = "https://example.com/{pkg_name}/{version}/"
                crates-io = "https://docs.rs/"
            "#,
        )
        .build();
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo \
            [..]bar=https://example.com/bar/1.0.0/[..]grimm=https://docs.rs/grimm/1.0.0/[..]",
        )
        .run();
    let queen = p.read_file("target/doc/foo/fn.queen.html");
    assert!(queen.contains(r#"href="https://example.com/bar/1.0.0/bar/struct.Queen.html""#));
    // The king example fails to link. Rustdoc seems to want the origin crate
    // name (baz) for re-exports. There are many issues in the issue tracker
    // for rustdoc re-exports, so I'm not sure, but I think this is maybe a
    // rustdoc issue. Alternatively, Cargo could provide mappings for all
    // transitive dependencies to fix this.
    let king = p.read_file("target/doc/foo/fn.king.html");
    assert!(king.contains(r#"-> King"#));

    let gold = p.read_file("target/doc/foo/fn.gold.html");
    assert!(gold.contains(r#"href="https://docs.rs/grimm/1.0.0/grimm/struct.Gold.html""#));
}

#[cargo_test]
fn multiple_versions() {
    // What happens when there are multiple versions.
    // NOTE: This is currently broken behavior. Rustdoc does not provide a way
    // to match renamed dependencies.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    Package::new("bar", "1.0.0")
        .file("src/lib.rs", "pub struct Spin;")
        .publish();
    Package::new("bar", "2.0.0")
        .file("src/lib.rs", "pub struct Straw;")
        .publish();
    let p = project()
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "foo"
                version = "0.1.0"
                edition = "2018"

                [dependencies]
                bar = "1.0"
                bar2 = {version="2.0", package="bar"}
            "#,
        )
        .file(
            "src/lib.rs",
            "
                pub fn fn1() -> bar::Spin {bar::Spin}
                pub fn fn2() -> bar2::Straw {bar2::Straw}
            ",
        )
        .build();
    docs_rs(&p);
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo \
            [..]bar=https://docs.rs/bar/1.0.0/[..]bar=https://docs.rs/bar/2.0.0/[..]",
        )
        .run();
    let fn1 = p.read_file("target/doc/foo/fn.fn1.html");
    // This should be 1.0.0, rustdoc seems to use the last entry when there
    // are duplicates.
    assert!(fn1.contains(r#"href="https://docs.rs/bar/2.0.0/bar/struct.Spin.html""#));
    let fn2 = p.read_file("target/doc/foo/fn.fn2.html");
    assert!(fn2.contains(r#"href="https://docs.rs/bar/2.0.0/bar/struct.Straw.html""#));
}

#[cargo_test]
fn rebuilds_when_changing() {
    // Make sure it rebuilds if the map changes.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    let p = basic_project();
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_does_not_contain("[..]--extern-html-root-url[..]")
        .run();

    docs_rs(&p);
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains("[..]--extern-html-root-url[..]")
        .run();
}
<commit_msg>Make rust-docs component optional for local testing.<commit_after>//! Tests for the -Zrustdoc-map feature.

use cargo_test_support::registry::Package;
use cargo_test_support::{is_nightly, paths, project, Project};

fn basic_project() -> Project {
    Package::new("bar", "1.0.0")
        .file("src/lib.rs", "pub struct Straw;")
        .publish();

    project()
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "foo"
                version = "0.1.0"
                edition = "2018"

                [dependencies]
                bar = "1.0"
            "#,
        )
        .file(
            "src/lib.rs",
            r#"
                pub fn myfun() -> Option<bar::Straw> {
                    None
                }
            "#,
        )
        .build()
}

fn docs_rs(p: &Project) {
    p.change_file(
        ".cargo/config",
        r#"
            [doc.extern-map.registries]
            crates-io = "https://docs.rs/"
        "#,
    );
}

#[cargo_test]
fn ignores_on_stable() {
    // Requires -Zrustdoc-map to use.
    let p = basic_project();
    docs_rs(&p);
    p.cargo("doc -v --no-deps")
        .with_stderr_does_not_contain("[..]--extern-html-root-url[..]")
        .run();
}

#[cargo_test]
fn simple() {
    // Basic test that it works with crates.io.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    let p = basic_project();
    docs_rs(&p);
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo [..]bar=https://docs.rs/bar/1.0.0/[..]",
        )
        .run();
    let myfun = p.read_file("target/doc/foo/fn.myfun.html");
    assert!(myfun.contains(r#"href="https://docs.rs/bar/1.0.0/bar/struct.Straw.html""#));
}

#[cargo_test]
fn std_docs() {
    // Mapping std docs somewhere else.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    if !cargo::util::is_ci() {
        // For local developers, skip this test if docs aren't installed.
        let docs = std::path::Path::new(&paths::sysroot()).join("share/doc/rust/html");
        if !docs.exists() {
            eprintln!(
                "documentation not found at {}, \
                skipping test (run `rustdoc component add rust-docs` to install",
                docs.display()
            );
            return;
        }
    }
    let p = basic_project();
    p.change_file(
        ".cargo/config",
        r#"
            [doc.extern-map]
            std = "local"
        "#,
    );
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains("[RUNNING] `rustdoc [..]--crate-name foo [..]std=file://[..]")
        .run();
    let myfun = p.read_file("target/doc/foo/fn.myfun.html");
    assert!(myfun.contains(r#"share/doc/rust/html/core/option/enum.Option.html""#));

    p.change_file(
        ".cargo/config",
        r#"
            [doc.extern-map]
            std = "https://example.com/rust/"
        "#,
    );
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo [..]std=https://example.com/rust/[..]",
        )
        .run();
    let myfun = p.read_file("target/doc/foo/fn.myfun.html");
    assert!(myfun.contains(r#"href="https://example.com/rust/core/option/enum.Option.html""#));
}

#[cargo_test]
fn renamed_dep() {
    // Handles renamed dependencies.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    Package::new("bar", "1.0.0")
        .file("src/lib.rs", "pub struct Straw;")
        .publish();

    let p = project()
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "foo"
                version = "0.1.0"
                edition = "2018"

                [dependencies]
                groovy = { version = "1.0", package = "bar" }
            "#,
        )
        .file(
            "src/lib.rs",
            r#"
                pub fn myfun() -> Option<groovy::Straw> {
                    None
                }
            "#,
        )
        .build();
    docs_rs(&p);
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo [..]bar=https://docs.rs/bar/1.0.0/[..]",
        )
        .run();
    let myfun = p.read_file("target/doc/foo/fn.myfun.html");
    assert!(myfun.contains(r#"href="https://docs.rs/bar/1.0.0/bar/struct.Straw.html""#));
}

#[cargo_test]
fn lib_name() {
    // Handles lib name != package name.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    Package::new("bar", "1.0.0")
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "bar"
                version = "1.0.0"

                [lib]
                name = "rumpelstiltskin"
            "#,
        )
        .file("src/lib.rs", "pub struct Straw;")
        .publish();

    let p = project()
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "foo"
                version = "0.1.0"

                [dependencies]
                bar = "1.0"
            "#,
        )
        .file(
            "src/lib.rs",
            r#"
                pub fn myfun() -> Option<rumpelstiltskin::Straw> {
                    None
                }
            "#,
        )
        .build();
    docs_rs(&p);
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo [..]rumpelstiltskin=https://docs.rs/bar/1.0.0/[..]",
        )
        .run();
    let myfun = p.read_file("target/doc/foo/fn.myfun.html");
    assert!(myfun.contains(r#"href="https://docs.rs/bar/1.0.0/rumpelstiltskin/struct.Straw.html""#));
}

#[cargo_test]
fn alt_registry() {
    // Supports other registry names.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    Package::new("bar", "1.0.0")
        .alternative(true)
        .file(
            "src/lib.rs",
            r#"
                extern crate baz;
                pub struct Queen;
                pub use baz::King;
            "#,
        )
        .registry_dep("baz", "1.0")
        .publish();
    Package::new("baz", "1.0.0")
        .alternative(true)
        .file("src/lib.rs", "pub struct King;")
        .publish();
    Package::new("grimm", "1.0.0")
        .file("src/lib.rs", "pub struct Gold;")
        .publish();

    let p = project()
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "foo"
                version = "0.1.0"
                edition = "2018"

                [dependencies]
                bar = { version = "1.0", registry="alternative" }
                grimm = "1.0"
            "#,
        )
        .file(
            "src/lib.rs",
            r#"
                pub fn queen() -> bar::Queen { bar::Queen }
                pub fn king() -> bar::King { bar::King }
                pub fn gold() -> grimm::Gold { grimm::Gold }
            "#,
        )
        .file(
            ".cargo/config",
            r#"
                [doc.extern-map.registries]
                alternative = "https://example.com/{pkg_name}/{version}/"
                crates-io = "https://docs.rs/"
            "#,
        )
        .build();
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo \
            [..]bar=https://example.com/bar/1.0.0/[..]grimm=https://docs.rs/grimm/1.0.0/[..]",
        )
        .run();
    let queen = p.read_file("target/doc/foo/fn.queen.html");
    assert!(queen.contains(r#"href="https://example.com/bar/1.0.0/bar/struct.Queen.html""#));
    // The king example fails to link. Rustdoc seems to want the origin crate
    // name (baz) for re-exports. There are many issues in the issue tracker
    // for rustdoc re-exports, so I'm not sure, but I think this is maybe a
    // rustdoc issue. Alternatively, Cargo could provide mappings for all
    // transitive dependencies to fix this.
    let king = p.read_file("target/doc/foo/fn.king.html");
    assert!(king.contains(r#"-> King"#));

    let gold = p.read_file("target/doc/foo/fn.gold.html");
    assert!(gold.contains(r#"href="https://docs.rs/grimm/1.0.0/grimm/struct.Gold.html""#));
}

#[cargo_test]
fn multiple_versions() {
    // What happens when there are multiple versions.
    // NOTE: This is currently broken behavior. Rustdoc does not provide a way
    // to match renamed dependencies.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    Package::new("bar", "1.0.0")
        .file("src/lib.rs", "pub struct Spin;")
        .publish();
    Package::new("bar", "2.0.0")
        .file("src/lib.rs", "pub struct Straw;")
        .publish();
    let p = project()
        .file(
            "Cargo.toml",
            r#"
                [package]
                name = "foo"
                version = "0.1.0"
                edition = "2018"

                [dependencies]
                bar = "1.0"
                bar2 = {version="2.0", package="bar"}
            "#,
        )
        .file(
            "src/lib.rs",
            "
                pub fn fn1() -> bar::Spin {bar::Spin}
                pub fn fn2() -> bar2::Straw {bar2::Straw}
            ",
        )
        .build();
    docs_rs(&p);
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains(
            "[RUNNING] `rustdoc [..]--crate-name foo \
            [..]bar=https://docs.rs/bar/1.0.0/[..]bar=https://docs.rs/bar/2.0.0/[..]",
        )
        .run();
    let fn1 = p.read_file("target/doc/foo/fn.fn1.html");
    // This should be 1.0.0, rustdoc seems to use the last entry when there
    // are duplicates.
    assert!(fn1.contains(r#"href="https://docs.rs/bar/2.0.0/bar/struct.Spin.html""#));
    let fn2 = p.read_file("target/doc/foo/fn.fn2.html");
    assert!(fn2.contains(r#"href="https://docs.rs/bar/2.0.0/bar/struct.Straw.html""#));
}

#[cargo_test]
fn rebuilds_when_changing() {
    // Make sure it rebuilds if the map changes.
    if !is_nightly() {
        // --extern-html-root-url is unstable
        return;
    }
    let p = basic_project();
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_does_not_contain("[..]--extern-html-root-url[..]")
        .run();

    docs_rs(&p);
    p.cargo("doc -v --no-deps -Zrustdoc-map")
        .masquerade_as_nightly_cargo()
        .with_stderr_contains("[..]--extern-html-root-url[..]")
        .run();
}
<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    storage::{BackupHandleRef, BackupStorage, FileHandle},
    ReadRecordBytes,
};
use anyhow::Result;
use bytes::Bytes;
use futures::stream::TryStreamExt;
use libra_crypto::HashValue;
use libra_types::{account_state_blob::AccountStateBlob, transaction::Version};
use std::mem::size_of;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
use tokio_util::compat::FuturesAsyncReadCompatExt;

pub struct BackupServiceClient {
    port: u16,
    client: reqwest::Client,
}

impl BackupServiceClient {
    pub fn new(port: u16) -> Self {
        Self {
            port,
            client: reqwest::Client::new(),
        }
    }

    async fn get(&self, path: &str) -> Result<impl AsyncRead> {
        Ok(self
            .client
            .get(&format!("http://localhost:{}/{}", self.port, path))
            .send()
            .await?
            .error_for_status()?
            .bytes_stream()
            .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e))
            .into_async_read()
            .compat())
    }

    pub async fn get_latest_state_root(&self) -> Result<(Version, HashValue)> {
        let mut buf = Vec::new();
        self.get("latest_state_root")
            .await?
            .read_to_end(&mut buf)
            .await?;
        Ok(lcs::from_bytes(&buf)?)
    }

    async fn get_account_range_proof(
        &self,
        key: HashValue,
        version: Version,
    ) -> Result<impl AsyncRead> {
        self.get(&format!("state_range_proof/{}/{:x}", version, key))
            .await
    }

    async fn get_state_snapshot(&self, version: Version) -> Result<impl AsyncRead> {
        self.get(&format!("state_snapshot/{}", version)).await
    }
}

fn backup_name(version: Version) -> String {
    format!("state_ver_{}", version)
}

fn chunk_name(first_idx: usize) -> String {
    format!("{}-.chunk", first_idx)
}

fn chunk_proof_name(first_idx: usize, last_idx: usize) -> String {
    format!("{}-{}.proof", first_idx, last_idx)
}

pub async fn backup_account_state(
    client: &BackupServiceClient,
    version: Version,
    storage: &impl BackupStorage,
    max_chunk_size: usize,
) -> Result<Vec<(FileHandle, FileHandle)>> {
    let backup_handle = storage.create_backup(&backup_name(version)).await?;

    let mut chunk_bytes = vec![];
    let mut ret = vec![];
    let mut current_idx: usize = 0;
    let mut chunk_first_idx: usize = 0;

    let mut state_snapshot_file = client.get_state_snapshot(version).await?;
    let mut prev_record_bytes: Option<Bytes> = None;
    while let Some(record_bytes) = state_snapshot_file.read_record_bytes().await? {
        if chunk_bytes.len() + size_of::<u32>() + record_bytes.len() > max_chunk_size {
            assert!(chunk_bytes.len() <= max_chunk_size);
            println!("Reached max_chunk_size.");

            let (chunk_handle, proof_handle) = write_chunk(
                client,
                version,
                storage,
                &backup_handle,
                &chunk_bytes,
                chunk_first_idx,
                current_idx,
                &prev_record_bytes.expect("max_chunk_size should be larger than account size."),
            )
            .await?;
            ret.push((chunk_handle, proof_handle));
            chunk_bytes = vec![];
            chunk_first_idx = current_idx + 1;
        }

        current_idx += 1;
        chunk_bytes.extend(&(record_bytes.len() as u32).to_be_bytes());
        chunk_bytes.extend(&record_bytes);
        prev_record_bytes = Some(record_bytes);
    }

    assert!(!chunk_bytes.is_empty());
    assert!(chunk_bytes.len() <= max_chunk_size);
    println!("Last chunk.");
    let (chunk_handle, proof_handle) = write_chunk(
        client,
        version,
        storage,
        &backup_handle,
        &chunk_bytes,
        chunk_first_idx,
        current_idx,
        &prev_record_bytes.expect("Should have at least one account."),
    )
    .await?;
    ret.push((chunk_handle, proof_handle));

    Ok(ret)
}

async fn write_chunk(
    client: &BackupServiceClient,
    version: u64,
    storage: &impl BackupStorage,
    backup_handle: &BackupHandleRef,
    chunk_bytes: &[u8],
    chunk_first_idx: usize,
    current_idx: usize,
    prev_record_bytes: &Bytes,
) -> Result<(FileHandle, FileHandle)> {
    let (prev_key, _): (HashValue, AccountStateBlob) = lcs::from_bytes(prev_record_bytes)?;
    println!("Asking proof for key: {:?}", prev_key,);
    let (chunk_handle, mut chunk_file) = storage
        .create_for_write(&backup_handle, &chunk_name(chunk_first_idx))
        .await?;
    chunk_file.write_all(&chunk_bytes).await?;
    let (proof_handle, mut proof_file) = storage
        .create_for_write(
            &backup_handle,
            &chunk_proof_name(chunk_first_idx, current_idx),
        )
        .await?;
    tokio::io::copy(
        &mut client.get_account_range_proof(prev_key, version).await?,
        &mut proof_file,
    )
    .await?;
    Ok((chunk_handle, proof_handle))
}
<commit_msg>[backup] BackupServiceClient explict no_proxy()<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    storage::{BackupHandleRef, BackupStorage, FileHandle},
    ReadRecordBytes,
};
use anyhow::Result;
use bytes::Bytes;
use futures::stream::TryStreamExt;
use libra_crypto::HashValue;
use libra_types::{account_state_blob::AccountStateBlob, transaction::Version};
use std::mem::size_of;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
use tokio_util::compat::FuturesAsyncReadCompatExt;

pub struct BackupServiceClient {
    port: u16,
    client: reqwest::Client,
}

impl BackupServiceClient {
    pub fn new(port: u16) -> Self {
        Self {
            port,
            client: reqwest::Client::builder()
                .no_proxy()
                .build()
                .expect("Http client should build."),
        }
    }

    async fn get(&self, path: &str) -> Result<impl AsyncRead> {
        Ok(self
            .client
            .get(&format!("http://localhost:{}/{}", self.port, path))
            .send()
            .await?
            .error_for_status()?
            .bytes_stream()
            .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e))
            .into_async_read()
            .compat())
    }

    pub async fn get_latest_state_root(&self) -> Result<(Version, HashValue)> {
        let mut buf = Vec::new();
        self.get("latest_state_root")
            .await?
            .read_to_end(&mut buf)
            .await?;
        Ok(lcs::from_bytes(&buf)?)
    }

    async fn get_account_range_proof(
        &self,
        key: HashValue,
        version: Version,
    ) -> Result<impl AsyncRead> {
        self.get(&format!("state_range_proof/{}/{:x}", version, key))
            .await
    }

    async fn get_state_snapshot(&self, version: Version) -> Result<impl AsyncRead> {
        self.get(&format!("state_snapshot/{}", version)).await
    }
}

fn backup_name(version: Version) -> String {
    format!("state_ver_{}", version)
}

fn chunk_name(first_idx: usize) -> String {
    format!("{}-.chunk", first_idx)
}

fn chunk_proof_name(first_idx: usize, last_idx: usize) -> String {
    format!("{}-{}.proof", first_idx, last_idx)
}

pub async fn backup_account_state(
    client: &BackupServiceClient,
    version: Version,
    storage: &impl BackupStorage,
    max_chunk_size: usize,
) -> Result<Vec<(FileHandle, FileHandle)>> {
    let backup_handle = storage.create_backup(&backup_name(version)).await?;

    let mut chunk_bytes = vec![];
    let mut ret = vec![];
    let mut current_idx: usize = 0;
    let mut chunk_first_idx: usize = 0;

    let mut state_snapshot_file = client.get_state_snapshot(version).await?;
    let mut prev_record_bytes: Option<Bytes> = None;
    while let Some(record_bytes) = state_snapshot_file.read_record_bytes().await? {
        if chunk_bytes.len() + size_of::<u32>() + record_bytes.len() > max_chunk_size {
            assert!(chunk_bytes.len() <= max_chunk_size);
            println!("Reached max_chunk_size.");

            let (chunk_handle, proof_handle) = write_chunk(
                client,
                version,
                storage,
                &backup_handle,
                &chunk_bytes,
                chunk_first_idx,
                current_idx,
                &prev_record_bytes.expect("max_chunk_size should be larger than account size."),
            )
            .await?;
            ret.push((chunk_handle, proof_handle));
            chunk_bytes = vec![];
            chunk_first_idx = current_idx + 1;
        }

        current_idx += 1;
        chunk_bytes.extend(&(record_bytes.len() as u32).to_be_bytes());
        chunk_bytes.extend(&record_bytes);
        prev_record_bytes = Some(record_bytes);
    }

    assert!(!chunk_bytes.is_empty());
    assert!(chunk_bytes.len() <= max_chunk_size);
    println!("Last chunk.");
    let (chunk_handle, proof_handle) = write_chunk(
        client,
        version,
        storage,
        &backup_handle,
        &chunk_bytes,
        chunk_first_idx,
        current_idx,
        &prev_record_bytes.expect("Should have at least one account."),
    )
    .await?;
    ret.push((chunk_handle, proof_handle));

    Ok(ret)
}

async fn write_chunk(
    client: &BackupServiceClient,
    version: u64,
    storage: &impl BackupStorage,
    backup_handle: &BackupHandleRef,
    chunk_bytes: &[u8],
    chunk_first_idx: usize,
    current_idx: usize,
    prev_record_bytes: &Bytes,
) -> Result<(FileHandle, FileHandle)> {
    let (prev_key, _): (HashValue, AccountStateBlob) = lcs::from_bytes(prev_record_bytes)?;
    println!("Asking proof for key: {:?}", prev_key,);
    let (chunk_handle, mut chunk_file) = storage
        .create_for_write(&backup_handle, &chunk_name(chunk_first_idx))
        .await?;
    chunk_file.write_all(&chunk_bytes).await?;
    let (proof_handle, mut proof_file) = storage
        .create_for_write(
            &backup_handle,
            &chunk_proof_name(chunk_first_idx, current_idx),
        )
        .await?;
    tokio::io::copy(
        &mut client.get_account_range_proof(prev_key, version).await?,
        &mut proof_file,
    )
    .await?;
    Ok((chunk_handle, proof_handle))
}
<|endoftext|>


<commit_before>use super::super::grid_loop::GridLoopField;
use super::super::{Grid, D, P};
use super::*;

use rand::Rng;
use FOUR_NEIGHBOURS;

pub struct GeneratorOption {
    pub disallow_dead_ends: bool,
    pub disallow_adjacent_clues: bool,
    pub technique: Technique,
    pub search_depth: i32,
    pub clue_lower_bound: Option<i32>,
    pub clue_upper_bound: Option<i32>,
}

fn check_clue_range(i: i32, options: &GeneratorOption) -> bool {
    if let Some(n) = options.clue_lower_bound {
        if !(n <= i) {
            return false;
        }
    }
    if let Some(n) = options.clue_upper_bound {
        if !(i <= n) {
            return false;
        }
    }
    true
}

pub fn generate<R: Rng>(
    height: i32,
    width: i32,
    options: &GeneratorOption,
    rng: &mut R,
) -> Option<Grid<Clue>> {
    let mut problem = Grid::new(height, width, Clue::NoClue);
    let mut current_score = 0i32;

    let n_steps = height * width * 10;
    let mut temperature = 10f64;
    let mut n_clues = 0;
    let mut has_clue = Grid::new(height, width, false);

    for _ in 0..n_steps {
        let mut update_cand = vec![];
        for y in 0..height {
            for x in 0..width {
                let pos = P(y, x);

                if options.disallow_adjacent_clues {
                    let mut flg = false;
                    for dy in -1..2 {
                        for dx in -1..2 {
                            if (dy != 0 || dx != 0)
                                && problem.get_or_default_p(pos + D(dy, dx), Clue::NoClue)
                                    != Clue::NoClue
                            {
                                flg = true;
                            }
                        }
                    }
                    if flg {
                        continue;
                    }
                }
                for i in 0..((y + 3) / 2) {
                    if !check_clue_range(i, options) {
                        continue;
                    }
                    update_cand.push((pos, Clue::Up(i)));
                }
                for i in 0..((x + 3) / 2) {
                    if !check_clue_range(i, options) {
                        continue;
                    }
                    update_cand.push((pos, Clue::Left(i)));
                }
                for i in 0..((height - y) / 2) {
                    if !check_clue_range(i, options) {
                        continue;
                    }
                    update_cand.push((pos, Clue::Down(i)));
                }
                for i in 0..((width - x + 2) / 2) {
                    if !check_clue_range(i, options) {
                        continue;
                    }
                    update_cand.push((pos, Clue::Right(i)));
                }
                if problem[pos] != Clue::NoClue {
                    update_cand.push((pos, Clue::NoClue));
                }
            }
        }
        rng.shuffle(&mut update_cand);

        let mut updated = false;

        for &(loc, clue) in &update_cand {
            let previous_clue = problem[loc];
            has_clue[loc] = clue != Clue::NoClue;
            if options.disallow_dead_ends && has_dead_end_nearby(loc, &has_clue) {
                has_clue[loc] = previous_clue != Clue::NoClue;
                continue;
            }

            problem[loc] = clue;

            let mut next_field = Field::new(&problem);
            next_field.set_technique(options.technique);
            next_field.trial_and_error(options.search_depth);

            let new_n_clues = n_clues - if previous_clue == Clue::NoClue { 0 } else { 1 }
                + if clue == Clue::NoClue { 0 } else { 1 };

            let score = next_field.grid_loop().num_decided_edges() - new_n_clues * 25;
            let update = !next_field.inconsistent()
                && (current_score < score
                    || rng.gen::<f64>() < ((score - current_score) as f64 / temperature).exp());
            if update {
                current_score = score;
                updated = true;
                n_clues = new_n_clues;

                if next_field.fully_solved() {
                    return Some(problem);
                }
                break;
            } else {
                problem[loc] = previous_clue;
                has_clue[loc] = previous_clue != Clue::NoClue;
            }
        }

        if !updated {
            break;
        }

        temperature *= 0.995f64;
    }

    None
}

fn has_dead_end_nearby(loc: P, has_clue: &Grid<bool>) -> bool {
    return is_dead_end(loc, has_clue)
        || is_dead_end(loc + D(-1, 0), has_clue)
        || is_dead_end(loc + D(1, 0), has_clue)
        || is_dead_end(loc + D(0, -1), has_clue)
        || is_dead_end(loc + D(0, 1), has_clue);
}
fn is_dead_end(pos: P, has_clue: &Grid<bool>) -> bool {
    if !has_clue.is_valid_p(pos) {
        return false;
    }
    let mut neighbour_count = 0;
    for &d in &FOUR_NEIGHBOURS {
        if !has_clue.get_or_default_p(pos + d, true) {
            neighbour_count += 1;
        }
    }
    return neighbour_count <= 1;
}
<commit_msg>Yajilin: support clue constraints in generator<commit_after>use super::super::grid_loop::GridLoopField;
use super::super::{Grid, D, P};
use super::*;

use rand::Rng;
use FOUR_NEIGHBOURS;

#[derive(Clone, Copy, Eq, PartialEq)]
pub enum ClueConstraint {
    Fixed(Clue),
    None,
    Any,
}

pub struct GeneratorOption {
    pub clue_constraint: Option<Grid<ClueConstraint>>,
    pub disallow_dead_ends: bool,
    pub disallow_adjacent_clues: bool,
    pub disallow_void_clue: bool,
    pub technique: Technique,
    pub search_depth: i32,
    pub clue_lower_bound: Option<i32>,
    pub clue_upper_bound: Option<i32>,
}

fn check_clue_range(i: i32, options: &GeneratorOption) -> bool {
    if let Some(n) = options.clue_lower_bound {
        if !(n <= i) {
            return false;
        }
    }
    if let Some(n) = options.clue_upper_bound {
        if !(i <= n) {
            return false;
        }
    }
    true
}

pub fn generate<R: Rng>(
    height: i32,
    width: i32,
    options: &GeneratorOption,
    rng: &mut R,
) -> Option<Grid<Clue>> {
    let mut problem = Grid::new(height, width, Clue::NoClue);
    let mut current_score = 0i32;

    let n_steps = height * width * 10;
    let mut temperature = 10f64;
    let mut n_clues = 0;
    let mut has_clue = Grid::new(height, width, false);

    if let Some(ref constraint) = options.clue_constraint {
        for y in 0..height {
            for x in 0..width {
                if let ClueConstraint::Fixed(c) = constraint[P(y, x)] {
                    problem[P(y, x)] = c;
                }
            }
        }
    }

    for _ in 0..n_steps {
        let mut update_cand = vec![];
        for y in 0..height {
            for x in 0..width {
                let pos = P(y, x);

                if let Some(ref constraint) = options.clue_constraint {
                    if constraint[pos] != ClueConstraint::Any {
                        continue;
                    }
                }

                if options.disallow_adjacent_clues {
                    let mut flg = false;
                    for dy in -1..2 {
                        for dx in -1..2 {
                            if (dy != 0 || dx != 0)
                                && problem.get_or_default_p(pos + D(dy, dx), Clue::NoClue)
                                    != Clue::NoClue
                            {
                                flg = true;
                            }
                        }
                    }
                    if flg {
                        continue;
                    }
                }
                for i in 0..((y + 3) / 2) {
                    if !check_clue_range(i, options) {
                        continue;
                    }
                    if options.disallow_void_clue && i == 0 && y == 0 {
                        continue;
                    }
                    update_cand.push((pos, Clue::Up(i)));
                }
                for i in 0..((x + 3) / 2) {
                    if !check_clue_range(i, options) {
                        continue;
                    }
                    if options.disallow_void_clue && i == 0 && x == 0 {
                        continue;
                    }
                    update_cand.push((pos, Clue::Left(i)));
                }
                for i in 0..((height - y) / 2) {
                    if !check_clue_range(i, options) {
                        continue;
                    }
                    if options.disallow_void_clue && i == 0 && y == height - 1 {
                        continue;
                    }
                    update_cand.push((pos, Clue::Down(i)));
                }
                for i in 0..((width - x + 2) / 2) {
                    if !check_clue_range(i, options) {
                        continue;
                    }
                    if options.disallow_void_clue && i == 0 && x == width - 1 {
                        continue;
                    }
                    update_cand.push((pos, Clue::Right(i)));
                }
                if problem[pos] != Clue::NoClue {
                    update_cand.push((pos, Clue::NoClue));
                }
            }
        }
        rng.shuffle(&mut update_cand);

        let mut updated = false;

        for &(loc, clue) in &update_cand {
            let previous_clue = problem[loc];
            has_clue[loc] = clue != Clue::NoClue;
            if options.disallow_dead_ends && has_dead_end_nearby(loc, &has_clue) {
                has_clue[loc] = previous_clue != Clue::NoClue;
                continue;
            }

            problem[loc] = clue;

            let mut next_field = Field::new(&problem);
            next_field.set_technique(options.technique);
            next_field.trial_and_error(options.search_depth);

            let new_n_clues = n_clues - if previous_clue == Clue::NoClue { 0 } else { 1 }
                + if clue == Clue::NoClue { 0 } else { 1 };

            let score = next_field.grid_loop().num_decided_edges() - new_n_clues * 25;
            let update = !next_field.inconsistent()
                && (current_score < score
                    || rng.gen::<f64>() < ((score - current_score) as f64 / temperature).exp());
            if update {
                current_score = score;
                updated = true;
                n_clues = new_n_clues;

                if next_field.fully_solved() {
                    return Some(problem);
                }
                break;
            } else {
                problem[loc] = previous_clue;
                has_clue[loc] = previous_clue != Clue::NoClue;
            }
        }

        if !updated {
            break;
        }

        temperature *= 0.995f64;
    }

    None
}

fn has_dead_end_nearby(loc: P, has_clue: &Grid<bool>) -> bool {
    return is_dead_end(loc, has_clue)
        || is_dead_end(loc + D(-1, 0), has_clue)
        || is_dead_end(loc + D(1, 0), has_clue)
        || is_dead_end(loc + D(0, -1), has_clue)
        || is_dead_end(loc + D(0, 1), has_clue);
}
fn is_dead_end(pos: P, has_clue: &Grid<bool>) -> bool {
    if !has_clue.is_valid_p(pos) {
        return false;
    }
    let mut neighbour_count = 0;
    for &d in &FOUR_NEIGHBOURS {
        if !has_clue.get_or_default_p(pos + d, true) {
            neighbour_count += 1;
        }
    }
    return neighbour_count <= 1;
}
<|endoftext|>