<commit_before>#![crate_name = "aws"]
#![crate_type = "lib"]

#![feature(collections)]
#![feature(core)]
#![feature(convert)]

extern crate curl;
#[cfg(unix)] extern crate openssl;
extern crate rustc_serialize as serialize;
extern crate time;
extern crate url;
extern crate ini;

#[macro_use] extern crate log;

pub mod credentials;
pub mod request;
pub mod signers;
<commit_msg>remove unused feature<commit_after>#![crate_name = "aws"]
#![crate_type = "lib"]

#![feature(collections)]
#![feature(convert)]

extern crate curl;
#[cfg(unix)] extern crate openssl;
extern crate rustc_serialize as serialize;
extern crate time;
extern crate url;
extern crate ini;

#[macro_use] extern crate log;

pub mod credentials;
pub mod request;
pub mod signers;
<|endoftext|>


<commit_before>//! Core I/O traits and combinators when working with Tokio.
//!
//! A description of the high-level I/O combinators can be [found online] in
//! addition to a description of the [low level details].
//!
//! [found online]: https://tokio.rs/docs/getting-started/core/
//! [low level details]: https://tokio.rs/docs/going-deeper/core-low-level/

#![deny(missing_docs, missing_debug_implementations)]
#![doc(html_root_url = "https://docs.rs/tokio-io/0.1")]

#[macro_use]
extern crate log;

#[macro_use]
extern crate futures;
extern crate bytes;

use std::io as std_io;
use std::io::Write;

use futures::{Async, Poll};
use futures::future::BoxFuture;
use futures::stream::BoxStream;

use bytes::{Buf, BufMut};

/// A convenience typedef around a `Future` whose error component is `io::Error`
pub type IoFuture<T> = BoxFuture<T, std_io::Error>;

/// A convenience typedef around a `Stream` whose error component is `io::Error`
pub type IoStream<T> = BoxStream<T, std_io::Error>;

/// A convenience macro for working with `io::Result<T>` from the `Read` and
/// `Write` traits.
///
/// This macro takes `io::Result<T>` as input, and returns `T` as the output. If
/// the input type is of the `Err` variant, then `Poll::NotReady` is returned if
/// it indicates `WouldBlock` or otherwise `Err` is returned.
#[macro_export]
macro_rules! try_nb {
    ($e:expr) => (match $e {
        Ok(t) => t,
        Err(ref e) if e.kind() == ::std::io::ErrorKind::WouldBlock => {
            return Ok(::futures::Async::NotReady)
        }
        Err(e) => return Err(e.into()),
    })
}

pub mod io;
pub mod codec;

mod copy;
mod flush;
mod framed;
mod framed_read;
mod framed_write;
mod length_delimited;
mod lines;
mod read;
mod read_exact;
mod read_to_end;
mod read_until;
mod shutdown;
mod split;
mod window;
mod write_all;

use codec::{Decoder, Encoder, Framed};
use split::{ReadHalf, WriteHalf};

/// A trait for readable objects which operated in an asynchronous and
/// futures-aware fashion.
///
/// This trait inherits from `io::Read` and indicates as a marker that an I/O
/// object is **nonblocking**, meaning that it will return an error instead of
/// blocking when bytes are read. Specifically this means that the `read`
/// function for traits that implement this type can have a few return values:
///
/// * `Ok(n)` means that `n` bytes of data was immediately read and placed into
///   the output buffer, where `n` == 0 implies that EOF has been reached.
/// * `Err(e) if e.kind() == ErrorKind::WouldBlock` means that no data was read
///   into the buffer provided. The I/O object is not currently readable but may
///   become readable in the future. Most importantly, **the current future's
///   task is scheduled to get unparked when the object is readable**. This
///   means that like `Future::poll` you'll receive a notification when the I/O
///   object is readable again.
/// * `Err(e)` for other errors are standard I/O errors coming from the
///   underlying object.
///
/// This trait importantly means that the `read` method only works in the
/// context of a future's task. The object may panic if used outside of a task.
pub trait AsyncRead: std_io::Read {
    /// Prepares an uninitialized buffer to be safe to pass to `read`. Returns
    /// `true` if the supplied buffer was zeroed out.
    ///
    /// While it would be highly unusual, implementations of [`io::Read`] are
    /// able to read data from the buffer passed as an argument. Because of
    /// this, the buffer passed to [`io::Read`] must be initialized memory. In
    /// situations where large numbers of buffers are used, constantly having to
    /// zero out buffers can be expensive.
    ///
    /// This function does any necessary work to prepare an uninitialized buffer
    /// to be safe to pass to `read`. If `read` guarantees to never attempt read
    /// data out of the supplied buffer, then `prepare_uninitialized_buffer`
    /// doesn't need to do any work.
    ///
    /// If this function returns `true`, then the memory has been zeroed out.
    /// This allows implementations of `AsyncRead` which are composed of
    /// multiple sub implementations to efficiently implement
    /// `prepare_uninitialized_buffer`.
    ///
    /// This function is called from [`read_buf`].
    ///
    /// [`io::Read`]: https://doc.rust-lang.org/std/io/trait.Read.html
    /// [`read_buf`]: #method.read_buf
    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [u8]) -> bool {
        for i in 0..buf.len() {
            buf[i] = 0;
        }

        true
    }

    /// Pull some bytes from this source into the specified `Buf`, returning
    /// how many bytes were read.
    ///
    /// The `buf` provided will have bytes read into it and the internal cursor
    /// will be advanced if any bytes were read. Note that this method typically
    /// will not reallocate the buffer provided.
    fn read_buf<B: BufMut>(&mut self, buf: &mut B) -> Poll<usize, std_io::Error> {
        if !buf.has_remaining_mut() {
            return Ok(Async::Ready(0));
        }

        unsafe {
            let n = {
                let b = buf.bytes_mut();

                self.prepare_uninitialized_buffer(b);

                try_nb!(self.read(b))
            };

            buf.advance_mut(n);
            Ok(Async::Ready(n))
        }
    }

    /// Provides a `Stream` and `Sink` interface for reading and writing to this
    /// `Io` object, using `Decode` and `Encode` to read and write the raw data.
    ///
    /// Raw I/O objects work with byte sequences, but higher-level code usually
    /// wants to batch these into meaningful chunks, called "frames". This
    /// method layers framing on top of an I/O object, by using the `Codec`
    /// traits to handle encoding and decoding of messages frames. Note that
    /// the incoming and outgoing frame types may be distinct.
    ///
    /// This function returns a *single* object that is both `Stream` and
    /// `Sink`; grouping this into a single object is often useful for layering
    /// things like gzip or TLS, which require both read and write access to the
    /// underlying object.
    ///
    /// If you want to work more directly with the streams and sink, consider
    /// calling `split` on the `Framed` returned by this method, which will
    /// break them into separate objects, allowing them to interact more easily.
    fn framed<T: Encoder + Decoder>(self, codec: T) -> Framed<Self, T>
        where Self: AsyncWrite + Sized,
    {
        framed::framed(self, codec)
    }

    /// Helper method for splitting this read/write object into two halves.
    ///
    /// The two halves returned implement the `Read` and `Write` traits,
    /// respectively.
    fn split(self) -> (ReadHalf<Self>, WriteHalf<Self>)
        where Self: AsyncWrite + Sized,
    {
        split::split(self)
    }
}

impl<T: ?Sized + AsyncRead> AsyncRead for Box<T> {
    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [u8]) -> bool {
        (**self).prepare_uninitialized_buffer(buf)
    }
}

impl<'a, T: ?Sized + AsyncRead> AsyncRead for &'a mut T {
    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [u8]) -> bool {
        (**self).prepare_uninitialized_buffer(buf)
    }
}

impl<'a> AsyncRead for &'a [u8] {
    unsafe fn prepare_uninitialized_buffer(&self, _buf: &mut [u8]) -> bool {
        false
    }
}

/// A trait for writable objects which operated in an asynchronous and
/// futures-aware fashion.
///
/// This trait inherits from `io::Write` and indicates that an I/O object is
/// **nonblocking**, meaning that it will return an error instead of blocking
/// when bytes are written. Specifically this means that the `write` function
/// for traits that implement this type can have a few return values:
///
/// * `Ok(n)` means that `n` bytes of data was immediately written .
/// * `Err(e) if e.kind() == ErrorKind::WouldBlock` means that no data was
///   written from the buffer provided. The I/O object is not currently
///   writable but may become writable in the future. Most importantly, **the
///   current future's task is scheduled to get unparked when the object is
///   readable**. This means that like `Future::poll` you'll receive a
///   notification when the I/O object is writable again.
/// * `Err(e)` for other errors are standard I/O errors coming from the
///   underlying object.
///
/// This trait importantly means that the `write` method only works in the
/// context of a future's task. The object may panic if used outside of a task.
pub trait AsyncWrite: std_io::Write {
    /// Initiates or attempts to shut down this writer, returning success when
    /// the I/O connection has completely shut down.
    ///
    /// This method is intended to be used for asynchronous shutdown of I/O
    /// connections. For example this is suitable for implementing shutdown of a
    /// TLS connection or calling `TcpStream::shutdown` on a proxied connection.
    ///
    /// This `shutdown` method is required by implementors of the
    /// `AsyncWrite` trait. Wrappers typically just want to proxy this call
    /// through to the wrapped type, and base types will typically implement
    /// shutdown logic here or just return `Ok(().into())`.
    ///
    /// # Return value
    ///
    /// This function returns a `Poll<(), io::Error>` classified as such:
    ///
    /// * `Ok(Async::Ready(()))` - indicates that the connection was
    ///   successfully shut down and is now safe to deallocate/drop/close
    ///   resources associated with it. This method means that the current task
    ///   will no longer receive any notifications due to this method and the
    ///   I/O object itself is likely no longer usable.
    ///
    /// * `Ok(Async::NotReady)` - indicates that shutdown is initiated but could
    ///   not complete just yet. This may mean that more I/O needs to happen to
    ///   continue this shutdown operation. The current task is scheduled to
    ///   receive a notification when it's otherwise ready to continue the
    ///   shutdown operation. When woken up this method should be called again.
    ///
    /// * `Err(e)` - indicates a fatal error has happened with shutdown,
    ///   indicating that the shutdown operation did not complete successfully.
    ///   This typically means that the I/O object is no longer usable.
    ///
    /// # Errors
    ///
    /// This function can return normal I/O errors through `Err`, described
    /// above. Additionally this method may also render the underlying
    /// `Write::write` method no longer usable (e.g. will return errors in the
    /// future). It's recommended that once `shutdown` is called the
    /// `write` method is no longer called.
    ///
    /// # Panics
    ///
    /// This function will panic if not called within the context of a future's
    /// task.
    fn shutdown(&mut self) -> Poll<(), std_io::Error>;

    /// Write a `Buf` into this value, returning how many bytes were written.
    ///
    /// Note that this method will advance the `buf` provided automatically by
    /// the number of bytes written.
    fn write_buf<B: Buf>(&mut self, buf: &mut B) -> Poll<usize, std_io::Error> {
        if !buf.has_remaining() {
            return Ok(Async::Ready(0));
        }

        let n = try_nb!(self.write(buf.bytes()));
        buf.advance(n);
        Ok(Async::Ready(n))
    }
}

impl<T: ?Sized + AsyncWrite> AsyncWrite for Box<T> {
    fn shutdown(&mut self) -> Poll<(), std_io::Error> {
        (**self).shutdown()
    }
}
impl<'a, T: ?Sized + AsyncWrite> AsyncWrite for &'a mut T {
    fn shutdown(&mut self) -> Poll<(), std_io::Error> {
        (**self).shutdown()
    }
}

impl AsyncRead for std_io::Repeat {
    unsafe fn prepare_uninitialized_buffer(&self, _: &mut [u8]) -> bool {
        false
    }
}

impl AsyncWrite for std_io::Sink {
    fn shutdown(&mut self) -> Poll<(), std_io::Error> {
        Ok(().into())
    }
}

// TODO: Implement `prepare_uninitialized_buffer` for `io::Take`.
// This is blocked on rust-lang/rust#27269
impl<T: AsyncRead> AsyncRead for std_io::Take<T> {
}

impl<T: AsyncWrite> AsyncWrite for std_io::BufWriter<T> {
    fn shutdown(&mut self) -> Poll<(), std_io::Error> {
        try_nb!(self.flush());
        self.get_mut().shutdown()
    }
}

impl<T: AsyncRead> AsyncRead for std_io::BufReader<T> {
    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [u8]) -> bool {
        self.get_ref().prepare_uninitialized_buffer(buf)
    }
}
<commit_msg>Implement AsyncRead for std::io::Chain<commit_after>//! Core I/O traits and combinators when working with Tokio.
//!
//! A description of the high-level I/O combinators can be [found online] in
//! addition to a description of the [low level details].
//!
//! [found online]: https://tokio.rs/docs/getting-started/core/
//! [low level details]: https://tokio.rs/docs/going-deeper/core-low-level/

#![deny(missing_docs, missing_debug_implementations)]
#![doc(html_root_url = "https://docs.rs/tokio-io/0.1")]

#[macro_use]
extern crate log;

#[macro_use]
extern crate futures;
extern crate bytes;

use std::io as std_io;
use std::io::Write;

use futures::{Async, Poll};
use futures::future::BoxFuture;
use futures::stream::BoxStream;

use bytes::{Buf, BufMut};

/// A convenience typedef around a `Future` whose error component is `io::Error`
pub type IoFuture<T> = BoxFuture<T, std_io::Error>;

/// A convenience typedef around a `Stream` whose error component is `io::Error`
pub type IoStream<T> = BoxStream<T, std_io::Error>;

/// A convenience macro for working with `io::Result<T>` from the `Read` and
/// `Write` traits.
///
/// This macro takes `io::Result<T>` as input, and returns `T` as the output. If
/// the input type is of the `Err` variant, then `Poll::NotReady` is returned if
/// it indicates `WouldBlock` or otherwise `Err` is returned.
#[macro_export]
macro_rules! try_nb {
    ($e:expr) => (match $e {
        Ok(t) => t,
        Err(ref e) if e.kind() == ::std::io::ErrorKind::WouldBlock => {
            return Ok(::futures::Async::NotReady)
        }
        Err(e) => return Err(e.into()),
    })
}

pub mod io;
pub mod codec;

mod copy;
mod flush;
mod framed;
mod framed_read;
mod framed_write;
mod length_delimited;
mod lines;
mod read;
mod read_exact;
mod read_to_end;
mod read_until;
mod shutdown;
mod split;
mod window;
mod write_all;

use codec::{Decoder, Encoder, Framed};
use split::{ReadHalf, WriteHalf};

/// A trait for readable objects which operated in an asynchronous and
/// futures-aware fashion.
///
/// This trait inherits from `io::Read` and indicates as a marker that an I/O
/// object is **nonblocking**, meaning that it will return an error instead of
/// blocking when bytes are read. Specifically this means that the `read`
/// function for traits that implement this type can have a few return values:
///
/// * `Ok(n)` means that `n` bytes of data was immediately read and placed into
///   the output buffer, where `n` == 0 implies that EOF has been reached.
/// * `Err(e) if e.kind() == ErrorKind::WouldBlock` means that no data was read
///   into the buffer provided. The I/O object is not currently readable but may
///   become readable in the future. Most importantly, **the current future's
///   task is scheduled to get unparked when the object is readable**. This
///   means that like `Future::poll` you'll receive a notification when the I/O
///   object is readable again.
/// * `Err(e)` for other errors are standard I/O errors coming from the
///   underlying object.
///
/// This trait importantly means that the `read` method only works in the
/// context of a future's task. The object may panic if used outside of a task.
pub trait AsyncRead: std_io::Read {
    /// Prepares an uninitialized buffer to be safe to pass to `read`. Returns
    /// `true` if the supplied buffer was zeroed out.
    ///
    /// While it would be highly unusual, implementations of [`io::Read`] are
    /// able to read data from the buffer passed as an argument. Because of
    /// this, the buffer passed to [`io::Read`] must be initialized memory. In
    /// situations where large numbers of buffers are used, constantly having to
    /// zero out buffers can be expensive.
    ///
    /// This function does any necessary work to prepare an uninitialized buffer
    /// to be safe to pass to `read`. If `read` guarantees to never attempt read
    /// data out of the supplied buffer, then `prepare_uninitialized_buffer`
    /// doesn't need to do any work.
    ///
    /// If this function returns `true`, then the memory has been zeroed out.
    /// This allows implementations of `AsyncRead` which are composed of
    /// multiple sub implementations to efficiently implement
    /// `prepare_uninitialized_buffer`.
    ///
    /// This function is called from [`read_buf`].
    ///
    /// [`io::Read`]: https://doc.rust-lang.org/std/io/trait.Read.html
    /// [`read_buf`]: #method.read_buf
    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [u8]) -> bool {
        for i in 0..buf.len() {
            buf[i] = 0;
        }

        true
    }

    /// Pull some bytes from this source into the specified `Buf`, returning
    /// how many bytes were read.
    ///
    /// The `buf` provided will have bytes read into it and the internal cursor
    /// will be advanced if any bytes were read. Note that this method typically
    /// will not reallocate the buffer provided.
    fn read_buf<B: BufMut>(&mut self, buf: &mut B) -> Poll<usize, std_io::Error> {
        if !buf.has_remaining_mut() {
            return Ok(Async::Ready(0));
        }

        unsafe {
            let n = {
                let b = buf.bytes_mut();

                self.prepare_uninitialized_buffer(b);

                try_nb!(self.read(b))
            };

            buf.advance_mut(n);
            Ok(Async::Ready(n))
        }
    }

    /// Provides a `Stream` and `Sink` interface for reading and writing to this
    /// `Io` object, using `Decode` and `Encode` to read and write the raw data.
    ///
    /// Raw I/O objects work with byte sequences, but higher-level code usually
    /// wants to batch these into meaningful chunks, called "frames". This
    /// method layers framing on top of an I/O object, by using the `Codec`
    /// traits to handle encoding and decoding of messages frames. Note that
    /// the incoming and outgoing frame types may be distinct.
    ///
    /// This function returns a *single* object that is both `Stream` and
    /// `Sink`; grouping this into a single object is often useful for layering
    /// things like gzip or TLS, which require both read and write access to the
    /// underlying object.
    ///
    /// If you want to work more directly with the streams and sink, consider
    /// calling `split` on the `Framed` returned by this method, which will
    /// break them into separate objects, allowing them to interact more easily.
    fn framed<T: Encoder + Decoder>(self, codec: T) -> Framed<Self, T>
        where Self: AsyncWrite + Sized,
    {
        framed::framed(self, codec)
    }

    /// Helper method for splitting this read/write object into two halves.
    ///
    /// The two halves returned implement the `Read` and `Write` traits,
    /// respectively.
    fn split(self) -> (ReadHalf<Self>, WriteHalf<Self>)
        where Self: AsyncWrite + Sized,
    {
        split::split(self)
    }
}

impl<T: ?Sized + AsyncRead> AsyncRead for Box<T> {
    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [u8]) -> bool {
        (**self).prepare_uninitialized_buffer(buf)
    }
}

impl<'a, T: ?Sized + AsyncRead> AsyncRead for &'a mut T {
    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [u8]) -> bool {
        (**self).prepare_uninitialized_buffer(buf)
    }
}

impl<'a> AsyncRead for &'a [u8] {
    unsafe fn prepare_uninitialized_buffer(&self, _buf: &mut [u8]) -> bool {
        false
    }
}

/// A trait for writable objects which operated in an asynchronous and
/// futures-aware fashion.
///
/// This trait inherits from `io::Write` and indicates that an I/O object is
/// **nonblocking**, meaning that it will return an error instead of blocking
/// when bytes are written. Specifically this means that the `write` function
/// for traits that implement this type can have a few return values:
///
/// * `Ok(n)` means that `n` bytes of data was immediately written .
/// * `Err(e) if e.kind() == ErrorKind::WouldBlock` means that no data was
///   written from the buffer provided. The I/O object is not currently
///   writable but may become writable in the future. Most importantly, **the
///   current future's task is scheduled to get unparked when the object is
///   readable**. This means that like `Future::poll` you'll receive a
///   notification when the I/O object is writable again.
/// * `Err(e)` for other errors are standard I/O errors coming from the
///   underlying object.
///
/// This trait importantly means that the `write` method only works in the
/// context of a future's task. The object may panic if used outside of a task.
pub trait AsyncWrite: std_io::Write {
    /// Initiates or attempts to shut down this writer, returning success when
    /// the I/O connection has completely shut down.
    ///
    /// This method is intended to be used for asynchronous shutdown of I/O
    /// connections. For example this is suitable for implementing shutdown of a
    /// TLS connection or calling `TcpStream::shutdown` on a proxied connection.
    ///
    /// This `shutdown` method is required by implementors of the
    /// `AsyncWrite` trait. Wrappers typically just want to proxy this call
    /// through to the wrapped type, and base types will typically implement
    /// shutdown logic here or just return `Ok(().into())`.
    ///
    /// # Return value
    ///
    /// This function returns a `Poll<(), io::Error>` classified as such:
    ///
    /// * `Ok(Async::Ready(()))` - indicates that the connection was
    ///   successfully shut down and is now safe to deallocate/drop/close
    ///   resources associated with it. This method means that the current task
    ///   will no longer receive any notifications due to this method and the
    ///   I/O object itself is likely no longer usable.
    ///
    /// * `Ok(Async::NotReady)` - indicates that shutdown is initiated but could
    ///   not complete just yet. This may mean that more I/O needs to happen to
    ///   continue this shutdown operation. The current task is scheduled to
    ///   receive a notification when it's otherwise ready to continue the
    ///   shutdown operation. When woken up this method should be called again.
    ///
    /// * `Err(e)` - indicates a fatal error has happened with shutdown,
    ///   indicating that the shutdown operation did not complete successfully.
    ///   This typically means that the I/O object is no longer usable.
    ///
    /// # Errors
    ///
    /// This function can return normal I/O errors through `Err`, described
    /// above. Additionally this method may also render the underlying
    /// `Write::write` method no longer usable (e.g. will return errors in the
    /// future). It's recommended that once `shutdown` is called the
    /// `write` method is no longer called.
    ///
    /// # Panics
    ///
    /// This function will panic if not called within the context of a future's
    /// task.
    fn shutdown(&mut self) -> Poll<(), std_io::Error>;

    /// Write a `Buf` into this value, returning how many bytes were written.
    ///
    /// Note that this method will advance the `buf` provided automatically by
    /// the number of bytes written.
    fn write_buf<B: Buf>(&mut self, buf: &mut B) -> Poll<usize, std_io::Error> {
        if !buf.has_remaining() {
            return Ok(Async::Ready(0));
        }

        let n = try_nb!(self.write(buf.bytes()));
        buf.advance(n);
        Ok(Async::Ready(n))
    }
}

impl<T: ?Sized + AsyncWrite> AsyncWrite for Box<T> {
    fn shutdown(&mut self) -> Poll<(), std_io::Error> {
        (**self).shutdown()
    }
}
impl<'a, T: ?Sized + AsyncWrite> AsyncWrite for &'a mut T {
    fn shutdown(&mut self) -> Poll<(), std_io::Error> {
        (**self).shutdown()
    }
}

impl AsyncRead for std_io::Repeat {
    unsafe fn prepare_uninitialized_buffer(&self, _: &mut [u8]) -> bool {
        false
    }
}

impl AsyncWrite for std_io::Sink {
    fn shutdown(&mut self) -> Poll<(), std_io::Error> {
        Ok(().into())
    }
}

// TODO: Implement `prepare_uninitialized_buffer` for `io::Take`.
// This is blocked on rust-lang/rust#27269
impl<T: AsyncRead> AsyncRead for std_io::Take<T> {
}

// TODO: Implement `prepare_uninitialized_buffer` when upstream exposes inner
// parts
impl<T, U> AsyncRead for std_io::Chain<T, U>
    where T: AsyncRead,
          U: AsyncRead,
{
}

impl<T: AsyncWrite> AsyncWrite for std_io::BufWriter<T> {
    fn shutdown(&mut self) -> Poll<(), std_io::Error> {
        try_nb!(self.flush());
        self.get_mut().shutdown()
    }
}

impl<T: AsyncRead> AsyncRead for std_io::BufReader<T> {
    unsafe fn prepare_uninitialized_buffer(&self, buf: &mut [u8]) -> bool {
        self.get_ref().prepare_uninitialized_buffer(buf)
    }
}
<|endoftext|>


<commit_before>#![feature(collections, unboxed_closures, core)]
extern crate itertools;
extern crate rustc_serialize;

use std::collections::{BTreeMap, HashMap, HashSet};
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::iter;
use std::ops;

use itertools::{GroupBy, Itertools};

/// A Document contains an id and content.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Eq, Debug, RustcEncodable, RustcDecodable)]
pub struct Document {
    id: String,
    content: String,
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document { id: id.into(), content: content.into(), }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn content(&self) -> &str {
        &self.content
    }
}

impl Hash for Document {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        state.write(self.id.as_bytes());
    }
}

impl PartialEq<Document> for Document {
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug)]
struct IndexedDocument {
    pub doc: Arc<Document>,
    pub highlighted: (usize, usize),
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

fn analyze_doc(doc: &str)
-> iter::FlatMap<
    iter::Filter<
        GroupBy<bool, std::str::CharIndices, fn(&(usize, char)) -> bool>,
        fn(&(bool, Vec<(usize, char)>)) -> bool>,
    iter::Map<ops::Range<usize>, Ngrams>,
    fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>
{
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
#[derive(Debug)]
pub struct InvertedIndex {
    index: BTreeMap<String, HashSet<IndexedDocument>>,
    docs: BTreeMap<String, Arc<Document>>,
}

impl InvertedIndex {
    pub fn new() -> InvertedIndex {
        InvertedIndex {
            index: BTreeMap::new(),
            docs: BTreeMap::new(),
        }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let doc = Arc::new(doc);
        let analyzed  = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, highlights) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&IndexedDocument { doc: doc.clone(), highlighted: highlights });
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            self.index.entry(ngram).or_insert_with(|| HashSet::new())
                .insert(IndexedDocument { doc: doc.clone(), highlighted: highlighted });
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    pub fn search(&self, query: &str) -> HashSet<SearchResult> {
        let map = query.split_whitespace()
            .flat_map(|word| self.index.get(&word.to_lowercase()))
            .flat_map(|docs| docs)
            .cloned()
            .fold(HashMap::new(), |mut map, search_result| {
                map.entry(search_result.doc)
                    .or_insert(Vec::new())
                    .push(search_result.highlighted);
                map
            });
        map.into_iter()
            .map(|(doc, mut highlights)| {
                highlights.sort();
                SearchResult { doc: doc, highlights: highlights }
            })
        .collect()
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug, RustcEncodable)]
pub struct SearchResult {
    doc: Arc<Document>,
    highlights: Vec<(usize, usize)>,
}

impl SearchResult {
    #[cfg(test)]
    fn new(doc: Arc<Document>, highlights: Vec<(usize, usize)>) -> SearchResult {
        SearchResult {
            doc: doc,
            highlights: highlights,
        }
    }

    pub fn doc(&self) -> &Arc<Document> {
        &self.doc
    }

    pub fn highlights(&self) -> &Vec<(usize, usize)> {
        &self.highlights
    }

    pub fn highlighted_content(&self) -> String {
        let &SearchResult {ref doc, ref highlights} = self;
        let content = doc.content();
        let mut begin_idx = 0;
        let mut parts = vec![];
        for &(begin, end) in highlights {
            parts.push(&content[begin_idx..begin]);
            parts.push("");
            parts.push(&content[begin..end]);
            parts.push("");
            begin_idx = end;
        }
        parts.push(&content[begin_idx..]);
        parts.into_iter().join("")
    }
}

#[test]
fn test_search1() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    index.index(doc1.clone());
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc2.clone());
    let search_results = index.search("today");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(25, 30)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 18)])
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());
}

#[test]
fn test_ngrams() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(6, 8), (25, 27)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 15)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());

}

#[test]
fn test_search2() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("2", "what to do today");
    let doc2 = Document::new("3", "hey today");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(5, 7), (11, 13)]),
        SearchResult::new(Arc::new(doc2), vec![(4, 6)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!(expected[0].highlighted_content(), "what to do today");
}

#[test]
fn test_unicode() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc.clone());
    let to_search = "å";
    let search_results = index.search(to_search);
    let &SearchResult { ref doc, ref highlights } = search_results.iter().next().unwrap();
    let (begin, end) = highlights[0];
    assert_eq!(&doc.content()[begin..end], to_search);
}

#[test]
fn test_update_doc() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc);
    let doc = Document::new("0", "different");
    index.index(doc);
    let search_results = index.search("å");
    assert!(search_results.is_empty());
    assert_eq!(index.docs.len(), 1);
}
<commit_msg>Avoid an allocation by returning an iterator over SearchResults instead of a set<commit_after>#![feature(collections, unboxed_closures, core)]
extern crate itertools;
extern crate rustc_serialize;

use std::collections::{BTreeMap, HashMap, HashSet};
use std::collections::hash_map::IntoIter;
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use std::iter;
use std::ops;

use itertools::{GroupBy, Itertools};

/// A Document contains an id and content.
/// Hashing and equality are based only on the id field.
#[derive(Clone, Eq, Debug, RustcEncodable, RustcDecodable)]
pub struct Document {
    id: String,
    content: String,
}

impl Document {
    /// Construct a new Document from an id and content.
    /// Both two arguments can be anything that can be turned into a String.
    pub fn new<S, T>(id: S, content: T) -> Document
        where S: Into<String>,
              T: Into<String>
    {
        Document { id: id.into(), content: content.into(), }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn content(&self) -> &str {
        &self.content
    }
}

impl Hash for Document {
    fn hash<H>(&self, state: &mut H) where H: Hasher {
        state.write(self.id.as_bytes());
    }
}

impl PartialEq<Document> for Document {
    fn eq(&self, other: &Document) -> bool {
        self.id == other.id
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug)]
struct IndexedDocument {
    pub doc: Arc<Document>,
    pub highlighted: (usize, usize),
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

fn analyze_doc(doc: &str)
-> iter::FlatMap<
    iter::Filter<
        GroupBy<bool, std::str::CharIndices, fn(&(usize, char)) -> bool>,
        fn(&(bool, Vec<(usize, char)>)) -> bool>,
    iter::Map<ops::Range<usize>, Ngrams>,
    fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>
{
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to sets of Documents.
#[derive(Debug)]
pub struct InvertedIndex {
    index: BTreeMap<String, HashSet<IndexedDocument>>,
    docs: BTreeMap<String, Arc<Document>>,
}

impl InvertedIndex {
    pub fn new() -> InvertedIndex {
        InvertedIndex {
            index: BTreeMap::new(),
            docs: BTreeMap::new(),
        }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let doc = Arc::new(doc);
        let analyzed  = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, highlights) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&IndexedDocument { doc: doc.clone(), highlighted: highlights });
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            self.index.entry(ngram).or_insert_with(|| HashSet::new())
                .insert(IndexedDocument { doc: doc.clone(), highlighted: highlighted });
        }
    }

    /// A basic search implementation that splits the query's content into whitespace-separated
    /// words, looks up the set of Documents for each word, and then concatenates the sets.
    pub fn search(&self, query: &str) -> SearchResults {
        let map = query.split_whitespace()
            .flat_map(|word| self.index.get(&word.to_lowercase()))
            .flat_map(|doc| doc)
            .cloned()
            .fold(HashMap::new(), |mut map, search_result| {
                map.entry(search_result.doc)
                    .or_insert(Vec::new())
                    .push(search_result.highlighted);
                map
            });
        let results = map.into_iter()
            .map(SearchResult::new_ as fn((Arc<Document>, Vec<(usize, usize)>)) -> SearchResult);
        SearchResults { it: results }
    }
}

pub struct SearchResults {
    it: iter::Map<
            IntoIter<Arc<Document>, Vec<(usize, usize)>>,
            fn((Arc<Document>, Vec<(usize, usize)>)) -> SearchResult>
}

impl Iterator for SearchResults {
    type Item = SearchResult;
    fn next(&mut self) -> Option<SearchResult> {
        self.it.next()
    }
}

#[derive(Clone, Eq, PartialEq, Hash, Debug, RustcEncodable)]
pub struct SearchResult {
    doc: Arc<Document>,
    highlights: Vec<(usize, usize)>,
}

impl SearchResult {
    fn new_((doc, highlights): (Arc<Document>, Vec<(usize, usize)>)) -> SearchResult {
        SearchResult::new(doc, highlights)
    }

    fn new(doc: Arc<Document>, mut highlights: Vec<(usize, usize)>) -> SearchResult {
        highlights.sort();
        SearchResult {
            doc: doc,
            highlights: highlights,
        }
    }

    pub fn doc(&self) -> &Arc<Document> {
        &self.doc
    }

    pub fn highlights(&self) -> &Vec<(usize, usize)> {
        &self.highlights
    }

    pub fn highlighted_content(&self) -> String {
        let &SearchResult {ref doc, ref highlights} = self;
        let content = doc.content();
        let mut begin_idx = 0;
        let mut parts = vec![];
        for &(begin, end) in highlights {
            parts.push(&content[begin_idx..begin]);
            parts.push("");
            parts.push(&content[begin..end]);
            parts.push("");
            begin_idx = end;
        }
        parts.push(&content[begin_idx..]);
        parts.into_iter().join("")
    }
}

#[test]
fn test_search1() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    index.index(doc1.clone());
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc2.clone());
    let search_results = index.search("today");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(25, 30)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 18)])
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());
}

#[test]
fn test_ngrams() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("1", "learn to program in rust today");
    let doc2 = Document::new("2", "what did you today do");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(6, 8), (25, 27)]),
        SearchResult::new(Arc::new(doc2), vec![(13, 15)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!("learn to program in rust today", expected[0].highlighted_content());

}

#[test]
fn test_search2() {
    let mut index = InvertedIndex::new();
    let doc1 = Document::new("2", "what to do today");
    let doc2 = Document::new("3", "hey today");
    index.index(doc1.clone());
    index.index(doc2.clone());
    let search_results = index.search("to");
    let expected = [
        SearchResult::new(Arc::new(doc1), vec![(5, 7), (11, 13)]),
        SearchResult::new(Arc::new(doc2), vec![(4, 6)]),
    ];
    assert_eq!(search_results, expected.iter().cloned().collect());
    assert_eq!(expected[0].highlighted_content(), "what to do today");
}

#[test]
fn test_unicode() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc.clone());
    let to_search = "å";
    let search_results = index.search(to_search);
    let &SearchResult { ref doc, ref highlights } = search_results.iter().next().unwrap();
    let (begin, end) = highlights[0];
    assert_eq!(&doc.content()[begin..end], to_search);
}

#[test]
fn test_update_doc() {
    let mut index = InvertedIndex::new();
    let doc = Document::new("0", "abc åäö");
    index.index(doc);
    let doc = Document::new("0", "different");
    index.index(doc);
    let search_results = index.search("å");
    assert!(search_results.is_empty());
    assert_eq!(index.docs.len(), 1);
}
<|endoftext|>


<commit_before>#![feature(if_let)]

use std::os;
use std::io::Command;
use std::io::process::InheritFd;
use std::default::Default;

/// Extra configuration to pass to gcc.
pub struct Config {
    /// Directories where gcc will look for header files.
    pub include_directories: Vec<Path>,
    /// Additional definitions (`-DKEY` or `-DKEY=VALUE`).
    pub definitions: Vec<(String, Option<String>)>,
}

impl Default for Config {
    fn default() -> Config {
        Config {
            include_directories: Vec::new(),
            definitions: Vec::new(),
        }
    }
}

/// Compile a library from the given set of input C files.
///
/// This will simply compile all files into object files and then assemble them
/// into the output. This will read the standard environment variables to detect
/// cross compilations and such.
///
/// # Example
///
/// ```no_run
/// use std::default::Default;
/// gcc::compile_library("libfoo.a", &Default::default(), &[
///     "foo.c",
///     "bar.c",
/// ]);
/// ```
pub fn compile_library(output: &str, config: &Config, files: &[&str]) {
    assert!(output.starts_with("lib"));
    assert!(output.ends_with(".a"));

    let target = os::getenv("TARGET").unwrap();
    let opt_level = os::getenv("OPT_LEVEL").unwrap();

    let mut cmd = Command::new(gcc());
    cmd.arg(format!("-O{}", opt_level));
    cmd.arg("-c");
    cmd.arg("-ffunction-sections").arg("-fdata-sections");
    cmd.args(cflags().as_slice());

    if target.as_slice().contains("i686") {
        cmd.arg("-m32");
    } else if target.as_slice().contains("x86_64") {
        cmd.arg("-m64");
    }

    if !target.as_slice().contains("i686") {
        cmd.arg("-fPIC");
    }

    for directory in config.include_directories.iter() {
        cmd.arg("-I").arg(directory);
    }

    for &(ref key, ref value) in config.definitions.iter() {
        if let &Some(ref value) = value {
            cmd.arg(format!("-D{}={}", key, value));
        } else {
            cmd.arg(format!("-D{}", key));
        }
    }

    let src = Path::new(os::getenv("CARGO_MANIFEST_DIR").unwrap());
    let dst = Path::new(os::getenv("OUT_DIR").unwrap());
    let mut objects = Vec::new();
    for file in files.iter() {
        let obj = dst.join(*file).with_extension("o");
        run(cmd.clone().arg(src.join(*file)).arg("-o").arg(&obj));
        objects.push(obj);
    }

    run(Command::new(ar()).arg("crus")
                          .arg(dst.join(output))
                          .args(objects.as_slice()));
    println!("cargo:rustc-flags=-L {} -l {}",
             dst.display(), output.slice(3, output.len() - 2));
}

fn run(cmd: &mut Command) {
    println!("running: {}", cmd);
    assert!(cmd.stdout(InheritFd(1))
               .stderr(InheritFd(2))
               .status()
               .unwrap()
               .success());

}

fn gcc() -> String {
    os::getenv("CC").unwrap_or(if cfg!(windows) {
        "gcc".to_string()
    } else {
        "cc".to_string()
    })
}

fn ar() -> String {
    os::getenv("AR").unwrap_or("ar".to_string())
}

fn cflags() -> Vec<String> {
    os::getenv("CFLAGS").unwrap_or(String::new())
       .as_slice().words().map(|s| s.to_string())
       .collect()
}
<commit_msg>Create output directory if it doesn't exist<commit_after>#![feature(if_let)]

use std::os;
use std::io::Command;
use std::io::process::InheritFd;
use std::default::Default;

/// Extra configuration to pass to gcc.
pub struct Config {
    /// Directories where gcc will look for header files.
    pub include_directories: Vec<Path>,
    /// Additional definitions (`-DKEY` or `-DKEY=VALUE`).
    pub definitions: Vec<(String, Option<String>)>,
}

impl Default for Config {
    fn default() -> Config {
        Config {
            include_directories: Vec::new(),
            definitions: Vec::new(),
        }
    }
}

/// Compile a library from the given set of input C files.
///
/// This will simply compile all files into object files and then assemble them
/// into the output. This will read the standard environment variables to detect
/// cross compilations and such.
///
/// # Example
///
/// ```no_run
/// use std::default::Default;
/// gcc::compile_library("libfoo.a", &Default::default(), &[
///     "foo.c",
///     "bar.c",
/// ]);
/// ```
pub fn compile_library(output: &str, config: &Config, files: &[&str]) {
    assert!(output.starts_with("lib"));
    assert!(output.ends_with(".a"));

    let target = os::getenv("TARGET").unwrap();
    let opt_level = os::getenv("OPT_LEVEL").unwrap();

    let mut cmd = Command::new(gcc());
    cmd.arg(format!("-O{}", opt_level));
    cmd.arg("-c");
    cmd.arg("-ffunction-sections").arg("-fdata-sections");
    cmd.args(cflags().as_slice());

    if target.as_slice().contains("i686") {
        cmd.arg("-m32");
    } else if target.as_slice().contains("x86_64") {
        cmd.arg("-m64");
    }

    if !target.as_slice().contains("i686") {
        cmd.arg("-fPIC");
    }

    for directory in config.include_directories.iter() {
        cmd.arg("-I").arg(directory);
    }

    for &(ref key, ref value) in config.definitions.iter() {
        if let &Some(ref value) = value {
            cmd.arg(format!("-D{}={}", key, value));
        } else {
            cmd.arg(format!("-D{}", key));
        }
    }

    let src = Path::new(os::getenv("CARGO_MANIFEST_DIR").unwrap());
    let dst = Path::new(os::getenv("OUT_DIR").unwrap());
    let mut objects = Vec::new();
    for file in files.iter() {
        let obj = dst.join(*file).with_extension("o");
        std::io::fs::mkdir_recursive(&obj.dir_path(), std::io::USER_RWX).unwrap();
        run(cmd.clone().arg(src.join(*file)).arg("-o").arg(&obj));
        objects.push(obj);
    }

    run(Command::new(ar()).arg("crus")
                          .arg(dst.join(output))
                          .args(objects.as_slice()));
    println!("cargo:rustc-flags=-L {} -l {}",
             dst.display(), output.slice(3, output.len() - 2));
}

fn run(cmd: &mut Command) {
    println!("running: {}", cmd);
    assert!(cmd.stdout(InheritFd(1))
               .stderr(InheritFd(2))
               .status()
               .unwrap()
               .success());

}

fn gcc() -> String {
    os::getenv("CC").unwrap_or(if cfg!(windows) {
        "gcc".to_string()
    } else {
        "cc".to_string()
    })
}

fn ar() -> String {
    os::getenv("AR").unwrap_or("ar".to_string())
}

fn cflags() -> Vec<String> {
    os::getenv("CFLAGS").unwrap_or(String::new())
       .as_slice().words().map(|s| s.to_string())
       .collect()
}
<|endoftext|>


<commit_before>#![feature(macro_rules, phase)]
#![crate_type = "dylib"]
#![crate_type = "rlib"]

// test harness access
#[cfg(test)]
extern crate test;

#[phase(plugin, link)]
extern crate log;

#[phase(plugin)]
extern crate serde_macros;

#[cfg(test)]
extern crate debug;

#[cfg(test)]
extern crate serialize;

pub mod de;
pub mod ser;
pub mod json;

//#[cfg(test)]
//pub mod bench_bytes;

#[cfg(test)]
pub mod bench_enum;

#[cfg(test)]
pub mod bench_struct;

#[cfg(test)]
pub mod bench_vec;

#[cfg(test)]
pub mod bench_map;

#[cfg(test)]
pub mod bench_log;

// an inner module so we can use serde_macros.
mod serde {
    pub use de;
    pub use ser;
}
<commit_msg>Export {S,Des}erializ{er,able} from serde module<commit_after>#![feature(macro_rules, phase)]
#![crate_type = "dylib"]
#![crate_type = "rlib"]

// test harness access
#[cfg(test)]
extern crate test;

#[phase(plugin, link)]
extern crate log;

#[phase(plugin)]
extern crate serde_macros;

#[cfg(test)]
extern crate debug;

#[cfg(test)]
extern crate serialize;

pub use ser::{Serializer, Serializable};
pub use de::{Deserializer, Deserializable};

pub mod de;
pub mod ser;
pub mod json;

//#[cfg(test)]
//pub mod bench_bytes;

#[cfg(test)]
pub mod bench_enum;

#[cfg(test)]
pub mod bench_struct;

#[cfg(test)]
pub mod bench_vec;

#[cfg(test)]
pub mod bench_map;

#[cfg(test)]
pub mod bench_log;

// an inner module so we can use serde_macros.
mod serde {
    pub use de;
    pub use ser;
}
<|endoftext|>


<commit_before>#![feature(plugin)]
#![plugin(dynasm)]
extern crate dynasmrt;
extern crate itertools;
extern crate crossbeam;
#[macro_use]
extern crate bitflags;
extern crate fnv;
extern crate num_bigint;
extern crate num_traits;

use std::borrow::Cow;
use std::error::Error;
use std::fmt::{self, Display, Formatter};
use std::borrow::Borrow;
use std::collections::HashMap;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use std::rc::Rc;
use std::str;

mod core;
mod label;
mod program;
mod parser;
mod assembler;
use program::{Command, SourceLoc, Integer, BigInteger};
use label::Label;

// re-export public api
pub use core::Interpreter;
#[cfg(target_arch = "x86_64")]
pub use core::debug_compile;

// This is here for visibility reasons
/// This struct contains the internal representation of a whitespace program.
#[derive(Debug, Clone)]
pub struct Program {
    source: Option<Vec<u8>>,
    commands: Vec<Command>,
    locs: Option<Vec<SourceLoc>>,
    source_is_whitespace: bool
}

// Also here for visibility reasons.
impl Program {
    fn compile(&mut self) -> Result<(), WsError> {
        let mut index_map = HashMap::<Rc<Label>, usize>::new();

        let locs = self.locs.as_ref().expect("Tried to compile a stripped program");

        for (index, (command, loc)) in self.commands.iter().zip(locs).enumerate() {
            if let Command::Label = *command {
                match index_map.entry(loc.label.clone().unwrap()) {
                    Occupied(_) => return Err(WsError::new(
                        WsErrorKind::ParseError(loc.line, loc.column, loc.span.start),
                        format!("Duplicate label: {}", loc.label.as_ref().unwrap())
                    )),
                    Vacant(e)   => e.insert(index + 1)
                };
            }
        }

        for (command, loc) in self.commands.iter_mut().zip(locs) {
            match *command {
                Command::Call {ref mut index} |
                Command::Jump {ref mut index} |
                Command::JumpIfZero {ref mut index} |
                Command::JumpIfNegative {ref mut index} => match index_map.entry(loc.label.clone().unwrap()) {
                    Occupied(e) => *index = *e.get(),
                    Vacant(_)   => return Err(WsError::new(
                        WsErrorKind::ParseError(loc.line, loc.column, loc.span.start),
                        format!("Undefined label: {}", loc.label.as_ref().unwrap())
                    ))
                },
                _ => ()
            };
        }

        Ok(())
    }
}

bitflags! {
    /// These are bitflag options to alter the behaviour of the interpreter.
    pub flags Options: u8 {
        /// Use wrapping arithmetric instead of detecting overflow and falling back
        const IGNORE_OVERFLOW    = 0b0000_0001,
        /// When an unknown key used to get an item from the heap, return 0 instead of causing an error.
        const UNCHECKED_HEAP     = 0b0000_0010,
        /// Don't fall back to biginteger interpretation on overflow, instead return an overflow error.
        const NO_FALLBACK        = 0b0001_0000,

        // features
        // const NEGATIVE_COPY   = 0b0000_1000,
    }
}

/// The common error type returned from all whitespacers execution functions.
#[derive(Debug)]
pub struct WsError {
    message: Cow<'static, str>,
    /// The kind of error that occurred.
    pub kind: WsErrorKind,
    location: Option<usize>,
    cause: Option<Box<Error>>
}

/// Simple information on what kind of error occurred.
#[derive(Debug, Clone)]
pub enum WsErrorKind {
    /// Compile-time parse error
    ParseError(usize, usize, usize), // line, column, index
    /// The stack was not of the correct size to execute an instruction.
    StackError,
    /// A missing key was requested from the heap.
    KeyError,
    /// The program tried to execute an instruction that doesn't exist (generally caused by control flow hitting the end of the program).
    InvalidIndex,
    /// The program tried to return but there was no location to return to on the callstack.
    CallStackError,
    /// Division or Modulo by zero.
    DivisionError,
    /// Something went wrong while trying to read from input or write to output.
    IOError,
    /// The program tried to read a number but no number was given.
    RuntimeParseError,
    // The following are often not reported to the user, rather they exist as signals
    // that indicate we need to switch to a bigint-based interpreter
    // Any resettable operation that results in a bigint having to be stored somewhere returns an Overflow
    /// An overflow occurred during an arithmetric operation. This will normally not be returned unless fallback is disabled.
    Overflow,
    /// An overflow occurred when a number input was requested. This is a bit of a special case, as the state cannot
    /// be rewound to before the number was parsed. Therefore, the key where the number will be read to, and the
    /// oversized integer that was parsed are returned in the error, and the location at which the error occurred
    /// is set to be the operation after the failed inum operation. Again, this will not be returned unless fallback is disabled.
    InumOverflow(Integer, BigInteger),
}

impl WsError {
    fn new<T: Into<Cow<'static, str>>>(kind: WsErrorKind, message: T) -> WsError {
        WsError {
            message: message.into(),
            kind: kind,
            location: None,
            cause: None
        }
    }

    fn wrap<E: Error + 'static, T: Into<Cow<'static, str>>>(error: E, kind: WsErrorKind, message: T) -> WsError {
        WsError {
            message: message.into(),
            kind: kind,
            location: None,
            cause: Some(Box::new(error))
        }
    }

    fn set_location(&mut self, index: usize) {
        self.location = Some(index);
    }

    /// Provide a nice error message using information stored in the program structure
    pub fn format_with_program(&self, program: &Program) -> String {
        if let Some(index) = self.location {
            if let Some(ref locs) = program.locs {
                let loc = &locs[index];
                if let Ok(text) = str::from_utf8(&program.source.as_ref().unwrap()[loc.span.clone()]) {
                    return format!("At command {} (line {}, column {}):\n{}\n\"{}\"",
                                   index + 1, loc.line, loc.column, self.message, text);
                } else {
                    return format!("At command {} (line {}, column {}):\n{}",
                                   index + 1, loc.line, loc.column, self.message);
                }
            }
            return format!("At command {}:\n{}", index + 1, self.message);
        }
        return format!("{}", self.message);
    }
}

impl Display for WsError {
    fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
        if let WsErrorKind::ParseError(line, column, index) = self.kind {
            write!(f, "At index {} (line {}, column {}):\n{}", index, line, column, self.message)
        } else if let Some(index) = self.location {
            write!(f, "At command {}:\n{}", index + 1, self.message)
        } else {
            f.write_str(&self.message)
        }
    }
}

impl Error for WsError {
    fn description(&self) -> &str {
        &self.message
    }

    fn cause(&self) -> Option<&Error> {
        self.cause.as_ref().map(|x| x.borrow())
    }
}

impl From<std::io::Error> for WsError {
    fn from(e: std::io::Error) -> WsError {
        let description = e.description().to_string();
        WsError::wrap(e, WsErrorKind::ParseError(0, 0, 0), description)
    }
}
<commit_msg>Fix an unintended panic on calling WsError::format_with_program when the program counter was out of range<commit_after>#![feature(plugin)]
#![plugin(dynasm)]
extern crate dynasmrt;
extern crate itertools;
extern crate crossbeam;
#[macro_use]
extern crate bitflags;
extern crate fnv;
extern crate num_bigint;
extern crate num_traits;

use std::borrow::Cow;
use std::error::Error;
use std::fmt::{self, Display, Formatter};
use std::borrow::Borrow;
use std::collections::HashMap;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use std::rc::Rc;
use std::str;

mod core;
mod label;
mod program;
mod parser;
mod assembler;
use program::{Command, SourceLoc, Integer, BigInteger};
use label::Label;

// re-export public api
pub use core::Interpreter;
#[cfg(target_arch = "x86_64")]
pub use core::debug_compile;

// This is here for visibility reasons
/// This struct contains the internal representation of a whitespace program.
#[derive(Debug, Clone)]
pub struct Program {
    source: Option<Vec<u8>>,
    commands: Vec<Command>,
    locs: Option<Vec<SourceLoc>>,
    source_is_whitespace: bool
}

// Also here for visibility reasons.
impl Program {
    fn compile(&mut self) -> Result<(), WsError> {
        let mut index_map = HashMap::<Rc<Label>, usize>::new();

        let locs = self.locs.as_ref().expect("Tried to compile a stripped program");

        for (index, (command, loc)) in self.commands.iter().zip(locs).enumerate() {
            if let Command::Label = *command {
                match index_map.entry(loc.label.clone().unwrap()) {
                    Occupied(_) => return Err(WsError::new(
                        WsErrorKind::ParseError(loc.line, loc.column, loc.span.start),
                        format!("Duplicate label: {}", loc.label.as_ref().unwrap())
                    )),
                    Vacant(e)   => e.insert(index + 1)
                };
            }
        }

        for (command, loc) in self.commands.iter_mut().zip(locs) {
            match *command {
                Command::Call {ref mut index} |
                Command::Jump {ref mut index} |
                Command::JumpIfZero {ref mut index} |
                Command::JumpIfNegative {ref mut index} => match index_map.entry(loc.label.clone().unwrap()) {
                    Occupied(e) => *index = *e.get(),
                    Vacant(_)   => return Err(WsError::new(
                        WsErrorKind::ParseError(loc.line, loc.column, loc.span.start),
                        format!("Undefined label: {}", loc.label.as_ref().unwrap())
                    ))
                },
                _ => ()
            };
        }

        Ok(())
    }
}

bitflags! {
    /// These are bitflag options to alter the behaviour of the interpreter.
    pub flags Options: u8 {
        /// Use wrapping arithmetric instead of detecting overflow and falling back
        const IGNORE_OVERFLOW    = 0b0000_0001,
        /// When an unknown key used to get an item from the heap, return 0 instead of causing an error.
        const UNCHECKED_HEAP     = 0b0000_0010,
        /// Don't fall back to biginteger interpretation on overflow, instead return an overflow error.
        const NO_FALLBACK        = 0b0001_0000,

        // features
        // const NEGATIVE_COPY   = 0b0000_1000,
    }
}

/// The common error type returned from all whitespacers execution functions.
#[derive(Debug)]
pub struct WsError {
    message: Cow<'static, str>,
    /// The kind of error that occurred.
    pub kind: WsErrorKind,
    location: Option<usize>,
    cause: Option<Box<Error>>
}

/// Simple information on what kind of error occurred.
#[derive(Debug, Clone)]
pub enum WsErrorKind {
    /// Compile-time parse error
    ParseError(usize, usize, usize), // line, column, index
    /// The stack was not of the correct size to execute an instruction.
    StackError,
    /// A missing key was requested from the heap.
    KeyError,
    /// The program tried to execute an instruction that doesn't exist (generally caused by control flow hitting the end of the program).
    InvalidIndex,
    /// The program tried to return but there was no location to return to on the callstack.
    CallStackError,
    /// Division or Modulo by zero.
    DivisionError,
    /// Something went wrong while trying to read from input or write to output.
    IOError,
    /// The program tried to read a number but no number was given.
    RuntimeParseError,
    // The following are often not reported to the user, rather they exist as signals
    // that indicate we need to switch to a bigint-based interpreter
    // Any resettable operation that results in a bigint having to be stored somewhere returns an Overflow
    /// An overflow occurred during an arithmetric operation. This will normally not be returned unless fallback is disabled.
    Overflow,
    /// An overflow occurred when a number input was requested. This is a bit of a special case, as the state cannot
    /// be rewound to before the number was parsed. Therefore, the key where the number will be read to, and the
    /// oversized integer that was parsed are returned in the error, and the location at which the error occurred
    /// is set to be the operation after the failed inum operation. Again, this will not be returned unless fallback is disabled.
    InumOverflow(Integer, BigInteger),
}

impl WsError {
    fn new<T: Into<Cow<'static, str>>>(kind: WsErrorKind, message: T) -> WsError {
        WsError {
            message: message.into(),
            kind: kind,
            location: None,
            cause: None
        }
    }

    fn wrap<E: Error + 'static, T: Into<Cow<'static, str>>>(error: E, kind: WsErrorKind, message: T) -> WsError {
        WsError {
            message: message.into(),
            kind: kind,
            location: None,
            cause: Some(Box::new(error))
        }
    }

    fn set_location(&mut self, index: usize) {
        self.location = Some(index);
    }

    /// Provide a nice error message using information stored in the program structure
    pub fn format_with_program(&self, program: &Program) -> String {
        if let Some(index) = self.location {
            if let Some(ref locs) = program.locs {
                if let Some(loc) = locs.get(index) {
                    if let Ok(text) = str::from_utf8(&program.source.as_ref().unwrap()[loc.span.clone()]) {
                        return format!("At command {} (line {}, column {}):\n{}\n\"{}\"",
                                       index + 1, loc.line, loc.column, self.message, text);
                    } else {
                        return format!("At command {} (line {}, column {}):\n{}",
                                       index + 1, loc.line, loc.column, self.message);
                    }
                } else {
                    return format!("At the end of the program:\n{}", self.message);
                }
            }
            return format!("At command {}:\n{}", index + 1, self.message);
        }
        return format!("{}", self.message);
    }
}

impl Display for WsError {
    fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
        if let WsErrorKind::ParseError(line, column, index) = self.kind {
            write!(f, "At index {} (line {}, column {}):\n{}", index, line, column, self.message)
        } else if let Some(index) = self.location {
            write!(f, "At command {}:\n{}", index + 1, self.message)
        } else {
            f.write_str(&self.message)
        }
    }
}

impl Error for WsError {
    fn description(&self) -> &str {
        &self.message
    }

    fn cause(&self) -> Option<&Error> {
        self.cause.as_ref().map(|x| x.borrow())
    }
}

impl From<std::io::Error> for WsError {
    fn from(e: std::io::Error) -> WsError {
        let description = e.description().to_string();
        WsError::wrap(e, WsErrorKind::ParseError(0, 0, 0), description)
    }
}
<|endoftext|>


<commit_before>
#![allow(dead_code)]

pub extern crate crypto;

use crypto::digest::Digest;

pub trait Hashable {
    fn to_bytes(&self) -> Vec<u8>;
}

impl Hashable for String {
    fn to_bytes(&self) -> Vec<u8> {
        self.clone().into_bytes()
    }
}

impl Hashable for u8 {
    fn to_bytes(&self) -> Vec<u8> {
        vec![*self]
    }
}

pub trait MerkleDigest {
    fn hash_bytes(&mut self, bytes: &Vec<u8>) -> Vec<u8>;
    fn combine_hashes(&mut self, left: &Vec<u8>, right: &Vec<u8>) -> Vec<u8>;
}

impl <D> MerkleDigest for D where D: Digest {

    fn hash_bytes(&mut self, bytes: &Vec<u8>) -> Vec<u8> {
        let mut hash = vec![0; self.output_bytes()];

        self.reset();
        self.input(bytes);
        self.result(&mut hash);
        self.reset();

        hash
    }

    fn combine_hashes(&mut self, left: &Vec<u8>, right: &Vec<u8>) -> Vec<u8> {
        let mut hash = vec![0; self.output_bytes()];

        self.reset();
        self.input(left);
        self.input(right);
        self.result(&mut hash);
        self.reset();

        hash
    }

}

enum Tree<T> {
    Leaf {
        hash: Vec<u8>,
        value: T
    },

    Node {
        hash: Vec<u8>,
        left: Box<Tree<T>>,
        right: Box<Tree<T>>
    }
}

impl <T> Tree<T> where T: Hashable {

    fn new(hash: Vec<u8>, value: T) -> Self {
        Tree::Leaf {
            hash: hash,
            value: value
        }
    }

    fn get_hash(&self) -> &Vec<u8> {
        match *self {
            Tree::Leaf { ref hash, value: _ }          => hash,
            Tree::Node { ref hash, left: _, right: _ } => hash
        }
    }
}

fn make_leaf<D, T>(digest: &mut D, value: T) -> Tree<T> where D: Digest, T: Hashable {
    let hash = digest.hash_bytes(&value.to_bytes());
    Tree::new(hash, value)
}

pub struct MerkleTree<D, T> where D: Digest, T: Hashable {
    digest: D,
    tree: Tree<T>,
    height: usize,
    count: usize
}

impl <D, T> MerkleTree<D, T> where D: Digest, T: Hashable {

    pub fn from_vec(mut digest: D, values: Vec<T>) -> Self {
        if values.is_empty() {
            panic!("Cannot build a Merkle tree from an empty vector.");
        }

        let count  = values.len();
        let height = (count as f32 + 1.0).log2().ceil() as usize;

        let mut cur = Vec::with_capacity(count);

        for v in values.into_iter().rev() {
            let leaf = make_leaf(&mut digest, v);
            cur.push(leaf);
        }

        cur.reverse();

        for i in 0 .. height {
            let mut next = Vec::with_capacity(height / 2 << i);

            while cur.len() > 0 {
                if cur.len() == 1 {
                    next.push(cur.remove(0));
                }
                else {
                    let left  = cur.remove(0);
                    let right = cur.remove(0);

                    let combined_hash = digest.combine_hashes(
                        left.get_hash(),
                        right.get_hash()
                    );

                    let node = Tree::Node {
                       hash: combined_hash,
                       left: Box::new(left),
                       right: Box::new(right)
                    };

                    next.push(node);
                }
            }

            cur = next;
        }

        assert!(cur.len() == 1);

        let tree = cur.remove(0);

        MerkleTree {
            digest: digest,
            tree: tree,
            height: height,
            count: count
        }
    }

    fn root_hash(&self) -> &Vec<u8> {
        self.tree.get_hash()
    }

}

#[cfg(test)]
mod tests;

<commit_msg>Remove type bounds on MerkleTree struct.<commit_after>
#![allow(dead_code)]

pub extern crate crypto;

use crypto::digest::Digest;

pub trait Hashable {
    fn to_bytes(&self) -> Vec<u8>;
}

impl Hashable for String {
    fn to_bytes(&self) -> Vec<u8> {
        self.clone().into_bytes()
    }
}

impl Hashable for u8 {
    fn to_bytes(&self) -> Vec<u8> {
        vec![*self]
    }
}

pub trait MerkleDigest {
    fn hash_bytes(&mut self, bytes: &Vec<u8>) -> Vec<u8>;
    fn combine_hashes(&mut self, left: &Vec<u8>, right: &Vec<u8>) -> Vec<u8>;
}

impl <D> MerkleDigest for D where D: Digest {

    fn hash_bytes(&mut self, bytes: &Vec<u8>) -> Vec<u8> {
        let mut hash = vec![0; self.output_bytes()];

        self.reset();
        self.input(bytes);
        self.result(&mut hash);
        self.reset();

        hash
    }

    fn combine_hashes(&mut self, left: &Vec<u8>, right: &Vec<u8>) -> Vec<u8> {
        let mut hash = vec![0; self.output_bytes()];

        self.reset();
        self.input(left);
        self.input(right);
        self.result(&mut hash);
        self.reset();

        hash
    }

}

enum Tree<T> {
    Leaf {
        hash: Vec<u8>,
        value: T
    },

    Node {
        hash: Vec<u8>,
        left: Box<Tree<T>>,
        right: Box<Tree<T>>
    }
}

impl <T> Tree<T> where T: Hashable {

    fn new(hash: Vec<u8>, value: T) -> Self {
        Tree::Leaf {
            hash: hash,
            value: value
        }
    }

    fn get_hash(&self) -> &Vec<u8> {
        match *self {
            Tree::Leaf { ref hash, value: _ }          => hash,
            Tree::Node { ref hash, left: _, right: _ } => hash
        }
    }
}

fn make_leaf<D, T>(digest: &mut D, value: T) -> Tree<T> where D: Digest, T: Hashable {
    let hash = digest.hash_bytes(&value.to_bytes());
    Tree::new(hash, value)
}

pub struct MerkleTree<D, T> {
    digest: D,
    tree: Tree<T>,
    height: usize,
    count: usize
}

impl <D, T> MerkleTree<D, T> where D: Digest, T: Hashable {

    pub fn from_vec(mut digest: D, values: Vec<T>) -> Self {
        if values.is_empty() {
            panic!("Cannot build a Merkle tree from an empty vector.");
        }

        let count  = values.len();
        let height = (count as f32 + 1.0).log2().ceil() as usize;

        let mut cur = Vec::with_capacity(count);

        for v in values.into_iter().rev() {
            let leaf = make_leaf(&mut digest, v);
            cur.push(leaf);
        }

        cur.reverse();

        for i in 0 .. height {
            let mut next = Vec::with_capacity(height / 2 << i);

            while cur.len() > 0 {
                if cur.len() == 1 {
                    next.push(cur.remove(0));
                }
                else {
                    let left  = cur.remove(0);
                    let right = cur.remove(0);

                    let combined_hash = digest.combine_hashes(
                        left.get_hash(),
                        right.get_hash()
                    );

                    let node = Tree::Node {
                       hash: combined_hash,
                       left: Box::new(left),
                       right: Box::new(right)
                    };

                    next.push(node);
                }
            }

            cur = next;
        }

        assert!(cur.len() == 1);

        let tree = cur.remove(0);

        MerkleTree {
            digest: digest,
            tree: tree,
            height: height,
            count: count
        }
    }

    fn root_hash(&self) -> &Vec<u8> {
        self.tree.get_hash()
    }

}

#[cfg(test)]
mod tests;

<|endoftext|>