<commit_before>use std::marker;
use std::cmp;
use std::mem;
use std::collections::HashMap;
use std::iter::FromIterator;

use solicit::http::client::RequestStream;
use solicit::http::client::ClientSession;
use solicit::http::session::Client;
use solicit::http::session::SessionState;
use solicit::http::session::DefaultSessionState;
use solicit::http::session::DefaultStream;
use solicit::http::session::StreamState;
use solicit::http::session::StreamDataError;
use solicit::http::session::StreamDataChunk;
use solicit::http::session::Stream as solicit_Stream;
use solicit::http::session::StreamIter;
use solicit::http::connection::HttpConnection;
use solicit::http::connection::SendStatus;
use solicit::http::connection::EndStream;
use solicit::http::connection::DataChunk;
use solicit::http::priority::SimplePrioritizer;
use solicit::http::frame::RawFrame;
use solicit::http::StreamId;
use solicit::http::HttpScheme;
use solicit::http::HttpError;
use solicit::http::Header;
use solicit::http::StaticHeader;

use futures;
use futures::Future;
use futures::stream::Stream;

use tokio_core;
use tokio_core::TcpStream;
use tokio_core::io as tokio_io;
use tokio_core::io::TaskIo;
use tokio_core::io::TaskIoRead;
use tokio_core::io::TaskIoWrite;
use tokio_core::LoopHandle;

use error::*;
use futures_misc::*;

use solicit_async::*;
use solicit_misc::*;

// TODO: make async
trait ResponseHandler : Send + 'static {
    fn headers(&mut self, headers: Vec<StaticHeader>);
    fn body_chunk(&mut self, chunk: Vec<u8>);
    fn trailers(&mut self, headers: Vec<StaticHeader>);
    fn end(&mut self);
}

enum ResponseState {
    Init,
    GotHeaders,
    GotBodyChunk,
    GotTrailers,
}

enum LastChunk {
    Empty,
    Headers(Vec<StaticHeader>),
    Chunk(Vec<u8>),
    Trailers(Vec<StaticHeader>),
}

struct GrpcHttp2ClientStream2<H : ResponseHandler> {
    state: StreamState,
    response_state: ResponseState,
    last_chunk: LastChunk,
    response_handler: Option<H>,
}

impl<H : ResponseHandler> solicit_Stream for GrpcHttp2ClientStream2<H> {
    fn set_headers<'n, 'v>(&mut self, headers: Vec<Header<'n, 'v>>) {
        let headers = headers.into_iter().map(|h| Header::new(h.name().to_owned(), h.value().to_owned())).collect();
        self.last_chunk = LastChunk::Headers(headers);
        self.response_state = match self.response_state {
            ResponseState::Init => ResponseState::GotHeaders,
            ResponseState::GotHeaders => panic!(),
            ResponseState::GotBodyChunk => ResponseState::GotTrailers,
            ResponseState::GotTrailers => panic!(),
        };
    }

    fn new_data_chunk(&mut self, data: &[u8]) {
        self.last_chunk = LastChunk::Chunk(data.to_owned());
    }

    fn set_state(&mut self, state: StreamState) {
        self.state = state;
    }

    fn get_data_chunk(&mut self, buf: &mut [u8]) -> Result<StreamDataChunk, StreamDataError> {
        unimplemented!()
    }

    fn state(&self) -> StreamState {
        self.state
    }
}

struct MySessionState<H : ResponseHandler> {
    streams: HashMap<StreamId, GrpcHttp2ClientStream2<H>>,
    next_stream_id: StreamId,
}

impl<H : ResponseHandler> MySessionState<H> {
    fn process_streams_after_handle_next_frame(&mut self) {
        let mut remove_ids = Vec::new();

        for (id, s) in &mut self.streams {
            let last_chunk = mem::replace(&mut s.last_chunk, LastChunk::Empty);
            match last_chunk {
                LastChunk::Empty => (),
                LastChunk::Chunk(chunk) => {
                    s.response_handler.as_mut().unwrap().body_chunk(chunk);
                }
                LastChunk::Headers(headers) => {
                    s.response_handler.as_mut().unwrap().headers(headers);
                }
                LastChunk::Trailers(headers) => {
                    s.response_handler.as_mut().unwrap().trailers(headers);
                }
            }

            if s.is_closed() {
                remove_ids.push(*id);
            }
        }

        Vec::from_iter(remove_ids.into_iter().map(|i| self.remove_stream(i).unwrap()));
    }
}

impl<H : ResponseHandler> SessionState for MySessionState<H> {
    type Stream = GrpcHttp2ClientStream2<H>;

    fn insert_outgoing(&mut self, stream: Self::Stream) -> StreamId {
        let id = self.next_stream_id;
        self.streams.insert(id, stream);
        self.next_stream_id += 2;
        id
    }

    fn insert_incoming(&mut self, stream_id: StreamId, stream: Self::Stream) -> Result<(), ()> {
        // TODO: assert parity
        // TODO: Assert that the stream IDs are monotonically increasing!
        self.streams.insert(stream_id, stream);
        Ok(())
    }

    #[inline]
    fn get_stream_ref(&self, stream_id: StreamId) -> Option<&Self::Stream> {
        self.streams.get(&stream_id)
    }

    #[inline]
    fn get_stream_mut(&mut self, stream_id: StreamId) -> Option<&mut Self::Stream> {
        self.streams.get_mut(&stream_id)
    }

    #[inline]
    fn remove_stream(&mut self, stream_id: StreamId) -> Option<Self::Stream> {
        self.streams.remove(&stream_id)
    }

    #[inline]
    fn iter(&mut self) -> StreamIter<GrpcHttp2ClientStream2<H>> {
        // https://github.com/mlalic/solicit/pull/34
        unimplemented!()
    }

    /// Number of currently active streams
    #[inline]
    fn len(&self) -> usize {
        self.streams.len()
    }
}

struct Inner<H : ResponseHandler> {
    host: String,
    conn: HttpConnection,
    call_tx: tokio_core::Sender<ToWriteMessage<H>>,
    session_state: MySessionState<H>,
}

struct HttpConnectionAsync<H : ResponseHandler> {
    inner: TaskRcMut<Inner<H>>,
}

struct StartRequestMessage<H : ResponseHandler> {
    headers: Vec<StaticHeader>,
    body: HttpStream<Vec<u8>>,
    response_handler: H,
}

struct BodyChunkMessage {
    stream_id: StreamId,
    chunk: Vec<u8>,
}

struct EndRequestMessage {
    stream_id: StreamId,
}

struct ReadToWriteMessage {
    buf: Vec<u8>,
}

enum ToWriteMessage<H : ResponseHandler> {
    Start(StartRequestMessage<H>),
    BodyChunk(BodyChunkMessage),
    End(EndRequestMessage),
    FromRead(ReadToWriteMessage),
}

struct WriteLoop<H : ResponseHandler> {
    write: TaskIoWrite<TcpStream>,
    inner: TaskRcMut<Inner<H>>,
}

impl<H : ResponseHandler> WriteLoop<H> {
    fn write_all(self, buf: Vec<u8>) -> HttpFuture<Self> {
        let WriteLoop { write, inner } = self;

        tokio_io::write_all(write, buf)
            .map(move |(write, _)| WriteLoop { write: write, inner: inner })
            .map_err(HttpError::from)
            .boxed()
    }

    fn process_from_read(self, message: ReadToWriteMessage) -> HttpFuture<Self> {
        self.write_all(message.buf)
    }

    fn process_start(self, start: StartRequestMessage<H>) -> HttpFuture<Self> {
        let StartRequestMessage { headers, body, response_handler } = start;
        let (buf, stream_id) = self.inner.with(move |inner: &mut Inner<H>| {

            let stream = GrpcHttp2ClientStream2 {
                state: StreamState::Open,
                response_handler: Some(response_handler),
                last_chunk: LastChunk::Empty,
                response_state: ResponseState::Init,
            };
            let stream_id = inner.session_state.insert_outgoing(stream);

            let send_buf = {
                let mut send_buf = VecSendFrame(Vec::new());

                inner.conn.sender(&mut send_buf).send_headers(headers, stream_id, EndStream::No).unwrap();

                (send_buf.0, stream_id)
            };

            send_buf
        });

        self.write_all(buf)
            .and_then(move |wl: WriteLoop<H>| {
                body.fold(wl, move |wl: WriteLoop<H>, chunk| {
                    wl.inner.with(|inner: &mut Inner<H>| {
                        inner.call_tx.send(ToWriteMessage::BodyChunk(BodyChunkMessage {
                            stream_id: stream_id,
                            chunk: chunk,
                        }))
                    });
                    futures::finished::<_, HttpError>(wl)
                })
            })
            .and_then(move |wl: WriteLoop<H>| {
                wl.inner.with(|inner: &mut Inner<H>| {
                    inner.call_tx.send(ToWriteMessage::End(EndRequestMessage {
                        stream_id: stream_id,
                    }));
                });
                futures::finished::<_, HttpError>(wl)
            })
            .boxed()
    }

    fn process_body_chunk(self, body_chunk: BodyChunkMessage) -> HttpFuture<Self> {
        let BodyChunkMessage { stream_id, chunk } = body_chunk;

        let buf = self.inner.with(move |inner: &mut Inner<H>| {
            {
                let stream = inner.session_state.get_stream_mut(stream_id);
                // TODO: check stream state
            }

            let mut send_buf = VecSendFrame(Vec::new());

            let mut pos = 0;
            const MAX_CHUNK_SIZE: usize = 8 * 1024;
            while pos < chunk.len() {
                let end = cmp::min(chunk.len(), pos + MAX_CHUNK_SIZE);

                let data_chunk = DataChunk::new_borrowed(&chunk[pos..end], stream_id, EndStream::No);

                inner.conn.sender(&mut send_buf).send_data(data_chunk).unwrap();

                pos = end;
            }

            send_buf.0
        });

        self.write_all(buf)
    }

    fn process_end(self, end: EndRequestMessage) -> HttpFuture<Self> {
        let EndRequestMessage { stream_id } = end;

        let buf = self.inner.with(move |inner: &mut Inner<H>| {
            let mut send_buf = VecSendFrame(Vec::new());

            let chunk = Vec::new();
            let data_chunk = DataChunk::new_borrowed(&chunk[..], stream_id, EndStream::Yes);

            inner.conn.sender(&mut send_buf).send_data(data_chunk).unwrap();

            send_buf.0
        });

        futures::finished(self).boxed()
    }

    fn process_message(self, message: ToWriteMessage<H>) -> HttpFuture<Self> {
        match message {
            ToWriteMessage::Start(start) => self.process_start(start),
            ToWriteMessage::BodyChunk(body_chunk) => self.process_body_chunk(body_chunk),
            ToWriteMessage::End(end) => self.process_end(end),
            ToWriteMessage::FromRead(from_read) => self.process_from_read(from_read),
        }
    }

    fn run(self, requests: tokio_core::Receiver<ToWriteMessage<H>>) -> HttpFuture<()> {
        let requests = requests.map_err(HttpError::from);
        requests
            .fold(self, move |wl, message: ToWriteMessage<H>| {
                wl.process_message(message)
            })
            .map(|_| ())
            .boxed()
    }
}

struct ReadLoop<H : ResponseHandler> {
    read: TaskIoRead<TcpStream>,
    inner: TaskRcMut<Inner<H>>,
}

impl<H : ResponseHandler> ReadLoop<H> {
    fn recv_raw_frame(self) -> HttpFuture<(Self, RawFrame<'static>)> {
        let ReadLoop { read, inner } = self;
        recv_raw_frame(read)
            .map(|(read, frame)| (ReadLoop { read: read, inner: inner}, frame))
            .map_err(HttpError::from)
            .boxed()
    }

    fn process_streams_after_handle_next_frame(self) -> HttpFuture<Self> {
        self.inner.with(|inner: &mut Inner<H>| {
            inner.session_state.process_streams_after_handle_next_frame();
        });

        futures::finished(self).boxed()
    }

    fn process_raw_frame(self, raw_frame: RawFrame) -> HttpFuture<Self> {
        self.inner.with(move |inner: &mut Inner<H>| {
            let mut send = VecSendFrame(Vec::new());
            {
                let mut session = ClientSession::new(&mut inner.session_state, &mut send);
                inner.conn.handle_next_frame(&mut OnceReceiveFrame::new(raw_frame), &mut session)
                    .unwrap();
            }

            if !send.0.is_empty() {
                inner.call_tx.send(ToWriteMessage::FromRead(ReadToWriteMessage { buf: send.0 }))
                    .expect("read to write");
            }
        });

        self.process_streams_after_handle_next_frame()
    }

    fn read_process_frame(self) -> HttpFuture<Self> {
        self.recv_raw_frame()
            .and_then(move |(rl, frame)| rl.process_raw_frame(frame))
            .boxed()
    }

    fn run(self) -> HttpFuture<()> {
        let stream = stream_repeat(());

        let future = stream.fold(self, |rl, _| {
            rl.read_process_frame()
        });

        future.map(|_| ()).boxed()
    }
}

impl<H : ResponseHandler> HttpConnectionAsync<H> {
    fn new(lh: LoopHandle, conn: TcpStream) -> HttpFuture<(Self, HttpFuture<()>)> {
        let (call_tx, call_rx) = lh.channel();

        let handshake = client_handshake(conn);
        handshake.and_then(move |conn| {
            let (read, write) = TaskIo::new(conn).split();

            let inner = TaskRcMut::new(Inner {
                host: "localhost".to_owned(), // TODO
                conn: HttpConnection::new(HttpScheme::Http),
                call_tx: call_tx,
                session_state: MySessionState {
                    next_stream_id: 1,
                    streams: HashMap::new(),
                }
            });

            let write_inner = inner.clone();
            let run_write = call_rx.map_err(HttpError::from).and_then(move |call_rx| WriteLoop { write: write, inner: write_inner }.run(call_rx));
            let run_read = ReadLoop { read: read, inner: inner.clone() }.run();

            let c = HttpConnectionAsync {
                inner: inner,
            };
            Ok((c, run_write.join(run_read).map(|_| ()).boxed()))
        }).boxed()
    }

    fn start_request(
        &self,
        headers: Vec<StaticHeader>,
        body: HttpStream<Vec<u8>>,
        response_handler: H)
            -> HttpFuture<()>
    {
        self.inner.with(|inner: &mut Inner<H>| {
            inner.call_tx.send(ToWriteMessage::Start(StartRequestMessage {
                headers: headers,
                body: body,
                response_handler: response_handler,
            }));
        });
        futures::finished(()).boxed()
    }
}
<commit_msg>-todo<commit_after>use std::marker;
use std::cmp;
use std::mem;
use std::collections::HashMap;
use std::iter::FromIterator;

use solicit::http::client::RequestStream;
use solicit::http::client::ClientSession;
use solicit::http::session::Client;
use solicit::http::session::SessionState;
use solicit::http::session::DefaultSessionState;
use solicit::http::session::DefaultStream;
use solicit::http::session::StreamState;
use solicit::http::session::StreamDataError;
use solicit::http::session::StreamDataChunk;
use solicit::http::session::Stream as solicit_Stream;
use solicit::http::session::StreamIter;
use solicit::http::connection::HttpConnection;
use solicit::http::connection::SendStatus;
use solicit::http::connection::EndStream;
use solicit::http::connection::DataChunk;
use solicit::http::priority::SimplePrioritizer;
use solicit::http::frame::RawFrame;
use solicit::http::StreamId;
use solicit::http::HttpScheme;
use solicit::http::HttpError;
use solicit::http::Header;
use solicit::http::StaticHeader;

use futures;
use futures::Future;
use futures::stream::Stream;

use tokio_core;
use tokio_core::TcpStream;
use tokio_core::io as tokio_io;
use tokio_core::io::TaskIo;
use tokio_core::io::TaskIoRead;
use tokio_core::io::TaskIoWrite;
use tokio_core::LoopHandle;

use error::*;
use futures_misc::*;

use solicit_async::*;
use solicit_misc::*;

// TODO: make async
trait ResponseHandler : Send + 'static {
    fn headers(&mut self, headers: Vec<StaticHeader>);
    fn body_chunk(&mut self, chunk: Vec<u8>);
    fn trailers(&mut self, headers: Vec<StaticHeader>);
    fn end(&mut self);
}

enum ResponseState {
    Init,
    GotHeaders,
    GotBodyChunk,
    GotTrailers,
}

enum LastChunk {
    Empty,
    Headers(Vec<StaticHeader>),
    Chunk(Vec<u8>),
    Trailers(Vec<StaticHeader>),
}

struct GrpcHttp2ClientStream2<H : ResponseHandler> {
    state: StreamState,
    response_state: ResponseState,
    last_chunk: LastChunk,
    response_handler: Option<H>,
}

impl<H : ResponseHandler> solicit_Stream for GrpcHttp2ClientStream2<H> {
    fn set_headers<'n, 'v>(&mut self, headers: Vec<Header<'n, 'v>>) {
        let headers = headers.into_iter().map(|h| Header::new(h.name().to_owned(), h.value().to_owned())).collect();
        self.last_chunk = LastChunk::Headers(headers);
        self.response_state = match self.response_state {
            ResponseState::Init => ResponseState::GotHeaders,
            ResponseState::GotHeaders => panic!(),
            ResponseState::GotBodyChunk => ResponseState::GotTrailers,
            ResponseState::GotTrailers => panic!(),
        };
    }

    fn new_data_chunk(&mut self, data: &[u8]) {
        self.last_chunk = LastChunk::Chunk(data.to_owned());
    }

    fn set_state(&mut self, state: StreamState) {
        self.state = state;
    }

    fn get_data_chunk(&mut self, buf: &mut [u8]) -> Result<StreamDataChunk, StreamDataError> {
        unimplemented!()
    }

    fn state(&self) -> StreamState {
        self.state
    }
}

struct MySessionState<H : ResponseHandler> {
    streams: HashMap<StreamId, GrpcHttp2ClientStream2<H>>,
    next_stream_id: StreamId,
}

impl<H : ResponseHandler> MySessionState<H> {
    fn process_streams_after_handle_next_frame(&mut self) {
        let mut remove_ids = Vec::new();

        for (id, s) in &mut self.streams {
            let last_chunk = mem::replace(&mut s.last_chunk, LastChunk::Empty);
            match last_chunk {
                LastChunk::Empty => (),
                LastChunk::Chunk(chunk) => {
                    s.response_handler.as_mut().unwrap().body_chunk(chunk);
                }
                LastChunk::Headers(headers) => {
                    s.response_handler.as_mut().unwrap().headers(headers);
                }
                LastChunk::Trailers(headers) => {
                    s.response_handler.as_mut().unwrap().trailers(headers);
                }
            }

            if s.is_closed() {
                remove_ids.push(*id);
            }
        }

        Vec::from_iter(remove_ids.into_iter().map(|i| self.remove_stream(i).unwrap()));
    }
}

impl<H : ResponseHandler> SessionState for MySessionState<H> {
    type Stream = GrpcHttp2ClientStream2<H>;

    fn insert_outgoing(&mut self, stream: Self::Stream) -> StreamId {
        let id = self.next_stream_id;
        self.streams.insert(id, stream);
        self.next_stream_id += 2;
        id
    }

    fn insert_incoming(&mut self, stream_id: StreamId, stream: Self::Stream) -> Result<(), ()> {
        // TODO: assert parity
        // TODO: Assert that the stream IDs are monotonically increasing!
        self.streams.insert(stream_id, stream);
        Ok(())
    }

    #[inline]
    fn get_stream_ref(&self, stream_id: StreamId) -> Option<&Self::Stream> {
        self.streams.get(&stream_id)
    }

    #[inline]
    fn get_stream_mut(&mut self, stream_id: StreamId) -> Option<&mut Self::Stream> {
        self.streams.get_mut(&stream_id)
    }

    #[inline]
    fn remove_stream(&mut self, stream_id: StreamId) -> Option<Self::Stream> {
        self.streams.remove(&stream_id)
    }

    #[inline]
    fn iter(&mut self) -> StreamIter<GrpcHttp2ClientStream2<H>> {
        // https://github.com/mlalic/solicit/pull/34
        unimplemented!()
    }

    /// Number of currently active streams
    #[inline]
    fn len(&self) -> usize {
        self.streams.len()
    }
}

struct Inner<H : ResponseHandler> {
    host: String,
    conn: HttpConnection,
    call_tx: tokio_core::Sender<ToWriteMessage<H>>,
    session_state: MySessionState<H>,
}

struct HttpConnectionAsync<H : ResponseHandler> {
    inner: TaskRcMut<Inner<H>>,
}

struct StartRequestMessage<H : ResponseHandler> {
    headers: Vec<StaticHeader>,
    body: HttpStream<Vec<u8>>,
    response_handler: H,
}

struct BodyChunkMessage {
    stream_id: StreamId,
    chunk: Vec<u8>,
}

struct EndRequestMessage {
    stream_id: StreamId,
}

struct ReadToWriteMessage {
    buf: Vec<u8>,
}

enum ToWriteMessage<H : ResponseHandler> {
    Start(StartRequestMessage<H>),
    BodyChunk(BodyChunkMessage),
    End(EndRequestMessage),
    FromRead(ReadToWriteMessage),
}

struct WriteLoop<H : ResponseHandler> {
    write: TaskIoWrite<TcpStream>,
    inner: TaskRcMut<Inner<H>>,
}

impl<H : ResponseHandler> WriteLoop<H> {
    fn write_all(self, buf: Vec<u8>) -> HttpFuture<Self> {
        let WriteLoop { write, inner } = self;

        tokio_io::write_all(write, buf)
            .map(move |(write, _)| WriteLoop { write: write, inner: inner })
            .map_err(HttpError::from)
            .boxed()
    }

    fn process_from_read(self, message: ReadToWriteMessage) -> HttpFuture<Self> {
        self.write_all(message.buf)
    }

    fn process_start(self, start: StartRequestMessage<H>) -> HttpFuture<Self> {
        let StartRequestMessage { headers, body, response_handler } = start;
        let (buf, stream_id) = self.inner.with(move |inner: &mut Inner<H>| {

            let stream = GrpcHttp2ClientStream2 {
                state: StreamState::Open,
                response_handler: Some(response_handler),
                last_chunk: LastChunk::Empty,
                response_state: ResponseState::Init,
            };
            let stream_id = inner.session_state.insert_outgoing(stream);

            let send_buf = {
                let mut send_buf = VecSendFrame(Vec::new());

                inner.conn.sender(&mut send_buf).send_headers(headers, stream_id, EndStream::No).unwrap();

                (send_buf.0, stream_id)
            };

            send_buf
        });

        self.write_all(buf)
            .and_then(move |wl: WriteLoop<H>| {
                body.fold(wl, move |wl: WriteLoop<H>, chunk| {
                    wl.inner.with(|inner: &mut Inner<H>| {
                        inner.call_tx.send(ToWriteMessage::BodyChunk(BodyChunkMessage {
                            stream_id: stream_id,
                            chunk: chunk,
                        }))
                    });
                    futures::finished::<_, HttpError>(wl)
                })
            })
            .and_then(move |wl: WriteLoop<H>| {
                wl.inner.with(|inner: &mut Inner<H>| {
                    inner.call_tx.send(ToWriteMessage::End(EndRequestMessage {
                        stream_id: stream_id,
                    }));
                });
                futures::finished::<_, HttpError>(wl)
            })
            .boxed()
    }

    fn process_body_chunk(self, body_chunk: BodyChunkMessage) -> HttpFuture<Self> {
        let BodyChunkMessage { stream_id, chunk } = body_chunk;

        let buf = self.inner.with(move |inner: &mut Inner<H>| {
            {
                let stream = inner.session_state.get_stream_mut(stream_id);
                // TODO: check stream state
            }

            let mut send_buf = VecSendFrame(Vec::new());

            let mut pos = 0;
            const MAX_CHUNK_SIZE: usize = 8 * 1024;
            while pos < chunk.len() {
                let end = cmp::min(chunk.len(), pos + MAX_CHUNK_SIZE);

                let data_chunk = DataChunk::new_borrowed(&chunk[pos..end], stream_id, EndStream::No);

                inner.conn.sender(&mut send_buf).send_data(data_chunk).unwrap();

                pos = end;
            }

            send_buf.0
        });

        self.write_all(buf)
    }

    fn process_end(self, end: EndRequestMessage) -> HttpFuture<Self> {
        let EndRequestMessage { stream_id } = end;

        let buf = self.inner.with(move |inner: &mut Inner<H>| {
            let mut send_buf = VecSendFrame(Vec::new());

            let chunk = Vec::new();
            let data_chunk = DataChunk::new_borrowed(&chunk[..], stream_id, EndStream::Yes);

            inner.conn.sender(&mut send_buf).send_data(data_chunk).unwrap();

            send_buf.0
        });

        futures::finished(self).boxed()
    }

    fn process_message(self, message: ToWriteMessage<H>) -> HttpFuture<Self> {
        match message {
            ToWriteMessage::Start(start) => self.process_start(start),
            ToWriteMessage::BodyChunk(body_chunk) => self.process_body_chunk(body_chunk),
            ToWriteMessage::End(end) => self.process_end(end),
            ToWriteMessage::FromRead(from_read) => self.process_from_read(from_read),
        }
    }

    fn run(self, requests: tokio_core::Receiver<ToWriteMessage<H>>) -> HttpFuture<()> {
        let requests = requests.map_err(HttpError::from);
        requests
            .fold(self, move |wl, message: ToWriteMessage<H>| {
                wl.process_message(message)
            })
            .map(|_| ())
            .boxed()
    }
}

struct ReadLoop<H : ResponseHandler> {
    read: TaskIoRead<TcpStream>,
    inner: TaskRcMut<Inner<H>>,
}

impl<H : ResponseHandler> ReadLoop<H> {
    fn recv_raw_frame(self) -> HttpFuture<(Self, RawFrame<'static>)> {
        let ReadLoop { read, inner } = self;
        recv_raw_frame(read)
            .map(|(read, frame)| (ReadLoop { read: read, inner: inner}, frame))
            .map_err(HttpError::from)
            .boxed()
    }

    fn process_streams_after_handle_next_frame(self) -> HttpFuture<Self> {
        self.inner.with(|inner: &mut Inner<H>| {
            inner.session_state.process_streams_after_handle_next_frame();
        });

        futures::finished(self).boxed()
    }

    fn process_raw_frame(self, raw_frame: RawFrame) -> HttpFuture<Self> {
        self.inner.with(move |inner: &mut Inner<H>| {
            let mut send = VecSendFrame(Vec::new());
            {
                let mut session = ClientSession::new(&mut inner.session_state, &mut send);
                inner.conn.handle_next_frame(&mut OnceReceiveFrame::new(raw_frame), &mut session)
                    .unwrap();
            }

            if !send.0.is_empty() {
                inner.call_tx.send(ToWriteMessage::FromRead(ReadToWriteMessage { buf: send.0 }))
                    .expect("read to write");
            }
        });

        self.process_streams_after_handle_next_frame()
    }

    fn read_process_frame(self) -> HttpFuture<Self> {
        self.recv_raw_frame()
            .and_then(move |(rl, frame)| rl.process_raw_frame(frame))
            .boxed()
    }

    fn run(self) -> HttpFuture<()> {
        let stream = stream_repeat(());

        let future = stream.fold(self, |rl, _| {
            rl.read_process_frame()
        });

        future.map(|_| ()).boxed()
    }
}

impl<H : ResponseHandler> HttpConnectionAsync<H> {
    fn new(lh: LoopHandle, conn: TcpStream, host: String) -> HttpFuture<(Self, HttpFuture<()>)> {
        let (call_tx, call_rx) = lh.channel();

        let handshake = client_handshake(conn);
        handshake.and_then(move |conn| {
            let (read, write) = TaskIo::new(conn).split();

            let inner = TaskRcMut::new(Inner {
                host: host.to_owned(),
                conn: HttpConnection::new(HttpScheme::Http),
                call_tx: call_tx,
                session_state: MySessionState {
                    next_stream_id: 1,
                    streams: HashMap::new(),
                }
            });

            let write_inner = inner.clone();
            let run_write = call_rx.map_err(HttpError::from).and_then(move |call_rx| WriteLoop { write: write, inner: write_inner }.run(call_rx));
            let run_read = ReadLoop { read: read, inner: inner.clone() }.run();

            let c = HttpConnectionAsync {
                inner: inner,
            };
            Ok((c, run_write.join(run_read).map(|_| ()).boxed()))
        }).boxed()
    }

    fn start_request(
        &self,
        headers: Vec<StaticHeader>,
        body: HttpStream<Vec<u8>>,
        response_handler: H)
            -> HttpFuture<()>
    {
        self.inner.with(|inner: &mut Inner<H>| {
            inner.call_tx.send(ToWriteMessage::Start(StartRequestMessage {
                headers: headers,
                body: body,
                response_handler: response_handler,
            }));
        });
        futures::finished(()).boxed()
    }
}
<|endoftext|>


<commit_before>use hyper::StatusCode;

use hyperx::{
    header::{ByteRangeSpec, ByteRangeSpec::FromTo, ContentRangeSpec, Range::Bytes},
    Headers,
};

use reqwest;
use reqwest::{header, Client};

use buffer::Buffer;
use reader::Reader;

use std::cmp;
use std::io::{Error, ErrorKind, Read, Seek, SeekFrom};
use std::str::FromStr;
use std::time::Instant;

fn get_head(filename: &str) -> Result<reqwest::Response, reqwest::Error> {
    if filename.contains(".amazonaws.com") {
        let range = vec![FromTo(0, 0)];

        let mut headers = Headers::new();
        headers.set(Bytes(range));
        let client = reqwest::Client::builder()
            .default_headers(headers.into())
            .build()
            .unwrap();

        client.get(filename).send()
    } else {
        let client = Client::new();
        client.head(filename).send()
    }
}

#[derive(Debug)]
struct ResponseData {
    body_data: Vec<u8>,
    file_size: u64,
}

fn get_data(filename: &str, range: Vec<ByteRangeSpec>) -> Result<ResponseData, String> {
    let mut headers = Headers::new();
    headers.set(Bytes(range));
    let client = reqwest::Client::builder()
        .default_headers(headers.into())
        .build()
        .unwrap();

    let mut response = match client.get(filename).send() {
        Ok(content) => content,
        Err(_msg) => return Err("bad request".to_string()),
    };

    let status = response.status();

    if !(status == StatusCode::OK || status == StatusCode::PARTIAL_CONTENT) {
        error!("ERROR {:?}", response);
        return Err("bad response status".to_string());
    }

    let mut body: Vec<u8> = vec![];
    let _result = response.copy_to(&mut body);

    let file_size = match get_content_range(&response) {
        Ok(length) => length.unwrap_or(0),
        Err(_msg) => return Err("bad response header".to_string()),
    };

    Ok(ResponseData {
        body_data: body,
        file_size,
    })
}

fn get_content_range(response: &reqwest::Response) -> Result<Option<u64>, String> {
    if let Some(content_range) = response.headers().get(header::CONTENT_RANGE) {
        let content_range_str = content_range
            .to_str()
            .map_err(|msg| format!("Error serializing header value to str: {}", msg))?;

        match ContentRangeSpec::from_str(content_range_str)
            .map_err(|msg| format!("Error parsing content range from str: {}", msg))?
        {
            ContentRangeSpec::Bytes {
                instance_length: length,
                ..
            } => Ok(length),
            ContentRangeSpec::Unregistered { .. } => {
                Err("Unregistered, actually unsupported".to_string())
            }
        }
    } else {
        Err("Missing content_range".to_string())
    }
}

#[derive(Debug)]
pub struct HttpReader {
    pub filename: String,
    pub file_size: Option<u64>,
    pub position: u64,
    pub buffer: Buffer,
}

pub fn exists(filename: &str) -> bool {
    match get_head(filename) {
        Ok(resp) => resp.status().is_success(),
        Err(_msg) => false,
    }
}

fn get_data_range(position: u64, size: usize, max_end_position: Option<u64>) -> Vec<ByteRangeSpec> {
    let start = position;
    let end = match (position, size) {
        (0, 0) => 0,
        (_, _) => match max_end_position {
            Some(max) => {
                let max_size = max - position;
                position + cmp::min((size - 1) as u64, max_size)
            }
            None => position + (size - 1) as u64,
        },
    };

    vec![FromTo(start, end)]
}

fn load_data(reader: &mut HttpReader, size: usize) -> Result<Option<Vec<u8>>, String> {
    let start = Instant::now();
    info!("make HTTP request with request {:?} bytes", size);

    let position = match reader.buffer.size {
        Some(_) => reader.buffer.position,
        None => reader.position,
    };

    if let Some(total_file_size) = reader.file_size {
        if position >= total_file_size {
            info!(
                "request range out of range: {} > {}",
                position, total_file_size
            );
            return Ok(None);
        }
    }

    let range = get_data_range(position, size, reader.buffer.max_end_position);
    let response = match get_data(&reader.filename, range) {
        Ok(data) => data,
        Err(msg) => return Err(msg),
    };

    let elapsed = start.elapsed();
    if elapsed.as_secs() > 0 {
        warn!("Request duration {} seconds", elapsed.as_secs());
    }

    let new_position = position + response.body_data.len() as u64;
    match reader.buffer.size {
        Some(_) => {
            reader.buffer.position = new_position;
        }
        None => {
            reader.position = new_position;
        }
    };
    Ok(Some(response.body_data))
}

impl Reader for HttpReader {
    fn new() -> HttpReader {
        HttpReader {
            filename: "".to_string(),
            file_size: None,
            position: 0,
            buffer: Buffer {
                size: None,
                position: 0,
                max_end_position: None,
                buffer: vec![],
            },
        }
    }

    fn open(&mut self, filename: &str) -> Result<(), String> {
        self.filename = filename.to_string();

        match get_head(filename) {
            Err(msg) => Err(msg.to_string()),
            Ok(response) => {
                let content_length = match get_content_range(&response) {
                    Ok(length) => length,
                    _ => response.content_length(),
                };

                self.file_size = content_length;
                Ok(())
            }
        }
    }

    fn get_position(&mut self) -> Result<u64, String> {
        Ok(self.position)
    }

    fn get_cache_size(&self) -> Option<usize> {
        self.buffer.size
    }

    fn set_cache_size(&mut self, cache_size: Option<usize>) {
        self.buffer.size = cache_size;
    }

    fn get_max_end_position(&self) -> Option<u64> {
        self.buffer.max_end_position
    }

    fn set_max_end_position(&mut self, max_end_position: Option<u64>) {
        self.buffer.max_end_position = max_end_position;
    }

    fn get_size(&mut self) -> Result<u64, String> {
        match self.file_size {
            Some(length) => Ok(length),
            None => Err("No length detected".to_string()),
        }
    }
}

impl Read for HttpReader {
    fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
        if self.buffer.get_cached_size() >= buf.len() {
            self.position += buf.len() as u64;
            if self.buffer.get_data(buf) {
                Ok(buf.len())
            } else {
                Err(Error::new(
                    ErrorKind::Other,
                    "unable to read data from HTTP cache",
                ))
            }
        } else if let Some(buffer_size) = self.buffer.size {
            let some_data =
                load_data(self, buffer_size).map_err(|msg| Error::new(ErrorKind::Other, msg))?;

            if let Some(data) = some_data {
                self.buffer.append_data(&data.to_vec());
                self.position += buf.len() as u64;
                if self.buffer.get_data(buf) {
                    return Ok(buf.len());
                } else {
                    return Err(Error::new(
                        ErrorKind::Other,
                        "unable to read data from HTTP cache",
                    ));
                }
            } else {
                Ok(0)
            }
        } else {
            let some_data =
                load_data(self, buf.len()).map_err(|msg| Error::new(ErrorKind::Other, msg))?;

            if let Some(data) = some_data {
                if data.len() >= buf.len() {
                    buf.clone_from_slice(&data);
                    Ok(data.len())
                } else {
                    Ok(0)
                }
            } else {
                Ok(0)
            }
        }
    }
}

impl Seek for HttpReader {
    fn seek(&mut self, seek_from: SeekFrom) -> Result<u64, Error> {
        match seek_from {
            SeekFrom::Current(offset) => {
                self.position += offset as u64;
                if self.buffer.size.is_some() {
                    if offset > 0 && self.buffer.get_cached_size() > offset as usize {
                        let mut skipped_data = vec![];
                        skipped_data.resize(offset as usize, 0);
                        let _skiped_data = self.buffer.get_data(&mut skipped_data);
                    } else {
                        self.buffer.reset();
                    }
                }
            }
            SeekFrom::Start(offset) => {
                self.buffer.reset();
                self.position = offset;
                if self.buffer.size.is_some() {
                    self.buffer.position = self.position;
                }
            }
            SeekFrom::End(offset) => {
                self.buffer.reset();
                match self.file_size {
                    Some(size) => {
                        self.position = size - offset as u64;
                        if self.buffer.size.is_some() {
                            self.buffer.position = self.position;
                        }
                    }
                    None => return Err(Error::new(ErrorKind::Other, "Missing file size")),
                }
            }
        }
        Ok(self.position)
    }
}
<commit_msg>remove return, as clippy mentionned it<commit_after>use hyper::StatusCode;

use hyperx::{
    header::{ByteRangeSpec, ByteRangeSpec::FromTo, ContentRangeSpec, Range::Bytes},
    Headers,
};

use reqwest;
use reqwest::{header, Client};

use buffer::Buffer;
use reader::Reader;

use std::cmp;
use std::io::{Error, ErrorKind, Read, Seek, SeekFrom};
use std::str::FromStr;
use std::time::Instant;

fn get_head(filename: &str) -> Result<reqwest::Response, reqwest::Error> {
    if filename.contains(".amazonaws.com") {
        let range = vec![FromTo(0, 0)];

        let mut headers = Headers::new();
        headers.set(Bytes(range));
        let client = reqwest::Client::builder()
            .default_headers(headers.into())
            .build()
            .unwrap();

        client.get(filename).send()
    } else {
        let client = Client::new();
        client.head(filename).send()
    }
}

#[derive(Debug)]
struct ResponseData {
    body_data: Vec<u8>,
    file_size: u64,
}

fn get_data(filename: &str, range: Vec<ByteRangeSpec>) -> Result<ResponseData, String> {
    let mut headers = Headers::new();
    headers.set(Bytes(range));
    let client = reqwest::Client::builder()
        .default_headers(headers.into())
        .build()
        .unwrap();

    let mut response = match client.get(filename).send() {
        Ok(content) => content,
        Err(_msg) => return Err("bad request".to_string()),
    };

    let status = response.status();

    if !(status == StatusCode::OK || status == StatusCode::PARTIAL_CONTENT) {
        error!("ERROR {:?}", response);
        return Err("bad response status".to_string());
    }

    let mut body: Vec<u8> = vec![];
    let _result = response.copy_to(&mut body);

    let file_size = match get_content_range(&response) {
        Ok(length) => length.unwrap_or(0),
        Err(_msg) => return Err("bad response header".to_string()),
    };

    Ok(ResponseData {
        body_data: body,
        file_size,
    })
}

fn get_content_range(response: &reqwest::Response) -> Result<Option<u64>, String> {
    if let Some(content_range) = response.headers().get(header::CONTENT_RANGE) {
        let content_range_str = content_range
            .to_str()
            .map_err(|msg| format!("Error serializing header value to str: {}", msg))?;

        match ContentRangeSpec::from_str(content_range_str)
            .map_err(|msg| format!("Error parsing content range from str: {}", msg))?
        {
            ContentRangeSpec::Bytes {
                instance_length: length,
                ..
            } => Ok(length),
            ContentRangeSpec::Unregistered { .. } => {
                Err("Unregistered, actually unsupported".to_string())
            }
        }
    } else {
        Err("Missing content_range".to_string())
    }
}

#[derive(Debug)]
pub struct HttpReader {
    pub filename: String,
    pub file_size: Option<u64>,
    pub position: u64,
    pub buffer: Buffer,
}

pub fn exists(filename: &str) -> bool {
    match get_head(filename) {
        Ok(resp) => resp.status().is_success(),
        Err(_msg) => false,
    }
}

fn get_data_range(position: u64, size: usize, max_end_position: Option<u64>) -> Vec<ByteRangeSpec> {
    let start = position;
    let end = match (position, size) {
        (0, 0) => 0,
        (_, _) => match max_end_position {
            Some(max) => {
                let max_size = max - position;
                position + cmp::min((size - 1) as u64, max_size)
            }
            None => position + (size - 1) as u64,
        },
    };

    vec![FromTo(start, end)]
}

fn load_data(reader: &mut HttpReader, size: usize) -> Result<Option<Vec<u8>>, String> {
    let start = Instant::now();
    info!("make HTTP request with request {:?} bytes", size);

    let position = match reader.buffer.size {
        Some(_) => reader.buffer.position,
        None => reader.position,
    };

    if let Some(total_file_size) = reader.file_size {
        if position >= total_file_size {
            info!(
                "request range out of range: {} > {}",
                position, total_file_size
            );
            return Ok(None);
        }
    }

    let range = get_data_range(position, size, reader.buffer.max_end_position);
    let response = match get_data(&reader.filename, range) {
        Ok(data) => data,
        Err(msg) => return Err(msg),
    };

    let elapsed = start.elapsed();
    if elapsed.as_secs() > 0 {
        warn!("Request duration {} seconds", elapsed.as_secs());
    }

    let new_position = position + response.body_data.len() as u64;
    match reader.buffer.size {
        Some(_) => {
            reader.buffer.position = new_position;
        }
        None => {
            reader.position = new_position;
        }
    };
    Ok(Some(response.body_data))
}

impl Reader for HttpReader {
    fn new() -> HttpReader {
        HttpReader {
            filename: "".to_string(),
            file_size: None,
            position: 0,
            buffer: Buffer {
                size: None,
                position: 0,
                max_end_position: None,
                buffer: vec![],
            },
        }
    }

    fn open(&mut self, filename: &str) -> Result<(), String> {
        self.filename = filename.to_string();

        match get_head(filename) {
            Err(msg) => Err(msg.to_string()),
            Ok(response) => {
                let content_length = match get_content_range(&response) {
                    Ok(length) => length,
                    _ => response.content_length(),
                };

                self.file_size = content_length;
                Ok(())
            }
        }
    }

    fn get_position(&mut self) -> Result<u64, String> {
        Ok(self.position)
    }

    fn get_cache_size(&self) -> Option<usize> {
        self.buffer.size
    }

    fn set_cache_size(&mut self, cache_size: Option<usize>) {
        self.buffer.size = cache_size;
    }

    fn get_max_end_position(&self) -> Option<u64> {
        self.buffer.max_end_position
    }

    fn set_max_end_position(&mut self, max_end_position: Option<u64>) {
        self.buffer.max_end_position = max_end_position;
    }

    fn get_size(&mut self) -> Result<u64, String> {
        match self.file_size {
            Some(length) => Ok(length),
            None => Err("No length detected".to_string()),
        }
    }
}

impl Read for HttpReader {
    fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
        if self.buffer.get_cached_size() >= buf.len() {
            self.position += buf.len() as u64;
            if self.buffer.get_data(buf) {
                Ok(buf.len())
            } else {
                Err(Error::new(
                    ErrorKind::Other,
                    "unable to read data from HTTP cache",
                ))
            }
        } else if let Some(buffer_size) = self.buffer.size {
            let some_data =
                load_data(self, buffer_size).map_err(|msg| Error::new(ErrorKind::Other, msg))?;

            if let Some(data) = some_data {
                self.buffer.append_data(&data.to_vec());
                self.position += buf.len() as u64;
                if self.buffer.get_data(buf) {
                    Ok(buf.len())
                } else {
                    Err(Error::new(
                        ErrorKind::Other,
                        "unable to read data from HTTP cache",
                    ))
                }
            } else {
                Ok(0)
            }
        } else {
            let some_data =
                load_data(self, buf.len()).map_err(|msg| Error::new(ErrorKind::Other, msg))?;

            if let Some(data) = some_data {
                if data.len() >= buf.len() {
                    buf.clone_from_slice(&data);
                    Ok(data.len())
                } else {
                    Ok(0)
                }
            } else {
                Ok(0)
            }
        }
    }
}

impl Seek for HttpReader {
    fn seek(&mut self, seek_from: SeekFrom) -> Result<u64, Error> {
        match seek_from {
            SeekFrom::Current(offset) => {
                self.position += offset as u64;
                if self.buffer.size.is_some() {
                    if offset > 0 && self.buffer.get_cached_size() > offset as usize {
                        let mut skipped_data = vec![];
                        skipped_data.resize(offset as usize, 0);
                        let _skiped_data = self.buffer.get_data(&mut skipped_data);
                    } else {
                        self.buffer.reset();
                    }
                }
            }
            SeekFrom::Start(offset) => {
                self.buffer.reset();
                self.position = offset;
                if self.buffer.size.is_some() {
                    self.buffer.position = self.position;
                }
            }
            SeekFrom::End(offset) => {
                self.buffer.reset();
                match self.file_size {
                    Some(size) => {
                        self.position = size - offset as u64;
                        if self.buffer.size.is_some() {
                            self.buffer.position = self.position;
                        }
                    }
                    None => return Err(Error::new(ErrorKind::Other, "Missing file size")),
                }
            }
        }
        Ok(self.position)
    }
}
<|endoftext|>


<commit_before>extern crate clap;

use clap::{App, Arg};

fn main() {

    // Option arguments are those that take an additional value, such as "-c value". In clap they
    // support three types of specification, those with short() as "-o some", or those with long()
    // as "--option value" or "--option=value"
    //
    // Options also support a multiple setting, which is discussed in the example below.
    let matches = App::new("MyApp")
                        // Regular App configuration goes here...

                        // Assume we an application that accepts an input file via the "-i file"
                        // or the "--input file" (as wel as "--input=file").
                        // Below every setting supported by option arguments is discussed.
                        // NOTE: You DO NOT need to specify each setting, only those which apply
                        // to your particular case.
                        .arg(Arg::with_name("input")
                                    .help("the input file to use") // Displayed when showing help info
                                    .takes_value(true)             // MUST be set to true in order to be an "option" argument
                                    .short("i")                    // This argument is triggered with "-i"
                                    .long("input")                 // This argument is triggered with "--input"
                                    .multiple(true)                // Set to true if you wish to allow multiple occurrences
                                                                   // such as "-i file -i other_file -i third_file"
                                    .required(true)                // By default this argument MUST be present
                                                                   // NOTE: mutual exclusions take precedence over
                                                                   // required arguments
                                    .requires("config")            // Says, "If the user uses "input", they MUST
                                                                   // also use this other 'config' arg too"
                                                                   // Can also specifiy a list using
                                                                   // requires_all(Vec<&str>)
                                    .conflicts_with("output")      // Opposite of requires(), says "if the
                                                                   // user uses -a, they CANNOT use 'output'"
                                                                   // also has a conflicts_with_all(Vec<&str>)
                        )
                        // NOTE: In order to compile this example, comment out conflicts_with()
                        // and requires() because we have not defined an "output" or "config"
                        // argument.
                        .get_matches();

    // We can find out whether or not "input" was used
    if matches.is_present("input") {
        println!("An input file was specified");
    }

    // We can also get the value for "input"
    //
    // NOTE: If we specified multiple(), this will only return the _FIRST_
    // occurrence
    if let Some(ref in_file) = matches.value_of("input") {
        println!("An input file: {}", in_file);
    }

    // If we specified the multiple() setting we can get all the values
    if let Some(in_v) = matches.values_of("input") {
        for in_file in in_v {
            println!("An input file: {}", in_file);
        }
    }

    // We can see how many times the option was used with the occurrences_of() method
    //
    // NOTE: Just like with flags, if we did not specify the multiple() setting this will only
    // return 1 no matter how many times the argument was used (unless it wasn't used at all, in
    // in which case 0 is returned)
    println!("The \"input\" argument was used {} times", matches.occurrences_of("input"));

    // Continued program logic goes here...
}
<commit_msg>Fix typo in 07_option_args.rs<commit_after>extern crate clap;

use clap::{App, Arg};

fn main() {

    // Option arguments are those that take an additional value, such as "-c value". In clap they
    // support three types of specification, those with short() as "-o some", or those with long()
    // as "--option value" or "--option=value"
    //
    // Options also support a multiple setting, which is discussed in the example below.
    let matches = App::new("MyApp")
                        // Regular App configuration goes here...

                        // Assume we an application that accepts an input file via the "-i file"
                        // or the "--input file" (as well as "--input=file").
                        // Below every setting supported by option arguments is discussed.
                        // NOTE: You DO NOT need to specify each setting, only those which apply
                        // to your particular case.
                        .arg(Arg::with_name("input")
                                    .help("the input file to use") // Displayed when showing help info
                                    .takes_value(true)             // MUST be set to true in order to be an "option" argument
                                    .short("i")                    // This argument is triggered with "-i"
                                    .long("input")                 // This argument is triggered with "--input"
                                    .multiple(true)                // Set to true if you wish to allow multiple occurrences
                                                                   // such as "-i file -i other_file -i third_file"
                                    .required(true)                // By default this argument MUST be present
                                                                   // NOTE: mutual exclusions take precedence over
                                                                   // required arguments
                                    .requires("config")            // Says, "If the user uses "input", they MUST
                                                                   // also use this other 'config' arg too"
                                                                   // Can also specifiy a list using
                                                                   // requires_all(Vec<&str>)
                                    .conflicts_with("output")      // Opposite of requires(), says "if the
                                                                   // user uses -a, they CANNOT use 'output'"
                                                                   // also has a conflicts_with_all(Vec<&str>)
                        )
                        // NOTE: In order to compile this example, comment out conflicts_with()
                        // and requires() because we have not defined an "output" or "config"
                        // argument.
                        .get_matches();

    // We can find out whether or not "input" was used
    if matches.is_present("input") {
        println!("An input file was specified");
    }

    // We can also get the value for "input"
    //
    // NOTE: If we specified multiple(), this will only return the _FIRST_
    // occurrence
    if let Some(ref in_file) = matches.value_of("input") {
        println!("An input file: {}", in_file);
    }

    // If we specified the multiple() setting we can get all the values
    if let Some(in_v) = matches.values_of("input") {
        for in_file in in_v {
            println!("An input file: {}", in_file);
        }
    }

    // We can see how many times the option was used with the occurrences_of() method
    //
    // NOTE: Just like with flags, if we did not specify the multiple() setting this will only
    // return 1 no matter how many times the argument was used (unless it wasn't used at all, in
    // in which case 0 is returned)
    println!("The \"input\" argument was used {} times", matches.occurrences_of("input"));

    // Continued program logic goes here...
}
<|endoftext|>


<commit_before>use std::collections::{BTreeMap, HashSet};
use std::hash::Hasher;
use std::iter;
use std::str::CharIndices;
use std::ops;

use itertools::{GroupBy, Itertools};

use util::*;
use super::*;
use Query::*;

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to postings.
#[derive(Debug)]
pub struct InvertedIndex {
    // Maps terms to their postings
    index: BTreeMap<String, PostingsMap>,
    // Maps doc ids to their docs
    docs: BTreeMap<String, Document>,
}

impl InvertedIndex {
    /// Constructs a new, empty InvertedIndex
    pub fn new() -> InvertedIndex {
        InvertedIndex { index: BTreeMap::new(), docs: BTreeMap::new() }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let analyzed = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, _) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&doc.id);
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            self.index
                .entry(ngram)
                .or_insert_with(BTreeMap::new)
                .entry(doc.id.clone())
                .or_insert_with(Vec::new)
                .search_coalesce(0, highlighted);
        }
    }

    /// Performs a search to the specification of the given query
    pub fn query(&self, query: &Query) -> Vec<SearchResult> {
        let postings = self.query_rec(query);
        self.compute_results(postings)
    }

    /// A helper method for performing a Match query
    pub fn search(&self, query: &str) -> Vec<SearchResult> {
        self.query(&Match(query))
    }

    fn postings(&self, query: &str) -> PostingsMap {
        let unique_terms: HashSet<_> = query.split_whitespace().map(str::to_lowercase).collect();
        unique_terms.into_iter().flat_map(|word| self.index.get(&word)).merge_postings()
    }

    fn query_rec(&self, query: &Query) -> PostingsMap {
        match *query {
            Match(query) => self.postings(query),
            And(queries) => {
                let postings: Vec<_> = queries.iter().map(|q| self.query_rec(q)).collect();
                postings.intersect_postings()
            }
            Or(queries) => queries.into_iter().map(|q| self.query_rec(q)).merge_postings(),
        }
    }

    fn compute_results(&self, postings: PostingsMap) -> Vec<SearchResult> {
        let mut results: Vec<_> = postings.into_iter()
                                          .map(|(doc_id, index_map)| {
                                              SearchResult::new(&self.docs[&doc_id],
                                                                index_map)
                                          })
                                          .collect();
        results.sort_by(|result1, result2| result2.score.partial_cmp(&result1.score).unwrap());
        results
    }
}

type Tokens<'a> = iter::FlatMap<
                iter::Filter<
                    GroupBy<bool, CharIndices<'a>, fn(&(usize, char)) -> bool>,
                    fn(&(bool, Vec<(usize, char)>)) -> bool>,
                iter::Map<ops::Range<usize>, Ngrams>,
                fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>;

fn analyze_doc(doc: &str) -> Tokens {
    doc.char_indices()
        .group_by(is_whitespace as fn(&(usize, char)) -> bool)
        .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
        .flat_map(ngrams)
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>,
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

#[cfg(test)]
mod test {
    use super::super::*;
    use Query::{And, Match, Or};
    use std::collections::BTreeMap;

    #[test]
    fn test_ngrams() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        index.index(doc1.clone());
        index.index(doc2.clone());
        let search_results = index.search("to");
        let expected: BTreeMap<_, _> = [(doc1.clone(), vec![(6, 8), (25, 27)]),
                                        (doc2, vec![(13, 15)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
        assert_eq!("learn to program in rust today",
                   search_results.iter()
                                 .find(|search_result| search_result.doc == &doc1)
                                 .unwrap()
                                 .highlight("", ""));

    }

    #[test]
    fn test_highlight() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("2", "Won\u{2019}t this split the ecosystem? Will everyone use?");
        index.index(doc1.clone());
        let expected = "Won\u{2019}t this split the *e*cosystem? Will *e*veryone use?";
        let search_results = index.search("e");
        assert_eq!(1, search_results.len());
        assert_eq!(search_results[0].highlight("*", "*"), expected);
    }

    #[test]
    fn test_unicode() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "嗨, 您好");
        index.index(doc.clone());
        let to_search = "您";
        let search_results = index.search(to_search);
        let &SearchResult { ref doc, ref highlights, .. } = search_results.iter().next().unwrap();
        let (begin, end) = highlights[0];
        assert_eq!(&doc.content()[begin..end], to_search);
    }

    #[test]
    fn test_update_doc() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "abc åäö");
        index.index(doc);
        let doc = Document::new("0", "different");
        index.index(doc);
        let search_results = index.search("å");
        assert!(search_results.is_empty());
        assert_eq!(index.docs.len(), 1);
    }

    #[test]
    fn test_ranking() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let doc2 = Document::new("1", "beast");
        index.index(doc2);
        let search_results = index.search("be");
        assert_eq!(index.docs.len(), 2);
        // "beat" should be first, since it's a closer match
        assert_eq!(search_results[0].doc, &doc);
    }

    #[test]
    fn test_duplicate_term() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be be");
        assert_eq!(search_results.len(), 1);
    }

    #[test]
    fn test_duplicate_term2() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be b");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_search() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("bE");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_index() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("be");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_and() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&And(&[Match("today"), Match("you")]));
        let expected: BTreeMap<_, _> = [(doc2, vec![(9, 12), (13, 18)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
    }

    #[test]
    fn test_and_or() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&Or(&[Match("you"), And(&[Match("today"), Match("you")])]));
        let expected: BTreeMap<_, _> = [(doc2, vec![(9, 12), (13, 18)]), (doc3, vec![(9, 12)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights, &expected[search_result.doc])
        }
    }
}
<commit_msg>rustfmt<commit_after>use std::collections::{BTreeMap, HashSet};
use std::hash::Hasher;
use std::iter;
use std::str::CharIndices;
use std::ops;

use itertools::{GroupBy, Itertools};

use util::*;
use super::*;
use Query::*;

/// A basic implementation of an `Index`, the inverted index is a data structure that maps
/// from words to postings.
#[derive(Debug)]
pub struct InvertedIndex {
    // Maps terms to their postings
    index: BTreeMap<String, PostingsMap>,
    // Maps doc ids to their docs
    docs: BTreeMap<String, Document>,
}

impl InvertedIndex {
    /// Constructs a new, empty InvertedIndex
    pub fn new() -> InvertedIndex {
        InvertedIndex {
            index: BTreeMap::new(),
            docs: BTreeMap::new(),
        }
    }

    /// A basic implementation of index, splits the document's content into whitespace-separated
    /// words, and inserts each word-document pair into the map.
    pub fn index(&mut self, doc: Document) {
        let analyzed = analyze_doc(doc.content());
        let previous_version = self.docs.insert(doc.id.clone(), doc.clone());
        if let Some(previous_version) = previous_version {
            let previous_analyzed = analyze_doc(previous_version.content());
            for (ngram, _) in previous_analyzed {
                let is_empty = {
                    let docs_for_ngram = self.index.get_mut(&ngram).unwrap();
                    docs_for_ngram.remove(&doc.id);
                    docs_for_ngram.is_empty()
                };
                if is_empty {
                    self.index.remove(&ngram);
                }
            }
        }

        for (ngram, highlighted) in analyzed {
            self.index
                .entry(ngram)
                .or_insert_with(BTreeMap::new)
                .entry(doc.id.clone())
                .or_insert_with(Vec::new)
                .search_coalesce(0, highlighted);
        }
    }

    /// Performs a search to the specification of the given query
    pub fn query(&self, query: &Query) -> Vec<SearchResult> {
        let postings = self.query_rec(query);
        self.compute_results(postings)
    }

    /// A helper method for performing a Match query
    pub fn search(&self, query: &str) -> Vec<SearchResult> {
        self.query(&Match(query))
    }

    fn postings(&self, query: &str) -> PostingsMap {
        let unique_terms: HashSet<_> = query.split_whitespace().map(str::to_lowercase).collect();
        unique_terms.into_iter().flat_map(|word| self.index.get(&word)).merge_postings()
    }

    fn query_rec(&self, query: &Query) -> PostingsMap {
        match *query {
            Match(query) => self.postings(query),
            And(queries) => {
                let postings: Vec<_> = queries.iter().map(|q| self.query_rec(q)).collect();
                postings.intersect_postings()
            }
            Or(queries) => queries.into_iter().map(|q| self.query_rec(q)).merge_postings(),
        }
    }

    fn compute_results(&self, postings: PostingsMap) -> Vec<SearchResult> {
        let mut results: Vec<_> = postings.into_iter()
                                          .map(|(doc_id, index_map)| {
                                              SearchResult::new(&self.docs[&doc_id], index_map)
                                          })
                                          .collect();
        results.sort_by(|result1, result2| result2.score.partial_cmp(&result1.score).unwrap());
        results
    }
}

type Tokens<'a> = iter::FlatMap<
                iter::Filter<
                    GroupBy<bool, CharIndices<'a>, fn(&(usize, char)) -> bool>,
                    fn(&(bool, Vec<(usize, char)>)) -> bool>,
                iter::Map<ops::Range<usize>, Ngrams>,
                fn((bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams>>;

fn analyze_doc(doc: &str) -> Tokens {
    doc.char_indices()
       .group_by(is_whitespace as fn(&(usize, char)) -> bool)
       .filter(not_whitespace as fn(&(bool, Vec<(usize, char)>)) -> bool)
       .flat_map(ngrams)
}

fn ngrams((_, chars): (bool, Vec<(usize, char)>)) -> iter::Map<ops::Range<usize>, Ngrams> {
    (1..chars.len() + 1).map(Ngrams::new(chars))
}

fn not_whitespace(&(is_whitespace, _): &(bool, Vec<(usize, char)>)) -> bool {
    !is_whitespace
}

fn is_whitespace(&(_, c): &(usize, char)) -> bool {
    c.is_whitespace()
}

struct Ngrams {
    chars: Vec<(usize, char)>,
}

impl Ngrams {
    fn new(chars: Vec<(usize, char)>) -> Ngrams {
        Ngrams { chars: chars }
    }
}

impl Fn<(usize,)> for Ngrams {
    extern "rust-call" fn call(&self, (to,): (usize,)) -> (String, (usize, usize)) {
        let word = self.chars[..to].iter().flat_map(|&(_, c)| c.to_lowercase()).collect();
        let start = self.chars[0].0;
        let (last_idx, last_char) = self.chars[to - 1];
        let finish = last_idx + last_char.len_utf8();
        (word, (start, finish))
    }
}

impl FnMut<(usize,)> for Ngrams {
    extern "rust-call" fn call_mut(&mut self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

impl FnOnce<(usize,)> for Ngrams {
    type Output = (String, (usize, usize));
    extern "rust-call" fn call_once(self, to: (usize,)) -> (String, (usize, usize)) {
        self.call(to)
    }
}

#[cfg(test)]
mod test {
    use super::super::*;
    use Query::{And, Match, Or};
    use std::collections::BTreeMap;

    #[test]
    fn test_ngrams() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        index.index(doc1.clone());
        index.index(doc2.clone());
        let search_results = index.search("to");
        let expected: BTreeMap<_, _> = [(doc1.clone(), vec![(6, 8), (25, 27)]),
                                        (doc2, vec![(13, 15)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights,
                       &expected[search_result.doc])
        }
        assert_eq!("learn to program in rust today",
                   search_results.iter()
                                 .find(|search_result| search_result.doc == &doc1)
                                 .unwrap()
                                 .highlight("", ""));

    }

    #[test]
    fn test_highlight() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("2",
                                 "Won\u{2019}t this split the ecosystem? Will everyone use?");
        index.index(doc1.clone());
        let expected = "Won\u{2019}t this split the *e*cosystem? Will *e*veryone use?";
        let search_results = index.search("e");
        assert_eq!(1, search_results.len());
        assert_eq!(search_results[0].highlight("*", "*"), expected);
    }

    #[test]
    fn test_unicode() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "嗨, 您好");
        index.index(doc.clone());
        let to_search = "您";
        let search_results = index.search(to_search);
        let &SearchResult { ref doc, ref highlights, .. } = search_results.iter().next().unwrap();
        let (begin, end) = highlights[0];
        assert_eq!(&doc.content()[begin..end], to_search);
    }

    #[test]
    fn test_update_doc() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "abc åäö");
        index.index(doc);
        let doc = Document::new("0", "different");
        index.index(doc);
        let search_results = index.search("å");
        assert!(search_results.is_empty());
        assert_eq!(index.docs.len(), 1);
    }

    #[test]
    fn test_ranking() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let doc2 = Document::new("1", "beast");
        index.index(doc2);
        let search_results = index.search("be");
        assert_eq!(index.docs.len(), 2);
        // "beat" should be first, since it's a closer match
        assert_eq!(search_results[0].doc, &doc);
    }

    #[test]
    fn test_duplicate_term() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be be");
        assert_eq!(search_results.len(), 1);
    }

    #[test]
    fn test_duplicate_term2() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "beat");
        index.index(doc.clone());
        let search_results = index.search("be b");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_search() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("bE");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_lowercase_index() {
        let mut index = InvertedIndex::new();
        let doc = Document::new("0", "BeAt");
        index.index(doc.clone());
        let search_results = index.search("be");
        assert_eq!(search_results.len(), 1);
        assert_eq!(search_results[0].highlights, vec![(0, 2)]);
    }

    #[test]
    fn test_and() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&And(&[Match("today"), Match("you")]));
        let expected: BTreeMap<_, _> = [(doc2, vec![(9, 12), (13, 18)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights,
                       &expected[search_result.doc])
        }
    }

    #[test]
    fn test_and_or() {
        let mut index = InvertedIndex::new();
        let doc1 = Document::new("1", "learn to program in rust today");
        let doc2 = Document::new("2", "what did you today do");
        let doc3 = Document::new("3", "what did you do yesterday");
        index.index(doc1.clone());
        index.index(doc2.clone());
        index.index(doc3.clone());
        let search_results = index.query(&Or(&[Match("you"),
                                               And(&[Match("today"), Match("you")])]));
        let expected: BTreeMap<_, _> = [(doc2, vec![(9, 12), (13, 18)]), (doc3, vec![(9, 12)])]
                                           .iter()
                                           .cloned()
                                           .collect();
        assert_eq!(search_results.len(), expected.len());
        for search_result in &search_results {
            assert_eq!(&search_result.highlights,
                       &expected[search_result.doc])
        }
    }
}
<|endoftext|>


<commit_before>//! Epoch-based memory reclamation.
//!
//! An interesting problem concurrent collections deal with comes from the remove operation.
//! Suppose that a thread removes an element from a lock-free map, while another thread is reading
//! that same element at the same time. The first thread must wait until the second thread stops
//! reading the element. Only then it is safe to destruct it.
//!
//! Programming languages that come with garbage collectors solve this problem trivially. The
//! garbage collector will destruct the removed element when no thread can hold a reference to it
//! anymore.
//!
//! This crate implements a basic memory reclamation mechanism, which is based on epochs. When an
//! element gets removed from a concurrent collection, it is inserted into a pile of garbage and
//! marked with the current epoch. Every time a thread accesses a collection, it checks the current
//! epoch, attempts to increment it, and destructs some garbage that became so old that no thread
//! can be referencing it anymore.
//!
//! That is the general mechanism behind epoch-based memory reclamation, but the details are a bit
//! more complicated. Anyhow, memory reclamation is designed to be fully automatic and something
//! users of concurrent collections don't have to worry much about.
//!
//! # Pointers
//!
//! Concurrent collections are built using atomic pointers. This module provides [`Atomic`], which
//! is just a shared atomic pointer to a heap-allocated object. Loading an [`Atomic`] yields a
//! [`Shared`], which is an epoch-protected pointer through which the loaded object can be safely
//! read.
//!
//! # Pinning
//!
//! Before an [`Atomic`] can be loaded, a participant must be [`pin`]ned. By pinning a participant
//! we declare that any object that gets removed from now on must not be destructed just
//! yet. Garbage collection of newly removed objects is suspended until the participant gets
//! unpinned.
//!
//! # Garbage
//!
//! Objects that get removed from concurrent collections must be stashed away until all currently
//! pinned participants get unpinned. Such objects can be stored into a thread-local or global
//! storage, where they are kept until the right time for their destruction comes.
//!
//! There is a global shared instance of garbage queue. You can [`defer`] the execution of an
//! arbitrary function until the global epoch is advanced enough. Most notably, concurrent data
//! structures may defer the deallocation of an object.
//!
//! # APIs
//!
//! For majority of use cases, just use the default garbage collector by invoking [`pin`]. If you
//! want to create your own garbage collector, use the [`Collector`] API.
//!
//! [`Atomic`]: struct.Atomic.html
//! [`Collector`]: struct.Collector.html
//! [`Shared`]: struct.Shared.html
//! [`pin`]: fn.pin.html
//! [`defer`]: fn.defer.html

#![warn(missing_docs)]
#![warn(missing_debug_implementations)]
#![cfg_attr(not(feature = "std"), no_std)]
#![cfg_attr(feature = "nightly", feature(cfg_target_has_atomic))]

#[macro_use]
extern crate cfg_if;
#[cfg(feature = "std")]
extern crate core;

cfg_if! {
    if #[cfg(feature = "alloc")] {
        extern crate alloc;
    } else if #[cfg(feature = "std")] {
        extern crate std as alloc;
    }
}

#[cfg_attr(feature = "nightly", cfg(target_has_atomic = "ptr"))]
cfg_if! {
    if #[cfg(any(feature = "alloc", feature = "std"))] {
        extern crate crossbeam_utils;
        #[macro_use]
        extern crate memoffset;
        #[macro_use]
        extern crate scopeguard;

        mod atomic;
        mod collector;
        mod deferred;
        mod epoch;
        mod guard;
        mod internal;
        mod sync;

        pub use self::atomic::{Atomic, CompareAndSetError, CompareAndSetOrdering, Owned, Pointer, Shared};
        pub use self::collector::{Collector, LocalHandle};
        pub use self::guard::{unprotected, Guard};
    }
}

cfg_if! {
    if #[cfg(feature = "std")] {
        #[macro_use]
        extern crate lazy_static;

        mod default;
        pub use self::default::{default_collector, is_pinned, pin};
    }
}
<commit_msg>fix broken doc link<commit_after>//! Epoch-based memory reclamation.
//!
//! An interesting problem concurrent collections deal with comes from the remove operation.
//! Suppose that a thread removes an element from a lock-free map, while another thread is reading
//! that same element at the same time. The first thread must wait until the second thread stops
//! reading the element. Only then it is safe to destruct it.
//!
//! Programming languages that come with garbage collectors solve this problem trivially. The
//! garbage collector will destruct the removed element when no thread can hold a reference to it
//! anymore.
//!
//! This crate implements a basic memory reclamation mechanism, which is based on epochs. When an
//! element gets removed from a concurrent collection, it is inserted into a pile of garbage and
//! marked with the current epoch. Every time a thread accesses a collection, it checks the current
//! epoch, attempts to increment it, and destructs some garbage that became so old that no thread
//! can be referencing it anymore.
//!
//! That is the general mechanism behind epoch-based memory reclamation, but the details are a bit
//! more complicated. Anyhow, memory reclamation is designed to be fully automatic and something
//! users of concurrent collections don't have to worry much about.
//!
//! # Pointers
//!
//! Concurrent collections are built using atomic pointers. This module provides [`Atomic`], which
//! is just a shared atomic pointer to a heap-allocated object. Loading an [`Atomic`] yields a
//! [`Shared`], which is an epoch-protected pointer through which the loaded object can be safely
//! read.
//!
//! # Pinning
//!
//! Before an [`Atomic`] can be loaded, a participant must be [`pin`]ned. By pinning a participant
//! we declare that any object that gets removed from now on must not be destructed just
//! yet. Garbage collection of newly removed objects is suspended until the participant gets
//! unpinned.
//!
//! # Garbage
//!
//! Objects that get removed from concurrent collections must be stashed away until all currently
//! pinned participants get unpinned. Such objects can be stored into a thread-local or global
//! storage, where they are kept until the right time for their destruction comes.
//!
//! There is a global shared instance of garbage queue. You can [`defer`] the execution of an
//! arbitrary function until the global epoch is advanced enough. Most notably, concurrent data
//! structures may defer the deallocation of an object.
//!
//! # APIs
//!
//! For majority of use cases, just use the default garbage collector by invoking [`pin`]. If you
//! want to create your own garbage collector, use the [`Collector`] API.
//!
//! [`Atomic`]: struct.Atomic.html
//! [`Collector`]: struct.Collector.html
//! [`Shared`]: struct.Shared.html
//! [`pin`]: fn.pin.html
//! [`defer`]: struct.Guard.html#method.defer

#![warn(missing_docs)]
#![warn(missing_debug_implementations)]
#![cfg_attr(not(feature = "std"), no_std)]
#![cfg_attr(feature = "nightly", feature(cfg_target_has_atomic))]

#[macro_use]
extern crate cfg_if;
#[cfg(feature = "std")]
extern crate core;

cfg_if! {
    if #[cfg(feature = "alloc")] {
        extern crate alloc;
    } else if #[cfg(feature = "std")] {
        extern crate std as alloc;
    }
}

#[cfg_attr(feature = "nightly", cfg(target_has_atomic = "ptr"))]
cfg_if! {
    if #[cfg(any(feature = "alloc", feature = "std"))] {
        extern crate crossbeam_utils;
        #[macro_use]
        extern crate memoffset;
        #[macro_use]
        extern crate scopeguard;

        mod atomic;
        mod collector;
        mod deferred;
        mod epoch;
        mod guard;
        mod internal;
        mod sync;

        pub use self::atomic::{Atomic, CompareAndSetError, CompareAndSetOrdering, Owned, Pointer, Shared};
        pub use self::collector::{Collector, LocalHandle};
        pub use self::guard::{unprotected, Guard};
    }
}

cfg_if! {
    if #[cfg(feature = "std")] {
        #[macro_use]
        extern crate lazy_static;

        mod default;
        pub use self::default::{default_collector, is_pinned, pin};
    }
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::io::Read;

/// A struct representing data contained in a gameboy cartridge (a.k.a. ROM).
/// Each Cartridge has an header with metadata (name, available hw on cart like rumble, ram, etc.)
/// Data contained in a cartridge is exposed as a Vec<u8> but should be accessed via the MMU.
#[derive(Debug)]
pub struct Cartridge {
    pub size: usize,
    pub data: Vec<u8>,
}

impl Cartridge {
    /// Creates a Cartridge by loading the file at the specified path.
    pub fn new_with_path(path: &str) -> Cartridge {
        Cartridge::new_with_vec(Cartridge::load_rom_file(path))
    }

    /// Creates a new Cartridge struct from a vector buffer.
    pub fn new_with_vec(data: Vec<u8>) -> Cartridge {
        Cartridge {
            size: data.len(),
            data: data,
        }
    }

    /// Loads binary data from a file into a vector buffer.
    fn load_rom_file(path: &str) -> Vec<u8> {
        let mut buf: Vec<u8> = vec![];

        if let Ok(mut file) = File::open(path) {
            match file.read_to_end(&mut buf) {
                Ok(_) => {},
                Err(e) => println!("{:?}", e),
            }
        }
        buf
    }
}
<commit_msg>Added some cartridge metadata processing<commit_after>use std::fs::File;
use std::io::Read;

use jeebie::utils::combine_as_u16;

/// A struct representing data contained in a gameboy cartridge (a.k.a. ROM).
/// Each Cartridge has an header with metadata (name, available hw on cart like rumble, ram, etc.)
/// Data contained in a cartridge is exposed as a Vec<u8> but should be accessed via the MMU.
#[derive(Debug)]
pub struct Cartridge {
    pub size: usize,
    pub name: String,
    pub data: Vec<u8>,
}

impl Cartridge {
    /// Creates a Cartridge by loading the file at the specified path.
    pub fn new_with_path(path: &str) -> Cartridge {
        Cartridge::new_with_vec(Cartridge::load_rom_file(path))
    }

    /// Creates a new Cartridge struct from a vector buffer.
    pub fn new_with_vec(data: Vec<u8>) -> Cartridge {
        let name_data : Vec<u8> = (&data[0x134..0x143]).to_vec();

        // read name data as ASCII string
        let name : String = name_data.into_iter()
            .map(| c | { c as char })
            .collect();

        let cgb_flag = data[0x143];

        let licensee_code_data : Vec<u8> = (&data[0x144..0x145]).to_vec();

        // read licensee code as ASCII string
        let name : String = licensee_code_data.into_iter()
            .map(| c | { c as char })
            .collect();

        let sgb_flag = data[0x146];

        let mbc_type = data[0x147];
        let rom_size = data[0x148];
        let ram_size = data[0x149];

        let dest_code = data[0x14A];
        let lic_code = data[0x14B];
        let version = data[0x14C];

        // only header_checksum is verified in real hardware
        let header_checksum = data[0x14D];
        let checksum = combine_as_u16(data[0x14E], data[0x14F]);

        // TODO: process the read data and save it in the cart struct
        Cartridge {
            size: data.len(),
            name: name,
            data: data,
        }
    }

    /// Loads binary data from a file into a vector buffer.
    fn load_rom_file(path: &str) -> Vec<u8> {
        let mut buf: Vec<u8> = vec![];

        if let Ok(mut file) = File::open(path) {
            match file.read_to_end(&mut buf) {
                Ok(_) => {},
                Err(e) => println!("{:?}", e),
            }
        }
        buf
    }
}
<|endoftext|>