<commit_before>//! VM implementation, traits and structs
//!
//! ### Lifecycle
//!
//! A VM can be started given a `Context` and a `BlockHeader`. The
//! user can then `fire` or `step` to run it. Those functions would
//! only fail if it needs some information (accounts in the current
//! block, or block hashes of previous blocks). If this happens, one
//! can use the function `commit_account` and `commit_blockhash` to
//! commit those information to the VM, and `fire` or `step` again
//! until it succeeds. The current VM status can always be obtained
//! using the `status` function.

mod memory;
mod stack;
mod pc;
mod storage;
mod params;
mod eval;
mod commit;
mod patch;
mod transaction;
pub mod errors;

pub use self::memory::{Memory, SeqMemory};
pub use self::stack::Stack;
pub use self::pc::{PC, Instruction};
pub use self::storage::Storage;
pub use self::params::*;
pub use self::patch::*;
pub use self::eval::{State, Machine, MachineStatus};
pub use self::commit::{AccountCommitment, Account, AccountState, BlockhashState};
pub use self::transaction::{Transaction, TransactionVM};

use std::collections::hash_map;
use utils::bigint::M256;
use utils::gas::Gas;
use utils::address::Address;
use self::errors::{RequireError, CommitError, MachineError};

#[derive(Debug, Clone)]
/// VM Status
pub enum VMStatus {
    /// A running VM.
    Running,
    /// VM is stopped with out errors.
    ExitedOk,
    /// VM is stopped due to an error. The state of the VM is before
    /// the last failing instruction.
    ExitedErr(MachineError),
}

pub trait VM {
    fn commit_account(&mut self, commitment: AccountCommitment) -> Result<(), CommitError>;
    fn commit_blockhash(&mut self, number: M256, hash: M256) -> Result<(), CommitError>;
    fn status(&self) -> VMStatus;
    fn step(&mut self) -> Result<(), RequireError>;
    fn fire(&mut self) -> Result<(), RequireError> {
        loop {
            match self.status() {
                VMStatus::Running => self.step()?,
                VMStatus::ExitedOk | VMStatus::ExitedErr(_) => return Ok(()),
            }
        }
    }
    fn accounts(&self) -> hash_map::Values<Address, Account>;
    fn out(&self) -> &[u8];
    fn available_gas(&self) -> Gas;
    fn used_gas(&self) -> Gas;
    fn refunded_gas(&self) -> Gas;
    fn logs(&self) -> &[Log];
}

/// A sequencial VM. It uses sequencial memory representation and hash
/// map storage for accounts.
pub type SeqContextVM = ContextVM<SeqMemory>;
pub type SeqTransactionVM = TransactionVM<SeqMemory>;

/// A VM that executes using a context and block information.
pub struct ContextVM<M> {
    machines: Vec<Machine<M>>,
    history: Vec<Context>
}

impl<M: Memory + Default> ContextVM<M> {
    /// Create a new VM using the given context, block header and patch.
    pub fn new(context: Context, block: BlockHeader, patch: &'static Patch) -> Self {
        let mut machines = Vec::new();
        machines.push(Machine::new(context, block, patch, 1));
        ContextVM {
            machines,
            history: Vec::new()
        }
    }

    pub fn with_states(context: Context, block: BlockHeader, patch: &'static Patch,
                       account_state: AccountState, blockhash_state: BlockhashState) -> Self {
        let mut machines = Vec::new();
        machines.push(Machine::with_states(context, block, patch, 1, account_state, blockhash_state));
        ContextVM {
            machines,
            history: Vec::new()
        }
    }

    /// Returns the call create history. Only used in testing.
    pub fn history(&self) -> &[Context] {
        self.history.as_slice()
    }
}

impl<M: Memory + Default> VM for ContextVM<M> {
    /// Commit an account information to this VM. This should only be
    /// used when receiving `RequireError`.
    fn commit_account(&mut self, commitment: AccountCommitment) -> Result<(), CommitError> {
        for machine in &mut self.machines {
            machine.commit_account(commitment.clone())?;
        }
        Ok(())
    }

    /// Commit a block hash to this VM. This should only be used when
    /// receiving `RequireError`.
    fn commit_blockhash(&mut self, number: M256, hash: M256) -> Result<(), CommitError> {
        for machine in &mut self.machines {
            machine.commit_blockhash(number, hash)?;
        }
        Ok(())
    }

    /// Returns the current status of the VM.
    fn status(&self) -> VMStatus {
        match self.machines[0].status() {
            MachineStatus::Running | MachineStatus::InvokeCreate(_) | MachineStatus::InvokeCall(_, _) => VMStatus::Running,
            MachineStatus::ExitedOk => VMStatus::ExitedOk,
            MachineStatus::ExitedErr(err) => VMStatus::ExitedErr(err.into()),
        }
    }

    /// Run one instruction and return. If it succeeds, VM status can
    /// still be `Running`. If the call stack has more than one items,
    /// this will only executes the last items' one single
    /// instruction.
    fn step(&mut self) -> Result<(), RequireError> {
        match self.machines.last().unwrap().status().clone() {
            MachineStatus::Running => {
                self.machines.last_mut().unwrap().step()
            },
            MachineStatus::ExitedOk | MachineStatus::ExitedErr(_) => {
                if self.machines.len() <= 1 {
                    Ok(())
                } else {
                    let finished = self.machines.pop().unwrap();
                    self.machines.last_mut().unwrap().apply_sub(finished);
                    Ok(())
                }
            },
            MachineStatus::InvokeCall(context, _) | MachineStatus::InvokeCreate(context) => {
                self.history.push(context.clone());
                let sub = self.machines.last().unwrap().derive(context);
                self.machines.push(sub);
                Ok(())
            },
        }
    }

    /// Run instructions until it reaches a `RequireError` or
    /// exits. If this function succeeds, the VM status can only be
    /// either `ExitedOk` or `ExitedErr`.
    fn fire(&mut self) -> Result<(), RequireError> {
        loop {
            match self.status() {
                VMStatus::Running => self.step()?,
                VMStatus::ExitedOk | VMStatus::ExitedErr(_) => return Ok(()),
            }
        }
    }

    /// Returns the changed or committed accounts information up to
    /// current execution status.
    fn accounts(&self) -> hash_map::Values<Address, Account> {
        self.machines[0].state().account_state.accounts()
    }

    /// Returns the out value, if any.
    fn out(&self) -> &[u8] {
        self.machines[0].state().out.as_slice()
    }

    /// Returns the available gas of this VM.
    fn available_gas(&self) -> Gas {
        self.machines[0].state().available_gas()
    }

    /// Returns the used gas of this VM.
    fn used_gas(&self) -> Gas {
        self.machines[0].state().used_gas
    }

    /// Returns the refunded gas of this VM.
    fn refunded_gas(&self) -> Gas {
        self.machines[0].state().refunded_gas
    }

    /// Returns logs to be appended to the current block if the user
    /// decided to accept the running status of this VM.
    fn logs(&self) -> &[Log] {
        self.machines[0].state().logs.as_slice()
    }
}
<commit_msg>Problem: VM total used gas should include memory gas<commit_after>//! VM implementation, traits and structs
//!
//! ### Lifecycle
//!
//! A VM can be started given a `Context` and a `BlockHeader`. The
//! user can then `fire` or `step` to run it. Those functions would
//! only fail if it needs some information (accounts in the current
//! block, or block hashes of previous blocks). If this happens, one
//! can use the function `commit_account` and `commit_blockhash` to
//! commit those information to the VM, and `fire` or `step` again
//! until it succeeds. The current VM status can always be obtained
//! using the `status` function.

mod memory;
mod stack;
mod pc;
mod storage;
mod params;
mod eval;
mod commit;
mod patch;
mod transaction;
pub mod errors;

pub use self::memory::{Memory, SeqMemory};
pub use self::stack::Stack;
pub use self::pc::{PC, Instruction};
pub use self::storage::Storage;
pub use self::params::*;
pub use self::patch::*;
pub use self::eval::{State, Machine, MachineStatus};
pub use self::commit::{AccountCommitment, Account, AccountState, BlockhashState};
pub use self::transaction::{Transaction, TransactionVM};

use std::collections::hash_map;
use utils::bigint::M256;
use utils::gas::Gas;
use utils::address::Address;
use self::errors::{RequireError, CommitError, MachineError};

#[derive(Debug, Clone)]
/// VM Status
pub enum VMStatus {
    /// A running VM.
    Running,
    /// VM is stopped with out errors.
    ExitedOk,
    /// VM is stopped due to an error. The state of the VM is before
    /// the last failing instruction.
    ExitedErr(MachineError),
}

pub trait VM {
    fn commit_account(&mut self, commitment: AccountCommitment) -> Result<(), CommitError>;
    fn commit_blockhash(&mut self, number: M256, hash: M256) -> Result<(), CommitError>;
    fn status(&self) -> VMStatus;
    fn step(&mut self) -> Result<(), RequireError>;
    fn fire(&mut self) -> Result<(), RequireError> {
        loop {
            match self.status() {
                VMStatus::Running => self.step()?,
                VMStatus::ExitedOk | VMStatus::ExitedErr(_) => return Ok(()),
            }
        }
    }
    fn accounts(&self) -> hash_map::Values<Address, Account>;
    fn out(&self) -> &[u8];
    fn available_gas(&self) -> Gas;
    fn used_gas(&self) -> Gas;
    fn refunded_gas(&self) -> Gas;
    fn logs(&self) -> &[Log];
}

/// A sequencial VM. It uses sequencial memory representation and hash
/// map storage for accounts.
pub type SeqContextVM = ContextVM<SeqMemory>;
pub type SeqTransactionVM = TransactionVM<SeqMemory>;

/// A VM that executes using a context and block information.
pub struct ContextVM<M> {
    machines: Vec<Machine<M>>,
    history: Vec<Context>
}

impl<M: Memory + Default> ContextVM<M> {
    /// Create a new VM using the given context, block header and patch.
    pub fn new(context: Context, block: BlockHeader, patch: &'static Patch) -> Self {
        let mut machines = Vec::new();
        machines.push(Machine::new(context, block, patch, 1));
        ContextVM {
            machines,
            history: Vec::new()
        }
    }

    pub fn with_states(context: Context, block: BlockHeader, patch: &'static Patch,
                       account_state: AccountState, blockhash_state: BlockhashState) -> Self {
        let mut machines = Vec::new();
        machines.push(Machine::with_states(context, block, patch, 1, account_state, blockhash_state));
        ContextVM {
            machines,
            history: Vec::new()
        }
    }

    /// Returns the call create history. Only used in testing.
    pub fn history(&self) -> &[Context] {
        self.history.as_slice()
    }
}

impl<M: Memory + Default> VM for ContextVM<M> {
    /// Commit an account information to this VM. This should only be
    /// used when receiving `RequireError`.
    fn commit_account(&mut self, commitment: AccountCommitment) -> Result<(), CommitError> {
        for machine in &mut self.machines {
            machine.commit_account(commitment.clone())?;
        }
        Ok(())
    }

    /// Commit a block hash to this VM. This should only be used when
    /// receiving `RequireError`.
    fn commit_blockhash(&mut self, number: M256, hash: M256) -> Result<(), CommitError> {
        for machine in &mut self.machines {
            machine.commit_blockhash(number, hash)?;
        }
        Ok(())
    }

    /// Returns the current status of the VM.
    fn status(&self) -> VMStatus {
        match self.machines[0].status() {
            MachineStatus::Running | MachineStatus::InvokeCreate(_) | MachineStatus::InvokeCall(_, _) => VMStatus::Running,
            MachineStatus::ExitedOk => VMStatus::ExitedOk,
            MachineStatus::ExitedErr(err) => VMStatus::ExitedErr(err.into()),
        }
    }

    /// Run one instruction and return. If it succeeds, VM status can
    /// still be `Running`. If the call stack has more than one items,
    /// this will only executes the last items' one single
    /// instruction.
    fn step(&mut self) -> Result<(), RequireError> {
        match self.machines.last().unwrap().status().clone() {
            MachineStatus::Running => {
                self.machines.last_mut().unwrap().step()
            },
            MachineStatus::ExitedOk | MachineStatus::ExitedErr(_) => {
                if self.machines.len() <= 1 {
                    Ok(())
                } else {
                    let finished = self.machines.pop().unwrap();
                    self.machines.last_mut().unwrap().apply_sub(finished);
                    Ok(())
                }
            },
            MachineStatus::InvokeCall(context, _) | MachineStatus::InvokeCreate(context) => {
                self.history.push(context.clone());
                let sub = self.machines.last().unwrap().derive(context);
                self.machines.push(sub);
                Ok(())
            },
        }
    }

    /// Run instructions until it reaches a `RequireError` or
    /// exits. If this function succeeds, the VM status can only be
    /// either `ExitedOk` or `ExitedErr`.
    fn fire(&mut self) -> Result<(), RequireError> {
        loop {
            match self.status() {
                VMStatus::Running => self.step()?,
                VMStatus::ExitedOk | VMStatus::ExitedErr(_) => return Ok(()),
            }
        }
    }

    /// Returns the changed or committed accounts information up to
    /// current execution status.
    fn accounts(&self) -> hash_map::Values<Address, Account> {
        self.machines[0].state().account_state.accounts()
    }

    /// Returns the out value, if any.
    fn out(&self) -> &[u8] {
        self.machines[0].state().out.as_slice()
    }

    /// Returns the available gas of this VM.
    fn available_gas(&self) -> Gas {
        self.machines[0].state().available_gas()
    }

    /// Returns the used gas of this VM.
    fn used_gas(&self) -> Gas {
        self.machines[0].state().memory_gas() + self.machines[0].state().used_gas
    }

    /// Returns the refunded gas of this VM.
    fn refunded_gas(&self) -> Gas {
        self.machines[0].state().refunded_gas
    }

    /// Returns logs to be appended to the current block if the user
    /// decided to accept the running status of this VM.
    fn logs(&self) -> &[Log] {
        self.machines[0].state().logs.as_slice()
    }
}
<|endoftext|>


<commit_before>use core::fmt;
use core::pin::Pin;
use futures_core::future::Future;
use futures_core::ready;
use futures_core::stream::{FusedStream, Stream};
use futures_core::task::{Context, Poll};
#[cfg(feature = "sink")]
use futures_sink::Sink;
use pin_project::pin_project;

// FIXME: docs, tests

/// Stream for the [`take_until`](super::StreamExt::take_until) method.
#[pin_project]
#[must_use = "streams do nothing unless polled"]
pub struct TakeUntil<St: Stream, Fut: Future> {
    #[pin]
    stream: St,
    /// Contains the inner Future on start and None once the inner Future is resolved
    /// or taken out by the user.
    #[pin]
    fut: Option<Fut>,
    /// Contains fut's return value once fut is resolved
    fut_result: Option<Fut::Output>,
    /// Whether the future was taken out by the user.
    free: bool,
}

impl<St, Fut> fmt::Debug for TakeUntil<St, Fut>
where
    St: Stream + fmt::Debug,
    St::Item: fmt::Debug,
    Fut: Future + fmt::Debug,
{
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("TakeUntil")
            .field("stream", &self.stream)
            .field("fut", &self.fut)
            .finish()
    }
}

impl<St, Fut> TakeUntil<St, Fut>
where
    St: Stream,
    Fut: Future,
{
    pub(super) fn new(stream: St, fut: Fut) -> Self {
        Self {
            stream,
            fut: Some(fut),
            fut_result: None,
            free: false,
        }
    }

    delegate_access_inner!(stream, St, ());

    /// Extract the stopping future out of the combinator.
    /// The future is returned only if it isn't resolved yet, ie. if the stream isn't stopped yet.
    /// Taking out the future means the combinator will be yielding
    /// elements from the wrapped stream without ever stopping it.
    pub fn take_future(&mut self) -> Option<Fut> {
        if self.fut.is_some() {
            self.free = true;
        }

        self.fut.take()
    }

    /// Once the stopping future is resolved, this method can be used
    /// to extract the value returned by the stopping future.
    ///
    /// This may be used to retrieve arbitrary data from the stopping
    /// future, for example a reason why the stream was stopped.
    ///
    /// This method will return `None` if the future isn't resovled yet,
    /// or if the result was already taken out.
    ///
    /// # Examples
    ///
    /// ```
    /// # futures::executor::block_on(async {
    /// use futures::future;
    /// use futures::stream::{self, StreamExt};
    /// use futures::task::Poll;
    ///
    /// let stream = stream::iter(1..=10);
    ///
    /// let mut i = 0;
    /// let stop_fut = future::poll_fn(|_cx| {
    ///     i += 1;
    ///     if i <= 5 {
    ///         Poll::Pending
    ///     } else {
    ///         Poll::Ready("reason")
    ///     }
    /// });
    ///
    /// let mut stream = stream.take_until(stop_fut);
    /// let _ = stream.by_ref().collect::<Vec<_>>().await;
    ///
    /// let result = stream.take_result().unwrap();
    /// assert_eq!(result, "reason");
    /// # });
    /// ```
    pub fn take_result(&mut self) -> Option<Fut::Output> {
        self.fut_result.take()
    }

    /// Whether the stream was stopped yet by the stopping future
    /// being resolved.
    pub fn is_stopped(&self) -> bool {
        !self.free && self.fut.is_none()
    }
}

impl<St, Fut> Stream for TakeUntil<St, Fut>
where
    St: Stream,
    Fut: Future,
{
    type Item = St::Item;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<St::Item>> {
        let mut this = self.project();

        if let Some(f) = this.fut.as_mut().as_pin_mut() {
            if let Poll::Ready(result) = f.poll(cx) {
                this.fut.set(None);
                *this.fut_result = Some(result);
            }
        }

        if !*this.free && this.fut.is_none() {
            // Future resolved, inner stream stopped
            Poll::Ready(None)
        } else {
            // Future either not resolved yet or taken out by the user
            let item = ready!(this.stream.poll_next(cx));
            if item.is_none() {
                this.fut.set(None);
            }
            Poll::Ready(item)
        }
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        if self.is_stopped() {
            return (0, Some(0));
        }

        self.stream.size_hint()
    }
}

impl<St, Fut> FusedStream for TakeUntil<St, Fut>
where
    St: Stream,
    Fut: Future,
{
    fn is_terminated(&self) -> bool {
        self.is_stopped()
    }
}

// Forwarding impl of Sink from the underlying stream
#[cfg(feature = "sink")]
impl<S, Fut, Item> Sink<Item> for TakeUntil<S, Fut>
where
    S: Stream + Sink<Item>,
    Fut: Future,
{
    type Error = S::Error;

    delegate_sink!(stream, Item);
}
<commit_msg>Fix typo in doc<commit_after>use core::fmt;
use core::pin::Pin;
use futures_core::future::Future;
use futures_core::ready;
use futures_core::stream::{FusedStream, Stream};
use futures_core::task::{Context, Poll};
#[cfg(feature = "sink")]
use futures_sink::Sink;
use pin_project::pin_project;

// FIXME: docs, tests

/// Stream for the [`take_until`](super::StreamExt::take_until) method.
#[pin_project]
#[must_use = "streams do nothing unless polled"]
pub struct TakeUntil<St: Stream, Fut: Future> {
    #[pin]
    stream: St,
    /// Contains the inner Future on start and None once the inner Future is resolved
    /// or taken out by the user.
    #[pin]
    fut: Option<Fut>,
    /// Contains fut's return value once fut is resolved
    fut_result: Option<Fut::Output>,
    /// Whether the future was taken out by the user.
    free: bool,
}

impl<St, Fut> fmt::Debug for TakeUntil<St, Fut>
where
    St: Stream + fmt::Debug,
    St::Item: fmt::Debug,
    Fut: Future + fmt::Debug,
{
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("TakeUntil")
            .field("stream", &self.stream)
            .field("fut", &self.fut)
            .finish()
    }
}

impl<St, Fut> TakeUntil<St, Fut>
where
    St: Stream,
    Fut: Future,
{
    pub(super) fn new(stream: St, fut: Fut) -> Self {
        Self {
            stream,
            fut: Some(fut),
            fut_result: None,
            free: false,
        }
    }

    delegate_access_inner!(stream, St, ());

    /// Extract the stopping future out of the combinator.
    /// The future is returned only if it isn't resolved yet, ie. if the stream isn't stopped yet.
    /// Taking out the future means the combinator will be yielding
    /// elements from the wrapped stream without ever stopping it.
    pub fn take_future(&mut self) -> Option<Fut> {
        if self.fut.is_some() {
            self.free = true;
        }

        self.fut.take()
    }

    /// Once the stopping future is resolved, this method can be used
    /// to extract the value returned by the stopping future.
    ///
    /// This may be used to retrieve arbitrary data from the stopping
    /// future, for example a reason why the stream was stopped.
    ///
    /// This method will return `None` if the future isn't resolved yet,
    /// or if the result was already taken out.
    ///
    /// # Examples
    ///
    /// ```
    /// # futures::executor::block_on(async {
    /// use futures::future;
    /// use futures::stream::{self, StreamExt};
    /// use futures::task::Poll;
    ///
    /// let stream = stream::iter(1..=10);
    ///
    /// let mut i = 0;
    /// let stop_fut = future::poll_fn(|_cx| {
    ///     i += 1;
    ///     if i <= 5 {
    ///         Poll::Pending
    ///     } else {
    ///         Poll::Ready("reason")
    ///     }
    /// });
    ///
    /// let mut stream = stream.take_until(stop_fut);
    /// let _ = stream.by_ref().collect::<Vec<_>>().await;
    ///
    /// let result = stream.take_result().unwrap();
    /// assert_eq!(result, "reason");
    /// # });
    /// ```
    pub fn take_result(&mut self) -> Option<Fut::Output> {
        self.fut_result.take()
    }

    /// Whether the stream was stopped yet by the stopping future
    /// being resolved.
    pub fn is_stopped(&self) -> bool {
        !self.free && self.fut.is_none()
    }
}

impl<St, Fut> Stream for TakeUntil<St, Fut>
where
    St: Stream,
    Fut: Future,
{
    type Item = St::Item;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<St::Item>> {
        let mut this = self.project();

        if let Some(f) = this.fut.as_mut().as_pin_mut() {
            if let Poll::Ready(result) = f.poll(cx) {
                this.fut.set(None);
                *this.fut_result = Some(result);
            }
        }

        if !*this.free && this.fut.is_none() {
            // Future resolved, inner stream stopped
            Poll::Ready(None)
        } else {
            // Future either not resolved yet or taken out by the user
            let item = ready!(this.stream.poll_next(cx));
            if item.is_none() {
                this.fut.set(None);
            }
            Poll::Ready(item)
        }
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        if self.is_stopped() {
            return (0, Some(0));
        }

        self.stream.size_hint()
    }
}

impl<St, Fut> FusedStream for TakeUntil<St, Fut>
where
    St: Stream,
    Fut: Future,
{
    fn is_terminated(&self) -> bool {
        self.is_stopped()
    }
}

// Forwarding impl of Sink from the underlying stream
#[cfg(feature = "sink")]
impl<S, Fut, Item> Sink<Item> for TakeUntil<S, Fut>
where
    S: Stream + Sink<Item>,
    Fut: Future,
{
    type Error = S::Error;

    delegate_sink!(stream, Item);
}
<|endoftext|>


<commit_before>extern crate html2text;
#[cfg(unix)] extern crate termion;
#[cfg(unix)] extern crate argparse;
#[cfg(unix)] extern crate unicode_width;
#[cfg(unix)]
mod top {
    use ::termion;
    use ::std;
    use ::html2text;
    use std::io::{self, Write};
    use html2text::render::text_renderer::{RichAnnotation,TaggedLine,TaggedLineElement};
    use argparse::{ArgumentParser, Store};
    use termion::input::TermRead;
    use termion::event::Key;
    use termion::raw::IntoRawMode;
    use termion::cursor::Goto;
    use termion::screen::AlternateScreen;
    use unicode_width::UnicodeWidthStr;

    fn to_style(tag: &Vec<RichAnnotation>) -> String {
        let mut style = String::new();

        for ann in tag {
            match *ann {
                RichAnnotation::Default => (),
                RichAnnotation::Link(_) => {
                    style.push_str(&format!("{}", termion::style::Underline));
                },
                RichAnnotation::Image => {
                    style.push_str(&format!("{}", termion::color::Fg(termion::color::LightBlue)));
                },
                RichAnnotation::Emphasis => {
                    style.push_str(&format!("{}", termion::color::Fg(termion::color::LightGreen)));
                },
                RichAnnotation::Strong => {
                    style.push_str(&format!("{}", termion::color::Fg(termion::color::LightGreen)));
                },
                RichAnnotation::Code => {
                    style.push_str(&format!("{}", termion::color::Fg(termion::color::LightYellow)));
                },
                RichAnnotation::Preformat(_is_cont) => {
                    style.push_str(&format!("{}", termion::color::Fg(termion::color::LightBlack)));
                },
            }
        }
        style
    }

    struct LinkMap {
        lines: Vec<Vec<Option<String>>>,  // lines[y][x] => Some(URL) or None
    }

    impl LinkMap {
        pub fn link_at(&self, x: usize, y: usize) -> Option<&str> {
            if let Some(ref linevec) = self.lines.get(y) {
                if let Some(&Some(ref text)) = linevec.get(x) {
                    return Some(&text)
                }
            }
            None
        }
    }

    fn link_from_tag(tag: &Vec<RichAnnotation>) -> Option<String> {
        let mut link = None;
        for annotation in tag {
            if let RichAnnotation::Link(ref text) = *annotation {
                link = Some(text.clone());
            }
        }
        link
    }

    fn find_links(lines: &Vec<TaggedLine<Vec<RichAnnotation>>>) -> LinkMap {
        use self::TaggedLineElement::Str;

        let mut map = Vec::new();
        for line in lines {
            let mut linevec = Vec::new();

            for tli in line.iter() {
                if let Str(ts) = tli {
                    let link = link_from_tag(&ts.tag);
                    for _ in 0..UnicodeWidthStr::width(ts.s.as_str()) {
                        linevec.push(link.clone());
                    }
                }
            }

            map.push(linevec);
        }
        LinkMap {
            lines: map,
        }
    }

    pub fn main() {
        let mut filename = String::new();
        {
            let mut ap = ArgumentParser::new();
            ap.refer(&mut filename)
              .add_argument("filename", Store, "Set HTML filename");
            ap.parse_args_or_exit();
        }

        let (width, height) = termion::terminal_size().unwrap();
        let (width, height) = (width as usize, height as usize);

        let mut file = std::fs::File::open(filename).expect("Tried to open file");
        let annotated = html2text::from_read_rich(&mut file, width as usize);

        let link_map = find_links(&annotated);

        let mut keys = io::stdin().keys();

        // max_y is the largest (0-based) index of a real document line.
        let max_y = annotated.len() - 1;
        // top_y is the (0-based) index of the document line shown at
        // the top of the visible screen.
        let mut top_y = 0;
        // doc_x and doc_y are the logical (0-based) x and y of the
        // cursor position within the document.
        let mut doc_x = 0;
        let mut doc_y = 0;

        let mut screen = AlternateScreen::from(
            io::stdout().into_raw_mode().unwrap());

        loop {
            // Sanity-check the current screen position. max_y should
            // be small enough that no blank lines beyond the end of
            // the document are visible on screen (except when the
            // document is shorter than a screenful); large enough
            // that the cursor isn't off the bottom of the visible
            // screen; and small enough that the cursor isn't off the
            // top.
            if max_y >= height-1 {
                top_y = std::cmp::min(top_y, max_y - (height-1));
            }
            if doc_y >= height-1 {
                top_y = std::cmp::max(top_y, doc_y - (height-1));
            }
            top_y = std::cmp::min(top_y, doc_y);

            let opt_url = link_map.link_at(doc_x, doc_y);
            let vis_y_limit = std::cmp::min(top_y + height, max_y + 1);
            write!(screen, "{}", termion::clear::All).unwrap();
            for (i, line) in annotated[top_y..vis_y_limit].iter().enumerate() {
                use self::TaggedLineElement::Str;

                write!(screen, "{}", Goto(1, i as u16 +1)).unwrap();
                for tli in line.iter() {
                    if let Str(ts) = tli {
                        let style = to_style(&ts.tag);
                        let link = link_from_tag(&ts.tag);
                        match (opt_url, link) {
                            (Some(ref t1), Some(ref t2)) if t1 == t2 => {
                                write!(screen, "{}", termion::style::Invert).unwrap();
                            },
                            _ => (),
                        }
                        write!(screen, "{}{}{}", style, ts.s, termion::style::Reset).unwrap();
                    }
                }
            }

            // 1-based screen coordinates
            let cursor_x = (doc_x + 1) as u16;
            let cursor_y = (doc_y - top_y + 1) as u16;
            write!(screen, "{}", Goto(cursor_x, cursor_y)).unwrap();

            screen.flush().unwrap();
            if let Some(Ok(k)) = keys.next() {
                match k {
                    Key::Char('q') => break,
                    Key::Char('j') | Key::Down => {
                        if doc_y < max_y {
                            doc_y += 1;
                        }
                    },
                    Key::Char('k') | Key::Up => {
                        if doc_y > 0 {
                            doc_y -= 1;
                        }
                    },
                    Key::Char('h') | Key::Left => {
                        if doc_x > 0 {
                            doc_x -= 1;
                        }
                    },
                    Key::Char('l') | Key::Right => {
                        if doc_x + 1 < width {
                            doc_x += 1;
                        }
                    },
                    Key::Char(' ') | Key::PageDown => {
                        // Ideally, move both the cursor and the top
                        // visible line down by a whole page
                        doc_y += height;
                        top_y += height;

                        // But bound the cursor within the document
                        doc_y = std::cmp::min(doc_y, max_y);

                        // And the standard bounds checking for top_y
                        // will take care of the rest of the special
                        // cases.
                    },
                    Key::PageUp => {
                        // Ideally, move both the cursor and the top
                        // visible line up by a whole page. But bound
                        // both at zero.
                        doc_y = std::cmp::max(doc_y, height) - height;
                        top_y = std::cmp::max(top_y, height) - height;
                    },
                    Key::Home => {
                        doc_y = 0;
                    },
                    Key::End => {
                        doc_y = max_y;
                    },
                    Key::Char('\t') => {
                    },
                    _ => {},
                }
            }
        }
    }
}

#[cfg(not(unix))]
mod top {
    pub fn main() {}
}

fn main() {
    top::main()
}
<commit_msg>Fragment support: follow intra-document hyperlinks in html2term.<commit_after>extern crate html2text;
#[cfg(unix)] extern crate termion;
#[cfg(unix)] extern crate argparse;
#[cfg(unix)] extern crate unicode_width;
#[cfg(unix)]
mod top {
    use ::termion;
    use ::std;
    use ::html2text;
    use std::io::{self, Write};
    use std::collections::HashMap;
    use html2text::render::text_renderer::{RichAnnotation,TaggedLine,TaggedLineElement};
    use argparse::{ArgumentParser, Store};
    use termion::input::TermRead;
    use termion::event::Key;
    use termion::raw::IntoRawMode;
    use termion::cursor::Goto;
    use termion::screen::AlternateScreen;
    use unicode_width::UnicodeWidthStr;

    fn to_style(tag: &Vec<RichAnnotation>) -> String {
        let mut style = String::new();

        for ann in tag {
            match *ann {
                RichAnnotation::Default => (),
                RichAnnotation::Link(_) => {
                    style.push_str(&format!("{}", termion::style::Underline));
                },
                RichAnnotation::Image => {
                    style.push_str(&format!("{}", termion::color::Fg(termion::color::LightBlue)));
                },
                RichAnnotation::Emphasis => {
                    style.push_str(&format!("{}", termion::color::Fg(termion::color::LightGreen)));
                },
                RichAnnotation::Strong => {
                    style.push_str(&format!("{}", termion::color::Fg(termion::color::LightGreen)));
                },
                RichAnnotation::Code => {
                    style.push_str(&format!("{}", termion::color::Fg(termion::color::LightYellow)));
                },
                RichAnnotation::Preformat(_is_cont) => {
                    style.push_str(&format!("{}", termion::color::Fg(termion::color::LightBlack)));
                },
            }
        }
        style
    }

    struct LinkMap {
        lines: Vec<Vec<Option<String>>>,  // lines[y][x] => Some(URL) or None
    }

    impl LinkMap {
        pub fn link_at(&self, x: usize, y: usize) -> Option<&str> {
            if let Some(ref linevec) = self.lines.get(y) {
                if let Some(&Some(ref text)) = linevec.get(x) {
                    return Some(&text)
                }
            }
            None
        }
    }

    fn link_from_tag(tag: &Vec<RichAnnotation>) -> Option<String> {
        let mut link = None;
        for annotation in tag {
            if let RichAnnotation::Link(ref text) = *annotation {
                link = Some(text.clone());
            }
        }
        link
    }

    fn find_links(lines: &Vec<TaggedLine<Vec<RichAnnotation>>>) -> LinkMap {
        use self::TaggedLineElement::Str;

        let mut map = Vec::new();
        for line in lines {
            let mut linevec = Vec::new();

            for tli in line.iter() {
                if let Str(ts) = tli {
                    let link = link_from_tag(&ts.tag);
                    for _ in 0..UnicodeWidthStr::width(ts.s.as_str()) {
                        linevec.push(link.clone());
                    }
                }
            }

            map.push(linevec);
        }
        LinkMap {
            lines: map,
        }
    }

    struct FragMap {
        start_xy: HashMap<String, (usize,usize)>,
    }

    fn find_frags(lines: &Vec<TaggedLine<Vec<RichAnnotation>>>) -> FragMap {
        use self::TaggedLineElement::*;

        let mut map = HashMap::new();
        let mut y = 0;
        for line in lines {
            let mut x = 0;
            for tli in line.iter() {
                match tli {
                    FragmentStart(fragname) => {
                        map.insert(fragname.to_string(), (x, y));
                    },
                    Str(ts) => {
                        x += UnicodeWidthStr::width(ts.s.as_str());
                    },
                }
            }
            y += 1;
        }
        FragMap {
            start_xy: map,
        }
    }

    pub fn main() {
        let mut filename = String::new();
        {
            let mut ap = ArgumentParser::new();
            ap.refer(&mut filename)
              .add_argument("filename", Store, "Set HTML filename");
            ap.parse_args_or_exit();
        }

        let (width, height) = termion::terminal_size().unwrap();
        let (width, height) = (width as usize, height as usize);

        let mut file = std::fs::File::open(filename).expect("Tried to open file");
        let annotated = html2text::from_read_rich(&mut file, width as usize);

        let link_map = find_links(&annotated);
        let frag_map = find_frags(&annotated);

        let mut keys = io::stdin().keys();

        // max_y is the largest (0-based) index of a real document line.
        let max_y = annotated.len() - 1;
        // top_y is the (0-based) index of the document line shown at
        // the top of the visible screen.
        let mut top_y = 0;
        // doc_x and doc_y are the logical (0-based) x and y of the
        // cursor position within the document.
        let mut doc_x = 0;
        let mut doc_y = 0;

        let mut screen = AlternateScreen::from(
            io::stdout().into_raw_mode().unwrap());

        loop {
            // Sanity-check the current screen position. max_y should
            // be small enough that no blank lines beyond the end of
            // the document are visible on screen (except when the
            // document is shorter than a screenful); large enough
            // that the cursor isn't off the bottom of the visible
            // screen; and small enough that the cursor isn't off the
            // top.
            if max_y >= height-1 {
                top_y = std::cmp::min(top_y, max_y - (height-1));
            }
            if doc_y >= height-1 {
                top_y = std::cmp::max(top_y, doc_y - (height-1));
            }
            top_y = std::cmp::min(top_y, doc_y);

            let opt_url = link_map.link_at(doc_x, doc_y);
            let vis_y_limit = std::cmp::min(top_y + height, max_y + 1);
            write!(screen, "{}", termion::clear::All).unwrap();
            for (i, line) in annotated[top_y..vis_y_limit].iter().enumerate() {
                use self::TaggedLineElement::Str;

                write!(screen, "{}", Goto(1, i as u16 +1)).unwrap();
                for tli in line.iter() {
                    if let Str(ts) = tli {
                        let style = to_style(&ts.tag);
                        let link = link_from_tag(&ts.tag);
                        match (opt_url, link) {
                            (Some(ref t1), Some(ref t2)) if t1 == t2 => {
                                write!(screen, "{}", termion::style::Invert).unwrap();
                            },
                            _ => (),
                        }
                        write!(screen, "{}{}{}", style, ts.s, termion::style::Reset).unwrap();
                    }
                }
            }

            // 1-based screen coordinates
            let cursor_x = (doc_x + 1) as u16;
            let cursor_y = (doc_y - top_y + 1) as u16;
            write!(screen, "{}", Goto(cursor_x, cursor_y)).unwrap();

            screen.flush().unwrap();
            if let Some(Ok(k)) = keys.next() {
                match k {
                    Key::Char('q') => break,
                    Key::Char('j') | Key::Down => {
                        if doc_y < max_y {
                            doc_y += 1;
                        }
                    },
                    Key::Char('k') | Key::Up => {
                        if doc_y > 0 {
                            doc_y -= 1;
                        }
                    },
                    Key::Char('h') | Key::Left => {
                        if doc_x > 0 {
                            doc_x -= 1;
                        }
                    },
                    Key::Char('l') | Key::Right => {
                        if doc_x + 1 < width {
                            doc_x += 1;
                        }
                    },
                    Key::Char(' ') | Key::PageDown => {
                        // Ideally, move both the cursor and the top
                        // visible line down by a whole page
                        doc_y += height;
                        top_y += height;

                        // But bound the cursor within the document
                        doc_y = std::cmp::min(doc_y, max_y);

                        // And the standard bounds checking for top_y
                        // will take care of the rest of the special
                        // cases.
                    },
                    Key::PageUp => {
                        // Ideally, move both the cursor and the top
                        // visible line up by a whole page. But bound
                        // both at zero.
                        doc_y = std::cmp::max(doc_y, height) - height;
                        top_y = std::cmp::max(top_y, height) - height;
                    },
                    Key::Home => {
                        doc_y = 0;
                    },
                    Key::End => {
                        doc_y = max_y;
                    },
                    Key::Char('\t') => {
                    },
                    Key::Char('\r') | Key::Char('\n') => {
                        if let Some(url) = opt_url {
                            if url.starts_with("#") {
                                let start = frag_map.start_xy.get(&url[1..]);
                                if let Some((x, y)) = start {
                                    doc_x = *x;
                                    doc_y = *y;
                                }
                            }
                        }
                    },
                    _ => {},
                }
            }
        }
    }
}

#[cfg(not(unix))]
mod top {
    pub fn main() {}
}

fn main() {
    top::main()
}
<|endoftext|>


<commit_before>extern crate pancurses;

use backend;
use event::{Event, Key};

use self::super::find_closest;
use theme::{Color, ColorStyle, Effect};

pub struct Concrete {
    window: pancurses::Window,
}

impl backend::Backend for Concrete {
    fn init() -> Self {
        ::std::env::set_var("ESCDELAY", "25");
        let window = pancurses::initscr();
        window.keypad(true);
        pancurses::noecho();
        pancurses::cbreak();
        pancurses::start_color();
        pancurses::curs_set(0);
        window.bkgd(pancurses::COLOR_PAIR(ColorStyle::Background.id() as u64));

        Concrete { window: window }
    }

    fn screen_size(&self) -> (usize, usize) {
        let (x, y) = self.window.get_max_yx();
        (x as usize, y as usize)
    }

    fn has_colors(&self) -> bool {
        pancurses::has_colors()
    }

    fn finish(&mut self) {
        pancurses::endwin();
    }

    fn init_color_style(&mut self, style: ColorStyle, foreground: &Color,
                        background: &Color) {
        pancurses::init_pair(style.id(),
                             find_closest(foreground) as i16,
                             find_closest(background) as i16);
    }

    fn with_color<F: FnOnce()>(&self, color: ColorStyle, f: F) {
        // TODO: pancurses doesn't have an `attr_get` equivalent
        // let mut current_style: pancurses::attr_t = 0;
        // let mut current_color: i16 = 0;
        // pancurses::attr_get(&mut current_style, &mut current_color);

        let style = pancurses::COLOR_PAIR(color.id() as u64);
        self.window.attron(style);
        f();
        self.window.attroff(style);
        // pancurses::attron(current_style);
    }

    fn with_effect<F: FnOnce()>(&self, effect: Effect, f: F) {
        let style = match effect {
            // A_REVERSE, taken from ncurses
            Effect::Reverse => 1 << (10 + 8u32),
            Effect::Simple => pancurses::A_NORMAL,
        };
        self.window.attron(style);
        f();
        self.window.attroff(style);
    }

    fn clear(&self) {
        self.window.clear();
    }

    fn refresh(&mut self) {
        self.window.refresh();
    }

    fn print_at(&self, (x, y): (usize, usize), text: &str) {
        self.window.mvaddstr(y as i32, x as i32, text);
    }

    fn poll_event(&self) -> Event {
        // TODO: there seems to not be any indication
        // of Ctrl/Alt/Shift in these :v
        if let Some(ev) = self.window.getch() {
            match ev {
                pancurses::Input::Character(c) => Event::Char(c),
                pancurses::Input::Unknown(i) => Event::Unknown(i),
                // TODO: I honestly have no fucking idea what KeyCodeYes is
                pancurses::Input::KeyCodeYes => Event::Refresh,
                pancurses::Input::KeyBreak => Event::Key(Key::PauseBreak),
                pancurses::Input::KeyDown => Event::Key(Key::Down),
                pancurses::Input::KeyUp => Event::Key(Key::Up),
                pancurses::Input::KeyLeft => Event::Key(Key::Left),
                pancurses::Input::KeyRight => Event::Key(Key::Right),
                pancurses::Input::KeyHome => Event::Key(Key::Home),
                pancurses::Input::KeyBackspace => Event::Key(Key::Backspace),
                pancurses::Input::KeyF0 => Event::Key(Key::F0),
                pancurses::Input::KeyF1 => Event::Key(Key::F1),
                pancurses::Input::KeyF2 => Event::Key(Key::F2),
                pancurses::Input::KeyF3 => Event::Key(Key::F3),
                pancurses::Input::KeyF4 => Event::Key(Key::F4),
                pancurses::Input::KeyF5 => Event::Key(Key::F5),
                pancurses::Input::KeyF6 => Event::Key(Key::F6),
                pancurses::Input::KeyF7 => Event::Key(Key::F7),
                pancurses::Input::KeyF8 => Event::Key(Key::F8),
                pancurses::Input::KeyF9 => Event::Key(Key::F9),
                pancurses::Input::KeyF10 => Event::Key(Key::F10),
                pancurses::Input::KeyF11 => Event::Key(Key::F11),
                pancurses::Input::KeyF12 => Event::Key(Key::F12),
                pancurses::Input::KeyF13 => Event::Shift(Key::F1),
                pancurses::Input::KeyF14 => Event::Shift(Key::F2),
                pancurses::Input::KeyF15 => Event::Shift(Key::F3),
                // TODO: not sure what those are
                pancurses::Input::KeyDL => Event::Refresh,
                pancurses::Input::KeyIL => Event::Key(Key::Insert),
                pancurses::Input::KeyDC => Event::Key(Key::Del),
                pancurses::Input::KeyIC => Event::Refresh,
                pancurses::Input::KeyEIC => Event::Refresh,
                pancurses::Input::KeyClear => Event::Refresh,
                pancurses::Input::KeyEOS => Event::Refresh,
                pancurses::Input::KeyEOL => Event::Refresh,
                pancurses::Input::KeySF => Event::Refresh,
                pancurses::Input::KeySR => Event::Refresh,
                pancurses::Input::KeyNPage => Event::Key(Key::PageDown),
                pancurses::Input::KeyPPage => Event::Key(Key::PageUp),
                pancurses::Input::KeySTab => Event::Shift(Key::Tab),
                pancurses::Input::KeyCTab => Event::Ctrl(Key::Tab),
                pancurses::Input::KeyCATab => Event::CtrlAlt(Key::Tab),
                pancurses::Input::KeyEnter => Event::Key(Key::Enter),
                pancurses::Input::KeySReset => Event::Refresh,
                pancurses::Input::KeyReset => Event::Refresh,
                pancurses::Input::KeyPrint => Event::Refresh,
                pancurses::Input::KeyLL => Event::Refresh,
                pancurses::Input::KeyAbort => Event::Refresh,
                pancurses::Input::KeySHelp => Event::Refresh,
                pancurses::Input::KeyLHelp => Event::Refresh,
                pancurses::Input::KeyBTab => Event::Shift(Key::Tab),
                pancurses::Input::KeyBeg => Event::Refresh,
                pancurses::Input::KeyCancel => Event::Refresh,
                pancurses::Input::KeyClose => Event::Refresh,
                pancurses::Input::KeyCommand => Event::Refresh,
                pancurses::Input::KeyCopy => Event::Refresh,
                pancurses::Input::KeyCreate => Event::Refresh,
                pancurses::Input::KeyEnd => Event::Refresh,
                pancurses::Input::KeyExit => Event::Refresh,
                pancurses::Input::KeyFind => Event::Refresh,
                pancurses::Input::KeyHelp => Event::Refresh,
                pancurses::Input::KeyMark => Event::Refresh,
                pancurses::Input::KeyMessage => Event::Refresh,
                pancurses::Input::KeyMove => Event::Refresh,
                pancurses::Input::KeyNext => Event::Refresh,
                pancurses::Input::KeyOpen => Event::Refresh,
                pancurses::Input::KeyOptions => Event::Refresh,
                pancurses::Input::KeyPrevious => Event::Refresh,
                pancurses::Input::KeyRedo => Event::Refresh,
                pancurses::Input::KeyReference => Event::Refresh,
                pancurses::Input::KeyRefresh => Event::Refresh,
                pancurses::Input::KeyReplace => Event::Refresh,
                pancurses::Input::KeyRestart => Event::Refresh,
                pancurses::Input::KeyResume => Event::Refresh,
                pancurses::Input::KeySave => Event::Refresh,
                pancurses::Input::KeySBeg => Event::Refresh,
                pancurses::Input::KeySCancel => Event::Refresh,
                pancurses::Input::KeySCommand => Event::Refresh,
                pancurses::Input::KeySCopy => Event::Refresh,
                pancurses::Input::KeySCreate => Event::Refresh,
                pancurses::Input::KeySDC => Event::Shift(Key::Del),
                pancurses::Input::KeySDL => Event::Refresh,
                pancurses::Input::KeySelect => Event::Refresh,
                pancurses::Input::KeySEnd => Event::Shift(Key::End),
                pancurses::Input::KeySEOL => Event::Refresh,
                pancurses::Input::KeySExit => Event::Refresh,
                pancurses::Input::KeySFind => Event::Refresh,
                pancurses::Input::KeySHome => Event::Shift(Key::Home),
                pancurses::Input::KeySIC => Event::Shift(Key::Insert),
                pancurses::Input::KeySLeft => Event::Refresh,
                pancurses::Input::KeySMessage => Event::Refresh,
                pancurses::Input::KeySMove => Event::Refresh,
                pancurses::Input::KeySNext => Event::Shift(Key::PageDown),
                pancurses::Input::KeySOptions => Event::Refresh,
                pancurses::Input::KeySPrevious => Event::Shift(Key::PageUp),
                pancurses::Input::KeySPrint => Event::Refresh,
                pancurses::Input::KeySRedo => Event::Refresh,
                pancurses::Input::KeySReplace => Event::Refresh,
                pancurses::Input::KeySRight => Event::Refresh,
                pancurses::Input::KeySResume => Event::Refresh,
                pancurses::Input::KeySSave => Event::Refresh,
                pancurses::Input::KeySSuspend => Event::Refresh,
                pancurses::Input::KeySUndo => Event::Refresh,
                pancurses::Input::KeySuspend => Event::Refresh,
                pancurses::Input::KeyUndo => Event::Refresh,
                pancurses::Input::KeyResize => Event::Refresh,
                pancurses::Input::KeyEvent => Event::Refresh,
                // TODO: mouse support
                pancurses::Input::KeyMouse => Event::Refresh,
                pancurses::Input::KeyA1 => Event::Refresh,
                pancurses::Input::KeyA3 => Event::Refresh,
                pancurses::Input::KeyB2 => Event::Key(Key::NumpadCenter),
                pancurses::Input::KeyC1 => Event::Refresh,
                pancurses::Input::KeyC3 => Event::Refresh,
            }
        } else {
            Event::Refresh
        }
    }

    fn set_refresh_rate(&mut self, fps: u32) {
        if fps == 0 {
            self.window.timeout(-1);
        } else {
            self.window.timeout(1000 / fps as i32);
        }
    }
}
<commit_msg>Hello yes this is typo man<commit_after>extern crate pancurses;

use backend;
use event::{Event, Key};

use self::super::find_closest;
use theme::{Color, ColorStyle, Effect};

pub struct Concrete {
    window: pancurses::Window,
}

impl backend::Backend for Concrete {
    fn init() -> Self {
        ::std::env::set_var("ESCDELAY", "25");
        let window = pancurses::initscr();
        window.keypad(true);
        pancurses::noecho();
        pancurses::cbreak();
        pancurses::start_color();
        pancurses::curs_set(0);
        window.bkgd(pancurses::COLOR_PAIR(ColorStyle::Background.id() as u64));

        Concrete { window: window }
    }

    fn screen_size(&self) -> (usize, usize) {
        let (x, y) = self.window.get_max_yx();
        (x as usize, y as usize)
    }

    fn has_colors(&self) -> bool {
        pancurses::has_colors()
    }

    fn finish(&mut self) {
        pancurses::endwin();
    }

    fn init_color_style(&mut self, style: ColorStyle, foreground: &Color,
                        background: &Color) {
        pancurses::init_pair(style.id(),
                             find_closest(foreground) as i16,
                             find_closest(background) as i16);
    }

    fn with_color<F: FnOnce()>(&self, color: ColorStyle, f: F) {
        // TODO: pancurses doesn't have an `attr_get` equivalent
        // let mut current_style: pancurses::attr_t = 0;
        // let mut current_color: i16 = 0;
        // pancurses::attr_get(&mut current_style, &mut current_color);

        let style = pancurses::COLOR_PAIR(color.id() as u64);
        self.window.attron(style);
        f();
        self.window.attroff(style);
        // pancurses::attron(current_style);
    }

    fn with_effect<F: FnOnce()>(&self, effect: Effect, f: F) {
        let style = match effect {
            // A_REVERSE, taken from ncurses
            Effect::Reverse => 1 << (10 + 8u32),
            Effect::Simple => pancurses::A_NORMAL,
        };
        self.window.attron(style);
        f();
        self.window.attroff(style);
    }

    fn clear(&self) {
        self.window.clear();
    }

    fn refresh(&mut self) {
        self.window.refresh();
    }

    fn print_at(&self, (x, y): (usize, usize), text: &str) {
        self.window.mvaddstr(y as i32, x as i32, text);
    }

    fn poll_event(&self) -> Event {
        // TODO: there seems to not be any indication
        // of Ctrl/Alt/Shift in these :v
        if let Some(ev) = self.window.getch() {
            match ev {
                pancurses::Input::Character(c) => Event::Char(c),
                pancurses::Input::Unknown(i) => Event::Unknown(i),
                // TODO: I honestly have no fucking idea what KeyCodeYes is
                pancurses::Input::KeyCodeYes => Event::Refresh,
                pancurses::Input::KeyBreak => Event::Key(Key::PauseBreak),
                pancurses::Input::KeyDown => Event::Key(Key::Down),
                pancurses::Input::KeyUp => Event::Key(Key::Up),
                pancurses::Input::KeyLeft => Event::Key(Key::Left),
                pancurses::Input::KeyRight => Event::Key(Key::Right),
                pancurses::Input::KeyHome => Event::Key(Key::Home),
                pancurses::Input::KeyBackspace => Event::Key(Key::Backspace),
                pancurses::Input::KeyF0 => Event::Key(Key::F0),
                pancurses::Input::KeyF1 => Event::Key(Key::F1),
                pancurses::Input::KeyF2 => Event::Key(Key::F2),
                pancurses::Input::KeyF3 => Event::Key(Key::F3),
                pancurses::Input::KeyF4 => Event::Key(Key::F4),
                pancurses::Input::KeyF5 => Event::Key(Key::F5),
                pancurses::Input::KeyF6 => Event::Key(Key::F6),
                pancurses::Input::KeyF7 => Event::Key(Key::F7),
                pancurses::Input::KeyF8 => Event::Key(Key::F8),
                pancurses::Input::KeyF9 => Event::Key(Key::F9),
                pancurses::Input::KeyF10 => Event::Key(Key::F10),
                pancurses::Input::KeyF11 => Event::Key(Key::F11),
                pancurses::Input::KeyF12 => Event::Key(Key::F12),
                pancurses::Input::KeyF13 => Event::Shift(Key::F1),
                pancurses::Input::KeyF14 => Event::Shift(Key::F2),
                pancurses::Input::KeyF15 => Event::Shift(Key::F3),
                // TODO: not sure what those are
                pancurses::Input::KeyDL => Event::Refresh,
                pancurses::Input::KeyIL => Event::Key(Key::Ins),
                pancurses::Input::KeyDC => Event::Key(Key::Del),
                pancurses::Input::KeyIC => Event::Refresh,
                pancurses::Input::KeyEIC => Event::Refresh,
                pancurses::Input::KeyClear => Event::Refresh,
                pancurses::Input::KeyEOS => Event::Refresh,
                pancurses::Input::KeyEOL => Event::Refresh,
                pancurses::Input::KeySF => Event::Refresh,
                pancurses::Input::KeySR => Event::Refresh,
                pancurses::Input::KeyNPage => Event::Key(Key::PageDown),
                pancurses::Input::KeyPPage => Event::Key(Key::PageUp),
                pancurses::Input::KeySTab => Event::Shift(Key::Tab),
                pancurses::Input::KeyCTab => Event::Ctrl(Key::Tab),
                pancurses::Input::KeyCATab => Event::CtrlAlt(Key::Tab),
                pancurses::Input::KeyEnter => Event::Key(Key::Enter),
                pancurses::Input::KeySReset => Event::Refresh,
                pancurses::Input::KeyReset => Event::Refresh,
                pancurses::Input::KeyPrint => Event::Refresh,
                pancurses::Input::KeyLL => Event::Refresh,
                pancurses::Input::KeyAbort => Event::Refresh,
                pancurses::Input::KeySHelp => Event::Refresh,
                pancurses::Input::KeyLHelp => Event::Refresh,
                pancurses::Input::KeyBTab => Event::Shift(Key::Tab),
                pancurses::Input::KeyBeg => Event::Refresh,
                pancurses::Input::KeyCancel => Event::Refresh,
                pancurses::Input::KeyClose => Event::Refresh,
                pancurses::Input::KeyCommand => Event::Refresh,
                pancurses::Input::KeyCopy => Event::Refresh,
                pancurses::Input::KeyCreate => Event::Refresh,
                pancurses::Input::KeyEnd => Event::Refresh,
                pancurses::Input::KeyExit => Event::Refresh,
                pancurses::Input::KeyFind => Event::Refresh,
                pancurses::Input::KeyHelp => Event::Refresh,
                pancurses::Input::KeyMark => Event::Refresh,
                pancurses::Input::KeyMessage => Event::Refresh,
                pancurses::Input::KeyMove => Event::Refresh,
                pancurses::Input::KeyNext => Event::Refresh,
                pancurses::Input::KeyOpen => Event::Refresh,
                pancurses::Input::KeyOptions => Event::Refresh,
                pancurses::Input::KeyPrevious => Event::Refresh,
                pancurses::Input::KeyRedo => Event::Refresh,
                pancurses::Input::KeyReference => Event::Refresh,
                pancurses::Input::KeyRefresh => Event::Refresh,
                pancurses::Input::KeyReplace => Event::Refresh,
                pancurses::Input::KeyRestart => Event::Refresh,
                pancurses::Input::KeyResume => Event::Refresh,
                pancurses::Input::KeySave => Event::Refresh,
                pancurses::Input::KeySBeg => Event::Refresh,
                pancurses::Input::KeySCancel => Event::Refresh,
                pancurses::Input::KeySCommand => Event::Refresh,
                pancurses::Input::KeySCopy => Event::Refresh,
                pancurses::Input::KeySCreate => Event::Refresh,
                pancurses::Input::KeySDC => Event::Shift(Key::Del),
                pancurses::Input::KeySDL => Event::Refresh,
                pancurses::Input::KeySelect => Event::Refresh,
                pancurses::Input::KeySEnd => Event::Shift(Key::End),
                pancurses::Input::KeySEOL => Event::Refresh,
                pancurses::Input::KeySExit => Event::Refresh,
                pancurses::Input::KeySFind => Event::Refresh,
                pancurses::Input::KeySHome => Event::Shift(Key::Home),
                pancurses::Input::KeySIC => Event::Shift(Key::Ins),
                pancurses::Input::KeySLeft => Event::Refresh,
                pancurses::Input::KeySMessage => Event::Refresh,
                pancurses::Input::KeySMove => Event::Refresh,
                pancurses::Input::KeySNext => Event::Shift(Key::PageDown),
                pancurses::Input::KeySOptions => Event::Refresh,
                pancurses::Input::KeySPrevious => Event::Shift(Key::PageUp),
                pancurses::Input::KeySPrint => Event::Refresh,
                pancurses::Input::KeySRedo => Event::Refresh,
                pancurses::Input::KeySReplace => Event::Refresh,
                pancurses::Input::KeySRight => Event::Refresh,
                pancurses::Input::KeySResume => Event::Refresh,
                pancurses::Input::KeySSave => Event::Refresh,
                pancurses::Input::KeySSuspend => Event::Refresh,
                pancurses::Input::KeySUndo => Event::Refresh,
                pancurses::Input::KeySuspend => Event::Refresh,
                pancurses::Input::KeyUndo => Event::Refresh,
                pancurses::Input::KeyResize => Event::Refresh,
                pancurses::Input::KeyEvent => Event::Refresh,
                // TODO: mouse support
                pancurses::Input::KeyMouse => Event::Refresh,
                pancurses::Input::KeyA1 => Event::Refresh,
                pancurses::Input::KeyA3 => Event::Refresh,
                pancurses::Input::KeyB2 => Event::Key(Key::NumpadCenter),
                pancurses::Input::KeyC1 => Event::Refresh,
                pancurses::Input::KeyC3 => Event::Refresh,
            }
        } else {
            Event::Refresh
        }
    }

    fn set_refresh_rate(&mut self, fps: u32) {
        if fps == 0 {
            self.window.timeout(-1);
        } else {
            self.window.timeout(1000 / fps as i32);
        }
    }
}
<|endoftext|>


<commit_before>// Iterates over every crate versions ever uploaded and (re-)renders their
// readme using the readme renderer from the cargo_registry crate.
//
// Warning: this can take a lot of time.

#![deny(warnings, clippy::all, rust_2018_idioms)]

#[macro_use]
extern crate serde;

use cargo_registry::{
    db,
    models::Version,
    render::readme_to_html,
    schema::{crates, readme_renderings, versions},
    Config,
};
use std::{io::Read, path::Path, thread};

use chrono::{TimeZone, Utc};
use diesel::{dsl::any, prelude::*};
use docopt::Docopt;
use flate2::read::GzDecoder;
use reqwest::Client;
use tar::{self, Archive};

const DEFAULT_PAGE_SIZE: usize = 25;
const USAGE: &str = "
Usage: render-readmes [options]
       render-readmes --help

Options:
    -h, --help         Show this message.
    --page-size NUM    How many versions should be queried and processed at a time.
    --older-than DATE  Only rerender readmes that are older than this date.
    --crate NAME       Only rerender readmes for the specified crate.
";

#[derive(Deserialize)]
struct Args {
    flag_page_size: Option<usize>,
    flag_older_than: Option<String>,
    flag_crate: Option<String>,
}

fn main() {
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.deserialize())
        .unwrap_or_else(|e| e.exit());
    let config = Config::default();
    let conn = db::connect_now().unwrap();

    let start_time = Utc::now();

    let older_than = if let Some(ref time) = args.flag_older_than {
        Utc.datetime_from_str(time, "%Y-%m-%d %H:%M:%S")
            .expect("Could not parse --older-than argument as a time")
    } else {
        start_time
    };
    let older_than = older_than.naive_utc();

    println!("Start time:                   {}", start_time);
    println!("Rendering readmes older than: {}", older_than);

    let mut query = versions::table
        .inner_join(crates::table)
        .left_outer_join(readme_renderings::table)
        .filter(
            readme_renderings::rendered_at
                .lt(older_than)
                .or(readme_renderings::version_id.is_null()),
        )
        .select(versions::id)
        .into_boxed();

    if let Some(crate_name) = args.flag_crate {
        println!("Rendering readmes for {}", crate_name);
        query = query.filter(crates::name.eq(crate_name));
    }

    let version_ids = query
        .load::<(i32)>(&conn)
        .expect("error loading version ids");

    let total_versions = version_ids.len();
    println!("Rendering {} versions", total_versions);

    let page_size = args.flag_page_size.unwrap_or(DEFAULT_PAGE_SIZE);

    let total_pages = total_versions / page_size;
    let total_pages = if total_versions % page_size == 0 {
        total_pages
    } else {
        total_pages + 1
    };

    let client = Client::new();

    for (page_num, version_ids_chunk) in version_ids.chunks(page_size).enumerate() {
        println!(
            "= Page {} of {} ==================================",
            page_num + 1,
            total_pages
        );

        let versions = versions::table
            .inner_join(crates::table)
            .filter(versions::id.eq(any(version_ids_chunk)))
            .select((versions::all_columns, crates::name))
            .load::<(Version, String)>(&conn)
            .expect("error loading versions");

        let mut tasks = Vec::with_capacity(page_size as usize);
        for (version, krate_name) in versions {
            let config = config.clone();
            Version::record_readme_rendering(version.id, &conn).unwrap_or_else(|_| {
                panic!(
                    "[{}-{}] Couldn't record rendering time",
                    krate_name, version.num
                )
            });
            let client = client.clone();
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", krate_name, version.num);
                let readme = get_readme(&config, &client, &version, &krate_name);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let content_length = readme.len() as u64;
                let content = std::io::Cursor::new(readme);
                let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
                config
                    .uploader
                    .upload(&client, &readme_path, content, content_length, "text/html")
                    .unwrap_or_else(|_| {
                        panic!(
                            "[{}-{}] Couldn't upload file to S3",
                            krate_name, version.num
                        )
                    });
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thread panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(
    config: &Config,
    client: &Client,
    version: &Version,
    krate_name: &str,
) -> Option<String> {
    let location = config
        .uploader
        .crate_location(krate_name, &version.num.to_string());

    let mut response = match client.get(&location).send() {
        Ok(r) => r,
        Err(err) => {
            println!(
                "[{}-{}] Unable to fetch crate: {}",
                krate_name, version.num, err
            );
            return None;
        }
    };

    if !response.status().is_success() {
        println!(
            "[{}-{}] Failed to get a 200 response: {}",
            krate_name,
            version.num,
            response.text().unwrap()
        );
        return None;
    }

    let reader = GzDecoder::new(response);
    let mut archive = Archive::new(reader);
    let mut entries = archive.entries().unwrap_or_else(|_| {
        panic!(
            "[{}-{}] Invalid tar archive entries",
            krate_name, version.num
        )
    });
    let manifest: Manifest = {
        let path = format!("{}-{}/Cargo.toml", krate_name, version.num);
        let contents = find_file_by_path(&mut entries, Path::new(&path), version, krate_name);
        toml::from_str(&contents).unwrap_or_else(|_| {
            panic!(
                "[{}-{}] Syntax error in manifest file",
                krate_name, version.num
            )
        })
    };

    let rendered = {
        let path = format!(
            "{}-{}/{}",
            krate_name, version.num, manifest.package.readme?
        );
        let contents = find_file_by_path(&mut entries, Path::new(&path), version, krate_name);
        readme_to_html(
            &contents,
            manifest
                .package
                .readme_file
                .as_ref()
                .map_or("README.md", |e| &**e),
            manifest.package.repository.as_ref().map(|e| &**e),
        )
    };
    return Some(rendered);

    #[derive(Deserialize)]
    struct Package {
        readme: Option<String>,
        readme_file: Option<String>,
        repository: Option<String>,
    }

    #[derive(Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<'_, R>,
    path: &Path,
    version: &Version,
    krate_name: &str,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                filepath == path
            }
        })
        .unwrap_or_else(|| {
            panic!(
                "[{}-{}] couldn't open file: {}",
                krate_name,
                version.num,
                path.display()
            )
        })
        .unwrap_or_else(|_| {
            panic!(
                "[{}-{}] file is not present: {}",
                krate_name,
                version.num,
                path.display()
            )
        });
    let mut contents = String::new();
    file.read_to_string(&mut contents).unwrap_or_else(|_| {
        panic!(
            "[{}-{}] Couldn't read file contents",
            krate_name, version.num
        )
    });
    contents
}
<commit_msg>Fix new clippy lint about unnecessary parens<commit_after>// Iterates over every crate versions ever uploaded and (re-)renders their
// readme using the readme renderer from the cargo_registry crate.
//
// Warning: this can take a lot of time.

#![deny(warnings, clippy::all, rust_2018_idioms)]

#[macro_use]
extern crate serde;

use cargo_registry::{
    db,
    models::Version,
    render::readme_to_html,
    schema::{crates, readme_renderings, versions},
    Config,
};
use std::{io::Read, path::Path, thread};

use chrono::{TimeZone, Utc};
use diesel::{dsl::any, prelude::*};
use docopt::Docopt;
use flate2::read::GzDecoder;
use reqwest::Client;
use tar::{self, Archive};

const DEFAULT_PAGE_SIZE: usize = 25;
const USAGE: &str = "
Usage: render-readmes [options]
       render-readmes --help

Options:
    -h, --help         Show this message.
    --page-size NUM    How many versions should be queried and processed at a time.
    --older-than DATE  Only rerender readmes that are older than this date.
    --crate NAME       Only rerender readmes for the specified crate.
";

#[derive(Deserialize)]
struct Args {
    flag_page_size: Option<usize>,
    flag_older_than: Option<String>,
    flag_crate: Option<String>,
}

fn main() {
    let args: Args = Docopt::new(USAGE)
        .and_then(|d| d.deserialize())
        .unwrap_or_else(|e| e.exit());
    let config = Config::default();
    let conn = db::connect_now().unwrap();

    let start_time = Utc::now();

    let older_than = if let Some(ref time) = args.flag_older_than {
        Utc.datetime_from_str(time, "%Y-%m-%d %H:%M:%S")
            .expect("Could not parse --older-than argument as a time")
    } else {
        start_time
    };
    let older_than = older_than.naive_utc();

    println!("Start time:                   {}", start_time);
    println!("Rendering readmes older than: {}", older_than);

    let mut query = versions::table
        .inner_join(crates::table)
        .left_outer_join(readme_renderings::table)
        .filter(
            readme_renderings::rendered_at
                .lt(older_than)
                .or(readme_renderings::version_id.is_null()),
        )
        .select(versions::id)
        .into_boxed();

    if let Some(crate_name) = args.flag_crate {
        println!("Rendering readmes for {}", crate_name);
        query = query.filter(crates::name.eq(crate_name));
    }

    let version_ids = query.load::<i32>(&conn).expect("error loading version ids");

    let total_versions = version_ids.len();
    println!("Rendering {} versions", total_versions);

    let page_size = args.flag_page_size.unwrap_or(DEFAULT_PAGE_SIZE);

    let total_pages = total_versions / page_size;
    let total_pages = if total_versions % page_size == 0 {
        total_pages
    } else {
        total_pages + 1
    };

    let client = Client::new();

    for (page_num, version_ids_chunk) in version_ids.chunks(page_size).enumerate() {
        println!(
            "= Page {} of {} ==================================",
            page_num + 1,
            total_pages
        );

        let versions = versions::table
            .inner_join(crates::table)
            .filter(versions::id.eq(any(version_ids_chunk)))
            .select((versions::all_columns, crates::name))
            .load::<(Version, String)>(&conn)
            .expect("error loading versions");

        let mut tasks = Vec::with_capacity(page_size as usize);
        for (version, krate_name) in versions {
            let config = config.clone();
            Version::record_readme_rendering(version.id, &conn).unwrap_or_else(|_| {
                panic!(
                    "[{}-{}] Couldn't record rendering time",
                    krate_name, version.num
                )
            });
            let client = client.clone();
            let handle = thread::spawn(move || {
                println!("[{}-{}] Rendering README...", krate_name, version.num);
                let readme = get_readme(&config, &client, &version, &krate_name);
                if readme.is_none() {
                    return;
                }
                let readme = readme.unwrap();
                let content_length = readme.len() as u64;
                let content = std::io::Cursor::new(readme);
                let readme_path = format!("readmes/{0}/{0}-{1}.html", krate_name, version.num);
                config
                    .uploader
                    .upload(&client, &readme_path, content, content_length, "text/html")
                    .unwrap_or_else(|_| {
                        panic!(
                            "[{}-{}] Couldn't upload file to S3",
                            krate_name, version.num
                        )
                    });
            });
            tasks.push(handle);
        }
        for handle in tasks {
            if let Err(err) = handle.join() {
                println!("Thread panicked: {:?}", err);
            }
        }
    }
}

/// Renders the readme of an uploaded crate version.
fn get_readme(
    config: &Config,
    client: &Client,
    version: &Version,
    krate_name: &str,
) -> Option<String> {
    let location = config
        .uploader
        .crate_location(krate_name, &version.num.to_string());

    let mut response = match client.get(&location).send() {
        Ok(r) => r,
        Err(err) => {
            println!(
                "[{}-{}] Unable to fetch crate: {}",
                krate_name, version.num, err
            );
            return None;
        }
    };

    if !response.status().is_success() {
        println!(
            "[{}-{}] Failed to get a 200 response: {}",
            krate_name,
            version.num,
            response.text().unwrap()
        );
        return None;
    }

    let reader = GzDecoder::new(response);
    let mut archive = Archive::new(reader);
    let mut entries = archive.entries().unwrap_or_else(|_| {
        panic!(
            "[{}-{}] Invalid tar archive entries",
            krate_name, version.num
        )
    });
    let manifest: Manifest = {
        let path = format!("{}-{}/Cargo.toml", krate_name, version.num);
        let contents = find_file_by_path(&mut entries, Path::new(&path), version, krate_name);
        toml::from_str(&contents).unwrap_or_else(|_| {
            panic!(
                "[{}-{}] Syntax error in manifest file",
                krate_name, version.num
            )
        })
    };

    let rendered = {
        let path = format!(
            "{}-{}/{}",
            krate_name, version.num, manifest.package.readme?
        );
        let contents = find_file_by_path(&mut entries, Path::new(&path), version, krate_name);
        readme_to_html(
            &contents,
            manifest
                .package
                .readme_file
                .as_ref()
                .map_or("README.md", |e| &**e),
            manifest.package.repository.as_ref().map(|e| &**e),
        )
    };
    return Some(rendered);

    #[derive(Deserialize)]
    struct Package {
        readme: Option<String>,
        readme_file: Option<String>,
        repository: Option<String>,
    }

    #[derive(Deserialize)]
    struct Manifest {
        package: Package,
    }
}

/// Search an entry by its path in a Tar archive.
fn find_file_by_path<R: Read>(
    entries: &mut tar::Entries<'_, R>,
    path: &Path,
    version: &Version,
    krate_name: &str,
) -> String {
    let mut file = entries
        .find(|entry| match *entry {
            Err(_) => false,
            Ok(ref file) => {
                let filepath = match file.path() {
                    Ok(p) => p,
                    Err(_) => return false,
                };
                filepath == path
            }
        })
        .unwrap_or_else(|| {
            panic!(
                "[{}-{}] couldn't open file: {}",
                krate_name,
                version.num,
                path.display()
            )
        })
        .unwrap_or_else(|_| {
            panic!(
                "[{}-{}] file is not present: {}",
                krate_name,
                version.num,
                path.display()
            )
        });
    let mut contents = String::new();
    file.read_to_string(&mut contents).unwrap_or_else(|_| {
        panic!(
            "[{}-{}] Couldn't read file contents",
            krate_name, version.num
        )
    });
    contents
}
<|endoftext|>