<commit_before>//! Dataflow analysis with arbitrary transfer functions.
//!
//! This module is a work in progress. You should instead use `BitDenotation` in
//! `librustc_mir/dataflow/mod.rs` and encode your transfer function as a [gen/kill set][gk]. In
//! doing so, your analysis will run faster and you will be able to generate graphviz diagrams for
//! debugging with no extra effort. The interface in this module is intended only for dataflow
//! problems that cannot be expressed using gen/kill sets.
//!
//! FIXME(ecstaticmorse): In the long term, the plan is to preserve the existing `BitDenotation`
//! interface, but make `Engine` and `ResultsCursor` the canonical way to perform and inspect a
//! dataflow analysis. This requires porting the graphviz debugging logic to this module, deciding
//! on a way to handle the `before` methods in `BitDenotation` and creating an adapter so that
//! gen-kill problems can still be evaluated efficiently. See the discussion in [#64566] for more
//! information.
//!
//! [gk]: https://en.wikipedia.org/wiki/Data-flow_analysis#Bit_vector_problems
//! [#64566]: https://github.com/rust-lang/rust/pull/64566

use std::borrow::Borrow;
use std::cmp::Ordering;
use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::{fs, io, ops};

use rustc::mir::{self, traversal, BasicBlock, Location};
use rustc::ty::{self, TyCtxt};
use rustc_data_structures::work_queue::WorkQueue;
use rustc_hir::def_id::DefId;
use rustc_index::bit_set::BitSet;
use rustc_index::vec::{Idx, IndexVec};
use rustc_span::symbol::sym;

use crate::dataflow::BottomValue;

mod graphviz;

/// A specific kind of dataflow analysis.
///
/// To run a dataflow analysis, one must set the initial state of the `START_BLOCK` via
/// `initialize_start_block` and define a transfer function for each statement or terminator via
/// the various `effect` methods. The entry set for all other basic blocks is initialized to
/// `Self::BOTTOM_VALUE`. The dataflow `Engine` then iteratively updates the various entry sets for
/// each block with the cumulative effects of the transfer functions of all preceding blocks.
///
/// You should use an `Engine` to actually run an analysis, and a `ResultsCursor` to inspect the
/// results of that analysis like so:
///
/// ```ignore(cross-crate-imports)
/// fn do_my_analysis(body: &mir::Body<'tcx>, dead_unwinds: &BitSet<BasicBlock>) {
///     // `MyAnalysis` implements `Analysis`.
///     let analysis = MyAnalysis::new();
///
///     let results = Engine::new(body, dead_unwinds, analysis).iterate_to_fixpoint();
///     let mut cursor = ResultsCursor::new(body, results);
///
///     for (_, statement_index) in body.block_data[START_BLOCK].statements.iter_enumerated() {
///         cursor.seek_after(Location { block: START_BLOCK, statement_index });
///         let state = cursor.get();
///         println!("{:?}", state);
///     }
/// }
/// ```
pub trait Analysis<'tcx>: BottomValue {
    /// The index type used to access the dataflow state.
    type Idx: Idx;

    /// A name, used for debugging, that describes this dataflow analysis.
    ///
    /// The name should be suitable as part of a filename, so avoid whitespace, slashes or periods
    /// and try to keep it short.
    const NAME: &'static str;

    /// How each element of your dataflow state will be displayed during debugging.
    ///
    /// By default, this is the `fmt::Debug` representation of `Self::Idx`.
    fn pretty_print_idx(&self, w: &mut impl io::Write, idx: Self::Idx) -> io::Result<()> {
        write!(w, "{:?}", idx)
    }

    /// The size of each bitvector allocated for each block.
    fn bits_per_block(&self, body: &mir::Body<'tcx>) -> usize;

    /// Mutates the entry set of the `START_BLOCK` to contain the initial state for dataflow
    /// analysis.
    fn initialize_start_block(&self, body: &mir::Body<'tcx>, state: &mut BitSet<Self::Idx>);

    /// Updates the current dataflow state with the effect of evaluating a statement.
    fn apply_statement_effect(
        &self,
        state: &mut BitSet<Self::Idx>,
        statement: &mir::Statement<'tcx>,
        location: Location,
    );

    /// Updates the current dataflow state with the effect of evaluating a terminator.
    ///
    /// Note that the effect of a successful return from a `Call` terminator should **not** be
    /// acounted for in this function. That should go in `apply_call_return_effect`. For example,
    /// in the `InitializedPlaces` analyses, the return place is not marked as initialized here.
    fn apply_terminator_effect(
        &self,
        state: &mut BitSet<Self::Idx>,
        terminator: &mir::Terminator<'tcx>,
        location: Location,
    );

    /// Updates the current dataflow state with the effect of a successful return from a `Call`
    /// terminator.
    ///
    /// This is separated from `apply_terminator_effect` to properly track state across
    /// unwind edges for `Call`s.
    fn apply_call_return_effect(
        &self,
        state: &mut BitSet<Self::Idx>,
        block: BasicBlock,
        func: &mir::Operand<'tcx>,
        args: &[mir::Operand<'tcx>],
        return_place: &mir::Place<'tcx>,
    );

    /// Applies the cumulative effect of an entire basic block to the dataflow state (except for
    /// `call_return_effect`, which is handled in the `Engine`).
    ///
    /// The default implementation calls `statement_effect` for every statement in the block before
    /// finally calling `terminator_effect`. However, some dataflow analyses are able to coalesce
    /// transfer functions for an entire block and apply them at once. Such analyses should
    /// override `block_effect`.
    fn apply_whole_block_effect(
        &self,
        state: &mut BitSet<Self::Idx>,
        block: BasicBlock,
        block_data: &mir::BasicBlockData<'tcx>,
    ) {
        for (statement_index, stmt) in block_data.statements.iter().enumerate() {
            let location = Location { block, statement_index };
            self.apply_statement_effect(state, stmt, location);
        }

        let location = Location { block, statement_index: block_data.statements.len() };
        self.apply_terminator_effect(state, block_data.terminator(), location);
    }

    /// Applies the cumulative effect of a sequence of statements (and possibly a terminator)
    /// within a single basic block.
    ///
    /// When called with `0..block_data.statements.len() + 1` as the statement range, this function
    /// is equivalent to `apply_whole_block_effect`.
    fn apply_partial_block_effect(
        &self,
        state: &mut BitSet<Self::Idx>,
        block: BasicBlock,
        block_data: &mir::BasicBlockData<'tcx>,
        mut range: ops::Range<usize>,
    ) {
        if range.is_empty() {
            return;
        }

        // The final location might be a terminator, so iterate through all statements until the
        // final one, then check to see whether the final one is a statement or terminator.
        //
        // This can't cause the range to wrap-around since we check that the range contains at
        // least one element above.
        range.end -= 1;
        let final_location = Location { block, statement_index: range.end };

        for statement_index in range {
            let location = Location { block, statement_index };
            let stmt = &block_data.statements[statement_index];
            self.apply_statement_effect(state, stmt, location);
        }

        if final_location.statement_index == block_data.statements.len() {
            let terminator = block_data.terminator();
            self.apply_terminator_effect(state, terminator, final_location);
        } else {
            let stmt = &block_data.statements[final_location.statement_index];
            self.apply_statement_effect(state, stmt, final_location);
        }
    }
}

#[derive(Clone, Copy, Debug)]
enum CursorPosition {
    AtBlockStart(BasicBlock),
    After(Location),
}

impl CursorPosition {
    fn block(&self) -> BasicBlock {
        match *self {
            Self::AtBlockStart(block) => block,
            Self::After(Location { block, .. }) => block,
        }
    }
}

type ResultsRefCursor<'a, 'mir, 'tcx, A> = ResultsCursor<'mir, 'tcx, A, &'a Results<'tcx, A>>;

/// Inspect the results of dataflow analysis.
///
/// This cursor has linear performance when visiting statements in a block in order. Visiting
/// statements within a block in reverse order is `O(n^2)`, where `n` is the number of statements
/// in that block.
pub struct ResultsCursor<'mir, 'tcx, A, R = Results<'tcx, A>>
where
    A: Analysis<'tcx>,
{
    body: &'mir mir::Body<'tcx>,
    results: R,
    state: BitSet<A::Idx>,

    pos: CursorPosition,

    /// Whether the effects of `apply_call_return_effect` are currently stored in `state`.
    ///
    /// This flag ensures that multiple calls to `seek_after_assume_call_returns` with the same
    /// target only result in one invocation of `apply_call_return_effect`.
    is_call_return_effect_applied: bool,
}

impl<'mir, 'tcx, A, R> ResultsCursor<'mir, 'tcx, A, R>
where
    A: Analysis<'tcx>,
    R: Borrow<Results<'tcx, A>>,
{
    /// Returns a new cursor for `results` that points to the start of the `START_BLOCK`.
    pub fn new(body: &'mir mir::Body<'tcx>, results: R) -> Self {
        ResultsCursor {
            body,
            pos: CursorPosition::AtBlockStart(mir::START_BLOCK),
            is_call_return_effect_applied: false,
            state: results.borrow().entry_sets[mir::START_BLOCK].clone(),
            results,
        }
    }

    pub fn analysis(&self) -> &A {
        &self.results.borrow().analysis
    }

    /// Resets the cursor to the start of the given `block`.
    pub fn seek_to_block_start(&mut self, block: BasicBlock) {
        self.state.overwrite(&self.results.borrow().entry_sets[block]);
        self.pos = CursorPosition::AtBlockStart(block);
        self.is_call_return_effect_applied = false;
    }

    /// Updates the cursor to hold the dataflow state immediately before `target`.
    pub fn seek_before(&mut self, target: Location) {
        assert!(target <= self.body.terminator_loc(target.block));

        if target.statement_index == 0 {
            self.seek_to_block_start(target.block);
        } else {
            self._seek_after(Location {
                block: target.block,
                statement_index: target.statement_index - 1,
            });
        }
    }

    /// Updates the cursor to hold the dataflow state at `target`.
    ///
    /// If `target` is a `Call` terminator, `apply_call_return_effect` will not be called. See
    /// `seek_after_assume_call_returns` if you wish to observe the dataflow state upon a
    /// successful return.
    pub fn seek_after(&mut self, target: Location) {
        assert!(target <= self.body.terminator_loc(target.block));

        // This check ensures the correctness of a call to `seek_after_assume_call_returns`
        // followed by one to `seek_after` with the same target.
        if self.is_call_return_effect_applied {
            self.seek_to_block_start(target.block);
        }

        self._seek_after(target);
    }

    /// Equivalent to `seek_after`, but also calls `apply_call_return_effect` if `target` is a
    /// `Call` terminator whose callee is convergent.
    pub fn seek_after_assume_call_returns(&mut self, target: Location) {
        assert!(target <= self.body.terminator_loc(target.block));

        self._seek_after(target);

        if target != self.body.terminator_loc(target.block) {
            return;
        }

        let term = self.body.basic_blocks()[target.block].terminator();
        if let mir::TerminatorKind::Call {
            destination: Some((return_place, _)), func, args, ..
        } = &term.kind
        {
            if !self.is_call_return_effect_applied {
                self.is_call_return_effect_applied = true;
                self.results.borrow().analysis.apply_call_return_effect(
                    &mut self.state,
                    target.block,
                    func,
                    args,
                    return_place,
                );
            }
        }
    }

    fn _seek_after(&mut self, target: Location) {
        let Location { block: target_block, statement_index: target_index } = target;

        if self.pos.block() != target_block {
            self.seek_to_block_start(target_block);
        }

        // If we're in the same block but after the target statement, we need to reset to the start
        // of the block.
        if let CursorPosition::After(Location { statement_index: curr_index, .. }) = self.pos {
            match curr_index.cmp(&target_index) {
                Ordering::Equal => return,
                Ordering::Less => {}
                Ordering::Greater => self.seek_to_block_start(target_block),
            }
        }

        // The cursor is now in the same block as the target location pointing at an earlier
        // statement.
        debug_assert_eq!(self.pos.block(), target_block);
        if let CursorPosition::After(Location { statement_index, .. }) = self.pos {
            debug_assert!(statement_index < target_index);
        }

        let first_unapplied_statement = match self.pos {
            CursorPosition::AtBlockStart(_) => 0,
            CursorPosition::After(Location { statement_index, .. }) => statement_index + 1,
        };

        let block_data = &self.body.basic_blocks()[target_block];
        self.results.borrow().analysis.apply_partial_block_effect(
            &mut self.state,
            target_block,
            block_data,
            first_unapplied_statement..target_index + 1,
        );

        self.pos = CursorPosition::After(target);
        self.is_call_return_effect_applied = false;
    }

    /// Gets the dataflow state at the current location.
    pub fn get(&self) -> &BitSet<A::Idx> {
        &self.state
    }
}

/// A completed dataflow analysis.
pub struct Results<'tcx, A>
where
    A: Analysis<'tcx>,
{
    analysis: A,
    entry_sets: IndexVec<BasicBlock, BitSet<A::Idx>>,
}

/// All information required to iterate a dataflow analysis to fixpoint.
pub struct Engine<'a, 'tcx, A>
where
    A: Analysis<'tcx>,
{
    analysis: A,
    bits_per_block: usize,
    tcx: TyCtxt<'tcx>,
    body: &'a mir::Body<'tcx>,
    def_id: DefId,
    dead_unwinds: &'a BitSet<BasicBlock>,
    entry_sets: IndexVec<BasicBlock, BitSet<A::Idx>>,
}

impl<A> Engine<'a, 'tcx, A>
where
    A: Analysis<'tcx>,
{
    pub fn new(
        tcx: TyCtxt<'tcx>,
        body: &'a mir::Body<'tcx>,
        def_id: DefId,
        dead_unwinds: &'a BitSet<BasicBlock>,
        analysis: A,
    ) -> Self {
        let bits_per_block = analysis.bits_per_block(body);

        let bottom_value_set = if A::BOTTOM_VALUE == true {
            BitSet::new_filled(bits_per_block)
        } else {
            BitSet::new_empty(bits_per_block)
        };

        let mut entry_sets = IndexVec::from_elem(bottom_value_set, body.basic_blocks());
        analysis.initialize_start_block(body, &mut entry_sets[mir::START_BLOCK]);

        Engine { analysis, bits_per_block, tcx, body, def_id, dead_unwinds, entry_sets }
    }

    pub fn iterate_to_fixpoint(mut self) -> Results<'tcx, A> {
        let mut temp_state = BitSet::new_empty(self.bits_per_block);

        let mut dirty_queue: WorkQueue<BasicBlock> =
            WorkQueue::with_none(self.body.basic_blocks().len());

        for (bb, _) in traversal::reverse_postorder(self.body) {
            dirty_queue.insert(bb);
        }

        // Add blocks that are not reachable from START_BLOCK to the work queue. These blocks will
        // be processed after the ones added above.
        for bb in self.body.basic_blocks().indices() {
            dirty_queue.insert(bb);
        }

        while let Some(bb) = dirty_queue.pop() {
            let bb_data = &self.body[bb];
            let on_entry = &self.entry_sets[bb];

            temp_state.overwrite(on_entry);
            self.analysis.apply_whole_block_effect(&mut temp_state, bb, bb_data);

            self.propagate_bits_into_graph_successors_of(
                &mut temp_state,
                (bb, bb_data),
                &mut dirty_queue,
            );
        }

        let Engine { tcx, body, def_id, analysis, entry_sets, .. } = self;

        let results = Results { analysis, entry_sets };

        let attrs = tcx.get_attrs(def_id);
        if let Some(path) = get_dataflow_graphviz_output_path(tcx, attrs, A::NAME) {
            let result = write_dataflow_graphviz_results(body, def_id, &path, &results);
            if let Err(e) = result {
                warn!("Failed to write dataflow results to {}: {}", path.display(), e);
            }
        }

        results
    }

    fn propagate_bits_into_graph_successors_of(
        &mut self,
        in_out: &mut BitSet<A::Idx>,
        (bb, bb_data): (BasicBlock, &'a mir::BasicBlockData<'tcx>),
        dirty_list: &mut WorkQueue<BasicBlock>,
    ) {
        match bb_data.terminator().kind {
            mir::TerminatorKind::Return
            | mir::TerminatorKind::Resume
            | mir::TerminatorKind::Abort
            | mir::TerminatorKind::GeneratorDrop
            | mir::TerminatorKind::Unreachable => {}

            mir::TerminatorKind::Goto { target }
            | mir::TerminatorKind::Assert { target, cleanup: None, .. }
            | mir::TerminatorKind::Yield { resume: target, drop: None, .. }
            | mir::TerminatorKind::Drop { target, location: _, unwind: None }
            | mir::TerminatorKind::DropAndReplace { target, value: _, location: _, unwind: None } =>
            {
                self.propagate_bits_into_entry_set_for(in_out, target, dirty_list);
            }

            mir::TerminatorKind::Yield { resume: target, drop: Some(drop), .. } => {
                self.propagate_bits_into_entry_set_for(in_out, target, dirty_list);
                self.propagate_bits_into_entry_set_for(in_out, drop, dirty_list);
            }

            mir::TerminatorKind::Assert { target, cleanup: Some(unwind), .. }
            | mir::TerminatorKind::Drop { target, location: _, unwind: Some(unwind) }
            | mir::TerminatorKind::DropAndReplace {
                target,
                value: _,
                location: _,
                unwind: Some(unwind),
            } => {
                self.propagate_bits_into_entry_set_for(in_out, target, dirty_list);
                if !self.dead_unwinds.contains(bb) {
                    self.propagate_bits_into_entry_set_for(in_out, unwind, dirty_list);
                }
            }

            mir::TerminatorKind::SwitchInt { ref targets, .. } => {
                for target in targets {
                    self.propagate_bits_into_entry_set_for(in_out, *target, dirty_list);
                }
            }

            mir::TerminatorKind::Call { cleanup, ref destination, ref func, ref args, .. } => {
                if let Some(unwind) = cleanup {
                    if !self.dead_unwinds.contains(bb) {
                        self.propagate_bits_into_entry_set_for(in_out, unwind, dirty_list);
                    }
                }

                if let Some((ref dest_place, dest_bb)) = *destination {
                    // N.B.: This must be done *last*, after all other
                    // propagation, as documented in comment above.
                    self.analysis.apply_call_return_effect(in_out, bb, func, args, dest_place);
                    self.propagate_bits_into_entry_set_for(in_out, dest_bb, dirty_list);
                }
            }

            mir::TerminatorKind::FalseEdges { real_target, imaginary_target } => {
                self.propagate_bits_into_entry_set_for(in_out, real_target, dirty_list);
                self.propagate_bits_into_entry_set_for(in_out, imaginary_target, dirty_list);
            }

            mir::TerminatorKind::FalseUnwind { real_target, unwind } => {
                self.propagate_bits_into_entry_set_for(in_out, real_target, dirty_list);
                if let Some(unwind) = unwind {
                    if !self.dead_unwinds.contains(bb) {
                        self.propagate_bits_into_entry_set_for(in_out, unwind, dirty_list);
                    }
                }
            }
        }
    }

    fn propagate_bits_into_entry_set_for(
        &mut self,
        in_out: &BitSet<A::Idx>,
        bb: BasicBlock,
        dirty_queue: &mut WorkQueue<BasicBlock>,
    ) {
        let entry_set = &mut self.entry_sets[bb];
        let set_changed = self.analysis.join(entry_set, &in_out);
        if set_changed {
            dirty_queue.insert(bb);
        }
    }
}

/// Looks for attributes like `#[rustc_mir(borrowck_graphviz_postflow="./path/to/suffix.dot")]` and
/// extracts the path with the given analysis name prepended to the suffix.
///
/// Returns `None` if no such attribute exists.
fn get_dataflow_graphviz_output_path(
    tcx: TyCtxt<'tcx>,
    attrs: ty::Attributes<'tcx>,
    analysis: &str,
) -> Option<PathBuf> {
    let mut rustc_mir_attrs = attrs
        .into_iter()
        .filter(|attr| attr.check_name(sym::rustc_mir))
        .flat_map(|attr| attr.meta_item_list().into_iter().flat_map(|v| v.into_iter()));

    let borrowck_graphviz_postflow =
        rustc_mir_attrs.find(|attr| attr.check_name(sym::borrowck_graphviz_postflow))?;

    let path_and_suffix = match borrowck_graphviz_postflow.value_str() {
        Some(p) => p,
        None => {
            tcx.sess.span_err(
                borrowck_graphviz_postflow.span(),
                "borrowck_graphviz_postflow requires a path",
            );

            return None;
        }
    };

    // Change "path/suffix.dot" to "path/analysis_name_suffix.dot"
    let mut ret = PathBuf::from(path_and_suffix.to_string());
    let suffix = ret.file_name().unwrap();

    let mut file_name: OsString = analysis.into();
    file_name.push("_");
    file_name.push(suffix);
    ret.set_file_name(file_name);

    Some(ret)
}

fn write_dataflow_graphviz_results<A: Analysis<'tcx>>(
    body: &mir::Body<'tcx>,
    def_id: DefId,
    path: &Path,
    results: &Results<'tcx, A>,
) -> io::Result<()> {
    debug!("printing dataflow results for {:?} to {}", def_id, path.display());

    let mut buf = Vec::new();
    let graphviz = graphviz::Formatter::new(body, def_id, results);

    dot::render(&graphviz, &mut buf)?;
    fs::write(path, buf)
}
<commit_msg>Remove old "generic" framework<commit_after><|endoftext|>


<commit_before>use codec_csl::CslCodec;
use provider::{
    async_trait::async_trait,
    eyre::Result,
    http_utils::{get_json_with, headers},
    once_cell::sync::Lazy,
    regex::Regex,
    stencila_schema::*,
    Provider, ProviderParsing, ProviderTrait,
};

/// A provider that identifies and enriches `Article` and other `CreativeWork` nodes
///
/// Uses the DOI content negotiation protocol to fetch schema.org JSON-LD or CSL JSON
/// which is then decoded to Stencila node types.
#[derive(Debug, Default)]
pub struct DoiProvider {}

const DOI_URL: &str = "https://doi.org/";

#[async_trait]
impl ProviderTrait for DoiProvider {
    fn spec() -> Provider {
        Provider::new("doi")
    }

    /// Parse a [`CreativeWork`] nodes from a string
    ///
    /// Examples of DOI strings that this methods attempts to parse:
    ///
    /// - 10.5334/jors.182
    /// - DOI: 10.5334/jors.182
    /// - http://doi.org/10.5334/jors.182
    /// - https://doi.org/10.5334/jors.182
    ///
    /// See https://www.crossref.org/blog/dois-and-matching-regular-expressions/
    /// for notes on DOI matching.
    fn parse(string: &str) -> Vec<ProviderParsing> {
        static REGEX: Lazy<Regex> = Lazy::new(|| {
            Regex::new(r"\b(10.\d{4,9}/[-._;()/:a-zA-Z0-9]+)\b").expect("Unable to create regex")
        });

        REGEX
            .captures_iter(string)
            .into_iter()
            .map(|captures| {
                let capture = captures.get(0).unwrap();

                let begin = capture.start();
                let end = capture.end();
                let node = Node::CreativeWork(CreativeWork {
                    identifiers: Some(vec![ThingIdentifiers::String(
                        [DOI_URL, capture.as_str()].concat(),
                    )]),
                    ..Default::default()
                });

                ProviderParsing { begin, end, node }
            })
            .collect()
    }

    /// Enrich a [`CreativeWork`] node using it's DOI
    ///
    /// If the node is a `CreativeWork` type with a DOI, then uses DOI content negotiation
    /// protocol to fetch CSL JSON to enrich properties of the node.
    async fn enrich(node: Node) -> Result<Node> {
        let url = match &node {
            Node::CreativeWork(CreativeWork { identifiers, .. })
            | Node::Article(Article { identifiers, .. }) => {
                identifiers.iter().flatten().find_map(|id| match id {
                    ThingIdentifiers::String(id) => {
                        if id.starts_with(DOI_URL) {
                            Some(id.clone())
                        } else {
                            None
                        }
                    }
                    ThingIdentifiers::PropertyValue(PropertyValue { value, .. }) => match value {
                        PropertyValueValue::String(id) => {
                            if id.starts_with(DOI_URL) {
                                Some(id.clone())
                            } else {
                                None
                            }
                        }
                        _ => None,
                    },
                })
            }
            _ => None,
        };

        let url = match url {
            Some(url) => url,
            None => return Ok(node),
        };

        let data = get_json_with(
            &url,
            &[(headers::ACCEPT, "application/vnd.citationstyles.csl+json")],
        )
        .await?;

        CslCodec::from_json(data)
    }
}

#[cfg(test)]
mod tests {
    use test_utils::assert_json_is;

    use super::*;

    #[tokio::test]
    async fn detect() -> Result<()> {
        // Test that the extracted DOI is correct
        for str in [
            "10.5334/jors.182",
            "   10.5334/jors.182  ",
            "DOI: 10.5334/jors.182",
            " doi:10.5334/jors.182 and more",
            "DOI: http://doi.org/10.5334/jors.182",
            "https://doi.org/10.5334/jors.182",
        ] {
            let detections = DoiProvider::detect(&Node::String(str.to_string())).await?;
            match &detections[0].node {
                Node::CreativeWork(CreativeWork { identifiers, .. }) => {
                    assert_json_is!(identifiers, ["https://doi.org/10.5334/jors.182"])
                }
                _ => bail!("Unexpected return type"),
            };
        }

        // Test that a collection of DOI's as entered "in the wild" are detected
        for str in [
            "10.1016/j.celrep.2013.10.048",
            "10.1016/j.stem.2013.04.008",
            "10.1038/nature08320",
            "10.1038/nbt.2249",
            "10.1038/s41587-019-0035-0",
            "10.1038/sdata.2018.13",
            "10.1101/gad.1616208",
            "10.1101/gr.223925.117",
            "10.1242/dev.156844",
            "10.3791/51609",
            "10.5281/zenodo.5137374",
            "DOI: 10.1002/0471142301.ns0321s44",
            "DOI: 10.1002/dad2.12156",
            "DOI: 10.1002/mds.28621",
            "DOI: 10.1007/978-1-62703-574-3_15",
            "DOI: 10.1016/j.nbd.2021.105482",
            "DOI: 10.1038/s41598-021-93105-y",
            "doi: 10.1093/bioinformatics/btab385",
            "DOI: 10.1126/sciadv.abg4922      ",
            "DOI: 10.1186/s12859-021-04307-0",
            "doi: 10.18129/B9.bioc.dasper",
            "doi: https://doi.org/10.1007/s00401-021-02343-x",
            "doi: https://doi.org/10.1016/j.isci.2021.102484 (pre-proof)",
            "doi: https://doi.org/10.1101/2021.07.11.451956",
            "doi:10.1038/s41467-021-22108-0",
            "doi.org/10.1101/2021.03.02.433576",
            "dx.doi.org/10.17504/protocols.io.bnhtmb6n",
            "dx.doi.org/10.17504/protocols.io.bxrjpm4n",
            "http://doi.org/10.5281/zenodo.5012149",
            "http://dx.doi.org/10.21769/BioProtoc.3888",
            "https://doi.org/10.1002/mds.28755",
            "https://doi.org/10.1016/j.stem.2018.09.009",
            "https://doi.org/10.1038/s41467-021-22108-0",
            "https://doi.org/10.1073/pnas.2025053118",
            "https://doi.org/10.1083/jcb.202010004",
            "https://doi.org/10.15454/FLANUP",
            "https://doi.org/10.21203/rs.3.rs-220057/v2",
            "https://doi.org/10.3390/ijms22052689",
            "https://doi.org/10.5281/zenodo.5011869",
            "https://dx.doi.org/10.17504/protocols.io.bazhif36",
        ] {
            let detections = DoiProvider::detect(&Node::String(str.to_string())).await?;
            assert_eq!(detections.len(), 1)
        }

        Ok(())
    }
}
<commit_msg>chore(DOI Provider): Add import back in<commit_after>use codec_csl::CslCodec;
use provider::{
    async_trait::async_trait,
    eyre::{bail, Result},
    http_utils::{get_json_with, headers},
    once_cell::sync::Lazy,
    regex::Regex,
    stencila_schema::*,
    Provider, ProviderParsing, ProviderTrait,
};

/// A provider that identifies and enriches `Article` and other `CreativeWork` nodes
///
/// Uses the DOI content negotiation protocol to fetch schema.org JSON-LD or CSL JSON
/// which is then decoded to Stencila node types.
#[derive(Debug, Default)]
pub struct DoiProvider {}

const DOI_URL: &str = "https://doi.org/";

#[async_trait]
impl ProviderTrait for DoiProvider {
    fn spec() -> Provider {
        Provider::new("doi")
    }

    /// Parse a [`CreativeWork`] nodes from a string
    ///
    /// Examples of DOI strings that this methods attempts to parse:
    ///
    /// - 10.5334/jors.182
    /// - DOI: 10.5334/jors.182
    /// - http://doi.org/10.5334/jors.182
    /// - https://doi.org/10.5334/jors.182
    ///
    /// See https://www.crossref.org/blog/dois-and-matching-regular-expressions/
    /// for notes on DOI matching.
    fn parse(string: &str) -> Vec<ProviderParsing> {
        static REGEX: Lazy<Regex> = Lazy::new(|| {
            Regex::new(r"\b(10.\d{4,9}/[-._;()/:a-zA-Z0-9]+)\b").expect("Unable to create regex")
        });

        REGEX
            .captures_iter(string)
            .into_iter()
            .map(|captures| {
                let capture = captures.get(0).unwrap();

                let begin = capture.start();
                let end = capture.end();
                let node = Node::CreativeWork(CreativeWork {
                    identifiers: Some(vec![ThingIdentifiers::String(
                        [DOI_URL, capture.as_str()].concat(),
                    )]),
                    ..Default::default()
                });

                ProviderParsing { begin, end, node }
            })
            .collect()
    }

    /// Enrich a [`CreativeWork`] node using it's DOI
    ///
    /// If the node is a `CreativeWork` type with a DOI, then uses DOI content negotiation
    /// protocol to fetch CSL JSON to enrich properties of the node.
    async fn enrich(node: Node) -> Result<Node> {
        let url = match &node {
            Node::CreativeWork(CreativeWork { identifiers, .. })
            | Node::Article(Article { identifiers, .. }) => {
                identifiers.iter().flatten().find_map(|id| match id {
                    ThingIdentifiers::String(id) => {
                        if id.starts_with(DOI_URL) {
                            Some(id.clone())
                        } else {
                            None
                        }
                    }
                    ThingIdentifiers::PropertyValue(PropertyValue { value, .. }) => match value {
                        PropertyValueValue::String(id) => {
                            if id.starts_with(DOI_URL) {
                                Some(id.clone())
                            } else {
                                None
                            }
                        }
                        _ => None,
                    },
                })
            }
            _ => None,
        };

        let url = match url {
            Some(url) => url,
            None => return Ok(node),
        };

        let data = get_json_with(
            &url,
            &[(headers::ACCEPT, "application/vnd.citationstyles.csl+json")],
        )
        .await?;

        CslCodec::from_json(data)
    }
}

#[cfg(test)]
mod tests {
    use test_utils::assert_json_is;

    use super::*;

    #[tokio::test]
    async fn detect() -> Result<()> {
        // Test that the extracted DOI is correct
        for str in [
            "10.5334/jors.182",
            "   10.5334/jors.182  ",
            "DOI: 10.5334/jors.182",
            " doi:10.5334/jors.182 and more",
            "DOI: http://doi.org/10.5334/jors.182",
            "https://doi.org/10.5334/jors.182",
        ] {
            let detections = DoiProvider::detect(&Node::String(str.to_string())).await?;
            match &detections[0].node {
                Node::CreativeWork(CreativeWork { identifiers, .. }) => {
                    assert_json_is!(identifiers, ["https://doi.org/10.5334/jors.182"])
                }
                _ => bail!("Unexpected return type"),
            };
        }

        // Test that a collection of DOI's as entered "in the wild" are detected
        for str in [
            "10.1016/j.celrep.2013.10.048",
            "10.1016/j.stem.2013.04.008",
            "10.1038/nature08320",
            "10.1038/nbt.2249",
            "10.1038/s41587-019-0035-0",
            "10.1038/sdata.2018.13",
            "10.1101/gad.1616208",
            "10.1101/gr.223925.117",
            "10.1242/dev.156844",
            "10.3791/51609",
            "10.5281/zenodo.5137374",
            "DOI: 10.1002/0471142301.ns0321s44",
            "DOI: 10.1002/dad2.12156",
            "DOI: 10.1002/mds.28621",
            "DOI: 10.1007/978-1-62703-574-3_15",
            "DOI: 10.1016/j.nbd.2021.105482",
            "DOI: 10.1038/s41598-021-93105-y",
            "doi: 10.1093/bioinformatics/btab385",
            "DOI: 10.1126/sciadv.abg4922      ",
            "DOI: 10.1186/s12859-021-04307-0",
            "doi: 10.18129/B9.bioc.dasper",
            "doi: https://doi.org/10.1007/s00401-021-02343-x",
            "doi: https://doi.org/10.1016/j.isci.2021.102484 (pre-proof)",
            "doi: https://doi.org/10.1101/2021.07.11.451956",
            "doi:10.1038/s41467-021-22108-0",
            "doi.org/10.1101/2021.03.02.433576",
            "dx.doi.org/10.17504/protocols.io.bnhtmb6n",
            "dx.doi.org/10.17504/protocols.io.bxrjpm4n",
            "http://doi.org/10.5281/zenodo.5012149",
            "http://dx.doi.org/10.21769/BioProtoc.3888",
            "https://doi.org/10.1002/mds.28755",
            "https://doi.org/10.1016/j.stem.2018.09.009",
            "https://doi.org/10.1038/s41467-021-22108-0",
            "https://doi.org/10.1073/pnas.2025053118",
            "https://doi.org/10.1083/jcb.202010004",
            "https://doi.org/10.15454/FLANUP",
            "https://doi.org/10.21203/rs.3.rs-220057/v2",
            "https://doi.org/10.3390/ijms22052689",
            "https://doi.org/10.5281/zenodo.5011869",
            "https://dx.doi.org/10.17504/protocols.io.bazhif36",
        ] {
            let detections = DoiProvider::detect(&Node::String(str.to_string())).await?;
            assert_eq!(detections.len(), 1)
        }

        Ok(())
    }
}
<|endoftext|>


<commit_before>use std::io::{Result, Write};

use env::Env;
use library;
use nameutil;
use super::ffi_type::*;
use super::super::general::version_condition;
use traits::*;

//used as glib:get-type in GLib-2.0.gir
const INTERN: &'static str= "intern";

pub fn generate_records_funcs<W: Write>(w: &mut W, env: &Env, records: &[&library::Record]) -> Result<()> {
    let intern_str = INTERN.to_string();
    for record in records {
        let glib_get_type = record.glib_get_type.as_ref().unwrap_or(&intern_str);
        try!(generate_object_funcs(w, env, &record.c_type,
            glib_get_type, &record.functions));
    }

    Ok(())
}

pub fn generate_classes_funcs<W: Write>(w: &mut W, env: &Env, classes: &[&library::Class]) -> Result<()> {
    for klass in classes {
        try!(generate_object_funcs(w, env, &klass.c_type,
            &klass.glib_get_type, &klass.functions));
    }

    Ok(())
}

pub fn generate_interfaces_funcs<W: Write>(w: &mut W, env: &Env, interfaces: &[&library::Interface]) -> Result<()> {
    for interface in interfaces {
        try!(generate_object_funcs(w, env,  &interface.c_type,
            &interface.glib_get_type, &interface.functions));
    }

    Ok(())
}

pub fn generate_other_funcs<W: Write>(w: &mut W, env: &Env, functions: &[library::Function]) -> Result<()> {
    generate_object_funcs(w, env,  "Other functions", INTERN, functions)
}

fn generate_object_funcs<W: Write>(w: &mut W, env: &Env, c_type: &str,
    glib_get_type: &str, functions: &[library::Function]) -> Result<()> {
    let write_get_type = glib_get_type != INTERN;
    if write_get_type || !functions.is_empty() {
        try!(writeln!(w, ""));
        try!(writeln!(w, "    //========================================================================="));
        try!(writeln!(w, "    // {}", c_type));
        try!(writeln!(w, "    //========================================================================="));
    }
    if write_get_type {
        try!(writeln!(w, "    pub fn {}() -> GType;", glib_get_type));
    }

    for func in functions {
        let (commented, sig) = function_signature(env, func, false);
        let comment = if commented { "//" } else { "" };
        try!(version_condition(w, &env.config.library_name,
            env.config.min_cfg_version, func.version, commented, 1));
        try!(writeln!(w, "    {}pub fn {}{};",
                      comment, func.c_identifier.as_ref().unwrap(), sig));
    }

    Ok(())
}

pub fn generate_callbacks<W: Write>(w: &mut W, env: &Env, callbacks: &[&library::Function]) -> Result<()> {
    for func in callbacks {
        let (commented, sig) = function_signature(env, func, true);
        let comment = if commented { "//" } else { "" };
        try!(writeln!(w, "{}pub type {} = Option<unsafe extern \"C\" fn{}>;",
                      comment, func.c_identifier.as_ref().unwrap(), sig));
    }

    Ok(())
}

pub fn function_signature(env: &Env, func: &library::Function, bare: bool) -> (bool, String) {
    let (mut commented, ret_str) = function_return_value(env, func);

    let mut parameter_strs: Vec<String> = Vec::new();
    for par in &func.parameters {
        let (c, par_str) = function_parameter(env, par, bare);
        parameter_strs.push(par_str);
        if c { commented = true; }
    }

    (commented, format!("({}){}", parameter_strs.connect(", "), ret_str))
}

fn function_return_value(env: &Env, func: &library::Function) -> (bool, String) {
    if func.ret.typ == Default::default() { return (false, String::new()) }
    let ffi_type = ffi_type(env, func.ret.typ, &func.ret.c_type);
    let commented = ffi_type.is_err();
    (commented, format!(" -> {}", ffi_type.as_str()))
}

fn function_parameter(env: &Env, par: &library::Parameter, bare: bool) -> (bool, String) {
    if let &library::Type::Fundamental(library::Fundamental::VarArgs) = env.library.type_(par.typ) {
        return (false, "...".into());
    }
    let ffi_type = ffi_type(env, par.typ, &par.c_type);
    let commented = ffi_type.is_err();
    let res = if bare {
        ffi_type.as_str().into()
    }
    else {
        format!("{}: {}", nameutil::mangle_keywords(&*par.name), ffi_type.as_str())
    };
    (commented, res)
}
<commit_msg>Auto merge of #110 - gkoz:utf8_shim, r=gkoz<commit_after>use std::collections::HashMap;
use std::io::{Result, Write};

use env::Env;
use library;
use nameutil;
use super::ffi_type::*;
use super::super::general::version_condition;
use traits::*;

//used as glib:get-type in GLib-2.0.gir
const INTERN: &'static str= "intern";

pub fn generate_records_funcs<W: Write>(w: &mut W, env: &Env, records: &[&library::Record]) -> Result<()> {
    let intern_str = INTERN.to_string();
    for record in records {
        let glib_get_type = record.glib_get_type.as_ref().unwrap_or(&intern_str);
        try!(generate_object_funcs(w, env, &record.c_type,
            glib_get_type, &record.functions));
    }

    Ok(())
}

pub fn generate_classes_funcs<W: Write>(w: &mut W, env: &Env, classes: &[&library::Class]) -> Result<()> {
    for klass in classes {
        try!(generate_object_funcs(w, env, &klass.c_type,
            &klass.glib_get_type, &klass.functions));
    }

    Ok(())
}

pub fn generate_interfaces_funcs<W: Write>(w: &mut W, env: &Env, interfaces: &[&library::Interface]) -> Result<()> {
    for interface in interfaces {
        try!(generate_object_funcs(w, env,  &interface.c_type,
            &interface.glib_get_type, &interface.functions));
    }

    Ok(())
}

pub fn generate_other_funcs<W: Write>(w: &mut W, env: &Env, functions: &[library::Function]) -> Result<()> {
    generate_object_funcs(w, env,  "Other functions", INTERN, functions)
}

fn generate_object_funcs<W: Write>(w: &mut W, env: &Env, c_type: &str,
    glib_get_type: &str, functions: &[library::Function]) -> Result<()> {
    let write_get_type = glib_get_type != INTERN;
    if write_get_type || !functions.is_empty() {
        try!(writeln!(w, ""));
        try!(writeln!(w, "    //========================================================================="));
        try!(writeln!(w, "    // {}", c_type));
        try!(writeln!(w, "    //========================================================================="));
    }
    if write_get_type {
        try!(writeln!(w, "    pub fn {}() -> GType;", glib_get_type));
    }

    for func in functions {
        let (commented, sig) = function_signature(env, func, false);
        let comment = if commented { "//" } else { "" };
        try!(version_condition(w, &env.config.library_name,
            env.config.min_cfg_version, func.version, commented, 1));
        let name = func.c_identifier.as_ref().unwrap();
        // since we work with gir-files from Linux, some function names need to be adjusted
        if let Some(win_name) = RENAME_ON_WINDOWS.get(&name[..]) {
            try!(writeln!(w, "    {}#[cfg(windows)]", comment));
            try!(writeln!(w, "    {}pub fn {}{};", comment, win_name, sig));
            try!(version_condition(w, &env.config.library_name,
                env.config.min_cfg_version, func.version, commented, 1));
            try!(writeln!(w, "    {}#[cfg(not(windows))]", comment));
        }
        try!(writeln!(w, "    {}pub fn {}{};", comment, name, sig));
    }

    Ok(())
}

lazy_static! {
    static ref RENAME_ON_WINDOWS: HashMap<&'static str, &'static str> = {
        let mut map = HashMap::new();
        [
            ("gdk_pixbuf_new_from_file", "gdk_pixbuf_new_from_file_utf8"),
            ("gdk_pixbuf_new_from_file_at_size", "gdk_pixbuf_new_from_file_at_size_utf8"),
            ("gdk_pixbuf_new_from_file_at_scale", "gdk_pixbuf_new_from_file_at_scale_utf8"),
            ("gdk_pixbuf_save", "gdk_pixbuf_save_utf8"),
            ("gdk_pixbuf_savev", "gdk_pixbuf_savev_utf8"),
            ("gdk_pixbuf_animation_new_from_file", "gdk_pixbuf_animation_new_from_file_utf8"),
        ].iter().map(|&(k, v)| map.insert(k, v)).count();
        map
    };
}

pub fn generate_callbacks<W: Write>(w: &mut W, env: &Env, callbacks: &[&library::Function]) -> Result<()> {
    for func in callbacks {
        let (commented, sig) = function_signature(env, func, true);
        let comment = if commented { "//" } else { "" };
        try!(writeln!(w, "{}pub type {} = Option<unsafe extern \"C\" fn{}>;",
                      comment, func.c_identifier.as_ref().unwrap(), sig));
    }

    Ok(())
}

pub fn function_signature(env: &Env, func: &library::Function, bare: bool) -> (bool, String) {
    let (mut commented, ret_str) = function_return_value(env, func);

    let mut parameter_strs: Vec<String> = Vec::new();
    for par in &func.parameters {
        let (c, par_str) = function_parameter(env, par, bare);
        parameter_strs.push(par_str);
        if c { commented = true; }
    }

    (commented, format!("({}){}", parameter_strs.connect(", "), ret_str))
}

fn function_return_value(env: &Env, func: &library::Function) -> (bool, String) {
    if func.ret.typ == Default::default() { return (false, String::new()) }
    let ffi_type = ffi_type(env, func.ret.typ, &func.ret.c_type);
    let commented = ffi_type.is_err();
    (commented, format!(" -> {}", ffi_type.as_str()))
}

fn function_parameter(env: &Env, par: &library::Parameter, bare: bool) -> (bool, String) {
    if let &library::Type::Fundamental(library::Fundamental::VarArgs) = env.library.type_(par.typ) {
        return (false, "...".into());
    }
    let ffi_type = ffi_type(env, par.typ, &par.c_type);
    let commented = ffi_type.is_err();
    let res = if bare {
        ffi_type.as_str().into()
    }
    else {
        format!("{}: {}", nameutil::mangle_keywords(&*par.name), ffi_type.as_str())
    };
    (commented, res)
}
<|endoftext|>


<commit_before>extern crate chrono;
extern crate yaml_rust;

use self::chrono::*;
use std::collections::BTreeMap;

#[derive(Debug)]
pub struct Bookmark {
    pub id: i32,
    pub name: String,
    pub url: String,
    pub time_created: DateTime<Local>,
    pub stamp: DateTime<Local>,
    pub rev_no: i32
}

pub enum QueryValue {
    Integer(i32),
    String(String),
    Date(DateTime<Local>)
}

impl<'a> Into<String> for &'a QueryValue {
    fn into(self) -> String {
        match self {
            &QueryValue::Integer(i) => format!("{}", i),
            &QueryValue::String(ref s) => format!("{}", s),
            &QueryValue::Date(t) => t.to_rfc2822()
        }
    }
}

impl Bookmark {
    pub fn to_yaml(self) -> String {
        let mut btree = BTreeMap::new();

        btree.insert("id", QueryValue::Integer(self.id));
        btree.insert("name", QueryValue::String(self.name));
        btree.insert("url", QueryValue::String(self.url));
        btree.insert("time_created", QueryValue::Date(self.time_created));
        btree.insert("stamp", QueryValue::Date(self.stamp));
        btree.insert("rev_no", QueryValue::Integer(self.rev_no));

        let mut yaml = String::new();
        yaml.push_str("Bookmark: \n");

        for (k, v) in btree {
            yaml.push_str("\t");
            yaml.push_str(k);
            yaml.push_str(": ");

            let s: String = (&v).into();
            yaml.push_str(s.as_str());
            yaml.push_str("\n");
        }

        yaml
    }

    pub fn to_btree(self) -> BTreeMap<String, QueryValue> {
        let mut btree : BTreeMap<String, QueryValue> = BTreeMap::new();

        btree.insert(String::from("id"), QueryValue::Integer(self.id));
        btree.insert(String::from("name"), QueryValue::String(self.name));
        btree.insert(String::from("url"), QueryValue::String(self.url));
        btree.insert(String::from("time_created"), QueryValue::Date(self.time_created));
        btree.insert(String::from("stamp"), QueryValue::Date(self.stamp));
        btree.insert(String::from("rev_no"), QueryValue::Integer(self.rev_no));

        btree
    }
}
<commit_msg>temporary Equals Bookmark<commit_after>extern crate chrono;
extern crate yaml_rust;

use self::chrono::*;
use std::collections::BTreeMap;

#[derive(Debug)]
pub struct Bookmark {
    pub id: i32,
    pub name: String,
    pub url: String,
    pub time_created: DateTime<Local>,
    pub stamp: DateTime<Local>,
    pub rev_no: i32
}

pub enum QueryValue {
    Integer(i32),
    String(String),
    Date(DateTime<Local>)
}

impl<'a> Into<String> for &'a QueryValue {
    fn into(self) -> String {
        match self {
            &QueryValue::Integer(i) => format!("{}", i),
            &QueryValue::String(ref s) => format!("{}", s),
            &QueryValue::Date(t) => t.to_rfc2822()
        }
    }
}

impl Bookmark {
    pub fn to_yaml(self) -> String {
        let mut btree = BTreeMap::new();

        btree.insert("id", QueryValue::Integer(self.id));
        btree.insert("name", QueryValue::String(self.name));
        btree.insert("url", QueryValue::String(self.url));
        btree.insert("time_created", QueryValue::Date(self.time_created));
        btree.insert("stamp", QueryValue::Date(self.stamp));
        btree.insert("rev_no", QueryValue::Integer(self.rev_no));

        let mut yaml = String::new();
        yaml.push_str("Bookmark: \n");

        for (k, v) in btree {
            yaml.push_str("\t");
            yaml.push_str(k);
            yaml.push_str(": ");

            let s: String = (&v).into();
            yaml.push_str(s.as_str());
            yaml.push_str("\n");
        }

        yaml
    }

    pub fn to_btree(self) -> BTreeMap<String, QueryValue> {
        let mut btree : BTreeMap<String, QueryValue> = BTreeMap::new();

        btree.insert(String::from("id"), QueryValue::Integer(self.id));
        btree.insert(String::from("name"), QueryValue::String(self.name));
        btree.insert(String::from("url"), QueryValue::String(self.url));
        btree.insert(String::from("time_created"), QueryValue::Date(self.time_created));
        btree.insert(String::from("stamp"), QueryValue::Date(self.stamp));
        btree.insert(String::from("rev_no"), QueryValue::Integer(self.rev_no));

        btree
    }

    pub fn equals(self, b: Bookmark) -> bool {

        if self.id != b.id {
            return false;
        }
        else if self.name != b.name {
            return false;
        }
        else if self.url != b.url {
            return false;
        }
        else if self.time_created != b.time_created {
            return false;
        }
        else if self.stamp != b.stamp {
            return false;
        }
        else if self.rev_no != b.rev_no {
            return false;
        }

        return true;
    }
}
<|endoftext|>


<commit_before>use colors;
use pixset;

#[derive(Clone, Copy, Debug, Default)]
pub struct Tile {
    pub fg: colors::Rgb,
    pub bg: colors::Rgb,
    pub pix: pixset::Pix,
}

// TODO default?
impl Tile {
    pub fn new() -> Self {
        Tile {
            fg: *colors::BLACK,
            bg: *colors::BLACK,
            pix: pixset::Pix::Empty,
        }
    }

    pub fn clear(&mut self) {
        // TODO gross
        let t = Tile::new();
        self.fg = t.fg;
        self.bg = t.bg;
        self.pix = t.pix;
    }
}
<commit_msg>feat(tile): add make method<commit_after>use colors;
use pixset;

#[derive(Clone, Copy, Debug, Default)]
pub struct Tile {
    pub fg: colors::Rgb,
    pub bg: colors::Rgb,
    pub pix: pixset::Pix,
}

// TODO default?
impl Tile {
    pub fn new() -> Self {
        Tile {
            fg: *colors::BLACK,
            bg: *colors::BLACK,
            pix: pixset::Pix::Empty,
        }
    }

    pub fn make(fg: colors::Rgb, bg: colors::Rgb, pix: pixset::Pix) -> Self {
        Tile {
            fg: fg,
            bg: bg,
            pix: pix,
        }
    }

    pub fn clear(&mut self) {
        // TODO gross
        let t = Tile::new();
        self.fg = t.fg;
        self.bg = t.bg;
        self.pix = t.pix;
    }
}
<|endoftext|>


<commit_before>use std::{
    env,
    ffi::OsStr,
    fs::{self, File},
    io::{BufReader, Cursor, Read},
    path::{Path, PathBuf},
    time::{Duration, SystemTime},
};

use anyhow::{ensure, Context, Result};
use log::debug;
use reqwest::{blocking::Client, Proxy};
use walkdir::{DirEntry, WalkDir};
use zip::ZipArchive;

use crate::types::PlatformType;

pub static TLDR_PAGES_DIR: &str = "tldr-pages";
static TLDR_OLD_PAGES_DIR: &str = "tldr-master";

#[derive(Debug)]
pub struct Cache {
    platform: PlatformType,
    cache_dir: PathBuf,
}

#[derive(Debug)]
pub struct PageLookupResult {
    pub page_path: PathBuf,
    pub patch_path: Option<PathBuf>,
}

impl PageLookupResult {
    pub fn with_page(page_path: PathBuf) -> Self {
        Self {
            page_path,
            patch_path: None,
        }
    }

    pub fn with_optional_patch(mut self, patch_path: Option<PathBuf>) -> Self {
        self.patch_path = patch_path;
        self
    }

    /// Create a buffered reader that sequentially reads from the page and the
    /// patch, as if they were concatenated.
    ///
    /// This will return an error if either the page file or the patch file
    /// cannot be opened.
    pub fn reader(&self) -> Result<BufReader<Box<dyn Read>>> {
        // Open page file
        let page_file = File::open(&self.page_path)
            .with_context(|| format!("Could not open page file at {}", self.page_path.display()))?;

        // Open patch file
        let patch_file_opt = match &self.patch_path {
            Some(path) => Some(
                File::open(path)
                    .with_context(|| format!("Could not open patch file at {}", path.display()))?,
            ),
            None => None,
        };

        // Create chained reader from file(s)
        //
        // Note: It might be worthwhile to create our own struct that accepts
        // the page and patch files and that will read them sequentially,
        // because it avoids the boxing below. However, the performance impact
        // would first need to be shown to be significant using a benchmark.
        Ok(BufReader::new(if let Some(patch_file) = patch_file_opt {
            Box::new(page_file.chain(&b"\n"[..]).chain(patch_file)) as Box<dyn Read>
        } else {
            Box::new(page_file) as Box<dyn Read>
        }))
    }
}

pub enum CacheFreshness {
    /// The cache is still fresh (less than MAX_CACHE_AGE old)
    Fresh,
    /// The cache is stale and should be updated
    Stale(Duration),
    /// The cache is missing
    Missing,
}

impl Cache {
    pub fn new<P>(platform: PlatformType, cache_dir: P) -> Result<Self>
    where
        P: Into<PathBuf>,
    {
        // Check whether `cache_dir` exists and is a directory
        let cache_dir = cache_dir.into();
        let (cache_dir_exists, cache_dir_is_dir) = cache_dir
            .metadata()
            .map_or((false, false), |md| (true, md.is_dir()));
        ensure!(
            !cache_dir_exists || cache_dir_is_dir,
            "Cache directory path `{}` is not a directory",
            cache_dir.display(),
        );

        // If necessary, create cache directory
        if !cache_dir_exists {
            // Try to create the complete directory path
            fs::create_dir_all(&cache_dir).with_context(|| {
                format!(
                    "Cache directory path `{}` cannot be created",
                    cache_dir.display(),
                )
            })?;
            eprintln!(
                "Successfully created cache directory path `{}`.",
                cache_dir.display(),
            );
        }

        Ok(Self {
            platform,
            cache_dir,
        })
    }

    /// Download the archive from the specified URL.
    fn download(archive_url: &str) -> Result<Vec<u8>> {
        let mut builder = Client::builder();
        if let Ok(ref host) = env::var("HTTP_PROXY") {
            if let Ok(proxy) = Proxy::http(host) {
                builder = builder.proxy(proxy);
            }
        }
        if let Ok(ref host) = env::var("HTTPS_PROXY") {
            if let Ok(proxy) = Proxy::https(host) {
                builder = builder.proxy(proxy);
            }
        }
        let client = builder
            .build()
            .context("Could not instantiate HTTP client")?;
        let mut resp = client
            .get(archive_url)
            .send()?
            .error_for_status()
            .with_context(|| format!("Could not download tldr pages from {}", archive_url))?;
        let mut buf: Vec<u8> = vec![];
        let bytes_downloaded = resp.copy_to(&mut buf)?;
        debug!("{} bytes downloaded", bytes_downloaded);
        Ok(buf)
    }

    /// Update the pages cache from the specified URL.
    pub fn update(&self, archive_url: &str) -> Result<()> {
        // First, download the compressed data
        let bytes: Vec<u8> = Self::download(archive_url)?;

        // Decompress the response body into an `Archive`
        let mut archive = ZipArchive::new(Cursor::new(bytes))
            .context("Could not decompress downloaded ZIP archive")?;

        // Determine paths
        let pages_dir = self.cache_dir.join(TLDR_PAGES_DIR);

        // Clear cache directory
        // Note: This is not the best solution. Ideally we would download the
        // archive to a temporary directory and then swap the two directories.
        // But renaming a directory doesn't work across filesystems and Rust
        // does not yet offer a recursive directory copying function. So for
        // now, we'll use this approach.
        self.clear()
            .context("Could not clear the cache directory")?;

        // Extract archive
        archive
            .extract(&pages_dir)
            .context("Could not unpack compressed data")?;

        Ok(())
    }

    /// Return the duration since the cache directory was last modified.
    pub fn last_update(&self) -> Option<Duration> {
        if let Ok(metadata) = fs::metadata(self.cache_dir.join(TLDR_PAGES_DIR)) {
            if let Ok(mtime) = metadata.modified() {
                let now = SystemTime::now();
                return now.duration_since(mtime).ok();
            };
        };
        None
    }

    /// Return the freshness of the cache (fresh, stale or missing).
    pub fn freshness(&self) -> CacheFreshness {
        match self.last_update() {
            Some(ago) if ago > crate::config::MAX_CACHE_AGE => CacheFreshness::Stale(ago),
            Some(_) => CacheFreshness::Fresh,
            None => CacheFreshness::Missing,
        }
    }

    /// Return the platform directory.
    fn get_platform_dir(&self) -> &'static str {
        match self.platform {
            PlatformType::Linux => "linux",
            PlatformType::OsX => "osx",
            PlatformType::SunOs => "sunos",
            PlatformType::Windows => "windows",
            PlatformType::Android => "android",
        }
    }

    /// Check for pages for a given platform in one of the given languages.
    fn find_page_for_platform(
        page_name: &str,
        cache_dir: &Path,
        platform: &str,
        language_dirs: &[String],
    ) -> Option<PathBuf> {
        language_dirs
            .iter()
            .map(|lang_dir| cache_dir.join(lang_dir).join(platform).join(page_name))
            .find(|path| path.exists() && path.is_file())
    }

    /// Look up custom patch (<name>.patch). If it exists, store it in a variable.
    fn find_patch(patch_name: &str, custom_pages_dir: Option<&Path>) -> Option<PathBuf> {
        custom_pages_dir
            .map(|custom_dir| custom_dir.join(patch_name))
            .filter(|path| path.exists() && path.is_file())
    }

    /// Search for a page and return the path to it.
    pub fn find_page(
        &self,
        name: &str,
        languages: &[String],
        custom_pages_dir: Option<&Path>,
    ) -> Option<PageLookupResult> {
        let page_filename = format!("{}.md", name);
        let patch_filename = format!("{}.patch", name);
        let custom_filename = format!("{}.page", name);

        // Determine directories
        let cache_dir = self.cache_dir.join(TLDR_PAGES_DIR);
        let lang_dirs: Vec<String> = languages
            .iter()
            .map(|lang| {
                if lang == "en" {
                    String::from("pages")
                } else {
                    format!("pages.{}", lang)
                }
            })
            .collect();

        // Look up custom page (<name>.page). If it exists, return it directly
        if let Some(config_dir) = custom_pages_dir {
            let custom_page = config_dir.join(custom_filename);
            if custom_page.exists() && custom_page.is_file() {
                return Some(PageLookupResult::with_page(custom_page));
            }
        }

        let patch_path = Self::find_patch(&patch_filename, custom_pages_dir);

        // Try to find a platform specific path next, append custom patch to it.
        let platform_dir = self.get_platform_dir();
        if let Some(page) =
            Self::find_page_for_platform(&page_filename, &cache_dir, platform_dir, &lang_dirs)
        {
            return Some(PageLookupResult::with_page(page).with_optional_patch(patch_path));
        }

        // Did not find platform specific results, fall back to "common"
        Self::find_page_for_platform(&page_filename, &cache_dir, "common", &lang_dirs)
            .map(|page| PageLookupResult::with_page(page).with_optional_patch(patch_path))
    }

    /// Return the available pages.
    pub fn list_pages(&self, custom_pages_dir: Option<&Path>) -> Vec<String> {
        // Determine platforms directory and platform
        let platforms_dir = self.cache_dir.join(TLDR_PAGES_DIR).join("pages");
        let platform_dir = self.get_platform_dir();

        // Closure that allows the WalkDir instance to traverse platform
        // specific and common page directories, but not others.
        let should_walk = |entry: &DirEntry| -> bool {
            let file_type = entry.file_type();
            let file_name = match entry.file_name().to_str() {
                Some(name) => name,
                None => return false,
            };
            if file_type.is_dir() {
                return file_name == "common" || file_name == platform_dir;
            } else if file_type.is_file() {
                return true;
            }
            false
        };

        let to_stem = |entry: DirEntry| -> Option<String> {
            entry
                .path()
                .file_stem()
                .and_then(OsStr::to_str)
                .map(str::to_string)
        };

        // Recursively walk through common and (if applicable) platform specific directory
        let mut pages = WalkDir::new(platforms_dir)
            .min_depth(1) // Skip root directory
            .into_iter()
            .filter_entry(should_walk) // Filter out pages for other architectures
            .filter_map(Result::ok) // Convert results to options, filter out errors
            .filter_map(|e| {
                let extension = e.path().extension().unwrap_or_default();
                if e.file_type().is_file() && extension == "md" {
                    to_stem(e)
                } else {
                    None
                }
            })
            .collect::<Vec<String>>();

        if let Some(custom_pages_dir) = custom_pages_dir {
            let is_page = |entry: &DirEntry| -> bool {
                let extension = entry.path().extension().unwrap_or_default();
                entry.file_type().is_file() && extension == "page"
            };

            let custom_pages = WalkDir::new(custom_pages_dir)
                .min_depth(1)
                .max_depth(1)
                .into_iter()
                .filter_entry(is_page)
                .filter_map(Result::ok)
                .filter_map(to_stem);

            pages.extend(custom_pages);
        }

        pages.sort();
        pages.dedup();
        pages
    }

    /// Delete the cache directory.
    pub fn clear(&self) -> Result<()> {
        // Check preconditions
        ensure!(
            self.cache_dir.exists(),
            "Cache path ({}) does not exist.",
            self.cache_dir.display(),
        );
        ensure!(
            self.cache_dir.is_dir(),
            "Cache path ({}) is not a directory.",
            self.cache_dir.display(),
        );

        // Delete old tldr-pages cache location as well if present
        // TODO: To be removed in the future
        for pages_dir_name in [TLDR_PAGES_DIR, TLDR_OLD_PAGES_DIR] {
            let pages_dir = self.cache_dir.join(pages_dir_name);

            if pages_dir.exists() {
                fs::remove_dir_all(&pages_dir).with_context(|| {
                    format!(
                        "Could not remove the cache directory at {}",
                        pages_dir.display()
                    )
                })?;
            }
        }

        Ok(())
    }
}

/// Unit Tests for cache module
#[cfg(test)]
mod tests {
    use super::*;

    use std::{
        fs::File,
        io::{Read, Write},
    };

    #[test]
    fn test_reader_with_patch() {
        // Write test files
        let dir = tempfile::tempdir().unwrap();
        let page_path = dir.path().join("test.page");
        let patch_path = dir.path().join("test.patch");
        {
            let mut f1 = File::create(&page_path).unwrap();
            f1.write_all(b"Hello\n").unwrap();
            let mut f2 = File::create(&patch_path).unwrap();
            f2.write_all(b"World").unwrap();
        }

        // Create chained reader from lookup result
        let lr = PageLookupResult::with_page(page_path).with_optional_patch(Some(patch_path));
        let mut reader = lr.reader().unwrap();

        // Read into a Vec
        let mut buf = Vec::new();
        reader.read_to_end(&mut buf).unwrap();

        assert_eq!(&buf, b"Hello\n\nWorld");
    }

    #[test]
    fn test_reader_without_patch() {
        // Write test file
        let dir = tempfile::tempdir().unwrap();
        let page_path = dir.path().join("test.page");
        {
            let mut f = File::create(&page_path).unwrap();
            f.write_all(b"Hello\n").unwrap();
        }

        // Create chained reader from lookup result
        let lr = PageLookupResult::with_page(page_path);
        let mut reader = lr.reader().unwrap();

        // Read into a Vec
        let mut buf = Vec::new();
        reader.read_to_end(&mut buf).unwrap();

        assert_eq!(&buf, b"Hello\n");
    }
}
<commit_msg>Cache: Add pages_dir helper method<commit_after>use std::{
    env,
    ffi::OsStr,
    fs::{self, File},
    io::{BufReader, Cursor, Read},
    path::{Path, PathBuf},
    time::{Duration, SystemTime},
};

use anyhow::{ensure, Context, Result};
use log::debug;
use reqwest::{blocking::Client, Proxy};
use walkdir::{DirEntry, WalkDir};
use zip::ZipArchive;

use crate::types::PlatformType;

pub static TLDR_PAGES_DIR: &str = "tldr-pages";
static TLDR_OLD_PAGES_DIR: &str = "tldr-master";

#[derive(Debug)]
pub struct Cache {
    platform: PlatformType,
    cache_dir: PathBuf,
}

#[derive(Debug)]
pub struct PageLookupResult {
    pub page_path: PathBuf,
    pub patch_path: Option<PathBuf>,
}

impl PageLookupResult {
    pub fn with_page(page_path: PathBuf) -> Self {
        Self {
            page_path,
            patch_path: None,
        }
    }

    pub fn with_optional_patch(mut self, patch_path: Option<PathBuf>) -> Self {
        self.patch_path = patch_path;
        self
    }

    /// Create a buffered reader that sequentially reads from the page and the
    /// patch, as if they were concatenated.
    ///
    /// This will return an error if either the page file or the patch file
    /// cannot be opened.
    pub fn reader(&self) -> Result<BufReader<Box<dyn Read>>> {
        // Open page file
        let page_file = File::open(&self.page_path)
            .with_context(|| format!("Could not open page file at {}", self.page_path.display()))?;

        // Open patch file
        let patch_file_opt = match &self.patch_path {
            Some(path) => Some(
                File::open(path)
                    .with_context(|| format!("Could not open patch file at {}", path.display()))?,
            ),
            None => None,
        };

        // Create chained reader from file(s)
        //
        // Note: It might be worthwhile to create our own struct that accepts
        // the page and patch files and that will read them sequentially,
        // because it avoids the boxing below. However, the performance impact
        // would first need to be shown to be significant using a benchmark.
        Ok(BufReader::new(if let Some(patch_file) = patch_file_opt {
            Box::new(page_file.chain(&b"\n"[..]).chain(patch_file)) as Box<dyn Read>
        } else {
            Box::new(page_file) as Box<dyn Read>
        }))
    }
}

pub enum CacheFreshness {
    /// The cache is still fresh (less than MAX_CACHE_AGE old)
    Fresh,
    /// The cache is stale and should be updated
    Stale(Duration),
    /// The cache is missing
    Missing,
}

impl Cache {
    pub fn new<P>(platform: PlatformType, cache_dir: P) -> Result<Self>
    where
        P: Into<PathBuf>,
    {
        // Check whether `cache_dir` exists and is a directory
        let cache_dir = cache_dir.into();
        let (cache_dir_exists, cache_dir_is_dir) = cache_dir
            .metadata()
            .map_or((false, false), |md| (true, md.is_dir()));
        ensure!(
            !cache_dir_exists || cache_dir_is_dir,
            "Cache directory path `{}` is not a directory",
            cache_dir.display(),
        );

        // If necessary, create cache directory
        if !cache_dir_exists {
            // Try to create the complete directory path
            fs::create_dir_all(&cache_dir).with_context(|| {
                format!(
                    "Cache directory path `{}` cannot be created",
                    cache_dir.display(),
                )
            })?;
            eprintln!(
                "Successfully created cache directory path `{}`.",
                cache_dir.display(),
            );
        }

        Ok(Self {
            platform,
            cache_dir,
        })
    }

    fn pages_dir(&self) -> PathBuf {
        self.cache_dir.join(TLDR_PAGES_DIR)
    }

    /// Download the archive from the specified URL.
    fn download(archive_url: &str) -> Result<Vec<u8>> {
        let mut builder = Client::builder();
        if let Ok(ref host) = env::var("HTTP_PROXY") {
            if let Ok(proxy) = Proxy::http(host) {
                builder = builder.proxy(proxy);
            }
        }
        if let Ok(ref host) = env::var("HTTPS_PROXY") {
            if let Ok(proxy) = Proxy::https(host) {
                builder = builder.proxy(proxy);
            }
        }
        let client = builder
            .build()
            .context("Could not instantiate HTTP client")?;
        let mut resp = client
            .get(archive_url)
            .send()?
            .error_for_status()
            .with_context(|| format!("Could not download tldr pages from {}", archive_url))?;
        let mut buf: Vec<u8> = vec![];
        let bytes_downloaded = resp.copy_to(&mut buf)?;
        debug!("{} bytes downloaded", bytes_downloaded);
        Ok(buf)
    }

    /// Update the pages cache from the specified URL.
    pub fn update(&self, archive_url: &str) -> Result<()> {
        // First, download the compressed data
        let bytes: Vec<u8> = Self::download(archive_url)?;

        // Decompress the response body into an `Archive`
        let mut archive = ZipArchive::new(Cursor::new(bytes))
            .context("Could not decompress downloaded ZIP archive")?;

        // Clear cache directory
        // Note: This is not the best solution. Ideally we would download the
        // archive to a temporary directory and then swap the two directories.
        // But renaming a directory doesn't work across filesystems and Rust
        // does not yet offer a recursive directory copying function. So for
        // now, we'll use this approach.
        self.clear()
            .context("Could not clear the cache directory")?;

        // Extract archive into pages dir
        archive
            .extract(&self.pages_dir())
            .context("Could not unpack compressed data")?;

        Ok(())
    }

    /// Return the duration since the cache directory was last modified.
    pub fn last_update(&self) -> Option<Duration> {
        if let Ok(metadata) = fs::metadata(self.pages_dir()) {
            if let Ok(mtime) = metadata.modified() {
                let now = SystemTime::now();
                return now.duration_since(mtime).ok();
            };
        };
        None
    }

    /// Return the freshness of the cache (fresh, stale or missing).
    pub fn freshness(&self) -> CacheFreshness {
        match self.last_update() {
            Some(ago) if ago > crate::config::MAX_CACHE_AGE => CacheFreshness::Stale(ago),
            Some(_) => CacheFreshness::Fresh,
            None => CacheFreshness::Missing,
        }
    }

    /// Return the platform directory.
    fn get_platform_dir(&self) -> &'static str {
        match self.platform {
            PlatformType::Linux => "linux",
            PlatformType::OsX => "osx",
            PlatformType::SunOs => "sunos",
            PlatformType::Windows => "windows",
            PlatformType::Android => "android",
        }
    }

    /// Check for pages for a given platform in one of the given languages.
    fn find_page_for_platform(
        page_name: &str,
        pages_dir: &Path,
        platform: &str,
        language_dirs: &[String],
    ) -> Option<PathBuf> {
        language_dirs
            .iter()
            .map(|lang_dir| pages_dir.join(lang_dir).join(platform).join(page_name))
            .find(|path| path.exists() && path.is_file())
    }

    /// Look up custom patch (<name>.patch). If it exists, store it in a variable.
    fn find_patch(patch_name: &str, custom_pages_dir: Option<&Path>) -> Option<PathBuf> {
        custom_pages_dir
            .map(|custom_dir| custom_dir.join(patch_name))
            .filter(|path| path.exists() && path.is_file())
    }

    /// Search for a page and return the path to it.
    pub fn find_page(
        &self,
        name: &str,
        languages: &[String],
        custom_pages_dir: Option<&Path>,
    ) -> Option<PageLookupResult> {
        let page_filename = format!("{}.md", name);
        let patch_filename = format!("{}.patch", name);
        let custom_filename = format!("{}.page", name);

        // Determine directory paths
        let pages_dir = self.pages_dir();
        let lang_dirs: Vec<String> = languages
            .iter()
            .map(|lang| {
                if lang == "en" {
                    String::from("pages")
                } else {
                    format!("pages.{}", lang)
                }
            })
            .collect();

        // Look up custom page (<name>.page). If it exists, return it directly
        if let Some(config_dir) = custom_pages_dir {
            let custom_page = config_dir.join(custom_filename);
            if custom_page.exists() && custom_page.is_file() {
                return Some(PageLookupResult::with_page(custom_page));
            }
        }

        let patch_path = Self::find_patch(&patch_filename, custom_pages_dir);

        // Try to find a platform specific path next, append custom patch to it.
        let platform_dir = self.get_platform_dir();
        if let Some(page) =
            Self::find_page_for_platform(&page_filename, &pages_dir, platform_dir, &lang_dirs)
        {
            return Some(PageLookupResult::with_page(page).with_optional_patch(patch_path));
        }

        // Did not find platform specific results, fall back to "common"
        Self::find_page_for_platform(&page_filename, &pages_dir, "common", &lang_dirs)
            .map(|page| PageLookupResult::with_page(page).with_optional_patch(patch_path))
    }

    /// Return the available pages.
    pub fn list_pages(&self, custom_pages_dir: Option<&Path>) -> Vec<String> {
        // Determine platforms directory and platform
        let platforms_dir = self.pages_dir().join("pages");
        let platform_dir = self.get_platform_dir();

        // Closure that allows the WalkDir instance to traverse platform
        // specific and common page directories, but not others.
        let should_walk = |entry: &DirEntry| -> bool {
            let file_type = entry.file_type();
            let file_name = match entry.file_name().to_str() {
                Some(name) => name,
                None => return false,
            };
            if file_type.is_dir() {
                return file_name == "common" || file_name == platform_dir;
            } else if file_type.is_file() {
                return true;
            }
            false
        };

        let to_stem = |entry: DirEntry| -> Option<String> {
            entry
                .path()
                .file_stem()
                .and_then(OsStr::to_str)
                .map(str::to_string)
        };

        // Recursively walk through common and (if applicable) platform specific directory
        let mut pages = WalkDir::new(platforms_dir)
            .min_depth(1) // Skip root directory
            .into_iter()
            .filter_entry(should_walk) // Filter out pages for other architectures
            .filter_map(Result::ok) // Convert results to options, filter out errors
            .filter_map(|e| {
                let extension = e.path().extension().unwrap_or_default();
                if e.file_type().is_file() && extension == "md" {
                    to_stem(e)
                } else {
                    None
                }
            })
            .collect::<Vec<String>>();

        if let Some(custom_pages_dir) = custom_pages_dir {
            let is_page = |entry: &DirEntry| -> bool {
                let extension = entry.path().extension().unwrap_or_default();
                entry.file_type().is_file() && extension == "page"
            };

            let custom_pages = WalkDir::new(custom_pages_dir)
                .min_depth(1)
                .max_depth(1)
                .into_iter()
                .filter_entry(is_page)
                .filter_map(Result::ok)
                .filter_map(to_stem);

            pages.extend(custom_pages);
        }

        pages.sort();
        pages.dedup();
        pages
    }

    /// Delete the cache directory.
    pub fn clear(&self) -> Result<()> {
        // Check preconditions
        ensure!(
            self.cache_dir.exists(),
            "Cache path ({}) does not exist.",
            self.cache_dir.display(),
        );
        ensure!(
            self.cache_dir.is_dir(),
            "Cache path ({}) is not a directory.",
            self.cache_dir.display(),
        );

        // Delete old tldr-pages cache location as well if present
        // TODO: To be removed in the future
        for pages_dir_name in [TLDR_PAGES_DIR, TLDR_OLD_PAGES_DIR] {
            let pages_dir = self.cache_dir.join(pages_dir_name);

            if pages_dir.exists() {
                fs::remove_dir_all(&pages_dir).with_context(|| {
                    format!(
                        "Could not remove the cache directory at {}",
                        pages_dir.display()
                    )
                })?;
            }
        }

        Ok(())
    }
}

/// Unit Tests for cache module
#[cfg(test)]
mod tests {
    use super::*;

    use std::{
        fs::File,
        io::{Read, Write},
    };

    #[test]
    fn test_reader_with_patch() {
        // Write test files
        let dir = tempfile::tempdir().unwrap();
        let page_path = dir.path().join("test.page");
        let patch_path = dir.path().join("test.patch");
        {
            let mut f1 = File::create(&page_path).unwrap();
            f1.write_all(b"Hello\n").unwrap();
            let mut f2 = File::create(&patch_path).unwrap();
            f2.write_all(b"World").unwrap();
        }

        // Create chained reader from lookup result
        let lr = PageLookupResult::with_page(page_path).with_optional_patch(Some(patch_path));
        let mut reader = lr.reader().unwrap();

        // Read into a Vec
        let mut buf = Vec::new();
        reader.read_to_end(&mut buf).unwrap();

        assert_eq!(&buf, b"Hello\n\nWorld");
    }

    #[test]
    fn test_reader_without_patch() {
        // Write test file
        let dir = tempfile::tempdir().unwrap();
        let page_path = dir.path().join("test.page");
        {
            let mut f = File::create(&page_path).unwrap();
            f.write_all(b"Hello\n").unwrap();
        }

        // Create chained reader from lookup result
        let lr = PageLookupResult::with_page(page_path);
        let mut reader = lr.reader().unwrap();

        // Read into a Vec
        let mut buf = Vec::new();
        reader.read_to_end(&mut buf).unwrap();

        assert_eq!(&buf, b"Hello\n");
    }
}
<|endoftext|>


<commit_before>use std::{convert};
use std::io::{Cursor, Read};

use byteorder::{self, ReadBytesExt};

#[derive(Debug, PartialEq, Eq)]
pub enum DeserializationError {
    UnexpectedTag,
    ShortData,
    ExtraData,
    IntegerOverflow,
    InvalidValue,
}

impl convert::From<byteorder::Error> for DeserializationError {
    fn from(e: byteorder::Error) -> DeserializationError {
        return match e {
            byteorder::Error::UnexpectedEOF => DeserializationError::ShortData,
            _ => panic!("Unexpected error!"),
        }
    }
}

pub type DeserializationResult<T> = Result<T, DeserializationError>;

pub struct Deserializer {
    reader: Cursor<Vec<u8>>,
}

impl Deserializer {
    pub fn new(data: Vec<u8>) -> Deserializer {
        return Deserializer{
            reader: Cursor::new(data),
        }
    }

    fn _read_length(&mut self) -> DeserializationResult<usize> {
        let b = try!(self.reader.read_u8());
        // TODO: handle lengths greater than 128
        assert!(b & 0x80 == 0);
        return Ok((b & 0x7f) as usize);
    }

    fn _read_with_tag<T, F>(&mut self, expected_tag: u8, body: F) -> DeserializationResult<T>
            where F: Fn(Vec<u8>) -> DeserializationResult<T> {
        let tag = try!(self.reader.read_u8());
        // TODO: only some of the bits in the first byte are for the tag
        if tag != expected_tag {
            return Err(DeserializationError::UnexpectedTag);
        }
        let length = try!(self._read_length());
        let mut data = vec![0; length];
        let n = self.reader.read(&mut data).unwrap();
        if n != length {
            return Err(DeserializationError::ShortData);
        }
        return body(data);
    }

    pub fn finish(self) -> DeserializationResult<()> {
        if self.reader.position() as usize != self.reader.get_ref().len() {
            return Err(DeserializationError::ExtraData);
        }
        return Ok(());
    }

    pub fn read_bool(&mut self) -> DeserializationResult<bool> {
        return self._read_with_tag(1, |data| {
            if data == b"\x00" {
                return Ok(false);
            } else if data == b"\xff" {
                return Ok(true)
            } else {
                return Err(DeserializationError::InvalidValue);
            }
        });
    }

    pub fn read_int(&mut self) -> DeserializationResult<i64> {
        return self._read_with_tag(2, |data| {
            if data.len() > 8 {
                return Err(DeserializationError::IntegerOverflow);
            }
            if data.len() > 1 {
                match (data[0], data[1] & 0x80) {
                    (0xff, 0x80) => return Err(DeserializationError::InvalidValue),
                    (0x00, 0x00) => return Err(DeserializationError::InvalidValue),
                    _ => {},
                }
            }
            let mut ret = 0;
            for b in data.iter() {
                ret <<= 8;
                ret |= *b as i64;
            }
            // Shift up and down in order to sign extend the result.
            ret <<= 64 - data.len() * 8;
            ret >>= 64 - data.len() * 8;
            return Ok(ret);
        });
    }
}

pub fn from_vec<F, T>(data: Vec<u8>, f: F) -> DeserializationResult<T>
        where F: Fn(&mut Deserializer) -> DeserializationResult<T> {
    let mut deserializer = Deserializer::new(data);
    let result = try!(f(&mut deserializer));
    try!(deserializer.finish());
    return Ok(result);
}

#[cfg(test)]
mod tests {
    use std::{fmt};

    use super::{Deserializer, DeserializationError, DeserializationResult, from_vec};

    fn assert_deserializes<T, F>(values: Vec<(DeserializationResult<T>, Vec<u8>)>, f: F)
            where T: Eq + fmt::Debug, F: Fn(&mut Deserializer) -> DeserializationResult<T> {
        for (expected, value) in values {
            let result = from_vec(value, &f);
            assert_eq!(result, expected);
        }
    }
    #[test]
    fn test_read_extra_data() {
        assert_deserializes(vec![
            (Err(DeserializationError::ExtraData), b"\x00".to_vec()),
        ], |_| {
            return Ok(());
        });
    }

    #[test]
    fn test_read_bool() {
        assert_deserializes(vec![
            (Ok(true), b"\x01\x01\xff".to_vec()),
            (Ok(false), b"\x01\x01\x00".to_vec()),
            (Err(DeserializationError::InvalidValue), b"\x01\x00".to_vec()),
            (Err(DeserializationError::InvalidValue), b"\x01\x01\x01".to_vec()),
            (Err(DeserializationError::InvalidValue), b"\x01\x02\x01\x01".to_vec()),
        ], |deserializer| {
            return deserializer.read_bool();
        });
    }

    #[test]
    fn test_read_int() {
        assert_deserializes(vec![
            (Ok(0), b"\x02\x01\x00".to_vec()),
            (Ok(127), b"\x02\x01\x7f".to_vec()),
            (Ok(128), b"\x02\x02\x00\x80".to_vec()),
            (Ok(256), b"\x02\x02\x01\x00".to_vec()),
            (Ok(-128), b"\x02\x01\x80".to_vec()),
            (Ok(-129), b"\x02\x02\xff\x7f".to_vec()),
            (Ok(-256), b"\x02\x02\xff\x00".to_vec()),
            (Err(DeserializationError::UnexpectedTag), b"\x03".to_vec()),
            (Err(DeserializationError::ShortData), b"\x02\x02\x00".to_vec()),
            (Err(DeserializationError::ShortData), b"".to_vec()),
            (Err(DeserializationError::ShortData), b"\x02".to_vec()),
            (
                Err(DeserializationError::IntegerOverflow),
                b"\x02\x09\x02\x00\x00\x00\x00\x00\x00\x00\x00".to_vec()
            ),
            (Err(DeserializationError::InvalidValue), b"\x02\x05\x00\x00\x00\x00\x01".to_vec()),
            (Err(DeserializationError::InvalidValue), b"\x02\x02\xff\x80".to_vec()),
        ], |deserializer| {
            return deserializer.read_int();
        });
    }
}
<commit_msg>shorter<commit_after>use std::{convert};
use std::io::{Cursor, Read};

use byteorder::{self, ReadBytesExt};

#[derive(Debug, PartialEq, Eq)]
pub enum DeserializationError {
    UnexpectedTag,
    ShortData,
    ExtraData,
    IntegerOverflow,
    InvalidValue,
}

impl convert::From<byteorder::Error> for DeserializationError {
    fn from(e: byteorder::Error) -> DeserializationError {
        return match e {
            byteorder::Error::UnexpectedEOF => DeserializationError::ShortData,
            _ => panic!("Unexpected error!"),
        }
    }
}

pub type DeserializationResult<T> = Result<T, DeserializationError>;

pub struct Deserializer {
    reader: Cursor<Vec<u8>>,
}

impl Deserializer {
    pub fn new(data: Vec<u8>) -> Deserializer {
        return Deserializer{
            reader: Cursor::new(data),
        }
    }

    fn _read_length(&mut self) -> DeserializationResult<usize> {
        let b = try!(self.reader.read_u8());
        // TODO: handle lengths greater than 128
        assert!(b & 0x80 == 0);
        return Ok((b & 0x7f) as usize);
    }

    fn _read_with_tag<T, F>(&mut self, expected_tag: u8, body: F) -> DeserializationResult<T>
            where F: Fn(Vec<u8>) -> DeserializationResult<T> {
        let tag = try!(self.reader.read_u8());
        // TODO: only some of the bits in the first byte are for the tag
        if tag != expected_tag {
            return Err(DeserializationError::UnexpectedTag);
        }
        let length = try!(self._read_length());
        let mut data = vec![0; length];
        let n = self.reader.read(&mut data).unwrap();
        if n != length {
            return Err(DeserializationError::ShortData);
        }
        return body(data);
    }

    pub fn finish(self) -> DeserializationResult<()> {
        if self.reader.position() as usize != self.reader.get_ref().len() {
            return Err(DeserializationError::ExtraData);
        }
        return Ok(());
    }

    pub fn read_bool(&mut self) -> DeserializationResult<bool> {
        return self._read_with_tag(1, |data| {
            if data == b"\x00" {
                return Ok(false);
            } else if data == b"\xff" {
                return Ok(true)
            } else {
                return Err(DeserializationError::InvalidValue);
            }
        });
    }

    pub fn read_int(&mut self) -> DeserializationResult<i64> {
        return self._read_with_tag(2, |data| {
            if data.len() > 8 {
                return Err(DeserializationError::IntegerOverflow);
            }
            if data.len() > 1 {
                match (data[0], data[1] & 0x80) {
                    (0xff, 0x80) | (0x00, 0x00) => return Err(DeserializationError::InvalidValue),
                    _ => {},
                }
            }
            let mut ret = 0;
            for b in data.iter() {
                ret <<= 8;
                ret |= *b as i64;
            }
            // Shift up and down in order to sign extend the result.
            ret <<= 64 - data.len() * 8;
            ret >>= 64 - data.len() * 8;
            return Ok(ret);
        });
    }
}

pub fn from_vec<F, T>(data: Vec<u8>, f: F) -> DeserializationResult<T>
        where F: Fn(&mut Deserializer) -> DeserializationResult<T> {
    let mut deserializer = Deserializer::new(data);
    let result = try!(f(&mut deserializer));
    try!(deserializer.finish());
    return Ok(result);
}

#[cfg(test)]
mod tests {
    use std::{fmt};

    use super::{Deserializer, DeserializationError, DeserializationResult, from_vec};

    fn assert_deserializes<T, F>(values: Vec<(DeserializationResult<T>, Vec<u8>)>, f: F)
            where T: Eq + fmt::Debug, F: Fn(&mut Deserializer) -> DeserializationResult<T> {
        for (expected, value) in values {
            let result = from_vec(value, &f);
            assert_eq!(result, expected);
        }
    }
    #[test]
    fn test_read_extra_data() {
        assert_deserializes(vec![
            (Err(DeserializationError::ExtraData), b"\x00".to_vec()),
        ], |_| {
            return Ok(());
        });
    }

    #[test]
    fn test_read_bool() {
        assert_deserializes(vec![
            (Ok(true), b"\x01\x01\xff".to_vec()),
            (Ok(false), b"\x01\x01\x00".to_vec()),
            (Err(DeserializationError::InvalidValue), b"\x01\x00".to_vec()),
            (Err(DeserializationError::InvalidValue), b"\x01\x01\x01".to_vec()),
            (Err(DeserializationError::InvalidValue), b"\x01\x02\x01\x01".to_vec()),
        ], |deserializer| {
            return deserializer.read_bool();
        });
    }

    #[test]
    fn test_read_int() {
        assert_deserializes(vec![
            (Ok(0), b"\x02\x01\x00".to_vec()),
            (Ok(127), b"\x02\x01\x7f".to_vec()),
            (Ok(128), b"\x02\x02\x00\x80".to_vec()),
            (Ok(256), b"\x02\x02\x01\x00".to_vec()),
            (Ok(-128), b"\x02\x01\x80".to_vec()),
            (Ok(-129), b"\x02\x02\xff\x7f".to_vec()),
            (Ok(-256), b"\x02\x02\xff\x00".to_vec()),
            (Err(DeserializationError::UnexpectedTag), b"\x03".to_vec()),
            (Err(DeserializationError::ShortData), b"\x02\x02\x00".to_vec()),
            (Err(DeserializationError::ShortData), b"".to_vec()),
            (Err(DeserializationError::ShortData), b"\x02".to_vec()),
            (
                Err(DeserializationError::IntegerOverflow),
                b"\x02\x09\x02\x00\x00\x00\x00\x00\x00\x00\x00".to_vec()
            ),
            (Err(DeserializationError::InvalidValue), b"\x02\x05\x00\x00\x00\x00\x01".to_vec()),
            (Err(DeserializationError::InvalidValue), b"\x02\x02\xff\x80".to_vec()),
        ], |deserializer| {
            return deserializer.read_int();
        });
    }
}
<|endoftext|>


<commit_before><commit_msg>Update unit tests<commit_after><|endoftext|>


<commit_before>extern crate image;
extern crate nalgebra;

use std::f64::consts::{PI_2, FRAC_PI_2};
use std::io::File;
use std::num::FloatMath;
use image::{GenericImage, Pixel};
use nalgebra::{DVec, Mat2, Pnt2, Rot2, Vec1, Vec2, Vec3};

fn draw(v: &DVec<f64>, path: &Path) {
    let width = v.len() as u32;
    let height = 128u32;
    let white = Pixel::from_channels(255, 255, 255, 255);
    let buffer = image::ImageBuffer::from_fn(width, height, |_, _| white);
    let mut img = image::DynamicImage::ImageRgba8(buffer);
    let red = Pixel::from_channels(255, 0, 0, 255);
    for i in range(0u32, width) {
        let half = (height / 2) as f64;
        let y = half * (1.0 + v[i as uint]);
        let y = nalgebra::clamp(y as u32, 1u32, height);
        img.put_pixel(i, height - y, red);
    }
    let out = File::create(path).unwrap();
    let _ = img.save(out, image::PNG);
}

fn main() {
    println!("24 days of Rust - nalgebra (day 14)");
    let v = Vec2::new(0.0f64, 1.0);
    // 90 degrees clockwise
    //  0, 1
    // -1, 0
    let rot = Mat2::new(0.0f64, -1.0, 1.0, 0.0);
    println!("{}", rot * v);
    let angle = FRAC_PI_2;
    let rot = Rot2::new(Vec1::new(angle));
    println!("{}", rot * v);
    let point = Pnt2::new(4.0f64, 4.0);
    println!("Translate from {} to {}", point, nalgebra::translate(&v, &point));

    let v1 = Vec3::new(2.0f64, 2.0, 0.0);
    let v2 = Vec3::new(2.0f64, -2.0, 0.0);
    if nalgebra::approx_eq(&0.0f64, &nalgebra::dot(&v1, &v2)) {
        println!("v1 is orthogonal to v2");
    }

    println!("{}", nalgebra::cross(&v1, &v2));
    println!("{}", nalgebra::cross(&v2, &v1));

    let sine = DVec::from_fn(512, |i: uint| {
        let t = i as f64 / 16.0f64;
        t.sin()
    });
    draw(&sine, &Path::new("out_sine.png"));

    let window = DVec::from_fn(512, |i: uint| {
        0.54f64 - 0.46 * (PI_2 * (i as f64) / 511f64).cos()
    });
    draw(&window, &Path::new("out_window.png"));

    draw(&(sine * window), &Path::new("out_windowed.png"));
}
<commit_msg>Refactored signal size.<commit_after>extern crate image;
extern crate nalgebra;

use std::f64::consts::{PI_2, FRAC_PI_2};
use std::io::File;
use std::num::FloatMath;
use image::{GenericImage, Pixel};
use nalgebra::{DVec, Mat2, Pnt2, Rot2, Vec1, Vec2, Vec3};

fn draw(v: &DVec<f64>, path: &Path) {
    let width = v.len() as u32;
    let height = 128u32;
    let white = Pixel::from_channels(255, 255, 255, 255);
    let buffer = image::ImageBuffer::from_fn(width, height, |_, _| white);
    let mut img = image::DynamicImage::ImageRgba8(buffer);
    let red = Pixel::from_channels(255, 0, 0, 255);
    for i in range(0u32, width) {
        let half = (height / 2) as f64;
        let y = half * (1.0 + v[i as uint]);
        let y = nalgebra::clamp(y as u32, 1u32, height);
        img.put_pixel(i, height - y, red);
    }
    let out = File::create(path).unwrap();
    let _ = img.save(out, image::PNG);
}

fn main() {
    println!("24 days of Rust - nalgebra (day 14)");
    let v = Vec2::new(0.0f64, 1.0);
    // 90 degrees clockwise
    //  0, 1
    // -1, 0
    let rot = Mat2::new(0.0f64, -1.0, 1.0, 0.0);
    println!("{}", rot * v);
    let angle = FRAC_PI_2;
    let rot = Rot2::new(Vec1::new(angle));
    println!("{}", rot * v);
    let point = Pnt2::new(4.0f64, 4.0);
    println!("Translate from {} to {}", point, nalgebra::translate(&v, &point));

    let v1 = Vec3::new(2.0f64, 2.0, 0.0);
    let v2 = Vec3::new(2.0f64, -2.0, 0.0);
    if nalgebra::approx_eq(&0.0f64, &nalgebra::dot(&v1, &v2)) {
        println!("v1 is orthogonal to v2");
    }

    println!("{}", nalgebra::cross(&v1, &v2));
    println!("{}", nalgebra::cross(&v2, &v1));

    const SIZE: uint = 512;
    let sine = DVec::from_fn(SIZE, |i: uint| {
        let t = i as f64 / 16.0f64;
        t.sin()
    });
    draw(&sine, &Path::new("out_sine.png"));

    let window = DVec::from_fn(SIZE, |i: uint| {
        0.54f64 - 0.46 * (PI_2 * (i as f64) / (SIZE - 1) as f64).cos()
    });
    draw(&window, &Path::new("out_window.png"));

    draw(&(sine * window), &Path::new("out_windowed.png"));
}
<|endoftext|>