<commit_before>use std::vec::Vec;
use std::string::String;

struct Compiler {
    pub lines: Vec<String>,
    pub instructions: Vec<String>,
}

impl Compiler {
    pub fn new() -> Self {
        Compiler {
            lines: Vec::new(),
            instructions: Vec::new(),
        }
    }

    pub fn parse(&mut self, assembly: &str) {
        self.split_lines(assembly);
        self.strip_comments();
    }

    fn strip_comments(&mut self) {
        let mut lines = self.lines.iter();
        while let Some(line) = lines.next() {
            let trimmed_line = line.trim();
            if !trimmed_line.is_empty() && !trimmed_line.starts_with(";") {
                let mut instruction = String::new();
                let stripped_instruction = match trimmed_line.find(';') {
                    Some(index) => trimmed_line.split_at(index).0,
                    None => trimmed_line,
                };
                instruction.push_str(stripped_instruction.trim());
                self.instructions.push(instruction);
            }
        }
    }

    fn split_lines(&mut self, assembly: &str) {
        let mut line = String::new();
        let mut chars = assembly.chars();

        while let Some(c) = chars.next() {
            match c {
                // Line Feed | Vertical Tab | Form Feed | Next Line | Line/Paragraph Separator
                '\u{000A}' | '\u{000B}' | '\u{000C}' | '\u{0085}' | '\u{2028}' | '\u{2029}' => {
                    self.lines.push(line);
                    line = String::new();
                }

                // Carriage Return
                '\u{000D}' => {
                    self.lines.push(line);
                    line = String::new();

                    // Carriage Return + Line Feed
                    match chars.next() {
                        Some(n) => {
                            if n != '\u{000A}' {
                                line.push(n);
                            }
                        }
                        None => {}
                    }
                }

                _ => {
                    line.push(c);
                }
            }
        }

        if !line.is_empty() {
            self.lines.push(line);
        }
    }
}

#[cfg(test)]
mod tests {
    use super::Compiler;

    #[test]
    fn it_splits_lines_by_line_feed() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\n0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_vertical_tab() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\x0B0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_form_feed() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\x0C0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_carriage_return() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\r0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_carriage_return_line_feed() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\r\n0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_next_line() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\u{0085}0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_line_separator() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\u{2028}0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_paragraph_separator() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\u{2029}0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_ignors_trailing_separators() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\r\n0 0 0 0\r\n");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_strips_single_line_comments() {
        let mut compiler = Compiler::new();
        compiler.parse("; single line comment\n0 0 0 0\n");

        assert_eq!(compiler.lines.len(), 2);
        assert_eq!(compiler.instructions.len(), 1);
    }

    #[test]
    fn it_strips_single_line_comments_starting_with_spaces() {
        let mut compiler = Compiler::new();
        compiler.parse(" ; single line comment\n0 0 0 0\n");

        assert_eq!(compiler.lines.len(), 2);
        assert_eq!(compiler.instructions.len(), 1);
    }

    #[test]
    fn it_strips_single_line_comments_starting_with_tabs() {
        let mut compiler = Compiler::new();
        compiler.parse("\t; single line comment\n0 0 0 0\n");

        assert_eq!(compiler.lines.len(), 2);
        assert_eq!(compiler.instructions.len(), 1);
    }

    #[test]
    fn it_strips_inline_comments() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0; inline comment\n");

        assert_eq!(compiler.lines.len(), 1);
        assert_eq!(compiler.instructions, vec!["0 0 0 0"]);
    }

    #[test]
    fn it_strips_inline_comments_starting_with_spaces() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0 ; inline comment\n");

        assert_eq!(compiler.lines.len(), 1);
        assert_eq!(compiler.instructions, vec!["0 0 0 0"]);
    }

    #[test]
    fn it_strips_inline_comments_starting_with_tabs() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\t; inline comment\n");

        assert_eq!(compiler.lines.len(), 1);
        assert_eq!(compiler.instructions, vec!["0 0 0 0"]);
    }
}
<commit_msg>Replaces String::new() with to_string().<commit_after>use std::vec::Vec;
use std::string::String;

struct Compiler {
    pub lines: Vec<String>,
    pub instructions: Vec<String>,
}

impl Compiler {
    pub fn new() -> Self {
        Compiler {
            lines: Vec::new(),
            instructions: Vec::new(),
        }
    }

    pub fn parse(&mut self, assembly: &str) {
        self.split_lines(assembly);
        self.strip_comments();
    }

    fn strip_comments(&mut self) {
        let mut lines = self.lines.iter();
        while let Some(line) = lines.next() {
            let trimmed_line = line.trim();
            if !trimmed_line.is_empty() && !trimmed_line.starts_with(";") {
                let instruction = match trimmed_line.find(';') {
                    Some(index) => trimmed_line.split_at(index).0,
                    None => trimmed_line,
                };
                self.instructions.push(instruction.trim().to_string());
            }
        }
    }

    fn split_lines(&mut self, assembly: &str) {
        let mut line = String::new();
        let mut chars = assembly.chars();

        while let Some(c) = chars.next() {
            match c {
                // Line Feed | Vertical Tab | Form Feed | Next Line | Line/Paragraph Separator
                '\u{000A}' | '\u{000B}' | '\u{000C}' | '\u{0085}' | '\u{2028}' | '\u{2029}' => {
                    self.lines.push(line);
                    line = String::new();
                }

                // Carriage Return
                '\u{000D}' => {
                    self.lines.push(line);
                    line = String::new();

                    // Carriage Return + Line Feed
                    match chars.next() {
                        Some(n) => {
                            if n != '\u{000A}' {
                                line.push(n);
                            }
                        }
                        None => {}
                    }
                }

                _ => {
                    line.push(c);
                }
            }
        }

        if !line.is_empty() {
            self.lines.push(line);
        }
    }
}

#[cfg(test)]
mod tests {
    use super::Compiler;

    #[test]
    fn it_splits_lines_by_line_feed() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\n0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_vertical_tab() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\x0B0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_form_feed() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\x0C0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_carriage_return() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\r0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_carriage_return_line_feed() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\r\n0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_next_line() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\u{0085}0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_line_separator() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\u{2028}0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_splits_lines_by_paragraph_separator() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\u{2029}0 0 0 0");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_ignors_trailing_separators() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\r\n0 0 0 0\r\n");

        assert_eq!(compiler.lines.len(), 2);
    }

    #[test]
    fn it_strips_single_line_comments() {
        let mut compiler = Compiler::new();
        compiler.parse("; single line comment\n0 0 0 0\n");

        assert_eq!(compiler.lines.len(), 2);
        assert_eq!(compiler.instructions.len(), 1);
    }

    #[test]
    fn it_strips_single_line_comments_starting_with_spaces() {
        let mut compiler = Compiler::new();
        compiler.parse(" ; single line comment\n0 0 0 0\n");

        assert_eq!(compiler.lines.len(), 2);
        assert_eq!(compiler.instructions.len(), 1);
    }

    #[test]
    fn it_strips_single_line_comments_starting_with_tabs() {
        let mut compiler = Compiler::new();
        compiler.parse("\t; single line comment\n0 0 0 0\n");

        assert_eq!(compiler.lines.len(), 2);
        assert_eq!(compiler.instructions.len(), 1);
    }

    #[test]
    fn it_strips_inline_comments() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0; inline comment\n");

        assert_eq!(compiler.lines.len(), 1);
        assert_eq!(compiler.instructions, vec!["0 0 0 0"]);
    }

    #[test]
    fn it_strips_inline_comments_starting_with_spaces() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0 ; inline comment\n");

        assert_eq!(compiler.lines.len(), 1);
        assert_eq!(compiler.instructions, vec!["0 0 0 0"]);
    }

    #[test]
    fn it_strips_inline_comments_starting_with_tabs() {
        let mut compiler = Compiler::new();
        compiler.parse("0 0 0 0\t; inline comment\n");

        assert_eq!(compiler.lines.len(), 1);
        assert_eq!(compiler.instructions, vec!["0 0 0 0"]);
    }
}
<|endoftext|>


<commit_before>use super::prelude::*;
use node_dispatch::{dispatch_block, dispatch_block_pair};
use std::hash::Hasher;
use stencila_schema::*;

/// Implements patching for `BlockContent`
///
/// Generates and applies `Replace` and `Transform` operations between variants of block content.
/// All other operations are passed through to variants.
impl Patchable for BlockContent {
    fn is_equal(&self, other: &Self) -> Result<()> {
        dispatch_block_pair!(self, other, bail!(Error::NotEqual), is_equal)
    }

    fn make_hash<H: Hasher>(&self, state: &mut H) {
        dispatch_block!(self, make_hash, state)
    }

    fn diff(&self, other: &Self, differ: &mut Differ) {
        dispatch_block_pair!(
            self,
            other,
            diff_transform(differ, self, other),
            diff,
            differ
        )
    }

    fn apply_add(&mut self, address: &mut Address, value: &Value) -> Result<()> {
        dispatch_block!(self, apply_add, address, value)
    }

    fn apply_remove(&mut self, address: &mut Address, items: usize) -> Result<()> {
        dispatch_block!(self, apply_remove, address, items)
    }

    fn apply_replace(&mut self, address: &mut Address, items: usize, value: &Value) -> Result<()> {
        if address.is_empty() {
            *self = Self::from_value(value)?;
            Ok(())
        } else {
            dispatch_block!(self, apply_replace, address, items, value)
        }
    }

    fn apply_move(&mut self, from: &mut Address, items: usize, to: &mut Address) -> Result<()> {
        dispatch_block!(self, apply_move, from, items, to)
    }

    fn apply_transform(&mut self, address: &mut Address, from: &str, to: &str) -> Result<()> {
        if address.is_empty() {
            assert_eq!(from, self.as_ref(), "Expected the same type");
            *self = apply_transform(self, to);
            Ok(())
        } else {
            dispatch_block!(self, apply_transform, address, from, to)
        }
    }
}

fn diff_transform(differ: &mut Differ, _from: &BlockContent, to: &BlockContent) {
    // TODO implement generation of `Transform` operations
    // Default is to generate a replace operation
    differ.replace(to)
}

fn apply_transform(_from: &BlockContent, _to: &str) -> BlockContent {
    // TODO implement application of `Transform` operations
    todo!()
}

// Implementations for `BlockContent` structs, including related structs
// (e.g. `Figure` vs `FigureSimple`, which are actually "works").

patchable_struct!(Heading, content, depth);

patchable_struct!(Paragraph, content);

patchable_struct!(MathBlock, math_language, text);

patchable_struct!(QuoteBlock, content);

patchable_struct!(CodeBlock, programming_language, text);
patchable_struct!(
    CodeChunk,
    programming_language,
    text,
    outputs,
    errors,
    label,
    caption,
    code_dependencies,
    code_dependents,
    compile_digest,
    execute_digest,
    execute_required,
    execute_status,
    execute_ended,
    execute_duration
);
patchable_variants!(
    CodeChunkCaption,
    CodeChunkCaption::VecBlockContent,
    CodeChunkCaption::String
);
patchable_struct!(CodeError, error_message, error_type, stack_trace);
patchable_variants!(
    CodeExecutableCodeDependencies,
    CodeExecutableCodeDependencies::CodeChunk,
    CodeExecutableCodeDependencies::Parameter
);
patchable_variants!(
    CodeExecutableCodeDependents,
    CodeExecutableCodeDependents::CodeChunk,
    CodeExecutableCodeDependents::CodeExpression
);
patchable_enum!(CodeChunkExecuteAuto);
patchable_enum!(CodeExecutableExecuteRequired);
patchable_enum!(CodeExecutableExecuteStatus);

patchable_struct!(List, items, order);
patchable_enum!(ListOrder);

patchable_struct!(ListItem, content);
patchable_variants!(
    ListItemContent,
    ListItemContent::VecBlockContent,
    ListItemContent::VecInlineContent
);

patchable_struct!(Table, label, caption, rows);
patchable_struct!(TableSimple, label, caption, rows);
patchable_variants!(
    TableCaption,
    TableCaption::VecBlockContent,
    TableCaption::String
);

patchable_struct!(TableRow, cells, row_type);
patchable_enum!(TableRowRowType);

patchable_struct!(TableCell, content, cell_type, colspan, rowspan);
patchable_enum!(TableCellCellType);
patchable_variants!(
    TableCellContent,
    TableCellContent::VecBlockContent,
    TableCellContent::VecInlineContent
);

patchable_struct!(Figure, label, caption, content);
patchable_struct!(FigureSimple, label, caption, content);
patchable_variants!(
    FigureCaption,
    FigureCaption::VecBlockContent,
    FigureCaption::String
);

patchable_struct!(Include, source);

patchable_struct!(ThematicBreak);

patchable_struct!(Claim, content, claim_type);
patchable_struct!(ClaimSimple, content, claim_type);
patchable_enum!(ClaimClaimType);

patchable_struct!(CollectionSimple);
<commit_msg>chore(HTML encoding): Add new properties for `CodeChunk` nodes<commit_after>use super::prelude::*;
use node_dispatch::{dispatch_block, dispatch_block_pair};
use std::hash::Hasher;
use stencila_schema::*;

/// Implements patching for `BlockContent`
///
/// Generates and applies `Replace` and `Transform` operations between variants of block content.
/// All other operations are passed through to variants.
impl Patchable for BlockContent {
    fn is_equal(&self, other: &Self) -> Result<()> {
        dispatch_block_pair!(self, other, bail!(Error::NotEqual), is_equal)
    }

    fn make_hash<H: Hasher>(&self, state: &mut H) {
        dispatch_block!(self, make_hash, state)
    }

    fn diff(&self, other: &Self, differ: &mut Differ) {
        dispatch_block_pair!(
            self,
            other,
            diff_transform(differ, self, other),
            diff,
            differ
        )
    }

    fn apply_add(&mut self, address: &mut Address, value: &Value) -> Result<()> {
        dispatch_block!(self, apply_add, address, value)
    }

    fn apply_remove(&mut self, address: &mut Address, items: usize) -> Result<()> {
        dispatch_block!(self, apply_remove, address, items)
    }

    fn apply_replace(&mut self, address: &mut Address, items: usize, value: &Value) -> Result<()> {
        if address.is_empty() {
            *self = Self::from_value(value)?;
            Ok(())
        } else {
            dispatch_block!(self, apply_replace, address, items, value)
        }
    }

    fn apply_move(&mut self, from: &mut Address, items: usize, to: &mut Address) -> Result<()> {
        dispatch_block!(self, apply_move, from, items, to)
    }

    fn apply_transform(&mut self, address: &mut Address, from: &str, to: &str) -> Result<()> {
        if address.is_empty() {
            assert_eq!(from, self.as_ref(), "Expected the same type");
            *self = apply_transform(self, to);
            Ok(())
        } else {
            dispatch_block!(self, apply_transform, address, from, to)
        }
    }
}

fn diff_transform(differ: &mut Differ, _from: &BlockContent, to: &BlockContent) {
    // TODO implement generation of `Transform` operations
    // Default is to generate a replace operation
    differ.replace(to)
}

fn apply_transform(_from: &BlockContent, _to: &str) -> BlockContent {
    // TODO implement application of `Transform` operations
    todo!()
}

// Implementations for `BlockContent` structs, including related structs
// (e.g. `Figure` vs `FigureSimple`, which are actually "works").

patchable_struct!(Heading, content, depth);

patchable_struct!(Paragraph, content);

patchable_struct!(MathBlock, math_language, text);

patchable_struct!(QuoteBlock, content);

patchable_struct!(CodeBlock, programming_language, text);
patchable_struct!(
    CodeChunk,
    programming_language,
    text,
    outputs,
    errors,
    label,
    caption,
    code_dependencies,
    code_dependents,
    compile_digest,
    execute_auto,
    execute_pure,
    execute_digest,
    execute_required,
    execute_status,
    execute_ended,
    execute_duration
);
patchable_variants!(
    CodeChunkCaption,
    CodeChunkCaption::VecBlockContent,
    CodeChunkCaption::String
);
patchable_struct!(CodeError, error_message, error_type, stack_trace);
patchable_variants!(
    CodeExecutableCodeDependencies,
    CodeExecutableCodeDependencies::CodeChunk,
    CodeExecutableCodeDependencies::Parameter
);
patchable_variants!(
    CodeExecutableCodeDependents,
    CodeExecutableCodeDependents::CodeChunk,
    CodeExecutableCodeDependents::CodeExpression
);
patchable_enum!(CodeChunkExecuteAuto);
patchable_enum!(CodeExecutableExecuteRequired);
patchable_enum!(CodeExecutableExecuteStatus);

patchable_struct!(List, items, order);
patchable_enum!(ListOrder);

patchable_struct!(ListItem, content);
patchable_variants!(
    ListItemContent,
    ListItemContent::VecBlockContent,
    ListItemContent::VecInlineContent
);

patchable_struct!(Table, label, caption, rows);
patchable_struct!(TableSimple, label, caption, rows);
patchable_variants!(
    TableCaption,
    TableCaption::VecBlockContent,
    TableCaption::String
);

patchable_struct!(TableRow, cells, row_type);
patchable_enum!(TableRowRowType);

patchable_struct!(TableCell, content, cell_type, colspan, rowspan);
patchable_enum!(TableCellCellType);
patchable_variants!(
    TableCellContent,
    TableCellContent::VecBlockContent,
    TableCellContent::VecInlineContent
);

patchable_struct!(Figure, label, caption, content);
patchable_struct!(FigureSimple, label, caption, content);
patchable_variants!(
    FigureCaption,
    FigureCaption::VecBlockContent,
    FigureCaption::String
);

patchable_struct!(Include, source);

patchable_struct!(ThematicBreak);

patchable_struct!(Claim, content, claim_type);
patchable_struct!(ClaimSimple, content, claim_type);
patchable_enum!(ClaimClaimType);

patchable_struct!(CollectionSimple);
<|endoftext|>


<commit_before>use eyre::Result;
use nom::{
    branch::alt,
    bytes::complete::{tag, take, take_until, take_while1},
    character::complete::{char, digit1, multispace0, multispace1},
    combinator::{map_res, not, peek},
    multi::{fold_many0, separated_list0},
    sequence::{delimited, preceded, tuple},
    IResult,
};
use stencila_schema::{
    Article, BlockContent, Cite, CiteGroup, CodeBlock, CodeFragment, Delete, Emphasis, Heading,
    InlineContent, Link, MathFragment, Node, Paragraph, Strong, Subscript, Superscript,
    ThematicBreak,
};

/// Decode a Markdown document to a `Node`
///
/// Intended for decoding an entire document, this function extracts
/// YAML front matter, parses the Markdown, and returns a `Node::Article` variant.
pub fn decode(md: &str) -> Result<Node> {
    let content = decode_fragment(md);

    let article = Article {
        content: Some(content),
        ..Default::default()
    };

    Ok(Node::Article(article))
}

/// Decode a Markdown fragment to a vector of `BlockContent`
///
/// Intended for decoding a fragment of Markdown (e.g. a Markdown cell in
/// a Jupyter Notebook) and inserting it into a larger document.
///
/// Uses the `pulldown_cmark` and transforms its `Event`s into
/// `Vec<BlockContent>`. Text is further parsed using `nom` based parsers
/// to handle the elements that `pulldown_cmark` does not handle (e.g. math).
pub fn decode_fragment(md: &str) -> Vec<BlockContent> {
    use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag};

    let parser = Parser::new_ext(md, Options::all());

    let mut inline_content: Vec<InlineContent> = Vec::new();
    let mut inline_content_marks: Vec<usize> = Vec::new();

    let mut text = String::new();

    let mut block_content: Vec<BlockContent> = Vec::new();
    for event in parser {
        match event {
            Event::Start(tag) => match tag {
                Tag::Heading(_) => inline_content.clear(),
                Tag::Paragraph => inline_content.clear(),
                Tag::CodeBlock(_) => (),
                Tag::Emphasis => inline_content_marks.push(inline_content.len()),
                Tag::Strong => inline_content_marks.push(inline_content.len()),
                Tag::Strikethrough => inline_content_marks.push(inline_content.len()),
                Tag::Link(_, _, _) => inline_content_marks.push(inline_content.len()),
                _ => println!("Start {:?}", tag),
            },
            Event::End(tag) => match tag {
                Tag::Heading(depth) => block_content.push(BlockContent::Heading(Heading {
                    depth: Some(Box::new(depth as i64)),
                    content: inline_content.clone(),
                    ..Default::default()
                })),
                Tag::Paragraph => block_content.push(BlockContent::Paragraph(Paragraph {
                    content: inline_content.clone(),
                    ..Default::default()
                })),
                Tag::CodeBlock(kind) => block_content.push(BlockContent::CodeBlock(CodeBlock {
                    text: text.clone(),
                    programming_language: match kind {
                        CodeBlockKind::Fenced(lang) => Some(Box::new(lang.to_string())),
                        _ => None,
                    },
                    ..Default::default()
                })),
                Tag::Emphasis => {
                    let n = inline_content_marks
                        .pop()
                        .expect("Unbalanced start and end");
                    let content = inline_content.split_off(n);
                    inline_content.push(InlineContent::Emphasis(Emphasis {
                        content,
                        ..Default::default()
                    }))
                }
                Tag::Strong => {
                    let n = inline_content_marks
                        .pop()
                        .expect("Unbalanced start and end");
                    let content = inline_content.split_off(n);
                    inline_content.push(InlineContent::Strong(Strong {
                        content,
                        ..Default::default()
                    }))
                }
                Tag::Strikethrough => {
                    let n = inline_content_marks
                        .pop()
                        .expect("Unbalanced start and end");
                    let content = inline_content.split_off(n);
                    inline_content.push(InlineContent::Delete(Delete {
                        content,
                        ..Default::default()
                    }))
                }
                Tag::Link(_link_type, url, title) => {
                    let n = inline_content_marks
                        .pop()
                        .expect("Unbalanced start and end");
                    let content = inline_content.split_off(n);
                    inline_content.push(InlineContent::Link(Link {
                        content,
                        target: url.to_string(),
                        title: Some(Box::new(title.to_string())),
                        ..Default::default()
                    }))
                }
                _ => println!(
                    "End {:?} {:?} {:?}",
                    tag, inline_content, inline_content_marks
                ),
            },
            Event::Text(value) => {
                let mut content = decode_inline_content(&value);
                inline_content.append(&mut content);
                text = value.to_string()
            }
            Event::Code(value) => {
                inline_content.push(InlineContent::CodeFragment(CodeFragment {
                    text: value.to_string(),
                    ..Default::default()
                }));
            }
            Event::Html(value) => {
                tracing::warn!("Unhandled Markdown element Html {}", value);
            }
            Event::FootnoteReference(value) => {
                tracing::warn!("Unhandled Markdown element FootnoteReference {}", value);
            }
            Event::SoftBreak => {
                tracing::warn!("Unhandled Markdown element SoftBreak");
            }
            Event::HardBreak => {
                tracing::warn!("Unhandled Markdown element HardBreak");
            }
            Event::Rule => block_content.push(BlockContent::ThematicBreak(ThematicBreak {
                ..Default::default()
            })),
            Event::TaskListMarker(value) => {
                tracing::warn!("Unhandled Markdown element TaskListMarker {}", value);
            }
        };
    }

    block_content
}

/// Decode a string into a vector of `InlineContent` nodes
///
/// This is the entry point into `nom` Markdown parsing functions.
/// It is infallible in that if there is a parse error,
/// the original input string is returned as the only item
/// in the vector (with a warning).
fn decode_inline_content(input: &str) -> Vec<InlineContent> {
    match inline_content(input) {
        Ok((_, content)) => content,
        Err(error) => {
            tracing::warn!("While parsing inline content: {}", error);
            vec![InlineContent::String(String::from(input))]
        }
    }
}

/// Parse a string into a vector of `InlineContent` nodes
///
/// Whilst accumulating, will combine adjacent `String` nodes.
/// This is necessary because of the catch all `character` parser.
fn inline_content(input: &str) -> IResult<&str, Vec<InlineContent>> {
    fold_many0(
        alt((
            cite_group,
            cite,
            math,
            subscript,
            superscript,
            string,
            character,
        )),
        Vec::new(),
        |mut vec: Vec<InlineContent>, node| {
            if let InlineContent::String(string) = &node {
                match vec.last_mut() {
                    Some(InlineContent::String(last)) => last.push_str(&string),
                    _ => vec.push(node),
                }
            } else {
                vec.push(node)
            }
            vec
        },
    )(input)
}

/// Parse a string into a `CiteGroup` node
///
/// Simply collects several `Cite` notes into a `CiteGroup`.
pub fn cite_group(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        delimited(
            char('['),
            separated_list0(delimited(multispace0, tag(";"), multispace0), cite),
            char(']'),
        ),
        |items: Vec<InlineContent>| -> Result<InlineContent> {
            Ok(InlineContent::CiteGroup(CiteGroup {
                items: items
                    .iter()
                    .filter_map(|item| match item {
                        InlineContent::Cite(cite) => Some(cite),
                        _ => None,
                    })
                    .cloned()
                    .collect(),
                ..Default::default()
            }))
        },
    )(input)
}

/// Parse a string into a `Cite` node
///
/// This attempts to follow Pandoc's citation handling as closely as possible
/// (see https://pandoc.org/MANUAL.html#citations).
///
/// The following properties of a `Cite` are parsed:
///   - [x] target
///   - [ ] citation_mode
///   - [ ] page_start
///   - [ ] page_end
///   - [ ] pagination
///   - [ ] citation_prefix
///   - [ ] citation_suffix
///   - [ ] citation_intent
pub fn cite(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        preceded(char('@'), take_while1(|chr: char| chr.is_alphanumeric())),
        |res: &str| -> Result<InlineContent> {
            let target = res.into();
            Ok(InlineContent::Cite(Cite {
                target,
                ..Default::default()
            }))
        },
    )(input)
}

/// Parse a string into an `InlineContent` node
///
/// This attempts to follow Pandoc's match parsing as closely as possible
/// (see https://pandoc.org/MANUAL.html#math).
pub fn math(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        delimited(
            // Pandoc: "opening $ must have a non-space character immediately to its right"
            tuple((char('$'), peek(not(multispace1)))),
            take_until("$"),
            // Pandoc: "the closing $ must have a non-space character immediately to its left,
            // and must not be followed immediately by a digit"
            tuple((peek(not(multispace1)), char('$'), peek(not(digit1)))),
        ),
        |res: &str| -> Result<InlineContent> {
            Ok(InlineContent::MathFragment(MathFragment {
                text: res.into(),
                ..Default::default()
            }))
        },
    )(input)
}

/// Parse a string into a `Subscript` node
pub fn subscript(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        delimited(char('~'), take_until("~"), char('~')),
        |res: &str| -> Result<InlineContent> {
            Ok(InlineContent::Subscript(Subscript {
                content: vec![InlineContent::String(res.into())],
                ..Default::default()
            }))
        },
    )(input)
}

/// Parse a string into a `Superscript` node
pub fn superscript(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        delimited(char('^'), take_until("^"), char('^')),
        |res: &str| -> Result<InlineContent> {
            Ok(InlineContent::Superscript(Superscript {
                content: vec![InlineContent::String(res.into())],
                ..Default::default()
            }))
        },
    )(input)
}

/// Accumulate characters into a `String` node
///
/// Will greedily take as many characters as possible, excluding those that appear at the
/// start of other inline parsers e.g. '$', '['
fn string(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        take_while1(|chr: char| chr != '^' && chr != '~' && chr != '$' && chr != '['),
        |res: &str| -> Result<InlineContent> { Ok(InlineContent::String(String::from(res))) },
    )(input)
}

/// Take a single character into a `String` node
///
/// Necessary so that the characters no consumed by `string` are not lost.
fn character(input: &str) -> IResult<&str, InlineContent> {
    map_res(take(1usize), |res: &str| -> Result<InlineContent> {
        Ok(InlineContent::String(String::from(res)))
    })(input)
}
<commit_msg>fix(Decode Markdown): Handle soft breaks and clean code<commit_after>use eyre::Result;
use nom::{
    branch::alt,
    bytes::complete::{tag, take, take_until, take_while1},
    character::complete::{char, digit1, multispace0, multispace1},
    combinator::{map_res, not, peek},
    multi::{fold_many0, separated_list0},
    sequence::{delimited, preceded, tuple},
    IResult,
};
use stencila_schema::{
    Article, BlockContent, Cite, CiteGroup, CodeBlock, CodeFragment, Delete, Emphasis, Heading,
    InlineContent, Link, MathFragment, Node, Paragraph, Strong, Subscript, Superscript,
    ThematicBreak,
};

/// Decode a Markdown document to a `Node`
///
/// Intended for decoding an entire document, this function extracts
/// YAML front matter, parses the Markdown, and returns a `Node::Article` variant.
pub fn decode(md: &str) -> Result<Node> {
    let content = decode_fragment(md);

    let article = Article {
        content: Some(content),
        ..Default::default()
    };

    Ok(Node::Article(article))
}

/// Decode a Markdown fragment to a vector of `BlockContent`
///
/// Intended for decoding a fragment of Markdown (e.g. a Markdown cell in
/// a Jupyter Notebook) and inserting it into a larger document.
///
/// Uses the `pulldown_cmark` and transforms its `Event`s into
/// `Vec<BlockContent>`. Text is further parsed using `nom` based parsers
/// to handle the elements that `pulldown_cmark` does not handle (e.g. math).
pub fn decode_fragment(md: &str) -> Vec<BlockContent> {
    use pulldown_cmark::{CodeBlockKind, Event, Options, Parser, Tag};

    // Holds the last text node
    let mut text = String::new();
    // Holds and manages inline content nodes
    struct Inlines {
        nodes: Vec<InlineContent>,
        marks: Vec<usize>,
    }

    impl Inlines {
        fn clear(&mut self) {
            self.nodes.clear();
            self.marks.clear()
        }

        fn mark(&mut self) {
            self.marks.push(self.nodes.len())
        }

        fn push(&mut self, node: InlineContent) {
            self.nodes.push(node)
        }

        fn append(&mut self, nodes: &mut Vec<InlineContent>) {
            self.nodes.append(nodes)
        }

        fn condense(mut nodes: Vec<InlineContent>) -> Vec<InlineContent> {
            let mut index = 1;
            while index < nodes.len() {
                // FIXME: Avoid this clone?
                let curr = nodes[index].clone();
                let prev = &mut nodes[index - 1];
                match (prev, curr) {
                    (InlineContent::String(prev), InlineContent::String(curr)) => {
                        if prev.ends_with(|chr: char| !chr.is_whitespace()) && curr == "\u{2029}" {
                            prev.push(' ');
                        } else {
                            prev.push_str(&curr);
                        }
                        nodes.remove(index);
                    }
                    _ => index += 1,
                }
            }
            nodes
        }

        fn take_tail(&mut self) -> Vec<InlineContent> {
            let n = self.marks.pop().expect("Unable to pop marks!");
            Inlines::condense(self.nodes.split_off(n))
        }

        fn take_all(&mut self) -> Vec<InlineContent> {
            Inlines::condense(self.nodes.split_off(0))
        }
    }

    let mut inlines = Inlines {
        nodes: Vec::new(),
        marks: Vec::new(),
    };

    // Holds the block content nodes
    let mut blocks: Vec<BlockContent> = Vec::new();

    let parser = Parser::new_ext(md, Options::all());
    for event in parser {
        match event {
            Event::Start(tag) => match tag {
                Tag::Heading(_) => inlines.clear(),
                Tag::Paragraph => inlines.clear(),
                Tag::CodeBlock(_) => (),
                Tag::Emphasis => inlines.mark(),
                Tag::Strong => inlines.mark(),
                Tag::Strikethrough => inlines.mark(),
                Tag::Link(_, _, _) => inlines.mark(),
                _ => println!("Start {:?}", tag),
            },
            Event::End(tag) => match tag {
                Tag::Heading(depth) => blocks.push(BlockContent::Heading(Heading {
                    depth: Some(Box::new(depth as i64)),
                    content: inlines.take_all(),
                    ..Default::default()
                })),
                Tag::Paragraph => blocks.push(BlockContent::Paragraph(Paragraph {
                    content: inlines.take_all(),
                    ..Default::default()
                })),
                Tag::CodeBlock(kind) => blocks.push(BlockContent::CodeBlock(CodeBlock {
                    text: text.clone(),
                    programming_language: match kind {
                        CodeBlockKind::Fenced(lang) => Some(Box::new(lang.to_string())),
                        _ => None,
                    },
                    ..Default::default()
                })),
                Tag::Emphasis => {
                    let content = inlines.take_tail();
                    inlines.push(InlineContent::Emphasis(Emphasis {
                        content,
                        ..Default::default()
                    }))
                }
                Tag::Strong => {
                    let content = inlines.take_tail();
                    inlines.push(InlineContent::Strong(Strong {
                        content,
                        ..Default::default()
                    }))
                }
                Tag::Strikethrough => {
                    let content = inlines.take_tail();
                    inlines.push(InlineContent::Delete(Delete {
                        content,
                        ..Default::default()
                    }))
                }
                Tag::Link(_link_type, url, title) => {
                    let content = inlines.take_tail();
                    inlines.push(InlineContent::Link(Link {
                        content,
                        target: url.to_string(),
                        title: Some(Box::new(title.to_string())),
                        ..Default::default()
                    }))
                }
                _ => (),
            },
            Event::Text(value) => {
                let mut content = decode_inline_content(&value);
                inlines.append(&mut content);
                text = value.to_string()
            }
            Event::Code(value) => {
                inlines.push(InlineContent::CodeFragment(CodeFragment {
                    text: value.to_string(),
                    ..Default::default()
                }));
            }
            Event::Html(value) => {
                tracing::warn!("Unhandled Markdown element Html {}", value);
            }
            Event::FootnoteReference(value) => {
                tracing::warn!("Unhandled Markdown element FootnoteReference {}", value);
            }
            Event::SoftBreak => {
                // A soft line break event occurs between lines of a multi-line paragraph
                // (between a `Text` event for each line). This inserts the Unicode soft break
                // character so that, when inlines are condensed a space can be added if
                // necessary.
                inlines.push(InlineContent::String("\u{2029}".to_string()))
            }
            Event::HardBreak => {
                tracing::warn!("Unhandled Markdown element HardBreak");
            }
            Event::Rule => blocks.push(BlockContent::ThematicBreak(ThematicBreak {
                ..Default::default()
            })),
            Event::TaskListMarker(value) => {
                tracing::warn!("Unhandled Markdown element TaskListMarker {}", value);
            }
        };
    }

    blocks
}

/// Decode a string into a vector of `InlineContent` nodes
///
/// This is the entry point into `nom` Markdown parsing functions.
/// It is infallible in that if there is a parse error,
/// the original input string is returned as the only item
/// in the vector (with a warning).
fn decode_inline_content(input: &str) -> Vec<InlineContent> {
    match inline_content(input) {
        Ok((_, content)) => content,
        Err(error) => {
            tracing::warn!("While parsing inline content: {}", error);
            vec![InlineContent::String(String::from(input))]
        }
    }
}

/// Parse a string into a vector of `InlineContent` nodes
///
/// Whilst accumulating, will combine adjacent `String` nodes.
/// This is necessary because of the catch all `character` parser.
fn inline_content(input: &str) -> IResult<&str, Vec<InlineContent>> {
    fold_many0(
        alt((
            cite_group,
            cite,
            math,
            subscript,
            superscript,
            string,
            character,
        )),
        Vec::new(),
        |mut vec: Vec<InlineContent>, node| {
            if let InlineContent::String(string) = &node {
                match vec.last_mut() {
                    Some(InlineContent::String(last)) => last.push_str(&string),
                    _ => vec.push(node),
                }
            } else {
                vec.push(node)
            }
            vec
        },
    )(input)
}

/// Parse a string into a `CiteGroup` node
///
/// Simply collects several `Cite` notes into a `CiteGroup`.
pub fn cite_group(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        delimited(
            char('['),
            separated_list0(delimited(multispace0, tag(";"), multispace0), cite),
            char(']'),
        ),
        |items: Vec<InlineContent>| -> Result<InlineContent> {
            Ok(InlineContent::CiteGroup(CiteGroup {
                items: items
                    .iter()
                    .filter_map(|item| match item {
                        InlineContent::Cite(cite) => Some(cite),
                        _ => None,
                    })
                    .cloned()
                    .collect(),
                ..Default::default()
            }))
        },
    )(input)
}

/// Parse a string into a `Cite` node
///
/// This attempts to follow Pandoc's citation handling as closely as possible
/// (see https://pandoc.org/MANUAL.html#citations).
///
/// The following properties of a `Cite` are parsed:
///   - [x] target
///   - [ ] citation_mode
///   - [ ] page_start
///   - [ ] page_end
///   - [ ] pagination
///   - [ ] citation_prefix
///   - [ ] citation_suffix
///   - [ ] citation_intent
pub fn cite(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        preceded(char('@'), take_while1(|chr: char| chr.is_alphanumeric())),
        |res: &str| -> Result<InlineContent> {
            let target = res.into();
            Ok(InlineContent::Cite(Cite {
                target,
                ..Default::default()
            }))
        },
    )(input)
}

/// Parse a string into an `InlineContent` node
///
/// This attempts to follow Pandoc's match parsing as closely as possible
/// (see https://pandoc.org/MANUAL.html#math).
pub fn math(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        delimited(
            // Pandoc: "opening $ must have a non-space character immediately to its right"
            tuple((char('$'), peek(not(multispace1)))),
            take_until("$"),
            // Pandoc: "the closing $ must have a non-space character immediately to its left,
            // and must not be followed immediately by a digit"
            tuple((peek(not(multispace1)), char('$'), peek(not(digit1)))),
        ),
        |res: &str| -> Result<InlineContent> {
            Ok(InlineContent::MathFragment(MathFragment {
                text: res.into(),
                ..Default::default()
            }))
        },
    )(input)
}

/// Parse a string into a `Subscript` node
pub fn subscript(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        delimited(char('~'), take_until("~"), char('~')),
        |res: &str| -> Result<InlineContent> {
            Ok(InlineContent::Subscript(Subscript {
                content: vec![InlineContent::String(res.into())],
                ..Default::default()
            }))
        },
    )(input)
}

/// Parse a string into a `Superscript` node
pub fn superscript(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        delimited(char('^'), take_until("^"), char('^')),
        |res: &str| -> Result<InlineContent> {
            Ok(InlineContent::Superscript(Superscript {
                content: vec![InlineContent::String(res.into())],
                ..Default::default()
            }))
        },
    )(input)
}

/// Accumulate characters into a `String` node
///
/// Will greedily take as many characters as possible, excluding those that appear at the
/// start of other inline parsers e.g. '$', '['
fn string(input: &str) -> IResult<&str, InlineContent> {
    map_res(
        take_while1(|chr: char| chr != '^' && chr != '~' && chr != '$' && chr != '['),
        |res: &str| -> Result<InlineContent> { Ok(InlineContent::String(String::from(res))) },
    )(input)
}

/// Take a single character into a `String` node
///
/// Necessary so that the characters no consumed by `string` are not lost.
fn character(input: &str) -> IResult<&str, InlineContent> {
    map_res(take(1usize), |res: &str| -> Result<InlineContent> {
        Ok(InlineContent::String(String::from(res)))
    })(input)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::utils::tests::snapshot_content;
    use insta::assert_json_snapshot;

    #[test]
    fn fragments() {
        snapshot_content("fragments/md/*.md", |content| {
            assert_json_snapshot!(decode_fragment(&content));
        });
    }
}
<|endoftext|>


<commit_before>use crate::{
    executor::Executor,
    message::{Delivery, DeliveryResult},
    types::ShortString,
    BasicProperties, Error, Result,
};
use crossbeam_channel::{Receiver, Sender};
use futures_core::stream::Stream;
use log::trace;
use parking_lot::{Mutex, MutexGuard};
use std::{
    fmt,
    future::Future,
    pin::Pin,
    sync::Arc,
    task::{Context, Poll, Waker},
};

pub trait ConsumerDelegate: Send + Sync {
    fn on_new_delivery(&self, delivery: DeliveryResult)
        -> Pin<Box<dyn Future<Output = ()> + Send>>;
    fn drop_prefetched_messages(&self) -> Pin<Box<dyn Future<Output = ()> + Send>> {
        Box::pin(async move {})
    }
}

impl<
        F: Future<Output = ()> + Send + 'static,
        DeliveryHandler: Fn(DeliveryResult) -> F + Send + Sync + 'static,
    > ConsumerDelegate for DeliveryHandler
{
    fn on_new_delivery(
        &self,
        delivery: DeliveryResult,
    ) -> Pin<Box<dyn Future<Output = ()> + Send>> {
        Box::pin(self(delivery))
    }
}

#[derive(Clone)]
pub struct Consumer {
    inner: Arc<Mutex<ConsumerInner>>,
}

impl Consumer {
    pub(crate) fn new(consumer_tag: ShortString, executor: Arc<dyn Executor>) -> Consumer {
        Consumer {
            inner: Arc::new(Mutex::new(ConsumerInner::new(consumer_tag, executor))),
        }
    }

    pub fn inner(&self) -> MutexGuard<'_, ConsumerInner> {
        self.inner.lock()
    }

    pub fn set_delegate<D: ConsumerDelegate + 'static>(&self, delegate: D) {
        let mut inner = self.inner();
        while let Some(delivery) = inner.next_delivery() {
            delegate.on_new_delivery(delivery);
        }
        inner.delegate = Some(Arc::new(Box::new(delegate)));
    }

    pub(crate) fn start_new_delivery(&mut self, delivery: Delivery) {
        self.inner().current_message = Some(delivery)
    }

    pub(crate) fn set_delivery_properties(&mut self, properties: BasicProperties) {
        if let Some(delivery) = self.inner().current_message.as_mut() {
            delivery.properties = properties;
        }
    }

    pub(crate) fn receive_delivery_content(&mut self, payload: Vec<u8>) {
        if let Some(delivery) = self.inner().current_message.as_mut() {
            delivery.receive_content(payload);
        }
    }

    pub(crate) fn new_delivery_complete(&mut self) -> Result<()> {
        let mut inner = self.inner();
        if let Some(delivery) = inner.current_message.take() {
            inner.new_delivery(delivery)?;
        }
        Ok(())
    }

    pub(crate) fn drop_prefetched_messages(&self) -> Result<()> {
        self.inner().drop_prefetched_messages()
    }

    pub(crate) fn cancel(&self) -> Result<()> {
        self.inner().cancel()
    }

    pub(crate) fn set_error(&self, error: Error) -> Result<()> {
        self.inner().set_error(error)
    }
}

pub struct ConsumerInner {
    current_message: Option<Delivery>,
    deliveries_in: Sender<DeliveryResult>,
    deliveries_out: Receiver<DeliveryResult>,
    task: Option<Waker>,
    tag: ShortString,
    delegate: Option<Arc<Box<dyn ConsumerDelegate>>>,
    executor: Arc<dyn Executor>,
}

pub struct ConsumerIterator {
    receiver: Receiver<DeliveryResult>,
}

impl Iterator for ConsumerIterator {
    type Item = Result<Delivery>;

    fn next(&mut self) -> Option<Self::Item> {
        self.receiver.recv().ok().and_then(Result::transpose)
    }
}

impl IntoIterator for Consumer {
    type Item = Result<Delivery>;
    type IntoIter = ConsumerIterator;

    fn into_iter(self) -> Self::IntoIter {
        ConsumerIterator {
            receiver: self.inner().deliveries_out.clone(),
        }
    }
}

impl fmt::Debug for Consumer {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let mut debug = f.debug_struct("Consumer");
        if let Some(inner) = self.inner.try_lock() {
            debug
                .field("tag", &inner.tag)
                .field("executor", &inner.executor)
                .field("task", &inner.task);
        }
        debug.finish()
    }
}

impl ConsumerInner {
    fn new(consumer_tag: ShortString, executor: Arc<dyn Executor>) -> Self {
        let (sender, receiver) = crossbeam_channel::unbounded();
        Self {
            current_message: None,
            deliveries_in: sender,
            deliveries_out: receiver,
            task: None,
            tag: consumer_tag,
            delegate: None,
            executor,
        }
    }

    pub fn next_delivery(&mut self) -> Option<DeliveryResult> {
        self.deliveries_out.try_recv().ok()
    }

    pub fn tag(&self) -> &ShortString {
        &self.tag
    }

    fn new_delivery(&mut self, delivery: Delivery) -> Result<()> {
        trace!("new_delivery; consumer_tag={}", self.tag);
        if let Some(delegate) = self.delegate.as_ref() {
            let delegate = delegate.clone();
            self.executor
                .spawn(delegate.on_new_delivery(Ok(Some(delivery))))?;
        } else {
            self.deliveries_in
                .send(Ok(Some(delivery)))
                .expect("failed to send delivery to consumer");
        }
        if let Some(task) = self.task.as_ref() {
            task.wake_by_ref();
        }
        Ok(())
    }

    fn drop_prefetched_messages(&mut self) -> Result<()> {
        trace!("drop_prefetched_messages; consumer_tag={}", self.tag);
        if let Some(delegate) = self.delegate.as_ref() {
            let delegate = delegate.clone();
            self.executor.spawn(delegate.drop_prefetched_messages())?;
        }
        while let Some(_) = self.next_delivery() {}
        Ok(())
    }

    fn cancel(&mut self) -> Result<()> {
        trace!("cancel; consumer_tag={}", self.tag);
        if let Some(delegate) = self.delegate.as_ref() {
            let delegate = delegate.clone();
            self.executor.spawn(delegate.on_new_delivery(Ok(None)))?;
        } else {
            self.deliveries_in
                .send(Ok(None))
                .expect("failed to send cancel to consumer");
        }
        if let Some(task) = self.task.take() {
            task.wake();
        }
        Ok(())
    }

    pub fn set_error(&mut self, error: Error) -> Result<()> {
        trace!("set_error; consumer_tag={}", self.tag);
        if let Some(delegate) = self.delegate.as_ref() {
            let delegate = delegate.clone();
            self.executor.spawn(delegate.on_new_delivery(Err(error)))?;
        } else {
            self.deliveries_in
                .send(Err(error))
                .expect("failed to send error to consumer");
        }
        self.cancel()
    }
}

impl Stream for Consumer {
    type Item = Result<Delivery>;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
        trace!("consumer poll_next");
        let mut inner = self.inner();
        trace!(
            "consumer poll; acquired inner lock, consumer_tag={}",
            inner.tag()
        );
        inner.task = Some(cx.waker().clone());
        if let Some(delivery) = inner.next_delivery() {
            match delivery {
                Ok(Some(delivery)) => {
                    trace!(
                        "delivery; consumer_tag={}, delivery_tag={:?}",
                        inner.tag(),
                        delivery.delivery_tag
                    );
                    Poll::Ready(Some(Ok(delivery)))
                }
                Ok(None) => {
                    trace!("consumer canceled; consumer_tag={}", inner.tag());
                    Poll::Ready(None)
                }
                Err(error) => Poll::Ready(Some(Err(error))),
            }
        } else {
            trace!("delivery; status=NotReady, consumer_tag={}", inner.tag());
            Poll::Pending
        }
    }
}

#[cfg(test)]
mod futures_tests {
    use super::*;
    use crate::executor::DefaultExecutor;

    use std::task::{Context, Poll};

    use futures_test::task::new_count_waker;
    use futures_util::stream::StreamExt;

    #[test]
    fn stream_on_cancel() {
        let (waker, awoken_count) = new_count_waker();
        let mut cx = Context::from_waker(&waker);

        let mut consumer = Consumer::new(
            ShortString::from("test-consumer"),
            Arc::new(DefaultExecutor::default()),
        );

        assert_eq!(awoken_count.get(), 0);
        assert_eq!(consumer.poll_next_unpin(&mut cx), Poll::Pending);

        consumer.cancel().unwrap();

        assert_eq!(awoken_count.get(), 1);
        assert_eq!(consumer.poll_next_unpin(&mut cx), Poll::Ready(None));
    }

    #[test]
    fn stream_on_error() {
        let (waker, awoken_count) = new_count_waker();
        let mut cx = Context::from_waker(&waker);

        let mut consumer = Consumer::new(
            ShortString::from("test-consumer"),
            Arc::new(DefaultExecutor::default()),
        );

        assert_eq!(awoken_count.get(), 0);
        assert_eq!(consumer.poll_next_unpin(&mut cx), Poll::Pending);

        consumer.set_error(Error::InvalidFrameReceived).unwrap();

        assert_eq!(awoken_count.get(), 1);
        assert_eq!(
            consumer.poll_next_unpin(&mut cx),
            Poll::Ready(Some(Err(Error::InvalidFrameReceived)))
        );
    }
}
<commit_msg>use another error in tests<commit_after>use crate::{
    executor::Executor,
    message::{Delivery, DeliveryResult},
    types::ShortString,
    BasicProperties, Error, Result,
};
use crossbeam_channel::{Receiver, Sender};
use futures_core::stream::Stream;
use log::trace;
use parking_lot::{Mutex, MutexGuard};
use std::{
    fmt,
    future::Future,
    pin::Pin,
    sync::Arc,
    task::{Context, Poll, Waker},
};

pub trait ConsumerDelegate: Send + Sync {
    fn on_new_delivery(&self, delivery: DeliveryResult)
        -> Pin<Box<dyn Future<Output = ()> + Send>>;
    fn drop_prefetched_messages(&self) -> Pin<Box<dyn Future<Output = ()> + Send>> {
        Box::pin(async move {})
    }
}

impl<
        F: Future<Output = ()> + Send + 'static,
        DeliveryHandler: Fn(DeliveryResult) -> F + Send + Sync + 'static,
    > ConsumerDelegate for DeliveryHandler
{
    fn on_new_delivery(
        &self,
        delivery: DeliveryResult,
    ) -> Pin<Box<dyn Future<Output = ()> + Send>> {
        Box::pin(self(delivery))
    }
}

#[derive(Clone)]
pub struct Consumer {
    inner: Arc<Mutex<ConsumerInner>>,
}

impl Consumer {
    pub(crate) fn new(consumer_tag: ShortString, executor: Arc<dyn Executor>) -> Consumer {
        Consumer {
            inner: Arc::new(Mutex::new(ConsumerInner::new(consumer_tag, executor))),
        }
    }

    pub fn inner(&self) -> MutexGuard<'_, ConsumerInner> {
        self.inner.lock()
    }

    pub fn set_delegate<D: ConsumerDelegate + 'static>(&self, delegate: D) {
        let mut inner = self.inner();
        while let Some(delivery) = inner.next_delivery() {
            delegate.on_new_delivery(delivery);
        }
        inner.delegate = Some(Arc::new(Box::new(delegate)));
    }

    pub(crate) fn start_new_delivery(&mut self, delivery: Delivery) {
        self.inner().current_message = Some(delivery)
    }

    pub(crate) fn set_delivery_properties(&mut self, properties: BasicProperties) {
        if let Some(delivery) = self.inner().current_message.as_mut() {
            delivery.properties = properties;
        }
    }

    pub(crate) fn receive_delivery_content(&mut self, payload: Vec<u8>) {
        if let Some(delivery) = self.inner().current_message.as_mut() {
            delivery.receive_content(payload);
        }
    }

    pub(crate) fn new_delivery_complete(&mut self) -> Result<()> {
        let mut inner = self.inner();
        if let Some(delivery) = inner.current_message.take() {
            inner.new_delivery(delivery)?;
        }
        Ok(())
    }

    pub(crate) fn drop_prefetched_messages(&self) -> Result<()> {
        self.inner().drop_prefetched_messages()
    }

    pub(crate) fn cancel(&self) -> Result<()> {
        self.inner().cancel()
    }

    pub(crate) fn set_error(&self, error: Error) -> Result<()> {
        self.inner().set_error(error)
    }
}

pub struct ConsumerInner {
    current_message: Option<Delivery>,
    deliveries_in: Sender<DeliveryResult>,
    deliveries_out: Receiver<DeliveryResult>,
    task: Option<Waker>,
    tag: ShortString,
    delegate: Option<Arc<Box<dyn ConsumerDelegate>>>,
    executor: Arc<dyn Executor>,
}

pub struct ConsumerIterator {
    receiver: Receiver<DeliveryResult>,
}

impl Iterator for ConsumerIterator {
    type Item = Result<Delivery>;

    fn next(&mut self) -> Option<Self::Item> {
        self.receiver.recv().ok().and_then(Result::transpose)
    }
}

impl IntoIterator for Consumer {
    type Item = Result<Delivery>;
    type IntoIter = ConsumerIterator;

    fn into_iter(self) -> Self::IntoIter {
        ConsumerIterator {
            receiver: self.inner().deliveries_out.clone(),
        }
    }
}

impl fmt::Debug for Consumer {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let mut debug = f.debug_struct("Consumer");
        if let Some(inner) = self.inner.try_lock() {
            debug
                .field("tag", &inner.tag)
                .field("executor", &inner.executor)
                .field("task", &inner.task);
        }
        debug.finish()
    }
}

impl ConsumerInner {
    fn new(consumer_tag: ShortString, executor: Arc<dyn Executor>) -> Self {
        let (sender, receiver) = crossbeam_channel::unbounded();
        Self {
            current_message: None,
            deliveries_in: sender,
            deliveries_out: receiver,
            task: None,
            tag: consumer_tag,
            delegate: None,
            executor,
        }
    }

    pub fn next_delivery(&mut self) -> Option<DeliveryResult> {
        self.deliveries_out.try_recv().ok()
    }

    pub fn tag(&self) -> &ShortString {
        &self.tag
    }

    fn new_delivery(&mut self, delivery: Delivery) -> Result<()> {
        trace!("new_delivery; consumer_tag={}", self.tag);
        if let Some(delegate) = self.delegate.as_ref() {
            let delegate = delegate.clone();
            self.executor
                .spawn(delegate.on_new_delivery(Ok(Some(delivery))))?;
        } else {
            self.deliveries_in
                .send(Ok(Some(delivery)))
                .expect("failed to send delivery to consumer");
        }
        if let Some(task) = self.task.as_ref() {
            task.wake_by_ref();
        }
        Ok(())
    }

    fn drop_prefetched_messages(&mut self) -> Result<()> {
        trace!("drop_prefetched_messages; consumer_tag={}", self.tag);
        if let Some(delegate) = self.delegate.as_ref() {
            let delegate = delegate.clone();
            self.executor.spawn(delegate.drop_prefetched_messages())?;
        }
        while let Some(_) = self.next_delivery() {}
        Ok(())
    }

    fn cancel(&mut self) -> Result<()> {
        trace!("cancel; consumer_tag={}", self.tag);
        if let Some(delegate) = self.delegate.as_ref() {
            let delegate = delegate.clone();
            self.executor.spawn(delegate.on_new_delivery(Ok(None)))?;
        } else {
            self.deliveries_in
                .send(Ok(None))
                .expect("failed to send cancel to consumer");
        }
        if let Some(task) = self.task.take() {
            task.wake();
        }
        Ok(())
    }

    pub fn set_error(&mut self, error: Error) -> Result<()> {
        trace!("set_error; consumer_tag={}", self.tag);
        if let Some(delegate) = self.delegate.as_ref() {
            let delegate = delegate.clone();
            self.executor.spawn(delegate.on_new_delivery(Err(error)))?;
        } else {
            self.deliveries_in
                .send(Err(error))
                .expect("failed to send error to consumer");
        }
        self.cancel()
    }
}

impl Stream for Consumer {
    type Item = Result<Delivery>;

    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
        trace!("consumer poll_next");
        let mut inner = self.inner();
        trace!(
            "consumer poll; acquired inner lock, consumer_tag={}",
            inner.tag()
        );
        inner.task = Some(cx.waker().clone());
        if let Some(delivery) = inner.next_delivery() {
            match delivery {
                Ok(Some(delivery)) => {
                    trace!(
                        "delivery; consumer_tag={}, delivery_tag={:?}",
                        inner.tag(),
                        delivery.delivery_tag
                    );
                    Poll::Ready(Some(Ok(delivery)))
                }
                Ok(None) => {
                    trace!("consumer canceled; consumer_tag={}", inner.tag());
                    Poll::Ready(None)
                }
                Err(error) => Poll::Ready(Some(Err(error))),
            }
        } else {
            trace!("delivery; status=NotReady, consumer_tag={}", inner.tag());
            Poll::Pending
        }
    }
}

#[cfg(test)]
mod futures_tests {
    use super::*;
    use crate::executor::DefaultExecutor;

    use std::task::{Context, Poll};

    use futures_test::task::new_count_waker;
    use futures_util::stream::StreamExt;

    #[test]
    fn stream_on_cancel() {
        let (waker, awoken_count) = new_count_waker();
        let mut cx = Context::from_waker(&waker);

        let mut consumer = Consumer::new(
            ShortString::from("test-consumer"),
            Arc::new(DefaultExecutor::default()),
        );

        assert_eq!(awoken_count.get(), 0);
        assert_eq!(consumer.poll_next_unpin(&mut cx), Poll::Pending);

        consumer.cancel().unwrap();

        assert_eq!(awoken_count.get(), 1);
        assert_eq!(consumer.poll_next_unpin(&mut cx), Poll::Ready(None));
    }

    #[test]
    fn stream_on_error() {
        let (waker, awoken_count) = new_count_waker();
        let mut cx = Context::from_waker(&waker);

        let mut consumer = Consumer::new(
            ShortString::from("test-consumer"),
            Arc::new(DefaultExecutor::default()),
        );

        assert_eq!(awoken_count.get(), 0);
        assert_eq!(consumer.poll_next_unpin(&mut cx), Poll::Pending);

        consumer.set_error(Error::ChannelsLimitReached).unwrap();

        assert_eq!(awoken_count.get(), 1);
        assert_eq!(
            consumer.poll_next_unpin(&mut cx),
            Poll::Ready(Some(Err(Error::ChannelsLimitReached)))
        );
    }
}
<|endoftext|>


<commit_before>//! Directory module for managing Algorithmia Data Directories
//!
//! # Examples
//!
//! ```no_run
//! use algorithmia::Algorithmia;
//! use std::fs::File;
//!
//! let client = Algorithmia::client("111112222233333444445555566");
//! let my_dir = client.dir(".my/my_dir");
//!
//! my_dir.create();
//! my_dir.put_file("/path/to/file");
//! ```

extern crate chrono;

use client::HttpClient;
use error::*;
use data::{self, DataFile, DeletedResult, XDataType};
use data::file::FileAdded;
use data::HasDataPath;
use json_helpers;

use std::io::Read;
use std::fs::File;
use std::path::Path;
use std::vec::IntoIter;
use std::ops::Deref;
use std::error::Error as StdError;

use hyper::header::ContentType;
use hyper::mime::{Mime, TopLevel, SubLevel};
use hyper::Url;
use rustc_serialize::{json, Decoder, Decodable};
use self::chrono::{DateTime, UTC};

/// Algorithmia Data Directory
pub struct DataDir {
    path: String,
    client: HttpClient,
}

#[derive(RustcDecodable, Debug)]
pub struct DirectoryUpdated {
    pub acl: Option<DataAcl>,
}

/// Response when deleting a new Directory
#[derive(RustcDecodable, Debug)]
pub struct DirectoryDeleted {
    // Omitting deleted.number and error.number for now
    pub result: DeletedResult,
}

#[derive(RustcDecodable, RustcEncodable, Debug)]
struct FolderItem {
    pub name: String,
    pub acl: Option<DataAcl>,
}

#[derive(Debug)]
struct FileItem {
    pub filename: String,
    pub size: u64,
    pub last_modified: DateTime<UTC>,
}

// Manual implemented Decodable: https://github.com/lifthrasiir/rust-chrono/issues/43
impl Decodable for FileItem {
    fn decode<D: Decoder>(d: &mut D) -> Result<FileItem, D::Error> {
        d.read_struct("root", 0, |d| {
            Ok(FileItem{
                filename: try!(d.read_struct_field("filename", 0, |d| Decodable::decode(d))),
                size: try!(d.read_struct_field("size", 0, |d| Decodable::decode(d))),
                last_modified: {
                    let json_str: String = try!(d.read_struct_field("last_modified", 0, |d| Decodable::decode(d)));
                    match json_str.parse() {
                        Ok(datetime) => datetime,
                        Err(err) => return Err(d.error(err.description())),
                    }
                },
            })
        })
    }
}

#[derive(RustcDecodable, RustcEncodable, Debug)]
pub struct DataAcl {
    pub read: Vec<String>
}

/// Response when querying an existing Directory
#[derive(RustcDecodable, Debug)]
struct DirectoryShow {
    pub acl: Option<DataAcl>,
    pub folders: Option<Vec<FolderItem>>,
    pub files: Option<Vec<FileItem>>,
    pub marker: Option<String>,
}

pub struct DirectoryListing<'a> {
    pub acl: Option<DataAcl>,
    dir: &'a DataDir,
    folders: IntoIter<FolderItem>,
    files: IntoIter<FileItem>,
    marker: Option<String>,
    query_count: u32,
}

impl <'a> DirectoryListing<'a> {
    fn new(dir: &'a DataDir) -> DirectoryListing<'a> {
        DirectoryListing {
            acl: None,
            dir: dir,
            folders: Vec::new().into_iter(),
            files: Vec::new().into_iter(),
            marker: None,
            query_count: 0,
        }
    }
}

pub struct DataFileEntry {
    pub size: u64,
    pub last_modified: DateTime<UTC>,
    file: DataFile,
}

impl Deref for DataFileEntry {
    type Target = DataFile;
    fn deref(&self) -> &DataFile {&self.file}
}

pub enum DirEntry {
    File(DataFileEntry),
    Dir(DataDir),
}

impl <'a> Iterator for DirectoryListing<'a> {
    type Item = Result<DirEntry, Error>;

    fn next(&mut self) -> Option<Self::Item> {
        match self.folders.next() {
            // Return folders first
            Some(d) => Some(Ok(DirEntry::Dir(self.dir.child(&d.name)))),
            None => match self.files.next() {
                // Return files second
                Some(f) => Some(Ok(DirEntry::File(
                    DataFileEntry{
                        size: f.size,
                        last_modified: f.last_modified,
                        file: self.dir.child(&f.filename),
                    }
                ))),
                None => {
                    // Query if there is another page of files/folders
                    if self.query_count == 0 || self.marker.is_some() {
                        match get_directory(self.dir, self.marker.clone()) {
                            Ok(ds) => {
                                self.query_count = self.query_count + 1;
                                self.folders = ds.folders.unwrap_or(Vec::new()).into_iter();
                                self.files = ds.files.unwrap_or(Vec::new()).into_iter();
                                self.marker = ds.marker;
                                self.next()
                            }
                            Err(err) => Some(Err(err)),
                        }
                    } else {
                        None
                    }
                }
            }
        }
    }
}

fn get_directory(dir: &DataDir, marker: Option<String>) -> Result<DirectoryShow, Error> {
    let url = match marker {
        Some(m) => Url::parse(&format!("{}?marker={}", dir.to_url(), m)).unwrap(),
        None => dir.to_url(),
    };

    let req = dir.client.get(url);
    let mut res = try!(req.send());

    if res.status.is_success() {
        if let Some(data_type) = res.headers.get::<XDataType>() {
            if "directory" != data_type.to_string() {
                return Err(Error::DataTypeError(format!("Expected directory, Received {}", data_type)));
            }
        }
    }

    let mut res_json = String::new();
    try!(res.read_to_string(&mut res_json));

    match res.status.is_success() {
        true => json_helpers::decode_to_result(res_json),
        false => Err(json_helpers::decode_to_error(res_json)),
    }
}

impl HasDataPath for DataDir {
    fn new(client: HttpClient, path: &str) -> Self { DataDir { client: client, path: data::parse_data_uri(path).to_string() } }
    fn path(&self) -> &str { &self.path }
    fn client(&self) -> &HttpClient { &self.client }
}

impl DataDir {
    /// Display Directory details if it exists
    ///
    /// # Examples
    /// ```no_run
    /// # use algorithmia::Algorithmia;
    /// # use algorithmia::data::{DirEntry, HasDataPath};
    /// let client = Algorithmia::client("111112222233333444445555566");
    /// let my_dir = client.dir(".my/my_dir");
    /// let dir_list = my_dir.list();
    /// for entry in dir_list {
    ///     match entry {
    ///         Ok(DirEntry::File(f)) => println!("File: {}", f.to_data_uri()),
    ///         Ok(DirEntry::Dir(d)) => println!("Dir: {}", d.to_data_uri()),
    ///         Err(err) => { println!("Error: {}", err); break; },
    ///     }
    /// };
    /// ```
    pub fn list(&self) -> DirectoryListing {
        DirectoryListing::new(self)
    }

    /// Create a Directory
    ///
    /// # Examples
    /// ```no_run
    /// # use algorithmia::Algorithmia;
    /// let client = Algorithmia::client("111112222233333444445555566");
    /// let my_dir = client.dir(".my/my_dir");
    /// match my_dir.create() {
    ///   Ok(_) => println!("Successfully created Directory"),
    ///   Err(e) => println!("Error created directory: {}", e),
    /// };
    /// ```
    pub fn create(&self) -> Result<(), Error> {
        let parent = try!(self.parent().ok_or(Error::DataPathError("has no parent".into())));
        let url = parent.to_url();

        // TODO: address complete abuse of this structure
        let input_data = FolderItem {
            name: try!(self.basename().ok_or(Error::DataPathError("has no basename".into()))).into(),
            acl: Some(DataAcl { read: vec![] }),
        };
        let raw_input = try!(json::encode(&input_data));

        // POST request
        let req = self.client.post(url)
            .header(ContentType(Mime(TopLevel::Application, SubLevel::Json, vec![])))
            .body(&raw_input);

        // Parse response
        let mut res = try!(req.send());

        match res.status.is_success() {
            true => Ok(()),
            false => {
                let mut res_json = String::new();
                try!(res.read_to_string(&mut res_json));
                Err(json_helpers::decode_to_error(res_json))
            }
        }
    }

    /// Delete a Directory
    ///
    /// # Examples
    /// ```no_run
    /// # use algorithmia::Algorithmia;
    /// let client = Algorithmia::client("111112222233333444445555566");
    /// let my_dir = client.dir(".my/my_dir");
    /// match my_dir.delete(false) {
    ///   Ok(_) => println!("Successfully deleted Directory"),
    ///   Err(err) => println!("Error deleting directory: {}", err),
    /// };
    /// ```
    pub fn delete(&self, force: bool) -> Result<DirectoryDeleted, Error> {
        // DELETE request
        let url_string = format!("{}?force={}", self.to_url(), force.to_string());
        let url = Url::parse(&url_string).unwrap();
        let req = self.client.delete(url);

        // Parse response
        let mut res = try!(req.send());
        let mut res_json = String::new();
        try!(res.read_to_string(&mut res_json));

        match res.status.is_success() {
            true => json_helpers::decode_to_result(res_json),
            false => Err(json_helpers::decode_to_error(res_json)),
        }
    }

    /// Upload a file to an existing Directory
    ///
    /// # Examples
    /// ```no_run
    /// # use algorithmia::Algorithmia;
    /// let client = Algorithmia::client("111112222233333444445555566");
    /// let my_dir = client.dir(".my/my_dir");
    ///
    /// match my_dir.put_file("/path/to/file") {
    ///   Ok(response) => println!("Successfully uploaded to: {}", response.result),
    ///   Err(err) => println!("Error uploading file: {}", err),
    /// };
    /// ```
    pub fn put_file<P: AsRef<Path>>(&self, file_path: P) -> Result<FileAdded, Error> {
        // FIXME: A whole lot of unwrap going on here...
        let path_ref = file_path.as_ref();
        let url_string = format!("{}/{}",
            self.to_url(),
            path_ref.file_name().unwrap().to_str().unwrap()
        );
        let url = Url::parse(&url_string).unwrap();

        let mut file = File::open(path_ref).unwrap();
        let req = self.client.put(url).body(&mut file);

        let mut res = try!(req.send());
        let mut res_json = String::new();
        try!(res.read_to_string(&mut res_json));

        match res.status.is_success() {
            true => json_helpers::decode_to_result(res_json),
            false => Err(json_helpers::decode_to_error(res_json)),
        }
    }

    pub fn child<T: HasDataPath>(&self, filename: &str) -> T {
        T::new(self.client.clone(), &format!("{}/{}", self.to_data_uri(), filename))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use data::HasDataPath;
    use Algorithmia;

    fn mock_client() -> Algorithmia { Algorithmia::client("") }

    #[test]
    fn test_to_url() {
        let mock_client = mock_client();
        let dir = DataDir::new(mock_client.http_client, "data://anowell/foo");
        assert_eq!(dir.to_url().serialize_path().unwrap(), "/v1/data/anowell/foo");
    }

    #[test]
    fn test_to_data_uri() {
        let dir = DataDir::new(mock_client().http_client, "/anowell/foo");
        assert_eq!(dir.to_data_uri(), "data://anowell/foo".to_string());
    }

    #[test]
    fn test_parent() {
        let dir = DataDir::new(mock_client().http_client, "data://anowell/foo");
        let expected = DataDir::new(mock_client().http_client, "data://anowell");
        assert_eq!(dir.parent().unwrap().path, expected.path);
    }
}<commit_msg>Make tests less brittle (to other changes that were removed)<commit_after>//! Directory module for managing Algorithmia Data Directories
//!
//! # Examples
//!
//! ```no_run
//! use algorithmia::Algorithmia;
//! use std::fs::File;
//!
//! let client = Algorithmia::client("111112222233333444445555566");
//! let my_dir = client.dir(".my/my_dir");
//!
//! my_dir.create();
//! my_dir.put_file("/path/to/file");
//! ```

extern crate chrono;

use client::HttpClient;
use error::*;
use data::{self, DataFile, DeletedResult, XDataType};
use data::file::FileAdded;
use data::HasDataPath;
use json_helpers;

use std::io::Read;
use std::fs::File;
use std::path::Path;
use std::vec::IntoIter;
use std::ops::Deref;
use std::error::Error as StdError;

use hyper::header::ContentType;
use hyper::mime::{Mime, TopLevel, SubLevel};
use hyper::Url;
use rustc_serialize::{json, Decoder, Decodable};
use self::chrono::{DateTime, UTC};

/// Algorithmia Data Directory
pub struct DataDir {
    path: String,
    client: HttpClient,
}

#[derive(RustcDecodable, Debug)]
pub struct DirectoryUpdated {
    pub acl: Option<DataAcl>,
}

/// Response when deleting a new Directory
#[derive(RustcDecodable, Debug)]
pub struct DirectoryDeleted {
    // Omitting deleted.number and error.number for now
    pub result: DeletedResult,
}

#[derive(RustcDecodable, RustcEncodable, Debug)]
struct FolderItem {
    pub name: String,
    pub acl: Option<DataAcl>,
}

#[derive(Debug)]
struct FileItem {
    pub filename: String,
    pub size: u64,
    pub last_modified: DateTime<UTC>,
}

// Manual implemented Decodable: https://github.com/lifthrasiir/rust-chrono/issues/43
impl Decodable for FileItem {
    fn decode<D: Decoder>(d: &mut D) -> Result<FileItem, D::Error> {
        d.read_struct("root", 0, |d| {
            Ok(FileItem{
                filename: try!(d.read_struct_field("filename", 0, |d| Decodable::decode(d))),
                size: try!(d.read_struct_field("size", 0, |d| Decodable::decode(d))),
                last_modified: {
                    let json_str: String = try!(d.read_struct_field("last_modified", 0, |d| Decodable::decode(d)));
                    match json_str.parse() {
                        Ok(datetime) => datetime,
                        Err(err) => return Err(d.error(err.description())),
                    }
                },
            })
        })
    }
}

#[derive(RustcDecodable, RustcEncodable, Debug)]
pub struct DataAcl {
    pub read: Vec<String>
}

/// Response when querying an existing Directory
#[derive(RustcDecodable, Debug)]
struct DirectoryShow {
    pub acl: Option<DataAcl>,
    pub folders: Option<Vec<FolderItem>>,
    pub files: Option<Vec<FileItem>>,
    pub marker: Option<String>,
}

pub struct DirectoryListing<'a> {
    pub acl: Option<DataAcl>,
    dir: &'a DataDir,
    folders: IntoIter<FolderItem>,
    files: IntoIter<FileItem>,
    marker: Option<String>,
    query_count: u32,
}

impl <'a> DirectoryListing<'a> {
    fn new(dir: &'a DataDir) -> DirectoryListing<'a> {
        DirectoryListing {
            acl: None,
            dir: dir,
            folders: Vec::new().into_iter(),
            files: Vec::new().into_iter(),
            marker: None,
            query_count: 0,
        }
    }
}

pub struct DataFileEntry {
    pub size: u64,
    pub last_modified: DateTime<UTC>,
    file: DataFile,
}

impl Deref for DataFileEntry {
    type Target = DataFile;
    fn deref(&self) -> &DataFile {&self.file}
}

pub enum DirEntry {
    File(DataFileEntry),
    Dir(DataDir),
}

impl <'a> Iterator for DirectoryListing<'a> {
    type Item = Result<DirEntry, Error>;

    fn next(&mut self) -> Option<Self::Item> {
        match self.folders.next() {
            // Return folders first
            Some(d) => Some(Ok(DirEntry::Dir(self.dir.child(&d.name)))),
            None => match self.files.next() {
                // Return files second
                Some(f) => Some(Ok(DirEntry::File(
                    DataFileEntry{
                        size: f.size,
                        last_modified: f.last_modified,
                        file: self.dir.child(&f.filename),
                    }
                ))),
                None => {
                    // Query if there is another page of files/folders
                    if self.query_count == 0 || self.marker.is_some() {
                        match get_directory(self.dir, self.marker.clone()) {
                            Ok(ds) => {
                                self.query_count = self.query_count + 1;
                                self.folders = ds.folders.unwrap_or(Vec::new()).into_iter();
                                self.files = ds.files.unwrap_or(Vec::new()).into_iter();
                                self.marker = ds.marker;
                                self.next()
                            }
                            Err(err) => Some(Err(err)),
                        }
                    } else {
                        None
                    }
                }
            }
        }
    }
}

fn get_directory(dir: &DataDir, marker: Option<String>) -> Result<DirectoryShow, Error> {
    let url = match marker {
        Some(m) => Url::parse(&format!("{}?marker={}", dir.to_url(), m)).unwrap(),
        None => dir.to_url(),
    };

    let req = dir.client.get(url);
    let mut res = try!(req.send());

    if res.status.is_success() {
        if let Some(data_type) = res.headers.get::<XDataType>() {
            if "directory" != data_type.to_string() {
                return Err(Error::DataTypeError(format!("Expected directory, Received {}", data_type)));
            }
        }
    }

    let mut res_json = String::new();
    try!(res.read_to_string(&mut res_json));

    match res.status.is_success() {
        true => json_helpers::decode_to_result(res_json),
        false => Err(json_helpers::decode_to_error(res_json)),
    }
}

impl HasDataPath for DataDir {
    fn new(client: HttpClient, path: &str) -> Self { DataDir { client: client, path: data::parse_data_uri(path).to_string() } }
    fn path(&self) -> &str { &self.path }
    fn client(&self) -> &HttpClient { &self.client }
}

impl DataDir {
    /// Display Directory details if it exists
    ///
    /// # Examples
    /// ```no_run
    /// # use algorithmia::Algorithmia;
    /// # use algorithmia::data::{DirEntry, HasDataPath};
    /// let client = Algorithmia::client("111112222233333444445555566");
    /// let my_dir = client.dir(".my/my_dir");
    /// let dir_list = my_dir.list();
    /// for entry in dir_list {
    ///     match entry {
    ///         Ok(DirEntry::File(f)) => println!("File: {}", f.to_data_uri()),
    ///         Ok(DirEntry::Dir(d)) => println!("Dir: {}", d.to_data_uri()),
    ///         Err(err) => { println!("Error: {}", err); break; },
    ///     }
    /// };
    /// ```
    pub fn list(&self) -> DirectoryListing {
        DirectoryListing::new(self)
    }

    /// Create a Directory
    ///
    /// # Examples
    /// ```no_run
    /// # use algorithmia::Algorithmia;
    /// let client = Algorithmia::client("111112222233333444445555566");
    /// let my_dir = client.dir(".my/my_dir");
    /// match my_dir.create() {
    ///   Ok(_) => println!("Successfully created Directory"),
    ///   Err(e) => println!("Error created directory: {}", e),
    /// };
    /// ```
    pub fn create(&self) -> Result<(), Error> {
        let parent = try!(self.parent().ok_or(Error::DataPathError("has no parent".into())));
        let url = parent.to_url();

        // TODO: address complete abuse of this structure
        let input_data = FolderItem {
            name: try!(self.basename().ok_or(Error::DataPathError("has no basename".into()))).into(),
            acl: Some(DataAcl { read: vec![] }),
        };
        let raw_input = try!(json::encode(&input_data));

        // POST request
        let req = self.client.post(url)
            .header(ContentType(Mime(TopLevel::Application, SubLevel::Json, vec![])))
            .body(&raw_input);

        // Parse response
        let mut res = try!(req.send());

        match res.status.is_success() {
            true => Ok(()),
            false => {
                let mut res_json = String::new();
                try!(res.read_to_string(&mut res_json));
                Err(json_helpers::decode_to_error(res_json))
            }
        }
    }

    /// Delete a Directory
    ///
    /// # Examples
    /// ```no_run
    /// # use algorithmia::Algorithmia;
    /// let client = Algorithmia::client("111112222233333444445555566");
    /// let my_dir = client.dir(".my/my_dir");
    /// match my_dir.delete(false) {
    ///   Ok(_) => println!("Successfully deleted Directory"),
    ///   Err(err) => println!("Error deleting directory: {}", err),
    /// };
    /// ```
    pub fn delete(&self, force: bool) -> Result<DirectoryDeleted, Error> {
        // DELETE request
        let url_string = format!("{}?force={}", self.to_url(), force.to_string());
        let url = Url::parse(&url_string).unwrap();
        let req = self.client.delete(url);

        // Parse response
        let mut res = try!(req.send());
        let mut res_json = String::new();
        try!(res.read_to_string(&mut res_json));

        match res.status.is_success() {
            true => json_helpers::decode_to_result(res_json),
            false => Err(json_helpers::decode_to_error(res_json)),
        }
    }

    /// Upload a file to an existing Directory
    ///
    /// # Examples
    /// ```no_run
    /// # use algorithmia::Algorithmia;
    /// let client = Algorithmia::client("111112222233333444445555566");
    /// let my_dir = client.dir(".my/my_dir");
    ///
    /// match my_dir.put_file("/path/to/file") {
    ///   Ok(response) => println!("Successfully uploaded to: {}", response.result),
    ///   Err(err) => println!("Error uploading file: {}", err),
    /// };
    /// ```
    pub fn put_file<P: AsRef<Path>>(&self, file_path: P) -> Result<FileAdded, Error> {
        // FIXME: A whole lot of unwrap going on here...
        let path_ref = file_path.as_ref();
        let url_string = format!("{}/{}",
            self.to_url(),
            path_ref.file_name().unwrap().to_str().unwrap()
        );
        let url = Url::parse(&url_string).unwrap();

        let mut file = File::open(path_ref).unwrap();
        let req = self.client.put(url).body(&mut file);

        let mut res = try!(req.send());
        let mut res_json = String::new();
        try!(res.read_to_string(&mut res_json));

        match res.status.is_success() {
            true => json_helpers::decode_to_result(res_json),
            false => Err(json_helpers::decode_to_error(res_json)),
        }
    }

    pub fn child<T: HasDataPath>(&self, filename: &str) -> T {
        T::new(self.client.clone(), &format!("{}/{}", self.to_data_uri(), filename))
    }
}

#[cfg(test)]
mod tests {
    use data::HasDataPath;
    use Algorithmia;

    fn mock_client() -> Algorithmia { Algorithmia::client("") }

    #[test]
    fn test_to_url() {
        let dir = mock_client().dir("data://anowell/foo");
        assert_eq!(dir.to_url().serialize_path().unwrap(), "/v1/data/anowell/foo");
    }

    #[test]
    fn test_to_data_uri() {
        let dir = mock_client().dir("/anowell/foo");
        assert_eq!(dir.to_data_uri(), "data://anowell/foo".to_string());
    }

    #[test]
    fn test_parent() {
        let dir = mock_client().dir("data://anowell/foo");
        let expected = mock_client().dir("data://anowell");
        assert_eq!(dir.parent().unwrap().path, expected.path);
    }
}<|endoftext|>


<commit_before>// Copyright 2017 tokio-jsonrpc Developers
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

use message::RPCError;

use serde::Serialize;
use serde_json::Value;
use futures::IntoFuture;

/// The server endpoint
///
/// This is usually implemented by the end application and provides the actual functionality of the
/// RPC server. It allows composition of more servers together.
///
/// In future it might be possible to generate servers with the help of some macros. Currently it
/// is up to the developer to handle conversion of parameters, etc.
pub trait Server {
    /// The successfull result of RPC call.
    type Success: Serialize;
    /// The result of the RPC call
    ///
    /// Once the future resolves, the value or error is sent to the client as the reply. The reply
    /// is wrapped automatically.
    type RPCCallResult: IntoFuture<Item = Self::Success, Error = RPCError>;
    /// The result of the RPC call
    ///
    /// As the client doesn't expect anything in return, both the success and error results are
    /// thrown away and therefore (). However, it still makes sense to distinguish success and
    /// error.
    type NotificationResult: IntoFuture<Item = (), Error = ()>;
    /// Called when the client requests something
    ///
    /// This is a callback from the [endpoint](struct.Endpoint.html) when the client requests
    /// something. If the method is unknown, it shall return `None`. This allows composition of
    /// servers.
    ///
    /// Conversion of parameters and handling of errors is up to the implementer of this trait.
    fn rpc(method: &str, params: &Option<Value>) -> Option<Self::RPCCallResult>;
    /// Called when the client sends a notification
    ///
    /// This is a callback from the [endpoint](struct.Endpoint.html) when the client requests
    /// something. If the method is unknown, it shall return `None`. This allows composition of
    /// servers.
    ///
    /// Conversion of parameters and handling of errors is up to the implementer of this trait.
    fn notification(method: &str, params: &Option<Value>) -> Option<Self::NotificationResult>;
}
<commit_msg>Endpoint: Provide the EmptyServer<commit_after>// Copyright 2017 tokio-jsonrpc Developers
//
// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
// http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

//! The endpoint of the JSON RPC connection
//!
//! This module helps building the endpoints of the connection. The endpoints act as both client
//! and server at the same time. If you want a client-only endpoint, use
//! [`EmptyServer`](struct.EmptyServer.html) as the server. If you want a server-only endpoint,
//! simply don't call any RPCs or notifications.

use message::RPCError;

use serde::Serialize;
use serde_json::Value;
use futures::IntoFuture;

/// The server endpoint
///
/// This is usually implemented by the end application and provides the actual functionality of the
/// RPC server. It allows composition of more servers together.
///
/// In future it might be possible to generate servers with the help of some macros. Currently it
/// is up to the developer to handle conversion of parameters, etc.
///
/// The default implementations of the callbacks return None, indicating that the given method is
/// not known. It allows implementing only rpcs or only notifications without having to worry about
/// the other callback. If you want a server that knows nothing at all, use
/// [`EmptyServer`](struct.EmptyServer.html).
pub trait Server {
    /// The successfull result of the RPC call.
    type Success: Serialize;
    /// The result of the RPC call
    ///
    /// Once the future resolves, the value or error is sent to the client as the reply. The reply
    /// is wrapped automatically.
    type RPCCallResult: IntoFuture<Item = Self::Success, Error = RPCError>;
    /// The result of the RPC call
    ///
    /// As the client doesn't expect anything in return, both the success and error results are
    /// thrown away and therefore (). However, it still makes sense to distinguish success and
    /// error.
    type NotificationResult: IntoFuture<Item = (), Error = ()>;
    /// Called when the client requests something
    ///
    /// This is a callback from the [endpoint](struct.Endpoint.html) when the client requests
    /// something. If the method is unknown, it shall return `None`. This allows composition of
    /// servers.
    ///
    /// Conversion of parameters and handling of errors is up to the implementer of this trait.
    fn rpc(_method: &str, _params: &Option<Value>) -> Option<Self::RPCCallResult> {
        None
    }
    /// Called when the client sends a notification
    ///
    /// This is a callback from the [endpoint](struct.Endpoint.html) when the client requests
    /// something. If the method is unknown, it shall return `None`. This allows composition of
    /// servers.
    ///
    /// Conversion of parameters and handling of errors is up to the implementer of this trait.
    fn notification(_method: &str, _params: &Option<Value>) -> Option<Self::NotificationResult> {
        None
    }
}

/// A RPC server that knows no methods
///
/// You can use this if you want to have a client-only [Endpoint](struct.Endpoint.html). It simply
/// refuses all the methods passed to it.
pub struct EmptyServer;

impl Server for EmptyServer {
    type Success = ();
    type RPCCallResult = Result<(), RPCError>;
    type NotificationResult = Result<(), ()>;
}
<|endoftext|>


<commit_before>use rand;
use rand::Rng;

pub mod constants;

pub fn get_random_bool() -> bool {
  let mut rnd = rand::thread_rng();
  rnd.gen()
}

pub fn get_tenth_bool() -> bool {
  let mut rnd = rand::thread_rng();
  rnd.gen::<f32>() < 0.05
}

<commit_msg>Do not warn about unused debug code<commit_after>use rand;
use rand::Rng;

pub mod constants;

pub fn get_random_bool() -> bool {
  let mut rnd = rand::thread_rng();
  rnd.gen()
}

#[allow(dead_code)]
pub fn get_weighted_random(weight: f32) -> bool {
  let mut rnd = rand::thread_rng();
  rnd.gen::<f32>() < weight
}

<|endoftext|>