<commit_before>#![feature(test)]

extern crate crossbeam_utils;
extern crate test;

use std::sync::Barrier;

use crossbeam_utils::atomic::AtomicCell;
use crossbeam_utils::thread;

#[bench]
fn load_u8(b: &mut test::Bencher) {
    let a = AtomicCell::new(0u8);
    let mut sum = 0;
    b.iter(|| sum += a.load());
    test::black_box(sum);
}

#[bench]
fn store_u8(b: &mut test::Bencher) {
    let a = AtomicCell::new(0u8);
    b.iter(|| a.store(1));
}

#[bench]
fn fetch_add_u8(b: &mut test::Bencher) {
    let a = AtomicCell::new(0u8);
    b.iter(|| a.fetch_add(1));
}

#[bench]
fn compare_and_swap_u8(b: &mut test::Bencher) {
    let a = AtomicCell::new(0u8);
    let mut i = 0;
    b.iter(|| {
        a.compare_and_swap(i, i.wrapping_add(1));
        i = i.wrapping_add(1);
    });
}

#[bench]
fn concurrent_load_u8(b: &mut test::Bencher) {
    const THREADS: usize = 2;
    const STEPS: usize = 1_000_000;

    let start = Barrier::new(THREADS + 1);
    let end = Barrier::new(THREADS + 1);
    let exit = AtomicCell::new(false);

    let a = AtomicCell::new(0u8);

    thread::scope(|scope| {
        for _ in 0..THREADS {
            scope.spawn(|| loop {
                start.wait();

                let mut sum = 0;
                for _ in 0..STEPS {
                    sum += a.load();
                }
                test::black_box(sum);

                end.wait();
                if exit.load() {
                    break;
                }
            });
        }

        start.wait();
        end.wait();

        b.iter(|| {
            start.wait();
            end.wait();
        });

        start.wait();
        exit.store(true);
        end.wait();
    }).unwrap();
}

#[bench]
fn load_usize(b: &mut test::Bencher) {
    let a = AtomicCell::new(0usize);
    let mut sum = 0;
    b.iter(|| sum += a.load());
    test::black_box(sum);
}

#[bench]
fn store_usize(b: &mut test::Bencher) {
    let a = AtomicCell::new(0usize);
    b.iter(|| a.store(1));
}

#[bench]
fn fetch_add_usize(b: &mut test::Bencher) {
    let a = AtomicCell::new(0usize);
    b.iter(|| a.fetch_add(1));
}

#[bench]
fn compare_and_swap_usize(b: &mut test::Bencher) {
    let a = AtomicCell::new(0usize);
    let mut i = 0;
    b.iter(|| {
        a.compare_and_swap(i, i.wrapping_add(1));
        i = i.wrapping_add(1);
    });
}

#[bench]
fn concurrent_load_usize(b: &mut test::Bencher) {
    const THREADS: usize = 2;
    const STEPS: usize = 1_000_000;

    let start = Barrier::new(THREADS + 1);
    let end = Barrier::new(THREADS + 1);
    let exit = AtomicCell::new(false);

    let a = AtomicCell::new(0usize);

    thread::scope(|scope| {
        for _ in 0..THREADS {
            scope.spawn(|| loop {
                start.wait();

                let mut sum = 0;
                for _ in 0..STEPS {
                    sum += a.load();
                }
                test::black_box(sum);

                end.wait();
                if exit.load() {
                    break;
                }
            });
        }

        start.wait();
        end.wait();

        b.iter(|| {
            start.wait();
            end.wait();
        });

        start.wait();
        exit.store(true);
        end.wait();
    }).unwrap();
}
<commit_msg>chmod -x *.rs<commit_after>#![feature(test)]

extern crate crossbeam_utils;
extern crate test;

use std::sync::Barrier;

use crossbeam_utils::atomic::AtomicCell;
use crossbeam_utils::thread;

#[bench]
fn load_u8(b: &mut test::Bencher) {
    let a = AtomicCell::new(0u8);
    let mut sum = 0;
    b.iter(|| sum += a.load());
    test::black_box(sum);
}

#[bench]
fn store_u8(b: &mut test::Bencher) {
    let a = AtomicCell::new(0u8);
    b.iter(|| a.store(1));
}

#[bench]
fn fetch_add_u8(b: &mut test::Bencher) {
    let a = AtomicCell::new(0u8);
    b.iter(|| a.fetch_add(1));
}

#[bench]
fn compare_and_swap_u8(b: &mut test::Bencher) {
    let a = AtomicCell::new(0u8);
    let mut i = 0;
    b.iter(|| {
        a.compare_and_swap(i, i.wrapping_add(1));
        i = i.wrapping_add(1);
    });
}

#[bench]
fn concurrent_load_u8(b: &mut test::Bencher) {
    const THREADS: usize = 2;
    const STEPS: usize = 1_000_000;

    let start = Barrier::new(THREADS + 1);
    let end = Barrier::new(THREADS + 1);
    let exit = AtomicCell::new(false);

    let a = AtomicCell::new(0u8);

    thread::scope(|scope| {
        for _ in 0..THREADS {
            scope.spawn(|| loop {
                start.wait();

                let mut sum = 0;
                for _ in 0..STEPS {
                    sum += a.load();
                }
                test::black_box(sum);

                end.wait();
                if exit.load() {
                    break;
                }
            });
        }

        start.wait();
        end.wait();

        b.iter(|| {
            start.wait();
            end.wait();
        });

        start.wait();
        exit.store(true);
        end.wait();
    }).unwrap();
}

#[bench]
fn load_usize(b: &mut test::Bencher) {
    let a = AtomicCell::new(0usize);
    let mut sum = 0;
    b.iter(|| sum += a.load());
    test::black_box(sum);
}

#[bench]
fn store_usize(b: &mut test::Bencher) {
    let a = AtomicCell::new(0usize);
    b.iter(|| a.store(1));
}

#[bench]
fn fetch_add_usize(b: &mut test::Bencher) {
    let a = AtomicCell::new(0usize);
    b.iter(|| a.fetch_add(1));
}

#[bench]
fn compare_and_swap_usize(b: &mut test::Bencher) {
    let a = AtomicCell::new(0usize);
    let mut i = 0;
    b.iter(|| {
        a.compare_and_swap(i, i.wrapping_add(1));
        i = i.wrapping_add(1);
    });
}

#[bench]
fn concurrent_load_usize(b: &mut test::Bencher) {
    const THREADS: usize = 2;
    const STEPS: usize = 1_000_000;

    let start = Barrier::new(THREADS + 1);
    let end = Barrier::new(THREADS + 1);
    let exit = AtomicCell::new(false);

    let a = AtomicCell::new(0usize);

    thread::scope(|scope| {
        for _ in 0..THREADS {
            scope.spawn(|| loop {
                start.wait();

                let mut sum = 0;
                for _ in 0..STEPS {
                    sum += a.load();
                }
                test::black_box(sum);

                end.wait();
                if exit.load() {
                    break;
                }
            });
        }

        start.wait();
        end.wait();

        b.iter(|| {
            start.wait();
            end.wait();
        });

        start.wait();
        exit.store(true);
        end.wait();
    }).unwrap();
}
<|endoftext|>


<commit_before>use regex::Regex;
use string::constants::UNACCONTABLE_WORDS;

macro_rules! special_cases{
    ($s:ident, $($singular: expr => $plural:expr), *) => {
        match &$s[..] {
            $(
                $singular => {
                    return $plural.to_string();
                },
            )*
            _ => ()
        }
    }
}

/// Converts a `String` to singularized `String`
///
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "foo_bars".to_string();
///     let expected_string: String = "foo_bar".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "oxen".to_string();
///     let expected_string: String = "ox".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "crates".to_string();
///     let expected_string: String = "crate".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "oxen".to_string();
///     let expected_string: String = "ox".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "boxes".to_string();
///     let expected_string: String = "box".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "vengeance".to_string();
///     let expected_string: String = "vengeance".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "yoga".to_string();
///     let expected_string: String = "yoga".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
///
pub fn to_singular(non_singular_string: String) -> String {
    if UNACCONTABLE_WORDS.contains(&non_singular_string.as_ref()) {
        non_singular_string
    } else {
        special_cases![non_singular_string,
            "oxen" => "ox",
            "boxes" => "box",
            "men" => "man",
            "women" => "woman",
            "dice" => "die",
            "yeses" => "yes",
            "feet" => "foot",
            "eaves" => "eave",
            "geese" => "goose",
            "teeth" => "tooth",
            "quizzes" => "quiz"
        ];
        for &(ref rule, replace) in RULES.iter().rev() {
            if let Some(c) = rule.captures(&non_singular_string) {
                if let Some(c) = c.get(1) {
                    return format!("{}{}", c.as_str(), replace);
                }
            }
        }

        format!("{}", non_singular_string)
    }
}

macro_rules! add_rule{
    ($r:ident, $rule:expr => $replace:expr) => {
        $r.push((Regex::new($rule).unwrap(), $replace));
    }
}

macro_rules! rules{
    ($r:ident; $($rule:expr => $replace:expr), *) => {
        $(
            add_rule!{$r, $rule => $replace}
        )*
    }
}

lazy_static!{
    static ref RULES: Vec<(Regex, &'static str)> = {
    let mut r = Vec::new();
    rules![r;
     r"(n)ews$" => "ews",
     r"(\w*)(o)es$" => "",
     r"(\w*)([ti])a$" => "um",
     r"((a)naly|(b)a|(d)iagno|(p)arenthe|(p)rogno|(s)ynop|(t)he)(sis|ses)$" => "sis",
     r"(^analy)(sis|ses)$" => "sis",
     r"(\w*)([^f])ves$" => "fe",
     r"(\w*)(hive)s$" => "",
     r"(\w*)(tive)s$" => "",
     r"(\w*)([lr])ves$" => "f",
     r"(\w*)([^aeiouy]|qu)ies$" => "y",
     r"(s)eries$" => "eries",
     r"(m)ovies$" => "ovie",
     r"(\w*)(x|ch|ss|sh)es$" => "",
     r"(m|l)ice$" => "ouse",
     r"(bus)(es)?$" => "",
     r"(shoe)s$" => "",
     r"(cris|test)(is|es)$" => "is",
     r"^(a)x[ie]s$" => "xis",
     r"(octop|vir)(us|i)$" => "us",
     r"(alias|status)(es)?$" => "",
     r"^(ox)en" => "",
     r"(vert|ind)ices$" => "ex",
     r"(matr)ices$" => "ix",
     r"(quiz)zes$" => "",
     r"(database)s$" => "",
     r"(\w*)s$" => "",
     r"(\w*)(ss)$" => ""
         ];
     r
    };
}

#[test]
fn singularize_string_if_a_regex_will_match() {
    let expected_string: String = "ox".to_string();
    let asserted_string: String = to_singular("oxen".to_string());
    assert!(expected_string == asserted_string);

}

#[test]
fn singularize_string_returns_none_option_if_no_match() {
    let expected_string: String = "bacon".to_string();
    let asserted_string: String = to_singular("bacon".to_string());

    assert!(expected_string == asserted_string);

}
<commit_msg>Fix singularization of ies suffix<commit_after>use regex::Regex;
use string::constants::UNACCONTABLE_WORDS;

macro_rules! special_cases{
    ($s:ident, $($singular: expr => $plural:expr), *) => {
        match &$s[..] {
            $(
                $singular => {
                    return $plural.to_string();
                },
            )*
            _ => ()
        }
    }
}

/// Converts a `String` to singularized `String`
///
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "foo_bars".to_string();
///     let expected_string: String = "foo_bar".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "oxen".to_string();
///     let expected_string: String = "ox".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "crates".to_string();
///     let expected_string: String = "crate".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "oxen".to_string();
///     let expected_string: String = "ox".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "boxes".to_string();
///     let expected_string: String = "box".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "vengeance".to_string();
///     let expected_string: String = "vengeance".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
/// ```
///     use inflector::string::singularize::to_singular;
///     let mock_string: String = "yoga".to_string();
///     let expected_string: String = "yoga".to_string();
///     let asserted_string: String = to_singular(mock_string);
///     assert!(asserted_string == expected_string);
///
/// ```
///
pub fn to_singular(non_singular_string: String) -> String {
    if UNACCONTABLE_WORDS.contains(&non_singular_string.as_ref()) {
        non_singular_string
    } else {
        special_cases![non_singular_string,
            "oxen" => "ox",
            "boxes" => "box",
            "men" => "man",
            "women" => "woman",
            "dice" => "die",
            "yeses" => "yes",
            "feet" => "foot",
            "eaves" => "eave",
            "geese" => "goose",
            "teeth" => "tooth",
            "quizzes" => "quiz"
        ];
        for &(ref rule, replace) in RULES.iter().rev() {
            if let Some(c) = rule.captures(&non_singular_string) {
                if let Some(c) = c.get(1) {
                    return format!("{}{}", c.as_str(), replace);
                }
            }
        }

        format!("{}", non_singular_string)
    }
}

macro_rules! add_rule{
    ($r:ident, $rule:expr => $replace:expr) => {
        $r.push((Regex::new($rule).unwrap(), $replace));
    }
}

macro_rules! rules{
    ($r:ident; $($rule:expr => $replace:expr), *) => {
        $(
            add_rule!{$r, $rule => $replace}
        )*
    }
}

lazy_static!{
    static ref RULES: Vec<(Regex, &'static str)> = {
    let mut r = Vec::new();
    rules![r;
     r"(\w*)s$" => "",
     r"(n)ews$" => "ews",
     r"(\w*)(o)es$" => "",
     r"(\w*)([ti])a$" => "um",
     r"((a)naly|(b)a|(d)iagno|(p)arenthe|(p)rogno|(s)ynop|(t)he)(sis|ses)$" => "sis",
     r"(^analy)(sis|ses)$" => "sis",
     r"(\w*)([^f])ves$" => "fe",
     r"(\w*)(hive)s$" => "",
     r"(\w*)(tive)s$" => "",
     r"(\w*)([lr])ves$" => "f",
     r"(\w*([^aeiouy]|qu))ies$" => "y",
     r"(s)eries$" => "eries",
     r"(m)ovies$" => "ovie",
     r"(\w*)(x|ch|ss|sh)es$" => "",
     r"(m|l)ice$" => "ouse",
     r"(bus)(es)?$" => "",
     r"(shoe)s$" => "",
     r"(cris|test)(is|es)$" => "is",
     r"^(a)x[ie]s$" => "xis",
     r"(octop|vir)(us|i)$" => "us",
     r"(alias|status)(es)?$" => "",
     r"^(ox)en" => "",
     r"(vert|ind)ices$" => "ex",
     r"(matr)ices$" => "ix",
     r"(quiz)zes$" => "",
     r"(database)s$" => "",
     r"(\w*)(ss)$" => ""
         ];
     r
    };
}

#[test]
fn singularize_ies_suffix() {
    assert_eq!("reply", to_singular("replies".to_owned()));
    assert_eq!("lady", to_singular("ladies".to_owned()));
    assert_eq!("soliloquy", to_singular("soliloquies".to_owned()));
}

#[test]
fn singularize_string_if_a_regex_will_match() {
    let expected_string: String = "ox".to_string();
    let asserted_string: String = to_singular("oxen".to_string());
    assert!(expected_string == asserted_string);

}

#[test]
fn singularize_string_returns_none_option_if_no_match() {
    let expected_string: String = "bacon".to_string();
    let asserted_string: String = to_singular("bacon".to_string());

    assert!(expected_string == asserted_string);

}
<|endoftext|>


<commit_before>extern crate handlebars;
extern crate rustc_serialize;
extern crate pulldown_cmark;

use renderer::html_handlebars::helpers;
use renderer::Renderer;
use book::MDBook;
use book::bookitem::BookItem;
use {utils, theme};

use std::path::{Path, PathBuf};
use std::fs::{self, File};
use std::error::Error;
use std::io::{self, Read, Write};
use std::collections::BTreeMap;

use self::handlebars::{Handlebars, JsonRender};
use self::rustc_serialize::json::{Json, ToJson};
use self::pulldown_cmark::{Parser, html};

pub struct HtmlHandlebars;

impl HtmlHandlebars {
    pub fn new() -> Self {
        HtmlHandlebars
    }
}

impl Renderer for HtmlHandlebars {
    fn render(&self, book: &MDBook) -> Result<(), Box<Error>> {
        debug!("[fn]: render");
        let mut handlebars = Handlebars::new();

        // Load theme
        let theme = theme::Theme::new(book.get_src());

        // Register template
        debug!("[*]: Register handlebars template");
        try!(handlebars.register_template_string("index", try!(String::from_utf8(theme.index))));

        // Register helpers
        debug!("[*]: Register handlebars helpers");
        handlebars.register_helper("toc", Box::new(helpers::toc::RenderToc));
        handlebars.register_helper("previous", Box::new(helpers::navigation::previous));
        handlebars.register_helper("next", Box::new(helpers::navigation::next));

        let mut data = try!(make_data(book));

        // Print version
        let mut print_content: String = String::new();

        // Check if dest directory exists
        debug!("[*]: Check if destination directory exists");
        match utils::create_path(book.get_dest()) {
            Err(_) => return Err(Box::new(io::Error::new(io::ErrorKind::Other, "Unexcpected error when constructing destination path"))),
            _ => {},
        };

        // Render a file for every entry in the book
        let mut index = true;
        for item in book.iter() {

            match item {
                &BookItem::Chapter(_, ref ch) | &BookItem::Affix(ref ch) => {
                    if ch.path != PathBuf::new() {

                        let path = book.get_src().join(&ch.path);

                        debug!("[*]: Opening file: {:?}", path);
                        let mut f = try!(File::open(&path));
                        let mut content: String = String::new();

                        debug!("[*]: Reading file");
                        try!(f.read_to_string(&mut content));

                        // Render markdown using the pulldown-cmark crate
                        content = render_html(&content);
                        print_content.push_str(&content);

                        // Remove content from previous file and render content for this one
                        data.remove("path");
                        match ch.path.to_str() {
                            Some(p) => { data.insert("path".to_string(), p.to_json()); },
                            None => return Err(Box::new(io::Error::new(io::ErrorKind::Other, "Could not convert path to str"))),
                        }

                        // Remove content from previous file and render content for this one
                        data.remove("content");
                        data.insert("content".to_string(), content.to_json());

                        // Remove path to root from previous file and render content for this one
                        data.remove("path_to_root");
                        data.insert("path_to_root".to_string(), utils::path_to_root(&ch.path).to_json());

                        // Rendere the handlebars template with the data
                        debug!("[*]: Render template");
                        let rendered = try!(handlebars.render("index", &data));

                        debug!("[*]: Create file {:?}", &book.get_dest().join(&ch.path).with_extension("html"));
                        // Write to file
                        let mut file = try!(utils::create_file(&book.get_dest().join(&ch.path).with_extension("html")));
                        output!("[*] Creating {:?} ✓", &book.get_dest().join(&ch.path).with_extension("html"));

                        try!(file.write_all(&rendered.into_bytes()));

                        // Create an index.html from the first element in SUMMARY.md
                        if index {
                            debug!("[*]: index.html");
                            try!(fs::copy(
                                book.get_dest().join(&ch.path.with_extension("html")),
                                book.get_dest().join("index.html")
                            ));

                            output!(
                                "[*] Creating index.html from {:?} ✓",
                                book.get_dest().join(&ch.path.with_extension("html"))
                                );
                            index = false;
                        }
                    }
                }
                _ => {}
            }
        }

        // Print version

        // Remove content from previous file and render content for this one
        data.remove("path");
        data.insert("path".to_string(), "print.md".to_json());

        // Remove content from previous file and render content for this one
        data.remove("content");
        data.insert("content".to_string(), print_content.to_json());

        // Remove path to root from previous file and render content for this one
        data.remove("path_to_root");
        data.insert("path_to_root".to_string(), utils::path_to_root(Path::new("print.md")).to_json());

        // Rendere the handlebars template with the data
        debug!("[*]: Render template");
        let rendered = try!(handlebars.render("index", &data));
        let mut file = try!(utils::create_file(&book.get_dest().join("print").with_extension("html")));
        try!(file.write_all(&rendered.into_bytes()));
        output!("[*] Creating print.html ✓");

        // Copy static files (js, css, images, ...)

        debug!("[*] Copy static files");
        // JavaScript
        let mut js_file = try!(File::create(book.get_dest().join("book.js")));
        try!(js_file.write_all(&theme.js));

        // Css
        let mut css_file = try!(File::create(book.get_dest().join("book.css")));
        try!(css_file.write_all(&theme.css));

        // JQuery local fallback
        let mut jquery = try!(File::create(book.get_dest().join("jquery.js")));
        try!(jquery.write_all(&theme.jquery));

        // Font Awesome local fallback
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/css/font-awesome").with_extension("css")));
        try!(font_awesome.write_all(theme::FONT_AWESOME));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/fontawesome-webfont.eot")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_EOT));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/fontawesome-webfont.svg")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_SVG));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/fontawesome-webfont.ttf")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_TTF));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/fontawesome-webfont.woff")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_WOFF));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/fontawesome-webfont.woff2")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_WOFF2));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/FontAwesome.ttf")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_TTF));

        // syntax highlighting
        let mut highlight_css = try!(File::create(book.get_dest().join("highlight.css")));
        try!(highlight_css.write_all(&theme.highlight_css));
        let mut highlight_js = try!(File::create(book.get_dest().join("highlight.js")));
        try!(highlight_js.write_all(&theme.highlight_js));

        Ok(())
    }
}

fn make_data(book: &MDBook) -> Result<BTreeMap<String,Json>, Box<Error>> {
    debug!("[fn]: make_data");

    let mut data  = BTreeMap::new();
    data.insert("language".to_string(), "en".to_json());
    data.insert("title".to_string(), book.get_title().to_json());

    let mut chapters = vec![];

    for item in book.iter() {
        // Create the data to inject in the template
        let mut chapter = BTreeMap::new();

        match item {
            &BookItem::Affix(ref ch) => {
                chapter.insert("name".to_string(), ch.name.to_json());
                match ch.path.to_str() {
                    Some(p) => { chapter.insert("path".to_string(), p.to_json()); },
                    None => return Err(Box::new(io::Error::new(io::ErrorKind::Other, "Could not convert path to str"))),
                }
            },
            &BookItem::Chapter(ref s, ref ch) => {
                chapter.insert("section".to_string(), s.to_json());
                chapter.insert("name".to_string(), ch.name.to_json());
                match ch.path.to_str() {
                    Some(p) => { chapter.insert("path".to_string(), p.to_json()); },
                    None => return Err(Box::new(io::Error::new(io::ErrorKind::Other, "Could not convert path to str"))),
                }
            },
            &BookItem::Spacer => {
                chapter.insert("spacer".to_string(), "_spacer_".to_json());
            }

        }

        chapters.push(chapter);
    }

    data.insert("chapters".to_string(), chapters.to_json());

    debug!("[*]: JSON constructed");
    Ok(data)
}

fn render_html(text: &str) -> String {
    let mut s = String::with_capacity(text.len() * 3 / 2);
    let p = Parser::new(&text);
    html::push_html(&mut s, p);
    s
}
<commit_msg>Fix #50 base url sometimes not correct for index.html<commit_after>extern crate handlebars;
extern crate rustc_serialize;
extern crate pulldown_cmark;

use renderer::html_handlebars::helpers;
use renderer::Renderer;
use book::MDBook;
use book::bookitem::BookItem;
use {utils, theme};

use std::path::{Path, PathBuf};
use std::fs::File;
use std::error::Error;
use std::io::{self, Read, Write};
use std::collections::BTreeMap;

use self::handlebars::{Handlebars, JsonRender};
use self::rustc_serialize::json::{Json, ToJson};
use self::pulldown_cmark::{Parser, html};

pub struct HtmlHandlebars;

impl HtmlHandlebars {
    pub fn new() -> Self {
        HtmlHandlebars
    }
}

impl Renderer for HtmlHandlebars {
    fn render(&self, book: &MDBook) -> Result<(), Box<Error>> {
        debug!("[fn]: render");
        let mut handlebars = Handlebars::new();

        // Load theme
        let theme = theme::Theme::new(book.get_src());

        // Register template
        debug!("[*]: Register handlebars template");
        try!(handlebars.register_template_string("index", try!(String::from_utf8(theme.index))));

        // Register helpers
        debug!("[*]: Register handlebars helpers");
        handlebars.register_helper("toc", Box::new(helpers::toc::RenderToc));
        handlebars.register_helper("previous", Box::new(helpers::navigation::previous));
        handlebars.register_helper("next", Box::new(helpers::navigation::next));

        let mut data = try!(make_data(book));

        // Print version
        let mut print_content: String = String::new();

        // Check if dest directory exists
        debug!("[*]: Check if destination directory exists");
        match utils::create_path(book.get_dest()) {
            Err(_) => return Err(Box::new(io::Error::new(io::ErrorKind::Other, "Unexcpected error when constructing destination path"))),
            _ => {},
        };

        // Render a file for every entry in the book
        let mut index = true;
        for item in book.iter() {

            match item {
                &BookItem::Chapter(_, ref ch) | &BookItem::Affix(ref ch) => {
                    if ch.path != PathBuf::new() {

                        let path = book.get_src().join(&ch.path);

                        debug!("[*]: Opening file: {:?}", path);
                        let mut f = try!(File::open(&path));
                        let mut content: String = String::new();

                        debug!("[*]: Reading file");
                        try!(f.read_to_string(&mut content));

                        // Render markdown using the pulldown-cmark crate
                        content = render_html(&content);
                        print_content.push_str(&content);

                        // Remove content from previous file and render content for this one
                        data.remove("path");
                        match ch.path.to_str() {
                            Some(p) => { data.insert("path".to_string(), p.to_json()); },
                            None => return Err(Box::new(io::Error::new(io::ErrorKind::Other, "Could not convert path to str"))),
                        }

                        // Remove content from previous file and render content for this one
                        data.remove("content");
                        data.insert("content".to_string(), content.to_json());

                        // Remove path to root from previous file and render content for this one
                        data.remove("path_to_root");
                        data.insert("path_to_root".to_string(), utils::path_to_root(&ch.path).to_json());

                        // Rendere the handlebars template with the data
                        debug!("[*]: Render template");
                        let rendered = try!(handlebars.render("index", &data));

                        debug!("[*]: Create file {:?}", &book.get_dest().join(&ch.path).with_extension("html"));
                        // Write to file
                        let mut file = try!(utils::create_file(&book.get_dest().join(&ch.path).with_extension("html")));
                        output!("[*] Creating {:?} ✓", &book.get_dest().join(&ch.path).with_extension("html"));

                        try!(file.write_all(&rendered.into_bytes()));

                        // Create an index.html from the first element in SUMMARY.md
                        if index {
                            debug!("[*]: index.html");

                            let mut index_file = try!(File::create(book.get_dest().join("index.html")));
                            let mut content = String::new();
                            let _source = try!(File::open(book.get_dest().join(&ch.path.with_extension("html"))))
                                                        .read_to_string(&mut content);

                            // This could cause a problem when someone displays code containing <base href=...>
                            // on the front page, however this case should be very very rare...
                            content = content.lines().filter(|line| !line.contains("<base href=")).collect();

                            try!(index_file.write_all(content.as_bytes()));

                            output!(
                                "[*] Creating index.html from {:?} ✓",
                                book.get_dest().join(&ch.path.with_extension("html"))
                                );
                            index = false;
                        }
                    }
                }
                _ => {}
            }
        }

        // Print version

        // Remove content from previous file and render content for this one
        data.remove("path");
        data.insert("path".to_string(), "print.md".to_json());

        // Remove content from previous file and render content for this one
        data.remove("content");
        data.insert("content".to_string(), print_content.to_json());

        // Remove path to root from previous file and render content for this one
        data.remove("path_to_root");
        data.insert("path_to_root".to_string(), utils::path_to_root(Path::new("print.md")).to_json());

        // Rendere the handlebars template with the data
        debug!("[*]: Render template");
        let rendered = try!(handlebars.render("index", &data));
        let mut file = try!(utils::create_file(&book.get_dest().join("print").with_extension("html")));
        try!(file.write_all(&rendered.into_bytes()));
        output!("[*] Creating print.html ✓");

        // Copy static files (js, css, images, ...)

        debug!("[*] Copy static files");
        // JavaScript
        let mut js_file = try!(File::create(book.get_dest().join("book.js")));
        try!(js_file.write_all(&theme.js));

        // Css
        let mut css_file = try!(File::create(book.get_dest().join("book.css")));
        try!(css_file.write_all(&theme.css));

        // JQuery local fallback
        let mut jquery = try!(File::create(book.get_dest().join("jquery.js")));
        try!(jquery.write_all(&theme.jquery));

        // Font Awesome local fallback
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/css/font-awesome").with_extension("css")));
        try!(font_awesome.write_all(theme::FONT_AWESOME));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/fontawesome-webfont.eot")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_EOT));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/fontawesome-webfont.svg")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_SVG));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/fontawesome-webfont.ttf")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_TTF));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/fontawesome-webfont.woff")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_WOFF));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/fontawesome-webfont.woff2")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_WOFF2));
        let mut font_awesome = try!(utils::create_file(&book.get_dest().join("_FontAwesome/fonts/FontAwesome.ttf")));
        try!(font_awesome.write_all(theme::FONT_AWESOME_TTF));

        // syntax highlighting
        let mut highlight_css = try!(File::create(book.get_dest().join("highlight.css")));
        try!(highlight_css.write_all(&theme.highlight_css));
        let mut highlight_js = try!(File::create(book.get_dest().join("highlight.js")));
        try!(highlight_js.write_all(&theme.highlight_js));

        Ok(())
    }
}

fn make_data(book: &MDBook) -> Result<BTreeMap<String,Json>, Box<Error>> {
    debug!("[fn]: make_data");

    let mut data  = BTreeMap::new();
    data.insert("language".to_string(), "en".to_json());
    data.insert("title".to_string(), book.get_title().to_json());

    let mut chapters = vec![];

    for item in book.iter() {
        // Create the data to inject in the template
        let mut chapter = BTreeMap::new();

        match item {
            &BookItem::Affix(ref ch) => {
                chapter.insert("name".to_string(), ch.name.to_json());
                match ch.path.to_str() {
                    Some(p) => { chapter.insert("path".to_string(), p.to_json()); },
                    None => return Err(Box::new(io::Error::new(io::ErrorKind::Other, "Could not convert path to str"))),
                }
            },
            &BookItem::Chapter(ref s, ref ch) => {
                chapter.insert("section".to_string(), s.to_json());
                chapter.insert("name".to_string(), ch.name.to_json());
                match ch.path.to_str() {
                    Some(p) => { chapter.insert("path".to_string(), p.to_json()); },
                    None => return Err(Box::new(io::Error::new(io::ErrorKind::Other, "Could not convert path to str"))),
                }
            },
            &BookItem::Spacer => {
                chapter.insert("spacer".to_string(), "_spacer_".to_json());
            }

        }

        chapters.push(chapter);
    }

    data.insert("chapters".to_string(), chapters.to_json());

    debug!("[*]: JSON constructed");
    Ok(data)
}

fn render_html(text: &str) -> String {
    let mut s = String::with_capacity(text.len() * 3 / 2);
    let p = Parser::new(&text);
    html::push_html(&mut s, p);
    s
}
<|endoftext|>


<commit_before>use rusqlite::Connection;

#[derive(Debug)]
pub struct JobDepend {
    pub id: i32,
    pub fk_job: i32,
    pub fk_variable: i32,
    pub sq_order: i32,
}

pub fn table_create_job_depend(conn: &Connection) -> &Connection {
    conn.execute(
        "CREATE TABLE JOBDEPEND (
                  id                INTEGER PRIMARY KEY ASC,
                  fk_job            INTEGER NOT NULL,
                  fk_variable       INTEGER NOT NULL,
                  sq_order          INTEGER NOT NULL,
                  UNIQUE (fk_job, fk_variable) ON CONFLICT REPLACE
                  FOREIGN KEY(fk_job) REFERENCES JOB(id) ON UPDATE CASCADE
                  FOREIGN KEY(fk_variable) REFERENCES VARIABLE_NAME(id) ON UPDATE CASCADE
                  )",
        &[],
    ).unwrap();
    return conn;
}

pub fn insert_job_depend(
    conn: &Connection,
    job: &i32,
    provider: &i32,
    sq_order: &i32,
) -> Result<i32, &'static str> {
    let me = JobDepend {
        id: 0,
        fk_job: *job,
        fk_variable: *provider,
        sq_order: *sq_order,
    };
    let job_depend_instance = conn.execute(
        "INSERT INTO JOBDEPEND (fk_job, fk_variable, sq_order)
                  VALUES (?1, ?2, ?3)",
        &[&me.fk_job, &me.fk_variable, &me.sq_order],
    );
    if job_depend_instance.is_err() {
        return Err("Failed trying to INSERT INTO JOBDEPEND");
    }
    job_depend_instance.unwrap();
    return Ok(0);
}

pub fn list_job_depend(conn: &Connection) -> Vec<JobDepend> {
    let mut stmt = conn.prepare("SELECT id, fk_job, fk_variable, sq_order  FROM JOBDEPEND")
        .unwrap();
    let wraped_fs_file_iter = stmt.query_map(&[], |row| JobDepend {
        id: row.get(0),
        fk_job: row.get(1),
        fk_variable: row.get(2),
        sq_order: row.get(3),
    });
    let mut items = Vec::<JobDepend>::new();
    if wraped_fs_file_iter.is_err() {
        return items;
    }
    let fs_file_iter = wraped_fs_file_iter.unwrap();
    for person in fs_file_iter {
        items.push(person.unwrap());
    }
    return items;
}

pub fn job_depend_list(conn: &Connection) {
    let mut stmt = conn.prepare("SELECT id, fk_job, fk_variable, sq_order FROM JOBDEPEND")
        .unwrap();
    let person_iter = stmt.query_map(&[], |row| JobDepend {
        id: row.get(0),
        fk_job: row.get(1),
        fk_variable: row.get(2),
        sq_order: row.get(3),
    }).unwrap();

    for person in person_iter {
        info!("Found job_depend {:?}", person.unwrap());
    }
}

pub fn pk_job_depend_by_all(
    conn: &Connection,
    fk_job: &i32,
    provider: &i32,
    sq_order: &i32,
) -> Result<i32, &'static str> {
    let mut output = 0;
    let mut stmt = conn.prepare("SELECT JOBDEPEND.id, JOBDEPEND.fk_job, JOBDEPEND.fk_variable, JOBDEPEND.sq_order  FROM JOBDEPEND WHERE JOBDEPEND.fk_job = ?1 AND JOBDEPEND.fk_variable=?2 AND JOBDEPEND.sq_order=?3 ").unwrap();
    let job_depend_iter = stmt.query_map(&[fk_job, provider, sq_order], |row| JobDepend {
        id: row.get(0),
        fk_job: row.get(1),
        fk_variable: row.get(2),
        sq_order: row.get(3),
    });
    if job_depend_iter.is_err() {
        return Err("Insert failed");
    }
    let result = job_depend_iter.unwrap();
    let mut found = 0;
    for person in result {
        let bill = person.unwrap();
        output = bill.id;
        found = 1;
    }
    if found != 0 {
        return Ok(output);
    }
    return Err("None found");
}

pub fn job_depend_count(conn: &Connection, fk_job: &i32) -> Result<i32, &'static str> {
    let count_prep_rc =
        conn.prepare("SELECT JOBDEPEND.id FROM JOBDEPEND WHERE JOBDEPEND.fk_job = ?1");
    match count_prep_rc {
        Ok(mut result) => {
            let result_row = result.query(&[fk_job]);
            let mut rox = result_row.unwrap();
            let mut counter = 0;
            while let Some(_) = rox.next() {
                counter += 1;
            }
            return Ok(counter);
        }
        Err(_) => {
            return Err("Failed");
        }
    }
}
<commit_msg>Beutify the rust code<commit_after>use rusqlite::Connection;

#[derive(Debug)]
pub struct JobDepend {
    pub id: i32,
    pub fk_job: i32,
    pub fk_variable: i32,
    pub sq_order: i32,
}

pub fn table_create_job_depend(conn: &Connection) -> &Connection {
    conn.execute(
        "CREATE TABLE JOBDEPEND (
                  id                INTEGER PRIMARY KEY ASC,
                  fk_job            INTEGER NOT NULL,
                  fk_variable       INTEGER NOT NULL,
                  sq_order          INTEGER NOT NULL,
                  UNIQUE (fk_job, fk_variable) ON CONFLICT REPLACE
                  FOREIGN KEY(fk_job) REFERENCES JOB(id) ON UPDATE CASCADE
                  FOREIGN KEY(fk_variable) REFERENCES VARIABLE_NAME(id) ON UPDATE CASCADE
                  )",
        &[],
    ).unwrap();
    return conn;
}

pub fn insert_job_depend(
    conn: &Connection,
    job: &i32,
    provider: &i32,
    sq_order: &i32,
) -> Result<i32, &'static str> {
    let me = JobDepend {
        id: 0,
        fk_job: *job,
        fk_variable: *provider,
        sq_order: *sq_order,
    };
    let job_depend_instance = conn.execute(
        "INSERT INTO JOBDEPEND (fk_job, fk_variable, sq_order)
                  VALUES (?1, ?2, ?3)",
        &[&me.fk_job, &me.fk_variable, &me.sq_order],
    );
    if job_depend_instance.is_err() {
        return Err("Failed trying to INSERT INTO JOBDEPEND");
    }
    job_depend_instance.unwrap();
    return Ok(0);
}

pub fn list_job_depend(conn: &Connection) -> Vec<JobDepend> {
    let mut stmt = conn
        .prepare("SELECT id, fk_job, fk_variable, sq_order  FROM JOBDEPEND")
        .unwrap();
    let wraped_fs_file_iter = stmt.query_map(&[], |row| JobDepend {
        id: row.get(0),
        fk_job: row.get(1),
        fk_variable: row.get(2),
        sq_order: row.get(3),
    });
    let mut items = Vec::<JobDepend>::new();
    if wraped_fs_file_iter.is_err() {
        return items;
    }
    let fs_file_iter = wraped_fs_file_iter.unwrap();
    for person in fs_file_iter {
        items.push(person.unwrap());
    }
    return items;
}

pub fn job_depend_list(conn: &Connection) {
    let mut stmt = conn
        .prepare("SELECT id, fk_job, fk_variable, sq_order FROM JOBDEPEND")
        .unwrap();
    let person_iter =
        stmt.query_map(&[], |row| JobDepend {
            id: row.get(0),
            fk_job: row.get(1),
            fk_variable: row.get(2),
            sq_order: row.get(3),
        }).unwrap();

    for person in person_iter {
        info!("Found job_depend {:?}", person.unwrap());
    }
}

pub fn pk_job_depend_by_all(
    conn: &Connection,
    fk_job: &i32,
    provider: &i32,
    sq_order: &i32,
) -> Result<i32, &'static str> {
    let mut output = 0;
    let mut stmt = conn.prepare("SELECT JOBDEPEND.id, JOBDEPEND.fk_job, JOBDEPEND.fk_variable, JOBDEPEND.sq_order  FROM JOBDEPEND WHERE JOBDEPEND.fk_job = ?1 AND JOBDEPEND.fk_variable=?2 AND JOBDEPEND.sq_order=?3 ").unwrap();
    let job_depend_iter = stmt.query_map(&[fk_job, provider, sq_order], |row| JobDepend {
        id: row.get(0),
        fk_job: row.get(1),
        fk_variable: row.get(2),
        sq_order: row.get(3),
    });
    if job_depend_iter.is_err() {
        return Err("Insert failed");
    }
    let result = job_depend_iter.unwrap();
    let mut found = 0;
    for person in result {
        let bill = person.unwrap();
        output = bill.id;
        found = 1;
    }
    if found != 0 {
        return Ok(output);
    }
    return Err("None found");
}

pub fn job_depend_count(conn: &Connection, fk_job: &i32) -> Result<i32, &'static str> {
    let count_prep_rc =
        conn.prepare("SELECT JOBDEPEND.id FROM JOBDEPEND WHERE JOBDEPEND.fk_job = ?1");
    match count_prep_rc {
        Ok(mut result) => {
            let result_row = result.query(&[fk_job]);
            let mut rox = result_row.unwrap();
            let mut counter = 0;
            while let Some(_) = rox.next() {
                counter += 1;
            }
            return Ok(counter);
        }
        Err(_) => {
            return Err("Failed");
        }
    }
}
<|endoftext|>


<commit_before>use crate::{
    background_jobs::Environment,
    models::VersionDownload,
    schema::{crates, metadata, version_downloads, versions},
};

use diesel::prelude::*;
use swirl::PerformError;

#[swirl::background_job]
pub fn update_downloads(env: &Environment) -> Result<(), PerformError> {
    let conn = env.connection()?;
    update(&conn)?;
    Ok(())
}

fn update(conn: &PgConnection) -> QueryResult<()> {
    use self::version_downloads::dsl::*;
    use diesel::dsl::now;
    use diesel::select;

    let rows = version_downloads
        .filter(processed.eq(false))
        .filter(downloads.ne(counted))
        .load(conn)?;

    println!("Updating {} versions", rows.len());
    collect(conn, &rows)?;
    println!("Finished updating versions");

    // Anything older than 24 hours ago will be frozen and will not be queried
    // against again.
    diesel::update(version_downloads)
        .set(processed.eq(true))
        .filter(date.lt(diesel::dsl::date(now)))
        .filter(downloads.eq(counted))
        .filter(processed.eq(false))
        .execute(conn)?;
    println!("Finished freezing old version_downloads");

    no_arg_sql_function!(refresh_recent_crate_downloads, ());
    select(refresh_recent_crate_downloads).execute(conn)?;
    println!("Finished running refresh_recent_crate_downloads");

    Ok(())
}

fn collect(conn: &PgConnection, rows: &[VersionDownload]) -> QueryResult<()> {
    use diesel::update;

    for download in rows {
        let amt = download.downloads - download.counted;

        conn.transaction::<_, diesel::result::Error, _>(|| {
            // increment the number of counted downloads
            update(version_downloads::table.find(download.id()))
                .set(version_downloads::counted.eq(version_downloads::counted + amt))
                .execute(conn)?;

            // Update the total number of version downloads
            let crate_id = update(versions::table.find(download.version_id))
                .set(versions::downloads.eq(versions::downloads + amt))
                .returning(versions::crate_id)
                .get_result::<i32>(conn)?;

            // Update the total number of crate downloads
            update(crates::table.find(crate_id))
                .set(crates::downloads.eq(crates::downloads + amt))
                .execute(conn)?;

            // Now that everything else for this crate is done, update the global counter of total
            // downloads
            update(metadata::table)
                .set(metadata::total_downloads.eq(metadata::total_downloads + i64::from(amt)))
                .execute(conn)?;

            Ok(())
        })?;
    }

    Ok(())
}

#[cfg(test)]
mod test {
    use super::*;
    use crate::{
        env,
        models::{Crate, NewCrate, NewUser, NewVersion, User, Version},
    };
    use std::collections::HashMap;

    fn conn() -> PgConnection {
        let conn = PgConnection::establish(&env("TEST_DATABASE_URL")).unwrap();
        conn.begin_test_transaction().unwrap();
        conn
    }

    fn user(conn: &PgConnection) -> User {
        NewUser::new(2, "login", None, None, "access_token")
            .create_or_update(None, conn)
            .unwrap()
    }

    fn crate_and_version(conn: &PgConnection, user_id: i32) -> (Crate, Version) {
        let krate = NewCrate {
            name: "foo",
            ..Default::default()
        }
        .create_or_update(conn, user_id, None)
        .unwrap();
        let version = NewVersion::new(
            krate.id,
            &semver::Version::parse("1.0.0").unwrap(),
            &HashMap::new(),
            None,
            None,
            0,
            user_id,
        )
        .unwrap();
        let version = version.save(conn, &[], "someone@example.com").unwrap();
        (krate, version)
    }

    #[test]
    fn increment() {
        use diesel::dsl::*;

        let conn = conn();
        let user = user(&conn);
        let (krate, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values(version_downloads::version_id.eq(version.id))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::date.eq(date(now - 1.day())),
                version_downloads::processed.eq(true),
            ))
            .execute(&conn)
            .unwrap();

        super::update(&conn).unwrap();
        let version_downloads = versions::table
            .find(version.id)
            .select(versions::downloads)
            .first(&conn);
        assert_eq!(Ok(1), version_downloads);
        let crate_downloads = crates::table
            .find(krate.id)
            .select(crates::downloads)
            .first(&conn);
        assert_eq!(Ok(1), crate_downloads);
        super::update(&conn).unwrap();
        let version_downloads = versions::table
            .find(version.id)
            .select(versions::downloads)
            .first(&conn);
        assert_eq!(Ok(1), version_downloads);
    }

    #[test]
    fn set_processed_true() {
        use diesel::dsl::*;

        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now - 2.days())),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        super::update(&conn).unwrap();
        let processed = version_downloads::table
            .filter(version_downloads::version_id.eq(version.id))
            .select(version_downloads::processed)
            .first(&conn);
        assert_eq!(Ok(true), processed);
    }

    #[test]
    fn dont_process_recent_row() {
        use diesel::dsl::*;
        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now)),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        super::update(&conn).unwrap();
        let processed = version_downloads::table
            .filter(version_downloads::version_id.eq(version.id))
            .select(version_downloads::processed)
            .first(&conn);
        assert_eq!(Ok(false), processed);
    }

    #[test]
    fn increment_a_little() {
        use diesel::dsl::*;
        use diesel::update;

        let conn = conn();
        let user = user(&conn);
        let (krate, version) = crate_and_version(&conn, user.id);
        update(versions::table)
            .set(versions::updated_at.eq(now - 2.hours()))
            .execute(&conn)
            .unwrap();
        update(crates::table)
            .set(crates::updated_at.eq(now - 2.hours()))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(1),
                version_downloads::date.eq(date(now)),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::date.eq(date(now - 1.day())),
            ))
            .execute(&conn)
            .unwrap();

        let version_before = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        let krate_before = Crate::all()
            .filter(crates::id.eq(krate.id))
            .first::<Crate>(&conn)
            .unwrap();
        super::update(&conn).unwrap();
        let version2 = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        assert_eq!(version2.downloads, 2);
        assert_eq!(version2.updated_at, version_before.updated_at);
        let krate2 = Crate::all()
            .filter(crates::id.eq(krate.id))
            .first::<Crate>(&conn)
            .unwrap();
        assert_eq!(krate2.downloads, 2);
        assert_eq!(krate2.updated_at, krate_before.updated_at);
        super::update(&conn).unwrap();
        let version3 = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        assert_eq!(version3.downloads, 2);
    }

    #[test]
    fn set_processed_no_set_updated_at() {
        use diesel::dsl::*;
        use diesel::update;

        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        update(versions::table)
            .set(versions::updated_at.eq(now - 2.days()))
            .execute(&conn)
            .unwrap();
        update(crates::table)
            .set(crates::updated_at.eq(now - 2.days()))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now - 2.days())),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();

        super::update(&conn).unwrap();
        let versions_changed = versions::table
            .select(versions::updated_at.ne(now - 2.days()))
            .get_result(&conn);
        let crates_changed = crates::table
            .select(crates::updated_at.ne(now - 2.days()))
            .get_result(&conn);
        assert_eq!(Ok(false), versions_changed);
        assert_eq!(Ok(false), crates_changed);
    }
}
<commit_msg>Auto merge of #2154 - jtgeibel:prod/reduce-dl-count-contention, r=sgrif<commit_after>use crate::{
    background_jobs::Environment,
    models::VersionDownload,
    schema::{crates, metadata, version_downloads, versions},
};

use diesel::prelude::*;
use swirl::PerformError;

#[swirl::background_job]
pub fn update_downloads(env: &Environment) -> Result<(), PerformError> {
    let conn = env.connection()?;
    update(&conn)?;
    Ok(())
}

fn update(conn: &PgConnection) -> QueryResult<()> {
    use self::version_downloads::dsl::*;
    use diesel::dsl::now;
    use diesel::select;

    let rows = version_downloads
        .filter(processed.eq(false))
        .filter(downloads.ne(counted))
        .load(conn)?;

    println!("Updating {} versions", rows.len());
    collect(conn, &rows)?;
    println!("Finished updating versions");

    // Anything older than 24 hours ago will be frozen and will not be queried
    // against again.
    diesel::update(version_downloads)
        .set(processed.eq(true))
        .filter(date.lt(diesel::dsl::date(now)))
        .filter(downloads.eq(counted))
        .filter(processed.eq(false))
        .execute(conn)?;
    println!("Finished freezing old version_downloads");

    no_arg_sql_function!(refresh_recent_crate_downloads, ());
    select(refresh_recent_crate_downloads).execute(conn)?;
    println!("Finished running refresh_recent_crate_downloads");

    Ok(())
}

fn collect(conn: &PgConnection, rows: &[VersionDownload]) -> QueryResult<()> {
    use diesel::update;

    for download in rows {
        let amt = download.downloads - download.counted;

        conn.transaction::<_, diesel::result::Error, _>(|| {
            // Update the total number of version downloads
            let crate_id = update(versions::table.find(download.version_id))
                .set(versions::downloads.eq(versions::downloads + amt))
                .returning(versions::crate_id)
                .get_result::<i32>(conn)?;

            // Update the total number of crate downloads
            update(crates::table.find(crate_id))
                .set(crates::downloads.eq(crates::downloads + amt))
                .execute(conn)?;

            // Update the global counter of total downloads
            update(metadata::table)
                .set(metadata::total_downloads.eq(metadata::total_downloads + i64::from(amt)))
                .execute(conn)?;

            // Record that these downloads have been propagated to the other tables.  This is done
            // last, immediately before the transaction is committed, to minimize lock contention
            // with counting new downloads.
            update(version_downloads::table.find(download.id()))
                .set(version_downloads::counted.eq(version_downloads::counted + amt))
                .execute(conn)?;

            Ok(())
        })?;
    }

    Ok(())
}

#[cfg(test)]
mod test {
    use super::*;
    use crate::{
        env,
        models::{Crate, NewCrate, NewUser, NewVersion, User, Version},
    };
    use std::collections::HashMap;

    fn conn() -> PgConnection {
        let conn = PgConnection::establish(&env("TEST_DATABASE_URL")).unwrap();
        conn.begin_test_transaction().unwrap();
        conn
    }

    fn user(conn: &PgConnection) -> User {
        NewUser::new(2, "login", None, None, "access_token")
            .create_or_update(None, conn)
            .unwrap()
    }

    fn crate_and_version(conn: &PgConnection, user_id: i32) -> (Crate, Version) {
        let krate = NewCrate {
            name: "foo",
            ..Default::default()
        }
        .create_or_update(conn, user_id, None)
        .unwrap();
        let version = NewVersion::new(
            krate.id,
            &semver::Version::parse("1.0.0").unwrap(),
            &HashMap::new(),
            None,
            None,
            0,
            user_id,
        )
        .unwrap();
        let version = version.save(conn, &[], "someone@example.com").unwrap();
        (krate, version)
    }

    #[test]
    fn increment() {
        use diesel::dsl::*;

        let conn = conn();
        let user = user(&conn);
        let (krate, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values(version_downloads::version_id.eq(version.id))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::date.eq(date(now - 1.day())),
                version_downloads::processed.eq(true),
            ))
            .execute(&conn)
            .unwrap();

        super::update(&conn).unwrap();
        let version_downloads = versions::table
            .find(version.id)
            .select(versions::downloads)
            .first(&conn);
        assert_eq!(Ok(1), version_downloads);
        let crate_downloads = crates::table
            .find(krate.id)
            .select(crates::downloads)
            .first(&conn);
        assert_eq!(Ok(1), crate_downloads);
        super::update(&conn).unwrap();
        let version_downloads = versions::table
            .find(version.id)
            .select(versions::downloads)
            .first(&conn);
        assert_eq!(Ok(1), version_downloads);
    }

    #[test]
    fn set_processed_true() {
        use diesel::dsl::*;

        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now - 2.days())),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        super::update(&conn).unwrap();
        let processed = version_downloads::table
            .filter(version_downloads::version_id.eq(version.id))
            .select(version_downloads::processed)
            .first(&conn);
        assert_eq!(Ok(true), processed);
    }

    #[test]
    fn dont_process_recent_row() {
        use diesel::dsl::*;
        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now)),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        super::update(&conn).unwrap();
        let processed = version_downloads::table
            .filter(version_downloads::version_id.eq(version.id))
            .select(version_downloads::processed)
            .first(&conn);
        assert_eq!(Ok(false), processed);
    }

    #[test]
    fn increment_a_little() {
        use diesel::dsl::*;
        use diesel::update;

        let conn = conn();
        let user = user(&conn);
        let (krate, version) = crate_and_version(&conn, user.id);
        update(versions::table)
            .set(versions::updated_at.eq(now - 2.hours()))
            .execute(&conn)
            .unwrap();
        update(crates::table)
            .set(crates::updated_at.eq(now - 2.hours()))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(1),
                version_downloads::date.eq(date(now)),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::date.eq(date(now - 1.day())),
            ))
            .execute(&conn)
            .unwrap();

        let version_before = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        let krate_before = Crate::all()
            .filter(crates::id.eq(krate.id))
            .first::<Crate>(&conn)
            .unwrap();
        super::update(&conn).unwrap();
        let version2 = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        assert_eq!(version2.downloads, 2);
        assert_eq!(version2.updated_at, version_before.updated_at);
        let krate2 = Crate::all()
            .filter(crates::id.eq(krate.id))
            .first::<Crate>(&conn)
            .unwrap();
        assert_eq!(krate2.downloads, 2);
        assert_eq!(krate2.updated_at, krate_before.updated_at);
        super::update(&conn).unwrap();
        let version3 = versions::table
            .find(version.id)
            .first::<Version>(&conn)
            .unwrap();
        assert_eq!(version3.downloads, 2);
    }

    #[test]
    fn set_processed_no_set_updated_at() {
        use diesel::dsl::*;
        use diesel::update;

        let conn = conn();
        let user = user(&conn);
        let (_, version) = crate_and_version(&conn, user.id);
        update(versions::table)
            .set(versions::updated_at.eq(now - 2.days()))
            .execute(&conn)
            .unwrap();
        update(crates::table)
            .set(crates::updated_at.eq(now - 2.days()))
            .execute(&conn)
            .unwrap();
        insert_into(version_downloads::table)
            .values((
                version_downloads::version_id.eq(version.id),
                version_downloads::downloads.eq(2),
                version_downloads::counted.eq(2),
                version_downloads::date.eq(date(now - 2.days())),
                version_downloads::processed.eq(false),
            ))
            .execute(&conn)
            .unwrap();

        super::update(&conn).unwrap();
        let versions_changed = versions::table
            .select(versions::updated_at.ne(now - 2.days()))
            .get_result(&conn);
        let crates_changed = crates::table
            .select(crates::updated_at.ne(now - 2.days()))
            .get_result(&conn);
        assert_eq!(Ok(false), versions_changed);
        assert_eq!(Ok(false), crates_changed);
    }
}
<|endoftext|>


<commit_before>// Issue #51162: A failed doctest was not printing its stdout/stderr
// FIXME: if/when the output of the test harness can be tested on its own, this test should be
// adapted to use that, and that normalize line can go away

// compile-flags:--test
// rustc-env:RUST_BACKTRACE=0
// normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR"
// failure-status: 101

// doctest fails at runtime
/// ```
/// println!("stdout 1");
/// eprintln!("stderr 1");
/// println!("stdout 2");
/// eprintln!("stderr 2");
/// panic!("oh no");
/// ```
pub struct SomeStruct;

// doctest fails at compile time
/// ```
/// no
/// ```
pub struct OtherStruct;
<commit_msg>Auto merge of #76445 - jyn514:doctests, r=Mark-Simulacrum,ollie27<commit_after>// Issue #51162: A failed doctest was not printing its stdout/stderr
// FIXME: if/when the output of the test harness can be tested on its own, this test should be
// adapted to use that, and that normalize line can go away

// compile-flags:--test --test-args --test-threads=1
// rustc-env:RUST_BACKTRACE=0
// normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR"
// failure-status: 101

// doctest fails at runtime
/// ```
/// println!("stdout 1");
/// eprintln!("stderr 1");
/// println!("stdout 2");
/// eprintln!("stderr 2");
/// panic!("oh no");
/// ```
pub struct SomeStruct;

// doctest fails at compile time
/// ```
/// no
/// ```
pub struct OtherStruct;
<|endoftext|>


<commit_before>//  * This file is part of the uutils coreutils package.
//  *
//  * (c) 2014 Vsevolod Velichko <torkvemada@sorokdva.net>
//  *
//  * For the full copyright and license information, please view the LICENSE
//  * file that was distributed with this source code.

// spell-checker:ignore (ToDO) subpath absto absfrom absbase

use clap::{crate_version, App, Arg};
use std::env;
use std::path::{Path, PathBuf};
use uucore::display::println_verbatim;
use uucore::fs::{canonicalize, MissingHandling, ResolveMode};
use uucore::InvalidEncodingHandling;

static ABOUT: &str = "Convert TO destination to the relative path from the FROM dir.
If FROM path is omitted, current working dir will be used.";

mod options {
    pub const DIR: &str = "DIR";
    pub const TO: &str = "TO";
    pub const FROM: &str = "FROM";
}

fn usage() -> String {
    format!("{} [-d DIR] TO [FROM]", uucore::execution_phrase())
}

pub fn uumain(args: impl uucore::Args) -> i32 {
    let args = args
        .collect_str(InvalidEncodingHandling::ConvertLossy)
        .accept_any();
    let usage = usage();

    let matches = uu_app().usage(&usage[..]).get_matches_from(args);

    let to = Path::new(matches.value_of(options::TO).unwrap()).to_path_buf(); // required
    let from = match matches.value_of(options::FROM) {
        Some(p) => Path::new(p).to_path_buf(),
        None => env::current_dir().unwrap(),
    };
    let absto = canonicalize(to, MissingHandling::Normal, ResolveMode::Logical).unwrap();
    let absfrom = canonicalize(from, MissingHandling::Normal, ResolveMode::Logical).unwrap();

    if matches.is_present(options::DIR) {
        let base = Path::new(&matches.value_of(options::DIR).unwrap()).to_path_buf();
        let absbase = canonicalize(base, MissingHandling::Normal, ResolveMode::Logical).unwrap();
        if !absto.as_path().starts_with(absbase.as_path())
            || !absfrom.as_path().starts_with(absbase.as_path())
        {
            println_verbatim(absto).unwrap();
            return 0;
        }
    }

    let mut suffix_pos = 0;
    for (f, t) in absfrom.components().zip(absto.components()) {
        if f == t {
            suffix_pos += 1;
        } else {
            break;
        }
    }

    let mut result = PathBuf::new();
    absfrom
        .components()
        .skip(suffix_pos)
        .map(|_| result.push(".."))
        .last();
    absto
        .components()
        .skip(suffix_pos)
        .map(|x| result.push(x.as_os_str()))
        .last();

    println_verbatim(result).unwrap();
    0
}

pub fn uu_app() -> App<'static, 'static> {
    App::new(uucore::util_name())
        .version(crate_version!())
        .about(ABOUT)
        .arg(
            Arg::with_name(options::DIR)
                .short("d")
                .takes_value(true)
                .help("If any of FROM and TO is not subpath of DIR, output absolute path instead of relative"),
        )
        .arg(
            Arg::with_name(options::TO)
                .required(true)
                .takes_value(true),
        )
        .arg(
            Arg::with_name(options::FROM)
                .takes_value(true),
        )
}
<commit_msg>relpath: return UResult from uumain() function<commit_after>//  * This file is part of the uutils coreutils package.
//  *
//  * (c) 2014 Vsevolod Velichko <torkvemada@sorokdva.net>
//  *
//  * For the full copyright and license information, please view the LICENSE
//  * file that was distributed with this source code.

// spell-checker:ignore (ToDO) subpath absto absfrom absbase

use clap::{crate_version, App, Arg};
use std::env;
use std::path::{Path, PathBuf};
use uucore::display::println_verbatim;
use uucore::error::{FromIo, UResult};
use uucore::fs::{canonicalize, MissingHandling, ResolveMode};
use uucore::InvalidEncodingHandling;

static ABOUT: &str = "Convert TO destination to the relative path from the FROM dir.
If FROM path is omitted, current working dir will be used.";

mod options {
    pub const DIR: &str = "DIR";
    pub const TO: &str = "TO";
    pub const FROM: &str = "FROM";
}

fn usage() -> String {
    format!("{} [-d DIR] TO [FROM]", uucore::execution_phrase())
}

#[uucore_procs::gen_uumain]
pub fn uumain(args: impl uucore::Args) -> UResult<()> {
    let args = args
        .collect_str(InvalidEncodingHandling::ConvertLossy)
        .accept_any();
    let usage = usage();

    let matches = uu_app().usage(&usage[..]).get_matches_from(args);

    let to = Path::new(matches.value_of(options::TO).unwrap()).to_path_buf(); // required
    let from = match matches.value_of(options::FROM) {
        Some(p) => Path::new(p).to_path_buf(),
        None => env::current_dir().unwrap(),
    };
    let absto = canonicalize(to, MissingHandling::Normal, ResolveMode::Logical)
        .map_err_context(String::new)?;
    let absfrom = canonicalize(from, MissingHandling::Normal, ResolveMode::Logical)
        .map_err_context(String::new)?;

    if matches.is_present(options::DIR) {
        let base = Path::new(&matches.value_of(options::DIR).unwrap()).to_path_buf();
        let absbase = canonicalize(base, MissingHandling::Normal, ResolveMode::Logical)
            .map_err_context(String::new)?;
        if !absto.as_path().starts_with(absbase.as_path())
            || !absfrom.as_path().starts_with(absbase.as_path())
        {
            return println_verbatim(absto).map_err_context(String::new);
        }
    }

    let mut suffix_pos = 0;
    for (f, t) in absfrom.components().zip(absto.components()) {
        if f == t {
            suffix_pos += 1;
        } else {
            break;
        }
    }

    let mut result = PathBuf::new();
    absfrom
        .components()
        .skip(suffix_pos)
        .map(|_| result.push(".."))
        .last();
    absto
        .components()
        .skip(suffix_pos)
        .map(|x| result.push(x.as_os_str()))
        .last();

    println_verbatim(result).map_err_context(String::new)
}

pub fn uu_app() -> App<'static, 'static> {
    App::new(uucore::util_name())
        .version(crate_version!())
        .about(ABOUT)
        .arg(
            Arg::with_name(options::DIR)
                .short("d")
                .takes_value(true)
                .help("If any of FROM and TO is not subpath of DIR, output absolute path instead of relative"),
        )
        .arg(
            Arg::with_name(options::TO)
                .required(true)
                .takes_value(true),
        )
        .arg(
            Arg::with_name(options::FROM)
                .takes_value(true),
        )
}
<|endoftext|>


<commit_before>//! The `parser` module contains a lot of useful functionality
//! to parse tokens from the lexer (and creating the parsetree
//! and the ast)
//! its an predictiv parser for a LL(1) grammar
//! for more info about the parser: look in the Compiler Dragonbook,
//! Chapter 4.4.4, "Nonrecursive Predictive Parsing"

use frontend::lexer::Token;
use frontend::ast;
use frontend::parsetree::{ParseTree, PNode};
use self::NonTerminalType::*;
use frontend::lexer::Token::{TokPassageName, TokText, TokNewLine,
    TokFormatItalicStart, TokFormatItalicEnd, TokFormatBoldStart, TokFormatBoldEnd,
    TokFormatMonoStart, TokFormatMonoEnd, TokPassageLink};

/*

----------------------------------------
Grammatik:

LL(1)
------------------------------------------------------------+-----------------------------------
S -> Passage S2                                             |
S2 -> S                                                     |
S2 -> ɛ                                                     |
Passage -> PassageName PassageContent                       | "add passage (name)", simple
PassageContent -> TextPassage PassageContent                | add text as child
PassageContent -> Formatting PassageContent                 |
PassageContent -> ɛ                                         |
Formating -> BoldFormatting                                 |
Formating -> ItalicFormatting                               |
BoldFormatting -> FormatBold BoldContent FormatBold         | add child bold
BoldContent -> TextPassage BoldContent                      | add text as child
BoldContent -> FormatItalic BoldItalicContent FormatItalic  | add child italic
BoldContent -> ɛ                                            |
ItalicFormatting -> FormatItalic ItalicContent FormatItalic | add child italic
ItalicContent -> TextPassage ItalicContent                  | add text as child
ItalicContent -> FormatBold BoldItalicContent FormatBold    | add child bold
ItalicContent -> ɛ                                          | add text as child
BoldItalicContent -> TextPassage                            |
------------------------------------------------------------+-----------------------------------

::Start
Hello World, ''Yeah''.
//Italic// is possible too.

*/

pub fn parse_tokens(tokens: Vec<Token>) -> ast::AST {
    let mut parser: Parser = Parser::new(tokens);
    parser.parsing();
    parser.ast
}

//==============================
// grammar

#[derive(Debug, Copy, Clone)]
pub enum NonTerminalType {
    S,
    Sf,
    Passage,
    Passagef,
    PassageContentNewline,
    PassageContentNonNewline,
    Formating,
    BoldFormatting,
    ItalicFormatting,
    MonoFormatting,
    MonoContent,
    Link
}

//==============================
// parser

struct Parser {
    tree: ParseTree,
    ast: ast::AST,
    stack: Stack,
    ast_path: Vec<usize>,
    tokens: Vec<Token>,
    lookahead: usize
}

impl Parser {
    pub fn new(tokens: Vec<Token>) -> Parser {
        Parser {
            tree: ParseTree::new(),
            ast: ast::AST::new(),
            stack: Stack::new(),
            ast_path: Vec::new(),
            tokens: tokens,
            lookahead: 0
        }
    }

    /// the predictive stack ll(1) parsing routine
    pub fn parsing(&mut self) {
        self.stack.push_start();

        while let Some(top) = self.stack.pop() {
            if self.tree.is_terminal(top.to_vec()) {
                self.next_token();
            } else {
                self.apply_grammar(top.to_vec());
            }
        }

        self.tree.print();
    }

    /// apply the ll(1) grammar
    /// the match-statement simulates the parsing-table behavior
    ///
    /// it creates the parse tree (from the ll(1) grammar)
    /// and the ast from the sdd
    fn apply_grammar(&mut self, top_path: Vec<usize>) {
        if let Some(token) = self.tokens.get_mut(self.lookahead) {

            let to_add_path: Vec<usize> = top_path.to_vec();

            // the frst item in the tuple is the current state and
            // the snd is the current lookup-token
            let state_first: (NonTerminalType, &Token) = (self.tree.get_non_terminal(top_path).clone(), token);

            let mut new_nodes = Vec::new();

            println!("match {:?}", state_first);
            match state_first {
                (S, &TokPassageName(_)) => {
                    new_nodes.push(PNode::new_non_terminal(Passage));
                    new_nodes.push(PNode::new_non_terminal(Sf));
                },
                (Sf, &TokPassageName(_)) => {
                    new_nodes.push(PNode::new_non_terminal(S));
                },
                (Passage, &TokPassageName(ref name)) => {
                    let new_token: Token = TokPassageName(name.clone());
                    new_nodes.push(PNode::new_terminal(new_token));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNewline));

                    // ast
                    self.ast_path.clear();
                    let ast_count_passages = self.ast.count_childs(self.ast_path.to_vec());
                    let new_token2: Token = TokPassageName(name.clone());
                    self.ast.add_passage(new_token2);
                    self.ast_path.push(ast_count_passages);
                },

                // PassageContentNewline / PassageContentNonNewline
                (PassageContentNewline, &TokText(ref text)) |
                (PassageContentNonNewline, &TokText(ref text)) => {
                    let new_token: Token = TokText(text.clone());
                    new_nodes.push(PNode::new_terminal(new_token));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNonNewline));

                    let new_token2: Token = TokText(text.clone());
                    self.ast.add_child(&self.ast_path, new_token2);
                },
                (PassageContentNewline, &TokFormatBoldStart) |
                (PassageContentNewline, &TokFormatItalicStart) |
                (PassageContentNewline, &TokFormatMonoStart) |
                (PassageContentNonNewline, &TokFormatBoldStart) |
                (PassageContentNonNewline, &TokFormatItalicStart) |
                (PassageContentNonNewline, &TokFormatMonoStart) => {
                    new_nodes.push(PNode::new_non_terminal(Formating));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNonNewline));
                },
                (PassageContentNewline, &TokPassageLink(_, _)) |
                (PassageContentNonNewline, &TokPassageLink(_, _)) => {
                    new_nodes.push(PNode::new_non_terminal(Link));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNewline));
                },
                (PassageContentNewline, &TokNewLine) |
                (PassageContentNonNewline, &TokNewLine) => {
                    new_nodes.push(PNode::new_terminal(TokNewLine));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNewline));

                    // ast
                    self.ast.add_child(&self.ast_path, TokNewLine);
                },
                (PassageContentNonNewline, &TokFormatBoldEnd) |
                (PassageContentNewline, &TokFormatBoldEnd) => {
                    // jump one ast-level higher
                    self.ast_path.pop();
                },
                (PassageContentNonNewline, &TokFormatItalicEnd) |
                (PassageContentNewline, &TokFormatItalicEnd) => {
                    // jump one ast-level higher
                    self.ast_path.pop();
                },

                // Formating
                (Formating, &TokFormatBoldStart) => {
                    new_nodes.push(PNode::new_non_terminal(BoldFormatting));
                },
                (Formating, &TokFormatItalicStart) => {
                    new_nodes.push(PNode::new_non_terminal(ItalicFormatting));
                },
                (Formating, &TokFormatMonoStart) => {
                    new_nodes.push(PNode::new_non_terminal(MonoFormatting));
                },

                // BoldFormatting
                (BoldFormatting, &TokFormatBoldStart) => {
                    new_nodes.push(PNode::new_terminal(TokFormatBoldStart));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNonNewline));
                    new_nodes.push(PNode::new_terminal(TokFormatBoldEnd));

                    //ast
                    let ast_count_passages = self.ast.count_childs(self.ast_path.to_vec());
                    let ast_token: Token = TokFormatBoldStart;
                    self.ast.add_child(&self.ast_path, ast_token);
                    self.ast_path.push(ast_count_passages);
                },

                // ItalicFormatting
                (ItalicFormatting, &TokFormatItalicStart) => {
                    new_nodes.push(PNode::new_terminal(TokFormatItalicStart));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNonNewline));
                    new_nodes.push(PNode::new_terminal(TokFormatItalicEnd));

                    //ast
                    let ast_count_passages = self.ast.count_childs(self.ast_path.to_vec());
                    let ast_token: Token = TokFormatItalicStart;
                    self.ast.add_child(&self.ast_path, ast_token);
                    self.ast_path.push(ast_count_passages);
                },

                // MonoFormatting
                (MonoFormatting, &TokFormatMonoStart) => {
                    new_nodes.push(PNode::new_terminal(TokFormatMonoStart));
                    new_nodes.push(PNode::new_non_terminal(MonoContent));
                    new_nodes.push(PNode::new_terminal(TokFormatMonoEnd));
                },

                // MonoContent
                (MonoContent, &TokText(ref text)) => {
                    let new_token: Token = TokText(text.clone());
                    new_nodes.push(PNode::new_terminal(new_token));
                    new_nodes.push(PNode::new_non_terminal(MonoContent));
                },
                (MonoContent, &TokNewLine) => {
                    new_nodes.push(PNode::new_terminal(TokNewLine));
                    new_nodes.push(PNode::new_non_terminal(MonoContent));
                },

                // Link
                (Link, &TokPassageLink(ref text, ref name)) => {
                    let new_token: Token = TokPassageLink(text.clone(), name.clone());
                    new_nodes.push(PNode::new_terminal(new_token));

                    // ast
                    let new_token2: Token = TokPassageLink(text.clone(), name.clone());
                    self.ast.add_child(&self.ast_path, new_token2);
                },

                _ => {

                }
            }

            // adds the new nodes to the tree
            // and adds the path in the tree to the stack
            let length = new_nodes.len().clone();
            self.tree.add_nodes(new_nodes, &to_add_path);
            self.stack.push_path(length as u8, to_add_path);

        } else {
            // no token left
            // only ɛ-productions could be here
            // these productions will be poped of the stack
        }
    }

    /// sets the lookahead to the next token
    fn next_token(&mut self) {
        self.lookahead = self.lookahead + 1;
    }
}

//==============================
// stack of the parser
struct Stack {
    data: Vec<Vec<usize>>
}

impl Stack {
    pub fn new() -> Stack {
        Stack { data: Vec::new() }
    }

    /// pushs the address of the first startsymbol to the stack
    fn push_start(&mut self) {
        self.data.push(vec![]);
    }

    /// save the path of the nodes in the tree to the stack
    /// the right part of the production
    /// should be on the stack in reverse order
    fn push_path(&mut self, count_elements: u8, to_add_path: Vec<usize>) {
        for i in 0..count_elements {
            let mut path: Vec<usize> = to_add_path.to_vec();
            path.push((count_elements-i-1) as usize);
            self.data.push(path);
        }
    }

    fn pop(&mut self) -> Option<Vec<usize>> {
        self.data.pop()
    }
}
<commit_msg>-add one comment<commit_after>//! The `parser` module contains a lot of useful functionality
//! to parse tokens from the lexer (and creating the parsetree
//! and the ast)
//! its an predictiv parser for a LL(1) grammar
//! for more info about the parser: look in the Compiler Dragonbook,
//! Chapter 4.4.4, "Nonrecursive Predictive Parsing"

use frontend::lexer::Token;
use frontend::ast;
use frontend::parsetree::{ParseTree, PNode};
use self::NonTerminalType::*;
use frontend::lexer::Token::{TokPassageName, TokText, TokNewLine,
    TokFormatItalicStart, TokFormatItalicEnd, TokFormatBoldStart, TokFormatBoldEnd,
    TokFormatMonoStart, TokFormatMonoEnd, TokPassageLink};

/*

----------------------------------------
Grammatik:

LL(1)
------------------------------------------------------------+-----------------------------------
S -> Passage S2                                             |
S2 -> S                                                     |
S2 -> ɛ                                                     |
Passage -> PassageName PassageContent                       | "add passage (name)", simple
PassageContent -> TextPassage PassageContent                | add text as child
PassageContent -> Formatting PassageContent                 |
PassageContent -> ɛ                                         |
Formating -> BoldFormatting                                 |
Formating -> ItalicFormatting                               |
BoldFormatting -> FormatBold BoldContent FormatBold         | add child bold
BoldContent -> TextPassage BoldContent                      | add text as child
BoldContent -> FormatItalic BoldItalicContent FormatItalic  | add child italic
BoldContent -> ɛ                                            |
ItalicFormatting -> FormatItalic ItalicContent FormatItalic | add child italic
ItalicContent -> TextPassage ItalicContent                  | add text as child
ItalicContent -> FormatBold BoldItalicContent FormatBold    | add child bold
ItalicContent -> ɛ                                          | add text as child
BoldItalicContent -> TextPassage                            |
------------------------------------------------------------+-----------------------------------

::Start
Hello World, ''Yeah''.
//Italic// is possible too.

*/

pub fn parse_tokens(tokens: Vec<Token>) -> ast::AST {
    let mut parser: Parser = Parser::new(tokens);
    parser.parsing();
    parser.ast
}

//==============================
// grammar

#[derive(Debug, Copy, Clone)]
pub enum NonTerminalType {
    S,
    Sf,
    Passage,
    Passagef,
    PassageContentNewline,
    PassageContentNonNewline,
    Formating,
    BoldFormatting,
    ItalicFormatting,
    MonoFormatting,
    MonoContent,
    Link
}

//==============================
// parser

struct Parser {
    tree: ParseTree,
    ast: ast::AST,
    stack: Stack,
    ast_path: Vec<usize>,
    tokens: Vec<Token>,
    lookahead: usize
}

impl Parser {
    pub fn new(tokens: Vec<Token>) -> Parser {
        Parser {
            tree: ParseTree::new(),
            ast: ast::AST::new(),
            stack: Stack::new(),
            ast_path: Vec::new(),
            tokens: tokens,
            lookahead: 0
        }
    }

    /// the predictive stack ll(1) parsing routine
    pub fn parsing(&mut self) {
        self.stack.push_start();

        while let Some(top) = self.stack.pop() {
            if self.tree.is_terminal(top.to_vec()) {
                self.next_token();
            } else {
                self.apply_grammar(top.to_vec());
            }
        }

        self.tree.print();
    }

    /// apply the ll(1) grammar
    /// the match-statement simulates the parsing-table behavior
    ///
    /// it creates the parse tree (from the ll(1) grammar)
    /// and the ast from the sdd
    fn apply_grammar(&mut self, top_path: Vec<usize>) {
        if let Some(token) = self.tokens.get_mut(self.lookahead) {

            let to_add_path: Vec<usize> = top_path.to_vec();

            // the frst item in the tuple is the current state and
            // the snd is the current lookup-token
            let state_first: (NonTerminalType, &Token) = (self.tree.get_non_terminal(top_path).clone(), token);

            let mut new_nodes = Vec::new();

            println!("match {:?}", state_first);
            match state_first {
                (S, &TokPassageName(_)) => {
                    new_nodes.push(PNode::new_non_terminal(Passage));
                    new_nodes.push(PNode::new_non_terminal(Sf));
                },
                (Sf, &TokPassageName(_)) => {
                    new_nodes.push(PNode::new_non_terminal(S));
                },
                (Passage, &TokPassageName(ref name)) => {
                    let new_token: Token = TokPassageName(name.clone());
                    new_nodes.push(PNode::new_terminal(new_token));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNewline));

                    // ast
                    self.ast_path.clear();
                    let ast_count_passages = self.ast.count_childs(self.ast_path.to_vec());
                    let new_token2: Token = TokPassageName(name.clone());
                    self.ast.add_passage(new_token2);
                    self.ast_path.push(ast_count_passages);
                },

                // PassageContentNewline / PassageContentNonNewline
                (PassageContentNewline, &TokText(ref text)) |
                (PassageContentNonNewline, &TokText(ref text)) => {
                    let new_token: Token = TokText(text.clone());
                    new_nodes.push(PNode::new_terminal(new_token));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNonNewline));

                    // ast
                    let new_token2: Token = TokText(text.clone());
                    self.ast.add_child(&self.ast_path, new_token2);
                },
                (PassageContentNewline, &TokFormatBoldStart) |
                (PassageContentNewline, &TokFormatItalicStart) |
                (PassageContentNewline, &TokFormatMonoStart) |
                (PassageContentNonNewline, &TokFormatBoldStart) |
                (PassageContentNonNewline, &TokFormatItalicStart) |
                (PassageContentNonNewline, &TokFormatMonoStart) => {
                    new_nodes.push(PNode::new_non_terminal(Formating));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNonNewline));
                },
                (PassageContentNewline, &TokPassageLink(_, _)) |
                (PassageContentNonNewline, &TokPassageLink(_, _)) => {
                    new_nodes.push(PNode::new_non_terminal(Link));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNewline));
                },
                (PassageContentNewline, &TokNewLine) |
                (PassageContentNonNewline, &TokNewLine) => {
                    new_nodes.push(PNode::new_terminal(TokNewLine));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNewline));

                    // ast
                    self.ast.add_child(&self.ast_path, TokNewLine);
                },
                (PassageContentNonNewline, &TokFormatBoldEnd) |
                (PassageContentNewline, &TokFormatBoldEnd) => {
                    // jump one ast-level higher
                    self.ast_path.pop();
                },
                (PassageContentNonNewline, &TokFormatItalicEnd) |
                (PassageContentNewline, &TokFormatItalicEnd) => {
                    // jump one ast-level higher
                    self.ast_path.pop();
                },

                // Formating
                (Formating, &TokFormatBoldStart) => {
                    new_nodes.push(PNode::new_non_terminal(BoldFormatting));
                },
                (Formating, &TokFormatItalicStart) => {
                    new_nodes.push(PNode::new_non_terminal(ItalicFormatting));
                },
                (Formating, &TokFormatMonoStart) => {
                    new_nodes.push(PNode::new_non_terminal(MonoFormatting));
                },

                // BoldFormatting
                (BoldFormatting, &TokFormatBoldStart) => {
                    new_nodes.push(PNode::new_terminal(TokFormatBoldStart));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNonNewline));
                    new_nodes.push(PNode::new_terminal(TokFormatBoldEnd));

                    //ast
                    let ast_count_passages = self.ast.count_childs(self.ast_path.to_vec());
                    let ast_token: Token = TokFormatBoldStart;
                    self.ast.add_child(&self.ast_path, ast_token);
                    self.ast_path.push(ast_count_passages);
                },

                // ItalicFormatting
                (ItalicFormatting, &TokFormatItalicStart) => {
                    new_nodes.push(PNode::new_terminal(TokFormatItalicStart));
                    new_nodes.push(PNode::new_non_terminal(PassageContentNonNewline));
                    new_nodes.push(PNode::new_terminal(TokFormatItalicEnd));

                    //ast
                    let ast_count_passages = self.ast.count_childs(self.ast_path.to_vec());
                    let ast_token: Token = TokFormatItalicStart;
                    self.ast.add_child(&self.ast_path, ast_token);
                    self.ast_path.push(ast_count_passages);
                },

                // MonoFormatting
                (MonoFormatting, &TokFormatMonoStart) => {
                    new_nodes.push(PNode::new_terminal(TokFormatMonoStart));
                    new_nodes.push(PNode::new_non_terminal(MonoContent));
                    new_nodes.push(PNode::new_terminal(TokFormatMonoEnd));
                },

                // MonoContent
                (MonoContent, &TokText(ref text)) => {
                    let new_token: Token = TokText(text.clone());
                    new_nodes.push(PNode::new_terminal(new_token));
                    new_nodes.push(PNode::new_non_terminal(MonoContent));
                },
                (MonoContent, &TokNewLine) => {
                    new_nodes.push(PNode::new_terminal(TokNewLine));
                    new_nodes.push(PNode::new_non_terminal(MonoContent));
                },

                // Link
                (Link, &TokPassageLink(ref text, ref name)) => {
                    let new_token: Token = TokPassageLink(text.clone(), name.clone());
                    new_nodes.push(PNode::new_terminal(new_token));

                    // ast
                    let new_token2: Token = TokPassageLink(text.clone(), name.clone());
                    self.ast.add_child(&self.ast_path, new_token2);
                },

                _ => {

                }
            }

            // adds the new nodes to the tree
            // and adds the path in the tree to the stack
            let length = new_nodes.len().clone();
            self.tree.add_nodes(new_nodes, &to_add_path);
            self.stack.push_path(length as u8, to_add_path);

        } else {
            // no token left
            // only ɛ-productions could be here
            // these productions will be poped of the stack
        }
    }

    /// sets the lookahead to the next token
    fn next_token(&mut self) {
        self.lookahead = self.lookahead + 1;
    }
}

//==============================
// stack of the parser
struct Stack {
    data: Vec<Vec<usize>>
}

impl Stack {
    pub fn new() -> Stack {
        Stack { data: Vec::new() }
    }

    /// pushs the address of the first startsymbol to the stack
    fn push_start(&mut self) {
        self.data.push(vec![]);
    }

    /// save the path of the nodes in the tree to the stack
    /// the right part of the production
    /// should be on the stack in reverse order
    fn push_path(&mut self, count_elements: u8, to_add_path: Vec<usize>) {
        for i in 0..count_elements {
            let mut path: Vec<usize> = to_add_path.to_vec();
            path.push((count_elements-i-1) as usize);
            self.data.push(path);
        }
    }

    fn pop(&mut self) -> Option<Vec<usize>> {
        self.data.pop()
    }
}
<|endoftext|>