<commit_before>extern crate ammonia;
extern crate elasticlunr;

use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::path::Path;

use pulldown_cmark::*;
use serde_json;
use self::elasticlunr::Index;

use book::{Book, BookItem};
use config::Search;
use errors::*;
use utils;
use theme::searcher;

/// Creates all files required for search.
pub fn create_files(search_config: &Search, destination: &Path, book: &Book) -> Result<()> {
    let mut index = Index::new(&["title", "body", "breadcrumbs"]);
    let mut doc_urls = Vec::with_capacity(book.sections.len());

    for item in book.iter() {
        render_item(&mut index, &search_config, &mut doc_urls, item)?;
    }

    let index = write_to_json(index, &search_config, doc_urls)?;
    debug!("Writing search index ✓");
    if index.len() > 10_000_000 {
        warn!("searchindex.json is very large ({} bytes)", index.len());
    }

    if search_config.copy_js {
        utils::fs::write_file(destination, "searchindex.json", index.as_bytes())?;
        utils::fs::write_file(destination, "searchindex.js", format!("window.search = {};", index).as_bytes())?;
        utils::fs::write_file(destination, "searcher.js", searcher::JS)?;
        utils::fs::write_file(destination, "mark.min.js", searcher::MARK_JS)?;
        utils::fs::write_file(destination, "elasticlunr.min.js", searcher::ELASTICLUNR_JS)?;
        debug!("Copying search files ✓");
    }

    Ok(())
}

/// Uses the given arguments to construct a search document, then inserts it to the given index.
fn add_doc(
    index: &mut Index,
    doc_urls: &mut Vec<String>,
    anchor_base: &str,
    section_id: &Option<String>,
    items: &[&str],
) {
    let doc_ref = if let &Some(ref id) = section_id {
        Cow::Owned(format!("{}#{}", anchor_base, id))
    } else {
        Cow::Borrowed(anchor_base.into())
    };
    let doc_ref = utils::collapse_whitespace(doc_ref.trim());
    let key = doc_urls.len();
    doc_urls.push(doc_ref.into());

    let items = items.iter().map(|&x| utils::collapse_whitespace(x.trim()));
    index.add_doc(&key.to_string(), items);
}

/// Renders markdown into flat unformatted text and adds it to the search index.
fn render_item(
    index: &mut Index,
    search_config: &Search,
    doc_urls: &mut Vec<String>,
    item: &BookItem,
) -> Result<()> {
    let chapter = match item {
        &BookItem::Chapter(ref ch) => ch,
        _ => return Ok(()),
    };

    let filepath = Path::new(&chapter.path).with_extension("html");
    let filepath = filepath
        .to_str()
        .chain_err(|| "Could not convert HTML path to str")?;
    let anchor_base = utils::fs::normalize_path(filepath);

    let mut opts = Options::empty();
    opts.insert(OPTION_ENABLE_TABLES);
    opts.insert(OPTION_ENABLE_FOOTNOTES);
    let p = Parser::new_ext(&chapter.content, opts);

    let mut in_header = false;
    let max_section_depth = search_config.heading_split_level as i32;
    let mut section_id = None;
    let mut heading = String::new();
    let mut body = String::new();
    let mut breadcrumbs = chapter.parent_names.clone();
    let mut footnote_numbers = HashMap::new();

    for event in p {
        match event {
            Event::Start(Tag::Header(i)) if i <= max_section_depth => {
                if heading.len() > 0 {
                    // Section finished, the next header is following now
                    // Write the data to the index, and clear it for the next section
                    add_doc(
                        index,
                        doc_urls,
                        &anchor_base,
                        &section_id,
                        &[&heading, &body, &breadcrumbs.join(" » ")],
                    );
                    section_id = None;
                    heading.clear();
                    body.clear();
                    breadcrumbs.pop();
                }

                in_header = true;
            }
            Event::End(Tag::Header(i)) if i <= max_section_depth => {
                in_header = false;
                section_id = Some(utils::id_from_content(&heading));
                breadcrumbs.push(heading.clone());
            }
            Event::Start(Tag::FootnoteDefinition(name)) => {
                let number = footnote_numbers.len() + 1;
                footnote_numbers.entry(name).or_insert(number);
            }
            Event::Start(_) | Event::End(_) | Event::SoftBreak | Event::HardBreak => {
                // Insert spaces where HTML output would usually seperate text
                // to ensure words don't get merged together
                if in_header {
                    heading.push(' ');
                } else {
                    body.push(' ');
                }
            }
            Event::Text(text) => {
                if in_header {
                    heading.push_str(&text);
                } else {
                    body.push_str(&text);
                }
            }
            Event::Html(html) | Event::InlineHtml(html) => {
                body.push_str(&clean_html(&html));
            }
            Event::FootnoteReference(name) => {
                let len = footnote_numbers.len() + 1;
                let number = footnote_numbers.entry(name).or_insert(len);
                body.push_str(&format!(" [{}] ", number));
            }
        }
    }

    if heading.len() > 0 {
        // Make sure the last section is added to the index
        add_doc(
            index,
            doc_urls,
            &anchor_base,
            &section_id,
            &[&heading, &body, &breadcrumbs.join(" » ")],
        );
    }

    Ok(())
}

fn write_to_json(index: Index, search_config: &Search, doc_urls: Vec<String>) -> Result<String> {
    use std::collections::BTreeMap;
    use self::elasticlunr::config::{SearchBool, SearchOptions, SearchOptionsField};

    #[derive(Serialize)]
    struct ResultsOptions {
        limit_results: u32,
        teaser_word_count: u32,
    }

    #[derive(Serialize)]
    struct SearchindexJson {
        /// The options used for displaying search results
        results_options: ResultsOptions,
        /// The searchoptions for elasticlunr.js
        search_options: SearchOptions,
        /// Used to lookup a document's URL from an integer document ref.
        doc_urls: Vec<String>,
        /// The index for elasticlunr.js
        index: elasticlunr::Index,
    }

    let mut fields = BTreeMap::new();
    let mut opt = SearchOptionsField::default();
    opt.boost = Some(search_config.boost_title);
    fields.insert("title".into(), opt);
    opt.boost = Some(search_config.boost_paragraph);
    fields.insert("body".into(), opt);
    opt.boost = Some(search_config.boost_hierarchy);
    fields.insert("breadcrumbs".into(), opt);

    let search_options = SearchOptions {
        bool: if search_config.use_boolean_and {
            SearchBool::And
        } else {
            SearchBool::Or
        },
        expand: search_config.expand,
        fields,
    };

    let results_options = ResultsOptions {
        limit_results: search_config.limit_results,
        teaser_word_count: search_config.teaser_word_count,
    };

    let json_contents = SearchindexJson {
        results_options,
        search_options,
        doc_urls,
        index,
    };

    // By converting to serde_json::Value as an intermediary, we use a
    // BTreeMap internally and can force a stable ordering of map keys.
    let json_contents = serde_json::to_value(&json_contents)?;
    let json_contents = serde_json::to_string(&json_contents)?;

    Ok(json_contents)
}

fn clean_html(html: &str) -> String {
    lazy_static! {
        static ref AMMONIA: ammonia::Builder<'static> = {
            let mut clean_content = HashSet::new();
            clean_content.insert("script");
            clean_content.insert("style");
            let mut builder = ammonia::Builder::new();
            builder
                .tags(HashSet::new())
                .tag_attributes(HashMap::new())
                .generic_attributes(HashSet::new())
                .link_rel(None)
                .allowed_classes(HashMap::new())
                .clean_content_tags(clean_content);
            builder
        };
    }
    AMMONIA.clean(html).to_string()
}
<commit_msg>Cleanup add_doc<commit_after>extern crate ammonia;
extern crate elasticlunr;

use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::path::Path;

use pulldown_cmark::*;
use serde_json;
use self::elasticlunr::Index;

use book::{Book, BookItem};
use config::Search;
use errors::*;
use utils;
use theme::searcher;

/// Creates all files required for search.
pub fn create_files(search_config: &Search, destination: &Path, book: &Book) -> Result<()> {
    let mut index = Index::new(&["title", "body", "breadcrumbs"]);
    let mut doc_urls = Vec::with_capacity(book.sections.len());

    for item in book.iter() {
        render_item(&mut index, &search_config, &mut doc_urls, item)?;
    }

    let index = write_to_json(index, &search_config, doc_urls)?;
    debug!("Writing search index ✓");
    if index.len() > 10_000_000 {
        warn!("searchindex.json is very large ({} bytes)", index.len());
    }

    if search_config.copy_js {
        utils::fs::write_file(destination, "searchindex.json", index.as_bytes())?;
        utils::fs::write_file(destination, "searchindex.js", format!("window.search = {};", index).as_bytes())?;
        utils::fs::write_file(destination, "searcher.js", searcher::JS)?;
        utils::fs::write_file(destination, "mark.min.js", searcher::MARK_JS)?;
        utils::fs::write_file(destination, "elasticlunr.min.js", searcher::ELASTICLUNR_JS)?;
        debug!("Copying search files ✓");
    }

    Ok(())
}

/// Uses the given arguments to construct a search document, then inserts it to the given index.
fn add_doc(
    index: &mut Index,
    doc_urls: &mut Vec<String>,
    anchor_base: &str,
    section_id: &Option<String>,
    items: &[&str],
) {
    let url = if let &Some(ref id) = section_id {
        Cow::Owned(format!("{}#{}", anchor_base, id))
    } else {
        Cow::Borrowed(anchor_base)
    };
    let url = utils::collapse_whitespace(url.trim());
    let doc_ref = doc_urls.len().to_string();
    doc_urls.push(url.into());

    let items = items.iter().map(|&x| utils::collapse_whitespace(x.trim()));
    index.add_doc(&doc_ref, items);
}

/// Renders markdown into flat unformatted text and adds it to the search index.
fn render_item(
    index: &mut Index,
    search_config: &Search,
    doc_urls: &mut Vec<String>,
    item: &BookItem,
) -> Result<()> {
    let chapter = match item {
        &BookItem::Chapter(ref ch) => ch,
        _ => return Ok(()),
    };

    let filepath = Path::new(&chapter.path).with_extension("html");
    let filepath = filepath
        .to_str()
        .chain_err(|| "Could not convert HTML path to str")?;
    let anchor_base = utils::fs::normalize_path(filepath);

    let mut opts = Options::empty();
    opts.insert(OPTION_ENABLE_TABLES);
    opts.insert(OPTION_ENABLE_FOOTNOTES);
    let p = Parser::new_ext(&chapter.content, opts);

    let mut in_header = false;
    let max_section_depth = search_config.heading_split_level as i32;
    let mut section_id = None;
    let mut heading = String::new();
    let mut body = String::new();
    let mut breadcrumbs = chapter.parent_names.clone();
    let mut footnote_numbers = HashMap::new();

    for event in p {
        match event {
            Event::Start(Tag::Header(i)) if i <= max_section_depth => {
                if heading.len() > 0 {
                    // Section finished, the next header is following now
                    // Write the data to the index, and clear it for the next section
                    add_doc(
                        index,
                        doc_urls,
                        &anchor_base,
                        &section_id,
                        &[&heading, &body, &breadcrumbs.join(" » ")],
                    );
                    section_id = None;
                    heading.clear();
                    body.clear();
                    breadcrumbs.pop();
                }

                in_header = true;
            }
            Event::End(Tag::Header(i)) if i <= max_section_depth => {
                in_header = false;
                section_id = Some(utils::id_from_content(&heading));
                breadcrumbs.push(heading.clone());
            }
            Event::Start(Tag::FootnoteDefinition(name)) => {
                let number = footnote_numbers.len() + 1;
                footnote_numbers.entry(name).or_insert(number);
            }
            Event::Start(_) | Event::End(_) | Event::SoftBreak | Event::HardBreak => {
                // Insert spaces where HTML output would usually seperate text
                // to ensure words don't get merged together
                if in_header {
                    heading.push(' ');
                } else {
                    body.push(' ');
                }
            }
            Event::Text(text) => {
                if in_header {
                    heading.push_str(&text);
                } else {
                    body.push_str(&text);
                }
            }
            Event::Html(html) | Event::InlineHtml(html) => {
                body.push_str(&clean_html(&html));
            }
            Event::FootnoteReference(name) => {
                let len = footnote_numbers.len() + 1;
                let number = footnote_numbers.entry(name).or_insert(len);
                body.push_str(&format!(" [{}] ", number));
            }
        }
    }

    if heading.len() > 0 {
        // Make sure the last section is added to the index
        add_doc(
            index,
            doc_urls,
            &anchor_base,
            &section_id,
            &[&heading, &body, &breadcrumbs.join(" » ")],
        );
    }

    Ok(())
}

fn write_to_json(index: Index, search_config: &Search, doc_urls: Vec<String>) -> Result<String> {
    use std::collections::BTreeMap;
    use self::elasticlunr::config::{SearchBool, SearchOptions, SearchOptionsField};

    #[derive(Serialize)]
    struct ResultsOptions {
        limit_results: u32,
        teaser_word_count: u32,
    }

    #[derive(Serialize)]
    struct SearchindexJson {
        /// The options used for displaying search results
        results_options: ResultsOptions,
        /// The searchoptions for elasticlunr.js
        search_options: SearchOptions,
        /// Used to lookup a document's URL from an integer document ref.
        doc_urls: Vec<String>,
        /// The index for elasticlunr.js
        index: elasticlunr::Index,
    }

    let mut fields = BTreeMap::new();
    let mut opt = SearchOptionsField::default();
    opt.boost = Some(search_config.boost_title);
    fields.insert("title".into(), opt);
    opt.boost = Some(search_config.boost_paragraph);
    fields.insert("body".into(), opt);
    opt.boost = Some(search_config.boost_hierarchy);
    fields.insert("breadcrumbs".into(), opt);

    let search_options = SearchOptions {
        bool: if search_config.use_boolean_and {
            SearchBool::And
        } else {
            SearchBool::Or
        },
        expand: search_config.expand,
        fields,
    };

    let results_options = ResultsOptions {
        limit_results: search_config.limit_results,
        teaser_word_count: search_config.teaser_word_count,
    };

    let json_contents = SearchindexJson {
        results_options,
        search_options,
        doc_urls,
        index,
    };

    // By converting to serde_json::Value as an intermediary, we use a
    // BTreeMap internally and can force a stable ordering of map keys.
    let json_contents = serde_json::to_value(&json_contents)?;
    let json_contents = serde_json::to_string(&json_contents)?;

    Ok(json_contents)
}

fn clean_html(html: &str) -> String {
    lazy_static! {
        static ref AMMONIA: ammonia::Builder<'static> = {
            let mut clean_content = HashSet::new();
            clean_content.insert("script");
            clean_content.insert("style");
            let mut builder = ammonia::Builder::new();
            builder
                .tags(HashSet::new())
                .tag_attributes(HashMap::new())
                .generic_attributes(HashSet::new())
                .link_rel(None)
                .allowed_classes(HashMap::new())
                .clean_content_tags(clean_content);
            builder
        };
    }
    AMMONIA.clean(html).to_string()
}
<|endoftext|>


<commit_before>// can you solve the passcode riddle https://www.youtube.com/watch?v=7Vd1dTBVbFg

//use std::cmp::Ord;
//use std::cmp::Eq;
use std::cmp::Ordering;

#[derive(Debug,Clone)]
struct Moo {
    x:u8,
    y:u8,
    z:u8,
    sum:u16,
}

impl std::cmp::PartialEq for Moo {
    fn eq(&self, other: &Self) -> bool {
        if (self.sum == other.sum)&&(self.x == other.x)&&(self.y==other.y)&&(self.z==other.z) {
            return true;
        } else {
            return false;
        }
    }
}

impl std::cmp::Eq for Moo {

}

impl std::cmp::PartialOrd for Moo {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        if self.sum > other.sum {
            return Option::Some(Ordering::Greater);
        }else if self.sum < other.sum {
            return Option::Some(Ordering::Less);
        } else {
            return Option::Some(Ordering::Equal);
        }
    }
}

impl std::cmp::Ord for Moo {
    fn cmp(&self, other: &Self) -> Ordering {
        if self.sum < other.sum {
            return Ordering::Less;
        }else if self.sum > other.sum {
            return Ordering::Greater;
        } else {
            return Ordering::Equal;
        }
    }
}

// this does accept multiple args like println!
macro_rules! print_err { //src: https://github.com/rust-lang/rfcs/issues/1078#issue-69693246
    ($($arg:tt)*) => (
        {
            use std::io::prelude::*;
            if let Err(e) = write!(&mut ::std::io::stderr(), "{}\n", format_args!($($arg)*)) {
                panic!("Failed to write to stderr.\
                    \nOriginal error output: {}\
                    \nSecondary error writing to stderr: {}", format!($($arg)*), e);
            }
        }
    )
}

use std::env;//can't use this in body because it says it's already used
macro_rules! dprintln {
//    ( $( $x:tt ),* ) => {
//    ( $( $x:expr ),* ) => { //see: https://doc.rust-lang.org/nightly/book/macros.html#matching
    ( $( $x:tt )* ) => { //see: https://doc.rust-lang.org/nightly/book/macros.html#matching
//src: https://doc.rust-lang.org/std/env/fn.var.html
        let key = "DEBUG";
        match env::var(key) {
            Ok(ref val) if val.as_str() != "0" => { print_err!($( $x )*);
            },//println!("{}: {:?}", key, val),
            //Err(e) => println!("couldn't interpret {}: {}", key, e),
            Err(_) | Ok(_) => {},
        }
    }
}

fn main() {
    const MAX:u8 = 9;
    //casting to different types! how nice is that ;-)
    //println!("{}",1u8 as u16+1u16 as u16);
//    let x:u16;
    let mut candidates: Vec<Moo>=vec![];
    for x in 1..1+MAX {//(1+MAX as u16) {
        for y in 1..MAX+1 { //1u16+MAX as u16 {
            for z in 1..MAX+1 {//1+(MAX as u16) {
                if (x<=y)&&(y<=z)
//                    &&(z != y) not yet!
                        && (x as u16 * y as u16 * z as u16) as u16 == 36 //clue1
                        {
                            let m:Moo = Moo {x:x, y:y,z:z,sum:(x+y+z) as u16};
                            candidates.push(m);//clue2 - storing all sums, for now
                    dprintln!("{} {} {} sum={}",x,y,z,x+y+z);
                } else {
//                    println!("{} {} {} rejected",x,y,z);
                }
            }
        }
    }
    //println!("{}",x);
    candidates.sort();
    dprintln!("Candidates(clue1): {:?}",candidates);
    //should keep only dups, because that's why she's unsure - and needed clue3
    //irrelevant block, ignore:
/*    let mut prev=0;
    for i in candidates.iter() {
        if (i.z != i.y) {//clue3 - largest number appears only once
            println!("{:?} clue3 ok", i);
            if prev == i.sum {
                println!("dup: {:?}", i);//coincidence that 229 is last!
            }
        }
        prev=i.sum;
    }*/

    let mut cand2:Vec<Moo> = Vec::new();
    for i in 0..candidates.len()-1 {
        if candidates[i].sum == candidates[i+1].sum {
            cand2.push(candidates[i].clone());
            cand2.push(candidates[i+1].clone());
        }
    }
    dprintln!("Candidates(after clue2): {:?}", cand2);//when she's unsure
    for i in cand2.iter() {
        if i.z != i.y {//clue3 - largest number appears only once
            println!("Found valid passcode {:?}", i);
        }
    }

    /*for x in 1..1+MAX {
        for y in 1..1+MAX {
            for z in 1..1+MAX {

            }
        }
    }*/
}
<commit_msg>rustfmt that<commit_after>// can you solve the passcode riddle https://www.youtube.com/watch?v=7Vd1dTBVbFg

// use std::cmp::Ord;
// use std::cmp::Eq;
use std::cmp::Ordering;

#[derive(Debug,Clone)]
struct Moo {
    x: u8,
    y: u8,
    z: u8,
    sum: u16,
}

impl std::cmp::PartialEq for Moo {
    fn eq(&self, other: &Self) -> bool {
        if (self.sum == other.sum) && (self.x == other.x) && (self.y == other.y) &&
           (self.z == other.z) {
            return true;
        } else {
            return false;
        }
    }
}

impl std::cmp::Eq for Moo {}

impl std::cmp::PartialOrd for Moo {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        if self.sum > other.sum {
            return Option::Some(Ordering::Greater);
        } else if self.sum < other.sum {
            return Option::Some(Ordering::Less);
        } else {
            return Option::Some(Ordering::Equal);
        }
    }
}

impl std::cmp::Ord for Moo {
    fn cmp(&self, other: &Self) -> Ordering {
        if self.sum < other.sum {
            return Ordering::Less;
        } else if self.sum > other.sum {
            return Ordering::Greater;
        } else {
            return Ordering::Equal;
        }
    }
}

// this does accept multiple args like println!
macro_rules! print_err { //src: https://github.com/rust-lang/rfcs/issues/1078#issue-69693246
    ($($arg:tt)*) => (
        {
            use std::io::prelude::*;
            if let Err(e) = write!(&mut ::std::io::stderr(), "{}\n", format_args!($($arg)*)) {
                panic!("Failed to write to stderr.\
                    \nOriginal error output: {}\
                    \nSecondary error writing to stderr: {}", format!($($arg)*), e);
            }
        }
    )
}

use std::env;//can't use this in body because it says it's already used
macro_rules! dprintln {
//    ( $( $x:tt ),* ) => {
//    ( $( $x:expr ),* ) => { //see: https://doc.rust-lang.org/nightly/book/macros.html#matching
    ( $( $x:tt )* ) => { //see: https://doc.rust-lang.org/nightly/book/macros.html#matching
//src: https://doc.rust-lang.org/std/env/fn.var.html
        let key = "DEBUG";
        match env::var(key) {
            Ok(ref val) if val.as_str() != "0" => { print_err!($( $x )*);
            },//println!("{}: {:?}", key, val),
            //Err(e) => println!("couldn't interpret {}: {}", key, e),
            Err(_) | Ok(_) => {},
        }
    }
}

fn main() {
    const MAX: u8 = 9;
    // casting to different types! how nice is that ;-)
    // println!("{}",1u8 as u16+1u16 as u16);
    //    let x:u16;
    let mut candidates: Vec<Moo> = vec![];
    for x in 1..1 + MAX {
        // (1+MAX as u16) {
        for y in 1..MAX + 1 {
            // 1u16+MAX as u16 {
            for z in 1..MAX + 1 {
                // 1+(MAX as u16) {
                if (x<=y)&&(y<=z)
//                    &&(z != y) not yet!
                        && (x as u16 * y as u16 * z as u16) as u16 == 36
                // clue1
                {
                    let m: Moo = Moo {
                        x: x,
                        y: y,
                        z: z,
                        sum: (x + y + z) as u16,
                    };
                    candidates.push(m);//clue2 - storing all sums, for now
                    dprintln!("{} {} {} sum={}", x, y, z, x + y + z);
                } else {
                    //                    println!("{} {} {} rejected",x,y,z);
                }
            }
        }
    }
    // println!("{}",x);
    candidates.sort();
    dprintln!("Candidates(clue1): {:?}", candidates);
    // should keep only dups, because that's why she's unsure - and needed clue3
    // irrelevant block, ignore:
    //    let mut prev=0;
    // for i in candidates.iter() {
    // if (i.z != i.y) {//clue3 - largest number appears only once
    // println!("{:?} clue3 ok", i);
    // if prev == i.sum {
    // println!("dup: {:?}", i);//coincidence that 229 is last!
    // }
    // }
    // prev=i.sum;
    // }

    let mut cand2: Vec<Moo> = Vec::new();
    for i in 0..candidates.len() - 1 {
        if candidates[i].sum == candidates[i + 1].sum {
            cand2.push(candidates[i].clone());
            cand2.push(candidates[i + 1].clone());
        }
    }
    dprintln!("Candidates(after clue2): {:?}", cand2);//when she's unsure
    for i in cand2.iter() {
        if i.z != i.y {
            // clue3 - largest number appears only once
            println!("Found valid passcode {:?}", i);
        }
    }

    // for x in 1..1+MAX {
    // for y in 1..1+MAX {
    // for z in 1..1+MAX {
    //
    // }
    // }
    // }
}
<|endoftext|>


<commit_before>// This file is part of the uutils coreutils package.
//
// (c) Jian Zeng <anonymousknight96@gmail.com>
//
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.

// spell-checker:ignore (vars) Passwd cstr fnam gecos ngroups egid

//! Get password/group file entry
//!
//! # Examples:
//!
//! ```
//! use uucore::entries::{self, Locate};
//!
//! let root_group = if cfg!(any(target_os = "linux", target_os = "android")) {
//!     "root"
//! } else {
//!     "wheel"
//! };
//!
//! assert_eq!("root", entries::uid2usr(0).unwrap());
//! assert_eq!(0, entries::usr2uid("root").unwrap());
//! assert!(entries::gid2grp(0).is_ok());
//! assert!(entries::grp2gid(root_group).is_ok());
//!
//! assert!(entries::Passwd::locate(0).is_ok());
//! assert!(entries::Passwd::locate("0").is_ok());
//! assert!(entries::Passwd::locate("root").is_ok());
//!
//! assert!(entries::Group::locate(0).is_ok());
//! assert!(entries::Group::locate("0").is_ok());
//! assert!(entries::Group::locate(root_group).is_ok());
//! ```

#[cfg(any(target_os = "freebsd", target_vendor = "apple"))]
use libc::time_t;
use libc::{c_char, c_int, gid_t, uid_t};
use libc::{getgrgid, getgrnam, getgroups, getpwnam, getpwuid, group, passwd};

use std::borrow::Cow;
use std::ffi::{CStr, CString};
use std::io::Error as IOError;
use std::io::ErrorKind;
use std::io::Result as IOResult;
use std::ptr;

extern "C" {
    fn getgrouplist(
        name: *const c_char,
        gid: gid_t,
        groups: *mut gid_t,
        ngroups: *mut c_int,
    ) -> c_int;
}

pub fn get_groups() -> IOResult<Vec<gid_t>> {
    let ngroups = unsafe { getgroups(0, ptr::null_mut()) };
    if ngroups == -1 {
        return Err(IOError::last_os_error());
    }
    let mut groups = Vec::with_capacity(ngroups as usize);
    let ngroups = unsafe { getgroups(ngroups, groups.as_mut_ptr()) };
    if ngroups == -1 {
        Err(IOError::last_os_error())
    } else {
        unsafe {
            groups.set_len(ngroups as usize);
        }
        Ok(groups)
    }
}

/// The list of group IDs returned from GNU's `groups` and GNU's `id --groups`
/// starts with the effective group ID (egid).
/// This is a wrapper for `get_groups()` to mimic this behavior.
///
/// If `arg_id` is `None` (default), `get_groups_gnu` moves the effective
/// group id (egid) to the first entry in the returned Vector.
/// If `arg_id` is `Some(x)`, `get_groups_gnu` moves the id with value `x`
/// to the first entry in the returned Vector. This might be necessary
/// for `id --groups --real` if `gid` and `egid` are not equal.
///
/// From: https://www.man7.org/linux/man-pages/man3/getgroups.3p.html
/// As implied by the definition of supplementary groups, the
/// effective group ID may appear in the array returned by
/// getgroups() or it may be returned only by getegid().  Duplication
/// may exist, but the application needs to call getegid() to be sure
/// of getting all of the information. Various implementation
/// variations and administrative sequences cause the set of groups
/// appearing in the result of getgroups() to vary in order and as to
/// whether the effective group ID is included, even when the set of
/// groups is the same (in the mathematical sense of ``set''). (The
/// history of a process and its parents could affect the details of
/// the result.)
pub fn get_groups_gnu(arg_id: Option<u32>) -> IOResult<Vec<gid_t>> {
    let mut groups = get_groups()?;
    let egid = arg_id.unwrap_or_else(crate::features::process::getegid);
    if !groups.is_empty() && *groups.first().unwrap() == egid {
        return Ok(groups);
    } else if let Some(index) = groups.iter().position(|&x| x == egid) {
        groups.remove(index);
    }
    groups.insert(0, egid);
    Ok(groups)
}

#[derive(Copy, Clone)]
pub struct Passwd {
    inner: passwd,
}

macro_rules! cstr2cow {
    ($v:expr) => {
        unsafe { CStr::from_ptr($v).to_string_lossy() }
    };
}

impl Passwd {
    /// AKA passwd.pw_name
    pub fn name(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_name)
    }

    /// AKA passwd.pw_uid
    pub fn uid(&self) -> uid_t {
        self.inner.pw_uid
    }

    /// AKA passwd.pw_gid
    pub fn gid(&self) -> gid_t {
        self.inner.pw_gid
    }

    /// AKA passwd.pw_gecos
    pub fn user_info(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_gecos)
    }

    /// AKA passwd.pw_shell
    pub fn user_shell(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_shell)
    }

    /// AKA passwd.pw_dir
    pub fn user_dir(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_dir)
    }

    /// AKA passwd.pw_passwd
    pub fn user_passwd(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_passwd)
    }

    /// AKA passwd.pw_class
    #[cfg(any(target_os = "freebsd", target_vendor = "apple"))]
    pub fn user_access_class(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_class)
    }

    /// AKA passwd.pw_change
    #[cfg(any(target_os = "freebsd", target_vendor = "apple"))]
    pub fn passwd_change_time(&self) -> time_t {
        self.inner.pw_change
    }

    /// AKA passwd.pw_expire
    #[cfg(any(target_os = "freebsd", target_vendor = "apple"))]
    pub fn expiration(&self) -> time_t {
        self.inner.pw_expire
    }

    pub fn as_inner(&self) -> &passwd {
        &self.inner
    }

    pub fn into_inner(self) -> passwd {
        self.inner
    }

    pub fn belongs_to(&self) -> Vec<gid_t> {
        let mut ngroups: c_int = 8;
        let mut groups = Vec::with_capacity(ngroups as usize);
        let gid = self.inner.pw_gid;
        let name = self.inner.pw_name;
        unsafe {
            if getgrouplist(name, gid, groups.as_mut_ptr(), &mut ngroups) == -1 {
                groups.resize(ngroups as usize, 0);
                getgrouplist(name, gid, groups.as_mut_ptr(), &mut ngroups);
            }
            groups.set_len(ngroups as usize);
        }
        groups.truncate(ngroups as usize);
        groups
    }
}

pub struct Group {
    inner: group,
}

impl Group {
    /// AKA group.gr_name
    pub fn name(&self) -> Cow<str> {
        cstr2cow!(self.inner.gr_name)
    }

    /// AKA group.gr_gid
    pub fn gid(&self) -> gid_t {
        self.inner.gr_gid
    }

    pub fn as_inner(&self) -> &group {
        &self.inner
    }

    pub fn into_inner(self) -> group {
        self.inner
    }
}

/// Fetch desired entry.
pub trait Locate<K> {
    fn locate(key: K) -> IOResult<Self>
    where
        Self: ::std::marker::Sized;
}

macro_rules! f {
    ($fnam:ident, $fid:ident, $t:ident, $st:ident) => {
        impl Locate<$t> for $st {
            fn locate(k: $t) -> IOResult<Self> {
                unsafe {
                    let data = $fid(k);
                    if !data.is_null() {
                        Ok($st {
                            inner: ptr::read(data as *const _),
                        })
                    } else {
                        Err(IOError::new(
                            ErrorKind::NotFound,
                            format!("No such id: {}", k),
                        ))
                    }
                }
            }
        }

        impl<'a> Locate<&'a str> for $st {
            fn locate(k: &'a str) -> IOResult<Self> {
                if let Ok(id) = k.parse::<$t>() {
                    let data = unsafe { $fid(id) };
                    if !data.is_null() {
                        Ok($st {
                            inner: unsafe { ptr::read(data as *const _) },
                        })
                    } else {
                        Err(IOError::new(
                            ErrorKind::NotFound,
                            format!("No such id: {}", id),
                        ))
                    }
                } else {
                    unsafe {
                        let data = $fnam(CString::new(k).unwrap().as_ptr());
                        if !data.is_null() {
                            Ok($st {
                                inner: ptr::read(data as *const _),
                            })
                        } else {
                            Err(IOError::new(
                                ErrorKind::NotFound,
                                format!("Not found: {}", k),
                            ))
                        }
                    }
                }
            }
        }
    };
}

f!(getpwnam, getpwuid, uid_t, Passwd);
f!(getgrnam, getgrgid, gid_t, Group);

#[inline]
pub fn uid2usr(id: uid_t) -> IOResult<String> {
    Passwd::locate(id).map(|p| p.name().into_owned())
}

#[inline]
pub fn gid2grp(id: gid_t) -> IOResult<String> {
    Group::locate(id).map(|p| p.name().into_owned())
}

#[inline]
pub fn usr2uid(name: &str) -> IOResult<uid_t> {
    Passwd::locate(name).map(|p| p.uid())
}

#[inline]
pub fn grp2gid(name: &str) -> IOResult<gid_t> {
    Group::locate(name).map(|p| p.gid())
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_entries_get_groups_gnu() {
        if let Ok(mut groups) = get_groups() {
            if let Some(last) = groups.pop() {
                groups.insert(0, last);
                assert_eq!(get_groups_gnu(Some(last)).unwrap(), groups);
            }
        }
    }
}
<commit_msg>uucore: entries: add documentation<commit_after>// This file is part of the uutils coreutils package.
//
// (c) Jian Zeng <anonymousknight96@gmail.com>
//
// For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code.

// spell-checker:ignore (vars) Passwd cstr fnam gecos ngroups egid

//! Get password/group file entry
//!
//! # Examples:
//!
//! ```
//! use uucore::entries::{self, Locate};
//!
//! let root_group = if cfg!(any(target_os = "linux", target_os = "android")) {
//!     "root"
//! } else {
//!     "wheel"
//! };
//!
//! assert_eq!("root", entries::uid2usr(0).unwrap());
//! assert_eq!(0, entries::usr2uid("root").unwrap());
//! assert!(entries::gid2grp(0).is_ok());
//! assert!(entries::grp2gid(root_group).is_ok());
//!
//! assert!(entries::Passwd::locate(0).is_ok());
//! assert!(entries::Passwd::locate("0").is_ok());
//! assert!(entries::Passwd::locate("root").is_ok());
//!
//! assert!(entries::Group::locate(0).is_ok());
//! assert!(entries::Group::locate("0").is_ok());
//! assert!(entries::Group::locate(root_group).is_ok());
//! ```

#[cfg(any(target_os = "freebsd", target_vendor = "apple"))]
use libc::time_t;
use libc::{c_char, c_int, gid_t, uid_t};
use libc::{getgrgid, getgrnam, getgroups, getpwnam, getpwuid, group, passwd};

use std::borrow::Cow;
use std::ffi::{CStr, CString};
use std::io::Error as IOError;
use std::io::ErrorKind;
use std::io::Result as IOResult;
use std::ptr;

extern "C" {
    fn getgrouplist(
        name: *const c_char,
        gid: gid_t,
        groups: *mut gid_t,
        ngroups: *mut c_int,
    ) -> c_int;
}

pub fn get_groups() -> IOResult<Vec<gid_t>> {
    let ngroups = unsafe { getgroups(0, ptr::null_mut()) };
    if ngroups == -1 {
        return Err(IOError::last_os_error());
    }
    let mut groups = Vec::with_capacity(ngroups as usize);
    let ngroups = unsafe { getgroups(ngroups, groups.as_mut_ptr()) };
    if ngroups == -1 {
        Err(IOError::last_os_error())
    } else {
        unsafe {
            groups.set_len(ngroups as usize);
        }
        Ok(groups)
    }
}

/// The list of group IDs returned from GNU's `groups` and GNU's `id --groups`
/// starts with the effective group ID (egid).
/// This is a wrapper for `get_groups()` to mimic this behavior.
///
/// If `arg_id` is `None` (default), `get_groups_gnu` moves the effective
/// group id (egid) to the first entry in the returned Vector.
/// If `arg_id` is `Some(x)`, `get_groups_gnu` moves the id with value `x`
/// to the first entry in the returned Vector. This might be necessary
/// for `id --groups --real` if `gid` and `egid` are not equal.
///
/// From: https://www.man7.org/linux/man-pages/man3/getgroups.3p.html
/// As implied by the definition of supplementary groups, the
/// effective group ID may appear in the array returned by
/// getgroups() or it may be returned only by getegid().  Duplication
/// may exist, but the application needs to call getegid() to be sure
/// of getting all of the information. Various implementation
/// variations and administrative sequences cause the set of groups
/// appearing in the result of getgroups() to vary in order and as to
/// whether the effective group ID is included, even when the set of
/// groups is the same (in the mathematical sense of ``set''). (The
/// history of a process and its parents could affect the details of
/// the result.)
pub fn get_groups_gnu(arg_id: Option<u32>) -> IOResult<Vec<gid_t>> {
    let mut groups = get_groups()?;
    let egid = arg_id.unwrap_or_else(crate::features::process::getegid);
    if !groups.is_empty() && *groups.first().unwrap() == egid {
        return Ok(groups);
    } else if let Some(index) = groups.iter().position(|&x| x == egid) {
        groups.remove(index);
    }
    groups.insert(0, egid);
    Ok(groups)
}

#[derive(Copy, Clone)]
pub struct Passwd {
    inner: passwd,
}

macro_rules! cstr2cow {
    ($v:expr) => {
        unsafe { CStr::from_ptr($v).to_string_lossy() }
    };
}

impl Passwd {
    /// AKA passwd.pw_name
    pub fn name(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_name)
    }

    /// AKA passwd.pw_uid
    pub fn uid(&self) -> uid_t {
        self.inner.pw_uid
    }

    /// AKA passwd.pw_gid
    pub fn gid(&self) -> gid_t {
        self.inner.pw_gid
    }

    /// AKA passwd.pw_gecos
    pub fn user_info(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_gecos)
    }

    /// AKA passwd.pw_shell
    pub fn user_shell(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_shell)
    }

    /// AKA passwd.pw_dir
    pub fn user_dir(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_dir)
    }

    /// AKA passwd.pw_passwd
    pub fn user_passwd(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_passwd)
    }

    /// AKA passwd.pw_class
    #[cfg(any(target_os = "freebsd", target_vendor = "apple"))]
    pub fn user_access_class(&self) -> Cow<str> {
        cstr2cow!(self.inner.pw_class)
    }

    /// AKA passwd.pw_change
    #[cfg(any(target_os = "freebsd", target_vendor = "apple"))]
    pub fn passwd_change_time(&self) -> time_t {
        self.inner.pw_change
    }

    /// AKA passwd.pw_expire
    #[cfg(any(target_os = "freebsd", target_vendor = "apple"))]
    pub fn expiration(&self) -> time_t {
        self.inner.pw_expire
    }

    pub fn as_inner(&self) -> &passwd {
        &self.inner
    }

    pub fn into_inner(self) -> passwd {
        self.inner
    }

    /// This is a wrapper function for `libc::getgrouplist`.
    ///
    /// From: https://man7.org/linux/man-pages/man3/getgrouplist.3.html
    /// If the user is a member of more than *ngroups groups, then
    /// getgrouplist() returns -1.  In this case, the value returned in
    /// *ngroups can be used to resize the buffer passed to a further
    /// call getgrouplist().
    pub fn belongs_to(&self) -> Vec<gid_t> {
        let mut ngroups: c_int = 8;
        let mut groups = Vec::with_capacity(ngroups as usize);
        let gid = self.inner.pw_gid;
        let name = self.inner.pw_name;
        unsafe {
            if getgrouplist(name, gid, groups.as_mut_ptr(), &mut ngroups) == -1 {
                groups.resize(ngroups as usize, 0);
                getgrouplist(name, gid, groups.as_mut_ptr(), &mut ngroups);
            }
            groups.set_len(ngroups as usize);
        }
        groups.truncate(ngroups as usize);
        groups
    }
}

pub struct Group {
    inner: group,
}

impl Group {
    /// AKA group.gr_name
    pub fn name(&self) -> Cow<str> {
        cstr2cow!(self.inner.gr_name)
    }

    /// AKA group.gr_gid
    pub fn gid(&self) -> gid_t {
        self.inner.gr_gid
    }

    pub fn as_inner(&self) -> &group {
        &self.inner
    }

    pub fn into_inner(self) -> group {
        self.inner
    }
}

/// Fetch desired entry.
pub trait Locate<K> {
    fn locate(key: K) -> IOResult<Self>
    where
        Self: ::std::marker::Sized;
}

macro_rules! f {
    ($fnam:ident, $fid:ident, $t:ident, $st:ident) => {
        impl Locate<$t> for $st {
            fn locate(k: $t) -> IOResult<Self> {
                unsafe {
                    let data = $fid(k);
                    if !data.is_null() {
                        Ok($st {
                            inner: ptr::read(data as *const _),
                        })
                    } else {
                        Err(IOError::new(
                            ErrorKind::NotFound,
                            format!("No such id: {}", k),
                        ))
                    }
                }
            }
        }

        impl<'a> Locate<&'a str> for $st {
            fn locate(k: &'a str) -> IOResult<Self> {
                if let Ok(id) = k.parse::<$t>() {
                    let data = unsafe { $fid(id) };
                    if !data.is_null() {
                        Ok($st {
                            inner: unsafe { ptr::read(data as *const _) },
                        })
                    } else {
                        Err(IOError::new(
                            ErrorKind::NotFound,
                            format!("No such id: {}", id),
                        ))
                    }
                } else {
                    unsafe {
                        let data = $fnam(CString::new(k).unwrap().as_ptr());
                        if !data.is_null() {
                            Ok($st {
                                inner: ptr::read(data as *const _),
                            })
                        } else {
                            Err(IOError::new(
                                ErrorKind::NotFound,
                                format!("Not found: {}", k),
                            ))
                        }
                    }
                }
            }
        }
    };
}

f!(getpwnam, getpwuid, uid_t, Passwd);
f!(getgrnam, getgrgid, gid_t, Group);

#[inline]
pub fn uid2usr(id: uid_t) -> IOResult<String> {
    Passwd::locate(id).map(|p| p.name().into_owned())
}

#[inline]
pub fn gid2grp(id: gid_t) -> IOResult<String> {
    Group::locate(id).map(|p| p.name().into_owned())
}

#[inline]
pub fn usr2uid(name: &str) -> IOResult<uid_t> {
    Passwd::locate(name).map(|p| p.uid())
}

#[inline]
pub fn grp2gid(name: &str) -> IOResult<gid_t> {
    Group::locate(name).map(|p| p.gid())
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_entries_get_groups_gnu() {
        if let Ok(mut groups) = get_groups() {
            if let Some(last) = groups.pop() {
                groups.insert(0, last);
                assert_eq!(get_groups_gnu(Some(last)).unwrap(), groups);
            }
        }
    }
}
<|endoftext|>


<commit_before>#[crate_id="github.com/sfackler/rust-phf#phf:0.0"];
#[crate_type="lib"];
#[feature(managed_boxes, macro_registrar)];

extern mod syntax;

use std::vec::VecIterator;

use syntax::ast;
use syntax::ast::{Name, TokenTree, LitStr, MutImmutable, Expr, ExprVec, ExprLit};
use syntax::codemap::Span;
use syntax::ext::base::{SyntaxExtension,
                        ExtCtxt,
                        MacResult,
                        MRExpr,
                        NormalTT,
                        SyntaxExpanderTT,
                        SyntaxExpanderTTExpanderWithoutContext};
use syntax::parse;
use syntax::parse::token;
use syntax::parse::token::{COMMA, EOF, FAT_ARROW};

pub struct PhfMap<T> {
    #[doc(hidden)]
    entries: &'static [(&'static str, T)],
}

impl<T> Container for PhfMap<T> {
    #[inline]
    fn len(&self) -> uint {
        self.entries.len()
    }
}

impl<T> Map<&'static str, T> for PhfMap<T> {
    #[inline]
    fn find<'a>(&'a self, key: & &'static str) -> Option<&'a T> {
        self.entries.bsearch(|&(val, _)| val.cmp(key)).map(|idx| {
            let (_, ref val) = self.entries[idx];
            val
        })
    }
}

impl<T> PhfMap<T> {
    #[inline]
    pub fn entries<'a>(&'a self) -> PhfMapEntries<'a, T> {
        PhfMapEntries { iter: self.entries.iter() }
    }

    #[inline]
    pub fn keys<'a>(&'a self) -> PhfMapKeys<'a, T> {
        PhfMapKeys { iter: self.entries() }
    }

    #[inline]
    pub fn values<'a>(&'a self) -> PhfMapValues<'a, T> {
        PhfMapValues { iter: self.entries() }
    }
}

pub struct PhfMapEntries<'a, T> {
    priv iter: VecIterator<'a, (&'static str, T)>,
}

impl<'a, T> Iterator<(&'static str, &'a T)> for PhfMapEntries<'a, T> {
    #[inline]
    fn next(&mut self) -> Option<(&'static str, &'a T)> {
        self.iter.next().map(|&(key, ref value)| (key, value))
    }
}

pub struct PhfMapKeys<'a, T> {
    priv iter: PhfMapEntries<'a, T>,
}

impl<'a, T> Iterator<&'static str> for PhfMapKeys<'a, T> {
    #[inline]
    fn next(&mut self) -> Option<&'static str> {
        self.iter.next().map(|(key, _)| key)
    }
}

pub struct PhfMapValues<'a, T> {
    priv iter: PhfMapEntries<'a, T>,
}

impl<'a, T> Iterator<&'a T> for PhfMapValues<'a, T> {
    #[inline]
    fn next(&mut self) -> Option<&'a T> {
        self.iter.next().map(|(_, value)| value)
    }
}

#[macro_registrar]
#[doc(hidden)]
pub fn macro_registrar(register: |Name, SyntaxExtension|) {
    register(token::intern("phf_map"),
             NormalTT(~SyntaxExpanderTT {
                expander: SyntaxExpanderTTExpanderWithoutContext(expand_mphf_map),
                span: None
             },
             None));
}

fn expand_mphf_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> MacResult {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(),
                                                tts.to_owned());
    let mut pairs = ~[];

    while parser.token != EOF {
        let key = parser.parse_expr();

        let key_str = match key.node {
            ExprLit(lit) => {
                match lit.node {
                    LitStr(s, _) => s,
                    _ => cx.span_fatal(key.span, "expected string literal"),
                }
            }
            _ => cx.span_fatal(key.span, "expected string literal"),
        };

        if !parser.eat(&FAT_ARROW) {
            cx.span_fatal(parser.span, "expected `=>`");
        }

        let value = parser.parse_expr();

        pairs.push((key_str, key, value));

        if !parser.eat(&COMMA) && parser.token != EOF {
            cx.span_fatal(parser.span, "expected `,`");
        }
    }

    pairs.sort_by(|&(ref a, _, _), &(ref b, _, _)| a.cmp(b));

    for window in pairs.windows(2) {
        let (ref a, ref a_expr, _) = window[0];
        let (ref b, ref b_expr, _) = window[1];
        if a == b {
            cx.span_err(sp, format!("duplicate key \"{}\"", *a));
            cx.span_err(a_expr.span, "one occurrence here");
            cx.span_err(b_expr.span, "one occurrence here");
        }
    }
    cx.parse_sess().span_diagnostic.handler().abort_if_errors();

    let entries = pairs.move_iter()
        .map(|(_, key, value)| quote_expr!(&*cx, ($key, $value)))
        .collect();
    let entries = @Expr {
        id: ast::DUMMY_NODE_ID,
        node: ExprVec(entries, MutImmutable),
        span: sp,
    };

    MRExpr(quote_expr!(cx, PhfMap { entries: &'static $entries }))
}
<commit_msg>Properly display duplicate keys<commit_after>#[crate_id="github.com/sfackler/rust-phf#phf:0.0"];
#[crate_type="lib"];
#[feature(managed_boxes, macro_registrar)];

extern mod syntax;

use std::vec::VecIterator;

use syntax::ast;
use syntax::ast::{Name, TokenTree, LitStr, MutImmutable, Expr, ExprVec, ExprLit};
use syntax::codemap::Span;
use syntax::ext::base::{SyntaxExtension,
                        ExtCtxt,
                        MacResult,
                        MRExpr,
                        NormalTT,
                        SyntaxExpanderTT,
                        SyntaxExpanderTTExpanderWithoutContext};
use syntax::parse;
use syntax::parse::token;
use syntax::parse::token::{COMMA, EOF, FAT_ARROW};

pub struct PhfMap<T> {
    #[doc(hidden)]
    entries: &'static [(&'static str, T)],
}

impl<T> Container for PhfMap<T> {
    #[inline]
    fn len(&self) -> uint {
        self.entries.len()
    }
}

impl<T> Map<&'static str, T> for PhfMap<T> {
    #[inline]
    fn find<'a>(&'a self, key: & &'static str) -> Option<&'a T> {
        self.entries.bsearch(|&(val, _)| val.cmp(key)).map(|idx| {
            let (_, ref val) = self.entries[idx];
            val
        })
    }
}

impl<T> PhfMap<T> {
    #[inline]
    pub fn entries<'a>(&'a self) -> PhfMapEntries<'a, T> {
        PhfMapEntries { iter: self.entries.iter() }
    }

    #[inline]
    pub fn keys<'a>(&'a self) -> PhfMapKeys<'a, T> {
        PhfMapKeys { iter: self.entries() }
    }

    #[inline]
    pub fn values<'a>(&'a self) -> PhfMapValues<'a, T> {
        PhfMapValues { iter: self.entries() }
    }
}

pub struct PhfMapEntries<'a, T> {
    priv iter: VecIterator<'a, (&'static str, T)>,
}

impl<'a, T> Iterator<(&'static str, &'a T)> for PhfMapEntries<'a, T> {
    #[inline]
    fn next(&mut self) -> Option<(&'static str, &'a T)> {
        self.iter.next().map(|&(key, ref value)| (key, value))
    }
}

pub struct PhfMapKeys<'a, T> {
    priv iter: PhfMapEntries<'a, T>,
}

impl<'a, T> Iterator<&'static str> for PhfMapKeys<'a, T> {
    #[inline]
    fn next(&mut self) -> Option<&'static str> {
        self.iter.next().map(|(key, _)| key)
    }
}

pub struct PhfMapValues<'a, T> {
    priv iter: PhfMapEntries<'a, T>,
}

impl<'a, T> Iterator<&'a T> for PhfMapValues<'a, T> {
    #[inline]
    fn next(&mut self) -> Option<&'a T> {
        self.iter.next().map(|(_, value)| value)
    }
}

#[macro_registrar]
#[doc(hidden)]
pub fn macro_registrar(register: |Name, SyntaxExtension|) {
    register(token::intern("phf_map"),
             NormalTT(~SyntaxExpanderTT {
                expander: SyntaxExpanderTTExpanderWithoutContext(expand_mphf_map),
                span: None
             },
             None));
}

fn expand_mphf_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> MacResult {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(),
                                                tts.to_owned());
    let mut pairs = ~[];

    while parser.token != EOF {
        let key = parser.parse_expr();

        let key_str = match key.node {
            ExprLit(lit) => {
                match lit.node {
                    LitStr(s, _) => s,
                    _ => cx.span_fatal(key.span, "expected string literal"),
                }
            }
            _ => cx.span_fatal(key.span, "expected string literal"),
        };

        if !parser.eat(&FAT_ARROW) {
            cx.span_fatal(parser.span, "expected `=>`");
        }

        let value = parser.parse_expr();

        pairs.push((key_str, key, value));

        if !parser.eat(&COMMA) && parser.token != EOF {
            cx.span_fatal(parser.span, "expected `,`");
        }
    }

    pairs.sort_by(|&(ref a, _, _), &(ref b, _, _)| a.cmp(b));
    check_for_duplicates(cx, sp, pairs);

    let entries = pairs.move_iter()
        .map(|(_, key, value)| quote_expr!(&*cx, ($key, $value)))
        .collect();
    let entries = @Expr {
        id: ast::DUMMY_NODE_ID,
        node: ExprVec(entries, MutImmutable),
        span: sp,
    };

    MRExpr(quote_expr!(cx, PhfMap { entries: &'static $entries }))
}

fn check_for_duplicates(cx: &mut ExtCtxt, sp: Span, entries: &[(@str, @Expr, @Expr)]) {
    let mut in_dup = false;
    for window in entries.windows(2) {
        let (a, a_expr, _) = window[0];
        let (b, b_expr, _) = window[1];
        if a == b {
            if !in_dup {
                cx.span_err(sp, format!("duplicate key \"{}\"", a));
                cx.span_err(a_expr.span, "one occurrence here");
                in_dup = true;
            }
            cx.span_err(b_expr.span, "one occurrence here");
        } else {
            in_dup = false;
        }
    }
    cx.parse_sess().span_diagnostic.handler().abort_if_errors();
}
<|endoftext|>


<commit_before>// See LICENSE file for copyright and license details.

use cgmath::vector::Vec2;

pub struct TileIterator {
  cursor: Vec2<i32>
}

impl TileIterator {
  pub fn new() -> TileIterator {
    TileIterator {
      cursor: Vec2::<i32>{x: 0, y: 0}
    }
  }
}

impl Iterator<Vec2<i32>> for TileIterator {
  fn next(&mut self) -> Option<Vec2<i32>> {
    let map_size = Vec2::<i32>{x: 3, y: 4};
    self.cursor.x += 1;
    if self.cursor.x == map_size.x {
      self.cursor.x = 0;
      self.cursor.y += 1;
    }
    if self.cursor.y == map_size.y {
      None
    } else {
      Some(self.cursor)
    }
  }
}

// vim: set tabstop=2 shiftwidth=2 softtabstop=2 expandtab:
<commit_msg>map.rs: Fixed TileIterator<commit_after>// See LICENSE file for copyright and license details.

use cgmath::vector::Vec2;

pub struct TileIterator {
  cursor: Vec2<i32>
}

impl TileIterator {
  pub fn new() -> TileIterator {
    TileIterator {
      cursor: Vec2::zero()
    }
  }
}

impl Iterator<Vec2<i32>> for TileIterator {
  fn next(&mut self) -> Option<Vec2<i32>> {
    let map_size = Vec2::<i32>{x: 3, y: 4};
    let current_pos = if self.cursor.y > map_size.y {
      None
    } else {
      Some(self.cursor)
    };
    self.cursor.x += 1;
    if self.cursor.x > map_size.x {
      self.cursor.x = 0;
      self.cursor.y += 1;
    }
    current_pos
  }
}

// vim: set tabstop=2 shiftwidth=2 softtabstop=2 expandtab:
<|endoftext|>


<commit_before>//! Auto-generate stub docs for the unstable book

#![deny(rust_2018_idioms)]
#![deny(warnings)]

use tidy::features::{Feature, Features, collect_lib_features, collect_lang_features};
use tidy::unstable_book::{collect_unstable_feature_names, collect_unstable_book_section_file_names,
                          PATH_STR, LANG_FEATURES_DIR, LIB_FEATURES_DIR};
use std::collections::BTreeSet;
use std::io::Write;
use std::fs::{self, File};
use std::env;
use std::path::Path;

/// A helper macro to `unwrap` a result except also print out details like:
///
/// * The file/line of the panic
/// * The expression that failed
/// * The error itself
macro_rules! t {
    ($e:expr) => (match $e {
        Ok(e) => e,
        Err(e) => panic!("{} failed with {}", stringify!($e), e),
    })
}

fn generate_stub_issue(path: &Path, name: &str, issue: u32) {
    let mut file = t!(File::create(path));
    t!(file.write_fmt(format_args!(include_str!("stub-issue.md"),
                                   name = name,
                                   issue = issue)));
}

fn generate_stub_no_issue(path: &Path, name: &str) {
    let mut file = t!(File::create(path));
    t!(file.write_fmt(format_args!(include_str!("stub-no-issue.md"),
                                   name = name)));
}

fn set_to_summary_str(set: &BTreeSet<String>, dir: &str
) -> String {
    set
        .iter()
        .map(|ref n| format!("    - [{}]({}/{}.md)",
                                      n.replace('-', "_"),
                                      dir,
                                      n))
        .fold("".to_owned(), |s, a| s + &a + "\n")
}

fn generate_summary(path: &Path, lang_features: &Features, lib_features: &Features) {
    let compiler_flags = collect_unstable_book_section_file_names(
        &path.join("compiler-flags"));

    let compiler_flags_str = set_to_summary_str(&compiler_flags,
                                                "compiler-flags");

    let unstable_lang_features = collect_unstable_feature_names(&lang_features);
    let unstable_lib_features = collect_unstable_feature_names(&lib_features);

    let lang_features_str = set_to_summary_str(&unstable_lang_features,
                                               LANG_FEATURES_DIR);
    let lib_features_str = set_to_summary_str(&unstable_lib_features,
                                              LIB_FEATURES_DIR);

    let mut file = t!(File::create(&path.join("SUMMARY.md")));
    t!(file.write_fmt(format_args!(include_str!("SUMMARY.md"),
                                   compiler_flags = compiler_flags_str,
                                   language_features = lang_features_str,
                                   library_features = lib_features_str)));

}

fn has_valid_tracking_issue(f: &Feature) -> bool {
    if let Some(n) = f.tracking_issue {
        if n > 0 {
            return true;
        }
    }
    false
}

fn generate_unstable_book_files(src :&Path, out: &Path, features :&Features) {
    let unstable_features = collect_unstable_feature_names(features);
    let unstable_section_file_names = collect_unstable_book_section_file_names(src);
    t!(fs::create_dir_all(&out));
    for feature_name in &unstable_features - &unstable_section_file_names {
        let feature_name_underscore = feature_name.replace('-', "_");
        let file_name = format!("{}.md", feature_name);
        let out_file_path = out.join(&file_name);
        let feature = &features[&feature_name_underscore];

        if has_valid_tracking_issue(&feature) {
            generate_stub_issue(&out_file_path,
                                &feature_name_underscore,
                                feature.tracking_issue.unwrap());
        } else {
            generate_stub_no_issue(&out_file_path, &feature_name_underscore);
        }
    }
}

fn copy_recursive(path: &Path, to: &Path) {
    for entry in t!(fs::read_dir(path)) {
        let e = t!(entry);
        let t = t!(e.metadata());
        let dest = &to.join(e.file_name());
        if t.is_file() {
            t!(fs::copy(&e.path(), dest));
        } else if t.is_dir() {
            t!(fs::create_dir_all(dest));
            copy_recursive(&e.path(), dest);
        }
    }
}

fn main() {
    let src_path_str = env::args_os().skip(1).next().expect("source path required");
    let dest_path_str = env::args_os().skip(2).next().expect("destination path required");
    let src_path = Path::new(&src_path_str);
    let dest_path = Path::new(&dest_path_str).join("src");

    let lang_features = collect_lang_features(src_path, &mut false);
    let lib_features = collect_lib_features(src_path).into_iter().filter(|&(ref name, _)| {
        !lang_features.contains_key(name)
    }).collect();

    let doc_src_path = src_path.join(PATH_STR);

    t!(fs::create_dir_all(&dest_path));

    generate_unstable_book_files(&doc_src_path.join(LANG_FEATURES_DIR),
                                 &dest_path.join(LANG_FEATURES_DIR),
                                 &lang_features);
    generate_unstable_book_files(&doc_src_path.join(LIB_FEATURES_DIR),
                                 &dest_path.join(LIB_FEATURES_DIR),
                                 &lib_features);

    copy_recursive(&doc_src_path, &dest_path);

    generate_summary(&dest_path, &lang_features, &lib_features);
}
<commit_msg>path -> from<commit_after>//! Auto-generate stub docs for the unstable book

#![deny(rust_2018_idioms)]
#![deny(warnings)]

use tidy::features::{Feature, Features, collect_lib_features, collect_lang_features};
use tidy::unstable_book::{collect_unstable_feature_names, collect_unstable_book_section_file_names,
                          PATH_STR, LANG_FEATURES_DIR, LIB_FEATURES_DIR};
use std::collections::BTreeSet;
use std::io::Write;
use std::fs::{self, File};
use std::env;
use std::path::Path;

/// A helper macro to `unwrap` a result except also print out details like:
///
/// * The file/line of the panic
/// * The expression that failed
/// * The error itself
macro_rules! t {
    ($e:expr) => (match $e {
        Ok(e) => e,
        Err(e) => panic!("{} failed with {}", stringify!($e), e),
    })
}

fn generate_stub_issue(path: &Path, name: &str, issue: u32) {
    let mut file = t!(File::create(path));
    t!(file.write_fmt(format_args!(include_str!("stub-issue.md"),
                                   name = name,
                                   issue = issue)));
}

fn generate_stub_no_issue(path: &Path, name: &str) {
    let mut file = t!(File::create(path));
    t!(file.write_fmt(format_args!(include_str!("stub-no-issue.md"),
                                   name = name)));
}

fn set_to_summary_str(set: &BTreeSet<String>, dir: &str
) -> String {
    set
        .iter()
        .map(|ref n| format!("    - [{}]({}/{}.md)",
                                      n.replace('-', "_"),
                                      dir,
                                      n))
        .fold("".to_owned(), |s, a| s + &a + "\n")
}

fn generate_summary(path: &Path, lang_features: &Features, lib_features: &Features) {
    let compiler_flags = collect_unstable_book_section_file_names(
        &path.join("compiler-flags"));

    let compiler_flags_str = set_to_summary_str(&compiler_flags,
                                                "compiler-flags");

    let unstable_lang_features = collect_unstable_feature_names(&lang_features);
    let unstable_lib_features = collect_unstable_feature_names(&lib_features);

    let lang_features_str = set_to_summary_str(&unstable_lang_features,
                                               LANG_FEATURES_DIR);
    let lib_features_str = set_to_summary_str(&unstable_lib_features,
                                              LIB_FEATURES_DIR);

    let mut file = t!(File::create(&path.join("SUMMARY.md")));
    t!(file.write_fmt(format_args!(include_str!("SUMMARY.md"),
                                   compiler_flags = compiler_flags_str,
                                   language_features = lang_features_str,
                                   library_features = lib_features_str)));

}

fn has_valid_tracking_issue(f: &Feature) -> bool {
    if let Some(n) = f.tracking_issue {
        if n > 0 {
            return true;
        }
    }
    false
}

fn generate_unstable_book_files(src :&Path, out: &Path, features :&Features) {
    let unstable_features = collect_unstable_feature_names(features);
    let unstable_section_file_names = collect_unstable_book_section_file_names(src);
    t!(fs::create_dir_all(&out));
    for feature_name in &unstable_features - &unstable_section_file_names {
        let feature_name_underscore = feature_name.replace('-', "_");
        let file_name = format!("{}.md", feature_name);
        let out_file_path = out.join(&file_name);
        let feature = &features[&feature_name_underscore];

        if has_valid_tracking_issue(&feature) {
            generate_stub_issue(&out_file_path,
                                &feature_name_underscore,
                                feature.tracking_issue.unwrap());
        } else {
            generate_stub_no_issue(&out_file_path, &feature_name_underscore);
        }
    }
}

fn copy_recursive(from: &Path, to: &Path) {
    for entry in t!(fs::read_dir(from)) {
        let e = t!(entry);
        let t = t!(e.metadata());
        let dest = &to.join(e.file_name());
        if t.is_file() {
            t!(fs::copy(&e.path(), dest));
        } else if t.is_dir() {
            t!(fs::create_dir_all(dest));
            copy_recursive(&e.path(), dest);
        }
    }
}

fn main() {
    let src_path_str = env::args_os().skip(1).next().expect("source path required");
    let dest_path_str = env::args_os().skip(2).next().expect("destination path required");
    let src_path = Path::new(&src_path_str);
    let dest_path = Path::new(&dest_path_str).join("src");

    let lang_features = collect_lang_features(src_path, &mut false);
    let lib_features = collect_lib_features(src_path).into_iter().filter(|&(ref name, _)| {
        !lang_features.contains_key(name)
    }).collect();

    let doc_src_path = src_path.join(PATH_STR);

    t!(fs::create_dir_all(&dest_path));

    generate_unstable_book_files(&doc_src_path.join(LANG_FEATURES_DIR),
                                 &dest_path.join(LANG_FEATURES_DIR),
                                 &lang_features);
    generate_unstable_book_files(&doc_src_path.join(LIB_FEATURES_DIR),
                                 &dest_path.join(LIB_FEATURES_DIR),
                                 &lib_features);

    copy_recursive(&doc_src_path, &dest_path);

    generate_summary(&dest_path, &lang_features, &lib_features);
}
<|endoftext|>


<commit_before>//! Emit diagnostics using the `annotate-snippets` library
//!
//! This is the equivalent of `./emitter.rs` but making use of the
//! [`annotate-snippets`][annotate_snippets] library instead of building the output ourselves.
//!
//! [annotate_snippets]: https://docs.rs/crate/annotate-snippets/

use syntax_pos::{SourceFile, MultiSpan, Loc};
use crate::{
    Level, CodeSuggestion, DiagnosticBuilder, Emitter,
    SourceMapperDyn, SubDiagnostic, DiagnosticId
};
use crate::emitter::FileWithAnnotatedLines;
use rustc_data_structures::sync::Lrc;
use crate::snippet::Line;
use annotate_snippets::snippet::*;
use annotate_snippets::display_list::DisplayList;
use annotate_snippets::formatter::DisplayListFormatter;

/// Generates diagnostics using annotate-snippet
pub struct AnnotateSnippetEmitterWriter {
    source_map: Option<Lrc<SourceMapperDyn>>,
    /// If true, hides the longer explanation text
    short_message: bool,
    /// If true, will normalize line numbers with `LL` to prevent noise in UI test diffs.
    ui_testing: bool,
}

impl Emitter for AnnotateSnippetEmitterWriter {
    /// The entry point for the diagnostics generation
    fn emit_diagnostic(&mut self, db: &DiagnosticBuilder<'_>) {
        let children = db.children.clone();
        let (primary_span, suggestions) = self.primary_span_formatted(&db);

        // FIXME(#59346): Add `fix_multispans_in_std_macros` function from emitter.rs

        self.emit_messages_default(&db.level,
                                   db.message(),
                                   &db.code,
                                   &primary_span,
                                   &children,
                                   &suggestions);
    }

    fn should_show_explain(&self) -> bool {
        !self.short_message
    }
}

/// Collects all the data needed to generate the data structures needed for the
/// `annotate-snippets` library.
struct DiagnosticConverter<'a> {
    source_map: Option<Lrc<SourceMapperDyn>>,
    level: Level,
    message: String,
    code: Option<DiagnosticId>,
    msp: MultiSpan,
    #[allow(dead_code)]
    children: &'a [SubDiagnostic],
    #[allow(dead_code)]
    suggestions: &'a [CodeSuggestion]
}

impl<'a>  DiagnosticConverter<'a> {
    /// Turns rustc Diagnostic information into a `annotate_snippets::snippet::Snippet`.
    fn to_annotation_snippet(&self) -> Option<Snippet> {
        if let Some(source_map) = &self.source_map {
            // Make sure our primary file comes first
            let primary_lo = if let Some(ref primary_span) =
                self.msp.primary_span().as_ref() {
                source_map.lookup_char_pos(primary_span.lo())
            } else {
                // FIXME(#59346): Not sure when this is the case and what
                // should be done if it happens
                return None
            };
            let annotated_files = FileWithAnnotatedLines::collect_annotations(
                &self.msp,
                &self.source_map
            );
            let slices = self.slices_for_files(annotated_files, primary_lo);

            Some(Snippet {
                title: Some(Annotation {
                    label: Some(self.message.to_string()),
                    id: self.code.clone().map(|c| {
                        match c {
                            DiagnosticId::Error(val) | DiagnosticId::Lint(val) => val
                        }
                    }),
                    annotation_type: Self::annotation_type_for_level(self.level),
                }),
                footer: vec![],
                slices,
            })
        } else {
            // FIXME(#59346): Is it ok to return None if there's no source_map?
            None
        }
    }

    fn slices_for_files(
        &self,
        annotated_files: Vec<FileWithAnnotatedLines>,
        primary_lo: Loc
    ) -> Vec<Slice> {
        // FIXME(#59346): Provide a test case where `annotated_files` is > 1
        annotated_files.iter().flat_map(|annotated_file| {
            annotated_file.lines.iter().map(|line| {
                let line_source = Self::source_string(annotated_file.file.clone(), &line);
                Slice {
                    source: line_source,
                    line_start: line.line_index,
                    origin: Some(primary_lo.file.name.to_string()),
                    // FIXME(#59346): Not really sure when `fold` should be true or false
                    fold: false,
                    annotations: line.annotations.iter().map(|a| {
                        self.annotation_to_source_annotation(a.clone())
                    }).collect(),
                }
            }).collect::<Vec<Slice>>()
        }).collect::<Vec<Slice>>()
    }

    /// Turns a `crate::snippet::Annotation` into a `SourceAnnotation`
    fn annotation_to_source_annotation(
        &self,
        annotation: crate::snippet::Annotation
    ) -> SourceAnnotation {
        SourceAnnotation {
            range: (annotation.start_col, annotation.end_col),
            label: annotation.label.unwrap_or("".to_string()),
            annotation_type: Self::annotation_type_for_level(self.level)
        }
    }

    /// Provides the source string for the given `line` of `file`
    fn source_string(
        file: Lrc<SourceFile>,
        line: &Line
    ) -> String {
        file.get_line(line.line_index - 1).map(|a| a.to_string()).unwrap_or(String::new())
    }

    /// Maps `Diagnostic::Level` to `snippet::AnnotationType`
    fn annotation_type_for_level(level: Level) -> AnnotationType {
        match level {
            Level::Bug | Level::Fatal | Level::Error => AnnotationType::Error,
            Level::Warning => AnnotationType::Warning,
            Level::Note => AnnotationType::Note,
            Level::Help => AnnotationType::Help,
            // FIXME(#59346): Not sure how to map these two levels
            Level::Cancelled | Level::FailureNote => AnnotationType::Error
        }
    }
}

impl AnnotateSnippetEmitterWriter {
    pub fn new(
        source_map: Option<Lrc<SourceMapperDyn>>,
        short_message: bool
    ) -> Self {
        Self {
            source_map,
            short_message,
            ui_testing: false,
        }
    }

    /// Allows to modify `Self` to enable or disable the `ui_testing` flag.
    ///
    /// If this is set to true, line numbers will be normalized as `LL` in the output.
    pub fn ui_testing(mut self, ui_testing: bool) -> Self {
        self.ui_testing = ui_testing;
        self
    }

    fn emit_messages_default(
        &mut self,
        level: &Level,
        message: String,
        code: &Option<DiagnosticId>,
        msp: &MultiSpan,
        children: &[SubDiagnostic],
        suggestions: &[CodeSuggestion]
    ) {
        let converter = DiagnosticConverter {
            source_map: self.source_map.clone(),
            level: level.clone(),
            message,
            code: code.clone(),
            msp: msp.clone(),
            children,
            suggestions
        };
        if let Some(snippet) = converter.to_annotation_snippet() {
            let dl = DisplayList::from(snippet);
            let dlf = DisplayListFormatter::new(true, self.ui_testing);
            // FIXME(#59346): Figure out if we can _always_ print to stderr or not.
            // `emitter.rs` has the `Destination` enum that lists various possible output
            // destinations.
            eprintln!("{}", dlf.format(&dl));
        };
    }
}
<commit_msg>annotate-snippet emitter: Update issue number<commit_after>//! Emit diagnostics using the `annotate-snippets` library
//!
//! This is the equivalent of `./emitter.rs` but making use of the
//! [`annotate-snippets`][annotate_snippets] library instead of building the output ourselves.
//!
//! [annotate_snippets]: https://docs.rs/crate/annotate-snippets/

use syntax_pos::{SourceFile, MultiSpan, Loc};
use crate::{
    Level, CodeSuggestion, DiagnosticBuilder, Emitter,
    SourceMapperDyn, SubDiagnostic, DiagnosticId
};
use crate::emitter::FileWithAnnotatedLines;
use rustc_data_structures::sync::Lrc;
use crate::snippet::Line;
use annotate_snippets::snippet::*;
use annotate_snippets::display_list::DisplayList;
use annotate_snippets::formatter::DisplayListFormatter;

/// Generates diagnostics using annotate-snippet
pub struct AnnotateSnippetEmitterWriter {
    source_map: Option<Lrc<SourceMapperDyn>>,
    /// If true, hides the longer explanation text
    short_message: bool,
    /// If true, will normalize line numbers with `LL` to prevent noise in UI test diffs.
    ui_testing: bool,
}

impl Emitter for AnnotateSnippetEmitterWriter {
    /// The entry point for the diagnostics generation
    fn emit_diagnostic(&mut self, db: &DiagnosticBuilder<'_>) {
        let children = db.children.clone();
        let (primary_span, suggestions) = self.primary_span_formatted(&db);

        // FIXME(#59346): Add `fix_multispans_in_std_macros` function from emitter.rs

        self.emit_messages_default(&db.level,
                                   db.message(),
                                   &db.code,
                                   &primary_span,
                                   &children,
                                   &suggestions);
    }

    fn should_show_explain(&self) -> bool {
        !self.short_message
    }
}

/// Collects all the data needed to generate the data structures needed for the
/// `annotate-snippets` library.
struct DiagnosticConverter<'a> {
    source_map: Option<Lrc<SourceMapperDyn>>,
    level: Level,
    message: String,
    code: Option<DiagnosticId>,
    msp: MultiSpan,
    #[allow(dead_code)]
    children: &'a [SubDiagnostic],
    #[allow(dead_code)]
    suggestions: &'a [CodeSuggestion]
}

impl<'a>  DiagnosticConverter<'a> {
    /// Turns rustc Diagnostic information into a `annotate_snippets::snippet::Snippet`.
    fn to_annotation_snippet(&self) -> Option<Snippet> {
        if let Some(source_map) = &self.source_map {
            // Make sure our primary file comes first
            let primary_lo = if let Some(ref primary_span) =
                self.msp.primary_span().as_ref() {
                source_map.lookup_char_pos(primary_span.lo())
            } else {
                // FIXME(#59346): Not sure when this is the case and what
                // should be done if it happens
                return None
            };
            let annotated_files = FileWithAnnotatedLines::collect_annotations(
                &self.msp,
                &self.source_map
            );
            let slices = self.slices_for_files(annotated_files, primary_lo);

            Some(Snippet {
                title: Some(Annotation {
                    label: Some(self.message.to_string()),
                    id: self.code.clone().map(|c| {
                        match c {
                            DiagnosticId::Error(val) | DiagnosticId::Lint(val) => val
                        }
                    }),
                    annotation_type: Self::annotation_type_for_level(self.level),
                }),
                footer: vec![],
                slices,
            })
        } else {
            // FIXME(#59346): Is it ok to return None if there's no source_map?
            None
        }
    }

    fn slices_for_files(
        &self,
        annotated_files: Vec<FileWithAnnotatedLines>,
        primary_lo: Loc
    ) -> Vec<Slice> {
        // FIXME(#64205): Provide a test case where `annotated_files` is > 1
        annotated_files.iter().flat_map(|annotated_file| {
            annotated_file.lines.iter().map(|line| {
                let line_source = Self::source_string(annotated_file.file.clone(), &line);
                Slice {
                    source: line_source,
                    line_start: line.line_index,
                    origin: Some(primary_lo.file.name.to_string()),
                    // FIXME(#59346): Not really sure when `fold` should be true or false
                    fold: false,
                    annotations: line.annotations.iter().map(|a| {
                        self.annotation_to_source_annotation(a.clone())
                    }).collect(),
                }
            }).collect::<Vec<Slice>>()
        }).collect::<Vec<Slice>>()
    }

    /// Turns a `crate::snippet::Annotation` into a `SourceAnnotation`
    fn annotation_to_source_annotation(
        &self,
        annotation: crate::snippet::Annotation
    ) -> SourceAnnotation {
        SourceAnnotation {
            range: (annotation.start_col, annotation.end_col),
            label: annotation.label.unwrap_or("".to_string()),
            annotation_type: Self::annotation_type_for_level(self.level)
        }
    }

    /// Provides the source string for the given `line` of `file`
    fn source_string(
        file: Lrc<SourceFile>,
        line: &Line
    ) -> String {
        file.get_line(line.line_index - 1).map(|a| a.to_string()).unwrap_or(String::new())
    }

    /// Maps `Diagnostic::Level` to `snippet::AnnotationType`
    fn annotation_type_for_level(level: Level) -> AnnotationType {
        match level {
            Level::Bug | Level::Fatal | Level::Error => AnnotationType::Error,
            Level::Warning => AnnotationType::Warning,
            Level::Note => AnnotationType::Note,
            Level::Help => AnnotationType::Help,
            // FIXME(#59346): Not sure how to map these two levels
            Level::Cancelled | Level::FailureNote => AnnotationType::Error
        }
    }
}

impl AnnotateSnippetEmitterWriter {
    pub fn new(
        source_map: Option<Lrc<SourceMapperDyn>>,
        short_message: bool
    ) -> Self {
        Self {
            source_map,
            short_message,
            ui_testing: false,
        }
    }

    /// Allows to modify `Self` to enable or disable the `ui_testing` flag.
    ///
    /// If this is set to true, line numbers will be normalized as `LL` in the output.
    pub fn ui_testing(mut self, ui_testing: bool) -> Self {
        self.ui_testing = ui_testing;
        self
    }

    fn emit_messages_default(
        &mut self,
        level: &Level,
        message: String,
        code: &Option<DiagnosticId>,
        msp: &MultiSpan,
        children: &[SubDiagnostic],
        suggestions: &[CodeSuggestion]
    ) {
        let converter = DiagnosticConverter {
            source_map: self.source_map.clone(),
            level: level.clone(),
            message,
            code: code.clone(),
            msp: msp.clone(),
            children,
            suggestions
        };
        if let Some(snippet) = converter.to_annotation_snippet() {
            let dl = DisplayList::from(snippet);
            let dlf = DisplayListFormatter::new(true, self.ui_testing);
            // FIXME(#59346): Figure out if we can _always_ print to stderr or not.
            // `emitter.rs` has the `Destination` enum that lists various possible output
            // destinations.
            eprintln!("{}", dlf.format(&dl));
        };
    }
}
<|endoftext|>


<commit_before>#![allow(deprecated)]
use std::path::PathBuf;
use std::{io, fs, slice};
use curl::easy::Easy;
use memmap::{Mmap, Protection};
use flate2::{Flush, Decompress};
use std::io::Write;
use utils::data::{DatasetIntfRef, DatasetIntf};
use std::rc::Rc;
use tensor::{Tensor, TensorKind};
use torch;
use std::marker::PhantomData;

type Sample<T> = (Tensor<T>, i64);
type CollatedSample<T> = (Tensor<T>, Tensor<i64>);

static URLS: [&str; 4] = ["http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz",
                          "http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz",
                          "http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz",
                          "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz"];
static RAW_FOLDER: &str = "raw";
static PROCESSED_FOLDER: &str = "processed";
static TRAINING_FILE: &str = "training.pt";
static TEST_FILE: &str = "test.pt";

fn create_dir_f(arg: PathBuf) -> io::Result<()> {
    let result = fs::create_dir(arg);
    if let Some(err) = result.err() {
        match err.kind() {
            io::ErrorKind::AlreadyExists => Ok(()),
            _ => Err(err),
        }
    } else {
        Ok(())
    }
}

fn check_exists(arg: &PathBuf) -> io::Result<bool> {
    let train = fs::metadata(arg.join(TRAINING_FILE))?;
    let test = fs::metadata(arg.join(TEST_FILE))?;

    return Ok(train.is_file() && test.is_file());
}

fn download(root: &String) -> io::Result<()> {
    let raw_path = PathBuf::from(root).join(RAW_FOLDER);
    let processed_path = PathBuf::from(root).join(PROCESSED_FOLDER);
    create_dir_f(raw_path.clone())?;
    create_dir_f(processed_path.clone())?;

    for url in URLS.iter() {
        println!("downloading {}", url);
        let paths: Vec<_> = url.split("/").collect();
        let fname: Vec<_> = paths[5].split("/").collect();
        let file_path = raw_path.join(fname[0]);
        let mut file = fs::OpenOptions::new()
            .write(true)
            .create(true)
            .open(file_path)?;
        let mut data = Vec::new();
        let mut handle = Easy::new();
        handle.url(url).unwrap();
        {
            let mut transfer = handle.transfer();
            transfer
                .write_function(|new_data| {
                                    data.extend_from_slice(new_data);
                                    Ok(new_data.len())
                                })
                .unwrap();
            transfer.perform().unwrap();
        }
        let mut output = Vec::with_capacity(data.len());
        // do we have a header?
        Decompress::new(false)
            .decompress_vec(data.as_slice(), &mut output, Flush::Finish)?;
        file.write_all(output.as_slice())?;
    }
    println!("Proceeding");

    let training_set =
        vec![read_image_file(raw_path.join("train-images-idx3-ubyte"))?,
                            read_label_file(raw_path.join("train-labels-idx1-ubyte"))?];
    let test_set = vec![read_image_file(raw_path.join("t10k-images-idx3-ubyte"))?,
                        read_label_file(raw_path.join("t10k-labels-idx1-ubyte"))?];

    torch::save(processed_path.join(TRAINING_FILE), &training_set)?;
    torch::save(processed_path.join(TEST_FILE), &test_set)?;

    Ok(())
}

fn read_image_file(path: PathBuf) -> io::Result<TensorKind> {
    let fp = Mmap::open_path(path, Protection::Read)?;
    let (length, nrows, ncols);
    {
        let data = unsafe { slice::from_raw_parts(fp.ptr() as *const u32, fp.len()) };
        assert_eq!(data[0], 2051);
        length = data[1] as usize;
        nrows = data[2] as usize;
        ncols = data[3] as usize;
    }
    let data = unsafe { fp.as_slice() };
    let mut images: Vec<Vec<u8>> = Vec::with_capacity(length);
    let mut idx = 16;
    for _ in 0..length {
        let mut img = Vec::with_capacity(nrows * ncols);
        for _ in 0..nrows * ncols {
            img.push(data[idx]);
            idx += 1;
        }
        images.push(img)
    }
    Ok(torch::byte_tensor_kind(images).view(&[-1, 28, 28]))
}

fn read_label_file(path: PathBuf) -> io::Result<TensorKind> {
    let fp = Mmap::open_path(path, Protection::Read)?;
    let length;
    {
        let data = unsafe { slice::from_raw_parts(fp.ptr() as *const u32, fp.len()) };
        assert_eq!(data[0], 2049);
        length = data[1] as usize;
    }
    let data = unsafe { fp.as_slice() };
    let mut labels: Vec<u8> = Vec::with_capacity(length);
    for i in 0..length {
        labels.push(data[8 + i])
    }
    Ok(torch::byte_tensor_kind(labels))
}

pub struct MNIST<T> {
    pub root: String,
    pub train: bool,
    pub data: Tensor<u8>,
    pub labels: Tensor<u8>,
    pub transform: Option<Xfrm>,
    phantom: PhantomData<T>,
}

#[builder(pattern="owned")]
#[derive(Builder)]
pub struct MNISTArgs {
    root: String,
    #[builder(default="true")]
    train: bool,
    #[builder(default="false")]
    download: bool,
}

type Xfrm = Box<fn(&TensorKind) -> TensorKind>;
impl MNISTArgsBuilder {
    pub fn done<T: Clone>(self, xfrm: Option<Xfrm>) -> DatasetIntfRef<CollatedSample<T>> {
        let args = self.build().unwrap();
        Rc::new(MNIST::new(args, xfrm))
    }
}

impl<T: 'static> MNIST<T> {
    pub fn build(root: &str) -> MNISTArgsBuilder {
        MNISTArgsBuilder::default().root(root.into())
    }
    pub fn new(args: MNISTArgs, xfrm: Option<Xfrm>) -> Self {
        if args.download {
            download(&args.root).expect("download failed");
        }
        let processed_path = PathBuf::from(&args.root).join(PROCESSED_FOLDER);
        if !check_exists(&processed_path).expect("Dataset not found, try downloading") {
            panic!("Dataset not found, use download=true to download it");
        }
        let mut v: Vec<TensorKind> = if args.train {
            torch::load(processed_path.join(TRAINING_FILE)).expect("torch load failed")
        } else {
            torch::load(processed_path.join(TEST_FILE)).expect("torch load failed")
        };
        let (data, labels) = (v.remove(0), v.remove(0));

        MNIST {
            root: args.root.clone(),
            train: args.train,
            data: data.into(),
            labels: labels.into(),
            transform: xfrm,
            phantom: PhantomData,
        }
    }
    fn index(&self, idx: usize) -> Sample<T> {
        let img = self.data.s(idx);
        let img = if let Some(ref transform) = self.transform {
            transform(&img.into()).into()
        } else {
            img.copy().cast::<T>()
        };
        (img, self.labels[idx].clone() as i64)
    }
}

impl<T> DatasetIntf for MNIST<T> {
    type Batch = CollatedSample<T>;
    fn len(&self) -> usize {
        if self.train { 60000 } else { 10000 }
    }
    fn collate(&self, sample: Vec<usize>) -> Self::Batch {
        let v: Vec<Sample<T>> = sample.into_iter().map(|i| self.index(i)).collect();
        let labels: Vec<i64> = v.iter().map(|&(_, ref t)| *t).collect();
        let imgs: Vec<Tensor<T>> = v.into_iter().map(|(d, _)| d).collect();
        // XXX should check for case of double
        let img_batch = torch::tensor(imgs);
        let label_batch = torch::long_tensor(labels);
        (img_batch, label_batch)
    }
}
<commit_msg>fix compile<commit_after>#![allow(deprecated)]
use std::path::PathBuf;
use std::{io, fs, slice};
use curl::easy::Easy;
use memmap::{Mmap, Protection};
use flate2::{Flush, Decompress};
use std::io::Write;
use utils::data::{DatasetIntfRef, DatasetIntf};
use std::rc::Rc;
use tensor::{Tensor, TensorKind};
use torch;
use std::marker::PhantomData;

type Sample<T> = (Tensor<T>, i64);
type CollatedSample<T> = (Tensor<T>, Tensor<i64>);

static URLS: [&str; 4] = ["http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz",
                          "http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz",
                          "http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz",
                          "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz"];
static RAW_FOLDER: &str = "raw";
static PROCESSED_FOLDER: &str = "processed";
static TRAINING_FILE: &str = "training.pt";
static TEST_FILE: &str = "test.pt";

fn create_dir_f(arg: PathBuf) -> io::Result<()> {
    let result = fs::create_dir(arg);
    if let Some(err) = result.err() {
        match err.kind() {
            io::ErrorKind::AlreadyExists => Ok(()),
            _ => Err(err),
        }
    } else {
        Ok(())
    }
}

fn check_exists(arg: &PathBuf) -> io::Result<bool> {
    let train = fs::metadata(arg.join(TRAINING_FILE))?;
    let test = fs::metadata(arg.join(TEST_FILE))?;

    return Ok(train.is_file() && test.is_file());
}

fn download(root: &String) -> io::Result<()> {
    let raw_path = PathBuf::from(root).join(RAW_FOLDER);
    let processed_path = PathBuf::from(root).join(PROCESSED_FOLDER);
    create_dir_f(raw_path.clone())?;
    create_dir_f(processed_path.clone())?;

    for url in URLS.iter() {
        println!("downloading {}", url);
        let paths: Vec<_> = url.split("/").collect();
        let fname: Vec<_> = paths[5].split("/").collect();
        let file_path = raw_path.join(fname[0]);
        let mut file = fs::OpenOptions::new()
            .write(true)
            .create(true)
            .open(file_path)?;
        let mut data = Vec::new();
        let mut handle = Easy::new();
        handle.url(url).unwrap();
        {
            let mut transfer = handle.transfer();
            transfer
                .write_function(|new_data| {
                                    data.extend_from_slice(new_data);
                                    Ok(new_data.len())
                                })
                .unwrap();
            transfer.perform().unwrap();
        }
        let mut output = Vec::with_capacity(data.len());
        // do we have a header?
        Decompress::new(false)
            .decompress_vec(data.as_slice(), &mut output, Flush::Finish)?;
        file.write_all(output.as_slice())?;
    }
    println!("Proceeding");

    let training_set =
        vec![read_image_file(raw_path.join("train-images-idx3-ubyte"))?,
                            read_label_file(raw_path.join("train-labels-idx1-ubyte"))?];
    let test_set = vec![read_image_file(raw_path.join("t10k-images-idx3-ubyte"))?,
                        read_label_file(raw_path.join("t10k-labels-idx1-ubyte"))?];

    torch::save(processed_path.join(TRAINING_FILE), &training_set)?;
    torch::save(processed_path.join(TEST_FILE), &test_set)?;

    Ok(())
}

fn read_image_file(path: PathBuf) -> io::Result<TensorKind> {
    let fp = Mmap::open_path(path, Protection::Read)?;
    let (length, nrows, ncols);
    {
        let data = unsafe { slice::from_raw_parts(fp.ptr() as *const u32, fp.len()) };
        assert_eq!(data[0], 2051);
        length = data[1] as usize;
        nrows = data[2] as usize;
        ncols = data[3] as usize;
    }
    let data = unsafe { fp.as_slice() };
    let mut images: Vec<Vec<u8>> = Vec::with_capacity(length);
    let mut idx = 16;
    for _ in 0..length {
        let mut img = Vec::with_capacity(nrows * ncols);
        for _ in 0..nrows * ncols {
            img.push(data[idx]);
            idx += 1;
        }
        images.push(img)
    }
    Ok(torch::byte_tensor_kind(images).view(&[-1, 28, 28]))
}

fn read_label_file(path: PathBuf) -> io::Result<TensorKind> {
    let fp = Mmap::open_path(path, Protection::Read)?;
    let length;
    {
        let data = unsafe { slice::from_raw_parts(fp.ptr() as *const u32, fp.len()) };
        assert_eq!(data[0], 2049);
        length = data[1] as usize;
    }
    let data = unsafe { fp.as_slice() };
    let mut labels: Vec<u8> = Vec::with_capacity(length);
    for i in 0..length {
        labels.push(data[8 + i])
    }
    Ok(torch::byte_tensor_kind(labels))
}

pub struct MNIST<T> {
    pub root: String,
    pub train: bool,
    pub data: Tensor<u8>,
    pub labels: Tensor<u8>,
    pub transform: Option<Xfrm>,
    phantom: PhantomData<T>,
}

#[builder(pattern="owned")]
#[derive(Builder)]
pub struct MNISTArgs {
    root: String,
    #[builder(default="true")]
    train: bool,
    #[builder(default="false")]
    download: bool,
}

type Xfrm = Box<fn(&TensorKind) -> TensorKind>;
impl MNISTArgsBuilder {
    pub fn done<T: Clone + 'static>(self, xfrm: Option<Xfrm>) -> DatasetIntfRef<CollatedSample<T>> {
        let args = self.build().unwrap();
        Rc::new(MNIST::new(args, xfrm))
    }
}

impl<T> MNIST<T> {
    pub fn build(root: &str) -> MNISTArgsBuilder {
        MNISTArgsBuilder::default().root(root.into())
    }
    pub fn new(args: MNISTArgs, xfrm: Option<Xfrm>) -> Self {
        if args.download {
            download(&args.root).expect("download failed");
        }
        let processed_path = PathBuf::from(&args.root).join(PROCESSED_FOLDER);
        if !check_exists(&processed_path).expect("Dataset not found, try downloading") {
            panic!("Dataset not found, use download=true to download it");
        }
        let mut v: Vec<TensorKind> = if args.train {
            torch::load(processed_path.join(TRAINING_FILE)).expect("torch load failed")
        } else {
            torch::load(processed_path.join(TEST_FILE)).expect("torch load failed")
        };
        let (data, labels) = (v.remove(0), v.remove(0));

        MNIST {
            root: args.root.clone(),
            train: args.train,
            data: data.into(),
            labels: labels.into(),
            transform: xfrm,
            phantom: PhantomData,
        }
    }
    fn index(&self, idx: usize) -> Sample<T> {
        let img = self.data.s(idx);
        let img = if let Some(ref transform) = self.transform {
            transform(&img.into()).into()
        } else {
            img.copy().cast::<T>()
        };
        (img, self.labels[idx].clone() as i64)
    }
}

impl<T> DatasetIntf for MNIST<T> {
    type Batch = CollatedSample<T>;
    fn len(&self) -> usize {
        if self.train { 60000 } else { 10000 }
    }
    fn collate(&self, sample: Vec<usize>) -> Self::Batch {
        let v: Vec<Sample<T>> = sample.into_iter().map(|i| self.index(i)).collect();
        let labels: Vec<i64> = v.iter().map(|&(_, ref t)| *t).collect();
        let imgs: Vec<Tensor<T>> = v.into_iter().map(|(d, _)| d).collect();
        // XXX should check for case of double
        let img_batch = torch::tensor(imgs);
        let label_batch = torch::long_tensor(labels);
        (img_batch, label_batch)
    }
}
<|endoftext|>