<commit_before>use std::rc::Rc;
use std::ops::Deref;
use std::path::{PathBuf, Path};
use std::collections::{HashSet, HashMap};
use std::fs::File;
use std::io::{BufReader, BufRead};
use cargo::core::{Workspace, Package};
use cargo::sources::PathSource;
use cargo::util::Config as CargoConfig;
use syn::{parse_file, Item, File as SFile};
use proc_macro2::Span;
use regex::Regex;
use config::Config;

/// Represents the results of analysis of a single file. Does not store the file
/// in question as this is expected to be maintained by the user.
#[derive(Clone, Debug)]
pub struct LineAnalysis {
    /// This represents lines that should be ignored in coverage
    /// but may be identifed as coverable in the DWARF tables
    pub ignore: HashSet<usize>,
    /// This represents lines that should be included in coverage
    /// But may be ignored.
    pub cover: HashSet<usize>,
}

/// When the LineAnalysis results are mapped to their files there needs to be
/// an easy way to get the information back. For the container used implement
/// this trait
pub trait SourceAnalysisQuery {
    fn should_ignore(&self, path: &Path, l:&usize) -> bool;
}

impl SourceAnalysisQuery for HashMap<PathBuf, LineAnalysis> {

    fn should_ignore(&self, path: &Path, l:&usize) -> bool {
        if self.contains_key(path) {
            self.get(path).unwrap().ignore.contains(l)
        } else {
            false
        }
    }

}

impl LineAnalysis {
    fn new() -> LineAnalysis {
        LineAnalysis {
            ignore: HashSet::new(),
            cover: HashSet::new()
        }
    }

    /// Adds the lines of the provided span to the ignore set
    pub fn ignore_span(&mut self, span: &Span) {
        for i in span.start().line..span.end().line {
            self.ignore.insert(i);
            if self.cover.contains(&i) {
                self.cover.remove(&i);
            }
        }
    }

    /// Adds the lines of the provided span to the cover set
    pub fn cover_span(&mut self, span: &Span) {
        for i in span.start().line..span.end().line {
            if !self.ignore.contains(&i) {
                self.cover.insert(i);
            }
        }
    }

    pub fn should_ignore(&self, line: &usize) -> bool {
        self.ignore.contains(line)
    }

    fn add_to_ignore(&mut self, lines: &[usize]) {
        for l in lines {
            self.ignore.insert(*l);
            if self.cover.contains(l) {
                self.cover.remove(l);
            }
        }
    }

    fn add_to_cover(&mut self, lines: &[usize]) {
        for l in lines {
            if !self.ignore.contains(l) {
                self.cover.insert(*l);
            }
        }
    }
}

/// Returns a list of files and line numbers to ignore (not indexes!)
pub fn get_line_analysis(project: &Workspace, config: &Config) -> HashMap<PathBuf, LineAnalysis> {
    let mut result: HashMap<PathBuf, LineAnalysis> = HashMap::new();
    // Members iterates over all non-virtual packages in the workspace
    for pkg in project.members() {
        if config.packages.is_empty() || config.packages.contains(&pkg.name().to_string()) {
            analyse_package(pkg, &config, project.config(), &mut result);
        }
    }
    result
}

fn analyse_lib_rs(file: &Path, result: &mut HashMap<PathBuf, LineAnalysis>) {
    if let Ok(f) = File::open(file) {
        let mut read_file = BufReader::new(f);
        if let Some(Ok(first)) = read_file.lines().nth(0) {
            if !(first.starts_with("pub") || first.starts_with("fn")) {
                let file = file.to_path_buf();
                if result.contains_key(&file) {
                    let l = result.get_mut(&file).unwrap();
                    l.add_to_ignore(&[1]);
                } else {
                    let mut l = LineAnalysis::new();
                    l.add_to_ignore(&[1]);
                    result.insert(file, l);
                }
            }
        }
    }
}

fn analyse_package(pkg: &Package,
                   config:&Config,
                   cargo_conf: &CargoConfig,
                   result: &mut HashMap<PathBuf, LineAnalysis>) {

    let mut src = PathSource::new(pkg.root(), pkg.package_id().source_id(), cargo_conf);
    if let Ok(package) = src.root_package() {
        for target in package.targets() {
            let path = target.src_path();
            let file = match path.to_str() {
                Some(s) => s,
                _ => continue
            };
            let skip_cause_test = config.ignore_tests &&
                                  path.starts_with(pkg.root().join("tests"));
            let skip_cause_example = path.starts_with(pkg.root().join("examples"));
            if !(skip_cause_test || skip_cause_example)  {
                let file = parse_file(file);
                if let Ok(file) = file {
                    let analysis = process_module(file);
                    // Check there's no conflict!
                    result.insert(path.to_path_buf(), analysis);
                }
            }
            // This could probably be done with the DWARF if I could find a discriminating factor
            // to why lib.rs:1 shows up as a real line!
            if file.ends_with("src/lib.rs") {
                analyse_lib_rs(path, result);
            }
        }
    }
}

fn process_module(file: SFile) -> LineAnalysis {
    let mut analysis = LineAnalysis::new();
    for item in &file.items {
        match item {
            Item::ExternCrate(i) => analysis.ignore_span(&i.extern_token.0),
            Item::Use(i) => analysis.ignore_span(&i.use_token.0),
            _ =>{}
        }
    }
    analysis
}

#[cfg(test)]
mod tests {
    use super::*;
    use syntex_syntax::parse::filemap_to_parser;
    use syntex_syntax::parse::parser::Parser;

    struct TestContext {
        conf: Config,
        codemap: Rc<CodeMap>,
        parse_session: ParseSess
    }

    impl TestContext {
        fn generate_parser(&self, filename: &str, src_string: &str) -> Parser {
            let filemap = self.codemap.new_filemap(filename.to_string(),
                                                   src_string.to_string());
            filemap_to_parser(&self.parse_session, filemap)
        }
    }

    impl Default for TestContext {
        fn default() -> TestContext {
            let codemap = Rc::new(CodeMap::new(FilePathMapping::empty()));
            let handler = Handler::with_tty_emitter(ColorConfig::Auto, false, false, Some(codemap.clone()));
            let parse_session = ParseSess::with_span_handler(handler, codemap.clone());
            TestContext {
                conf: Config::default(),
                codemap: codemap,
                parse_session: parse_session
            }
        }
    }

    fn parse_crate(ctx: &TestContext, parser: &mut Parser) -> Vec<usize>  {
        let krate = parser.parse_crate_mod();
        assert!(krate.is_ok());
        let krate = krate.unwrap();
        let unused: HashSet<PathBuf> = HashSet::new();
        let mut visitor = CoverageVisitor::from_session(&ctx.parse_session, &unused, &ctx.conf);
        visitor.visit_mod(&krate.module, krate.span, &krate.attrs, NodeId::new(0));
        visitor.lines.iter().map(|x| x.1).collect::<Vec<_>>()
    }

    #[test]
    fn filter_str_literals() {
        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("literals.rs", "fn test() {\nwriteln!(#\"test\n\ttest\n\ttest\"#);\n}\n");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.len() > 1);
        assert!(lines.contains(&3));
        assert!(lines.contains(&4));

        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("literals.rs", "fn test() {\nwrite(\"test\ntest\ntest\");\n}\nfn write(s:&str){}");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.len() > 1);
        assert!(lines.contains(&3));
        assert!(lines.contains(&4));

        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("literals.rs", "\n\nfn test() {\nwriteln!(\n#\"test\"#\n);\n}\n");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.contains(&5));
    }

    #[test]
    fn filter_struct_members() {
        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("struct_test.rs", "#[derive(Debug)]\npub struct Struct {\npub i: i32,\nj:String,\n}");
        let lines = parse_crate(&ctx, &mut parser);

        assert_eq!(lines.len(), 3);
        assert!(lines.contains(&1));
        assert!(lines.contains(&3));
        assert!(lines.contains(&4));
    }

    #[test]
    fn filter_struct_consts() {
        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("struct_test.rs",
                                             "struct T{x:String, y:i32}\nfn test()-> T {\nT{\nx:\"hello\".to_string(),\ny:4,\n}\n}");

        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.contains(&5));
    }

    #[test]
    fn filter_mods() {
        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("test.rs", "mod foo {\nfn double(x:i32)->i32 {\n x*2\n}\n}");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(!lines.contains(&3));

        let mut parser = ctx.generate_parser("test.rs", "mod foo{}");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.contains(&1));
    }

    #[test]
    fn filter_macros() {
        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("test.rs", "\n\nfn unused() {\nunimplemented!();\n}");

        let lines = parse_crate(&ctx, &mut parser);
        // Braces should be ignored so number could be higher
        assert!(lines.len() >= 1);
        assert!(lines.contains(&4));

        let mut parser = ctx.generate_parser("test.rs", "fn unused() {\nunreachable!();\n}");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.len() >= 1);
        assert!(lines.contains(&2));

        let mut parser = ctx.generate_parser("test.rs", "fn unused() {\nprintln!(\"text\");\n}");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(!lines.contains(&2));
    }

    #[test]
    fn filter_tests() {
        let ctx = TestContext::default();
        let src_string = "#[cfg(test)]\nmod tests {\n fn boo(){\nassert!(true);\n}\n}";
        let mut parser = ctx.generate_parser("test.rs", src_string);
        let lines = parse_crate(&ctx, &mut parser);
        assert!(!lines.contains(&4));

        let mut ctx = TestContext::default();
        ctx.conf.ignore_tests = true;
        let mut parser = ctx.generate_parser("test.rs", src_string);
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.contains(&4));

        let ctx = TestContext::default();
        let src_string = "#[test]\nfn mytest() { \n assert!(true);\n}";
        let mut parser = ctx.generate_parser("test.rs", src_string);
        let lines = parse_crate(&ctx, &mut parser);
        assert!(!lines.contains(&2));
        assert!(!lines.contains(&3));

        let mut ctx = TestContext::default();
        ctx.conf.ignore_tests = true;
        let mut parser = ctx.generate_parser("test.rs", src_string);
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.contains(&2));
        assert!(lines.contains(&3));

    }
}
<commit_msg>STarted work on ignore test modules/functions<commit_after>use std::path::{PathBuf, Path};
use std::collections::{HashSet, HashMap};
use std::fs::File;
use std::io::{BufReader, BufRead};
use cargo::core::{Workspace, Package};
use cargo::sources::PathSource;
use cargo::util::Config as CargoConfig;
use syn::{parse_file, Item, ItemMod, Ident, File as SFile};
use proc_macro2::Span;
use regex::Regex;
use config::Config;

/// Represents the results of analysis of a single file. Does not store the file
/// in question as this is expected to be maintained by the user.
#[derive(Clone, Debug)]
pub struct LineAnalysis {
    /// This represents lines that should be ignored in coverage
    /// but may be identifed as coverable in the DWARF tables
    pub ignore: HashSet<usize>,
    /// This represents lines that should be included in coverage
    /// But may be ignored.
    pub cover: HashSet<usize>,
}

/// When the LineAnalysis results are mapped to their files there needs to be
/// an easy way to get the information back. For the container used implement
/// this trait
pub trait SourceAnalysisQuery {
    fn should_ignore(&self, path: &Path, l:&usize) -> bool;
}

impl SourceAnalysisQuery for HashMap<PathBuf, LineAnalysis> {

    fn should_ignore(&self, path: &Path, l:&usize) -> bool {
        if self.contains_key(path) {
            self.get(path).unwrap().ignore.contains(l)
        } else {
            false
        }
    }

}

impl LineAnalysis {
    fn new() -> LineAnalysis {
        LineAnalysis {
            ignore: HashSet::new(),
            cover: HashSet::new()
        }
    }

    /// Adds the lines of the provided span to the ignore set
    pub fn ignore_span(&mut self, span: &Span) {
        for i in span.start().line..span.end().line {
            self.ignore.insert(i);
            if self.cover.contains(&i) {
                self.cover.remove(&i);
            }
        }
    }

    /// Adds the lines of the provided span to the cover set
    pub fn cover_span(&mut self, span: &Span) {
        for i in span.start().line..span.end().line {
            if !self.ignore.contains(&i) {
                self.cover.insert(i);
            }
        }
    }

    pub fn should_ignore(&self, line: &usize) -> bool {
        self.ignore.contains(line)
    }

    fn add_to_ignore(&mut self, lines: &[usize]) {
        for l in lines {
            self.ignore.insert(*l);
            if self.cover.contains(l) {
                self.cover.remove(l);
            }
        }
    }

    fn add_to_cover(&mut self, lines: &[usize]) {
        for l in lines {
            if !self.ignore.contains(l) {
                self.cover.insert(*l);
            }
        }
    }
}

/// Returns a list of files and line numbers to ignore (not indexes!)
pub fn get_line_analysis(project: &Workspace, config: &Config) -> HashMap<PathBuf, LineAnalysis> {
    let mut result: HashMap<PathBuf, LineAnalysis> = HashMap::new();
    // Members iterates over all non-virtual packages in the workspace
    for pkg in project.members() {
        if config.packages.is_empty() || config.packages.contains(&pkg.name().to_string()) {
            analyse_package(pkg, &config, project.config(), &mut result);
        }
    }
    result
}

fn analyse_lib_rs(file: &Path, result: &mut HashMap<PathBuf, LineAnalysis>) {
    if let Ok(f) = File::open(file) {
        let mut read_file = BufReader::new(f);
        if let Some(Ok(first)) = read_file.lines().nth(0) {
            if !(first.starts_with("pub") || first.starts_with("fn")) {
                let file = file.to_path_buf();
                if result.contains_key(&file) {
                    let l = result.get_mut(&file).unwrap();
                    l.add_to_ignore(&[1]);
                } else {
                    let mut l = LineAnalysis::new();
                    l.add_to_ignore(&[1]);
                    result.insert(file, l);
                }
            }
        }
    }
}

fn analyse_package(pkg: &Package,
                   config:&Config,
                   cargo_conf: &CargoConfig,
                   result: &mut HashMap<PathBuf, LineAnalysis>) {

    let mut src = PathSource::new(pkg.root(), pkg.package_id().source_id(), cargo_conf);
    if let Ok(package) = src.root_package() {
        for target in package.targets() {
            let path = target.src_path();
            let file = match path.to_str() {
                Some(s) => s,
                _ => continue
            };
            let skip_cause_test = config.ignore_tests &&
                                  path.starts_with(pkg.root().join("tests"));
            let skip_cause_example = path.starts_with(pkg.root().join("examples"));
            if !(skip_cause_test || skip_cause_example)  {
                let file = parse_file(file);
                if let Ok(file) = file {
                    let analysis = process_module(file, config);
                    // Check there's no conflict!
                    result.insert(path.to_path_buf(), analysis);
                }
            }
            // This could probably be done with the DWARF if I could find a discriminating factor
            // to why lib.rs:1 shows up as a real line!
            if file.ends_with("src/lib.rs") {
                analyse_lib_rs(path, result);
            }
        }
    }
}

fn process_module(file: SFile, config: &Config) -> LineAnalysis {
    let mut analysis = LineAnalysis::new();
    for item in &file.items {
        match item {
            Item::ExternCrate(i) => analysis.ignore_span(&i.extern_token.0),
            Item::Use(i) => analysis.ignore_span(&i.use_token.0),
            Item::Mod(i) => visit_mod(i, &mut analysis, config),
            _ =>{}
        }
    }
    analysis
}

fn visit_mod(module: &ItemMod, analysis: &mut LineAnalysis, config: &Config) {
    // Need to read the nested meta.. But this should work for fns
    let test_mod = module.attrs.iter()
                               .map(|ref x| x.interpret_meta())
                               .filter(|ref x| x.is_some())
                               .map(|x| x.unwrap())
                               .any(|x| x.name() == Ident::from("test"));
    if test_mod && config.ignore_tests {
        analysis.ignore_span(&module.mod_token.0);
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use syntex_syntax::parse::filemap_to_parser;
    use syntex_syntax::parse::parser::Parser;

    struct TestContext {
        conf: Config,
        codemap: Rc<CodeMap>,
        parse_session: ParseSess
    }

    impl TestContext {
        fn generate_parser(&self, filename: &str, src_string: &str) -> Parser {
            let filemap = self.codemap.new_filemap(filename.to_string(),
                                                   src_string.to_string());
            filemap_to_parser(&self.parse_session, filemap)
        }
    }

    impl Default for TestContext {
        fn default() -> TestContext {
            let codemap = Rc::new(CodeMap::new(FilePathMapping::empty()));
            let handler = Handler::with_tty_emitter(ColorConfig::Auto, false, false, Some(codemap.clone()));
            let parse_session = ParseSess::with_span_handler(handler, codemap.clone());
            TestContext {
                conf: Config::default(),
                codemap: codemap,
                parse_session: parse_session
            }
        }
    }

    fn parse_crate(ctx: &TestContext, parser: &mut Parser) -> Vec<usize>  {
        let krate = parser.parse_crate_mod();
        assert!(krate.is_ok());
        let krate = krate.unwrap();
        let unused: HashSet<PathBuf> = HashSet::new();
        let mut visitor = CoverageVisitor::from_session(&ctx.parse_session, &unused, &ctx.conf);
        visitor.visit_mod(&krate.module, krate.span, &krate.attrs, NodeId::new(0));
        visitor.lines.iter().map(|x| x.1).collect::<Vec<_>>()
    }

    #[test]
    fn filter_str_literals() {
        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("literals.rs", "fn test() {\nwriteln!(#\"test\n\ttest\n\ttest\"#);\n}\n");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.len() > 1);
        assert!(lines.contains(&3));
        assert!(lines.contains(&4));

        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("literals.rs", "fn test() {\nwrite(\"test\ntest\ntest\");\n}\nfn write(s:&str){}");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.len() > 1);
        assert!(lines.contains(&3));
        assert!(lines.contains(&4));

        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("literals.rs", "\n\nfn test() {\nwriteln!(\n#\"test\"#\n);\n}\n");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.contains(&5));
    }

    #[test]
    fn filter_struct_members() {
        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("struct_test.rs", "#[derive(Debug)]\npub struct Struct {\npub i: i32,\nj:String,\n}");
        let lines = parse_crate(&ctx, &mut parser);

        assert_eq!(lines.len(), 3);
        assert!(lines.contains(&1));
        assert!(lines.contains(&3));
        assert!(lines.contains(&4));
    }

    #[test]
    fn filter_struct_consts() {
        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("struct_test.rs",
                                             "struct T{x:String, y:i32}\nfn test()-> T {\nT{\nx:\"hello\".to_string(),\ny:4,\n}\n}");

        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.contains(&5));
    }

    #[test]
    fn filter_mods() {
        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("test.rs", "mod foo {\nfn double(x:i32)->i32 {\n x*2\n}\n}");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(!lines.contains(&3));

        let mut parser = ctx.generate_parser("test.rs", "mod foo{}");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.contains(&1));
    }

    #[test]
    fn filter_macros() {
        let ctx = TestContext::default();
        let mut parser = ctx.generate_parser("test.rs", "\n\nfn unused() {\nunimplemented!();\n}");

        let lines = parse_crate(&ctx, &mut parser);
        // Braces should be ignored so number could be higher
        assert!(lines.len() >= 1);
        assert!(lines.contains(&4));

        let mut parser = ctx.generate_parser("test.rs", "fn unused() {\nunreachable!();\n}");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.len() >= 1);
        assert!(lines.contains(&2));

        let mut parser = ctx.generate_parser("test.rs", "fn unused() {\nprintln!(\"text\");\n}");
        let lines = parse_crate(&ctx, &mut parser);
        assert!(!lines.contains(&2));
    }

    #[test]
    fn filter_tests() {
        let ctx = TestContext::default();
        let src_string = "#[cfg(test)]\nmod tests {\n fn boo(){\nassert!(true);\n}\n}";
        let mut parser = ctx.generate_parser("test.rs", src_string);
        let lines = parse_crate(&ctx, &mut parser);
        assert!(!lines.contains(&4));

        let mut ctx = TestContext::default();
        ctx.conf.ignore_tests = true;
        let mut parser = ctx.generate_parser("test.rs", src_string);
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.contains(&4));

        let ctx = TestContext::default();
        let src_string = "#[test]\nfn mytest() { \n assert!(true);\n}";
        let mut parser = ctx.generate_parser("test.rs", src_string);
        let lines = parse_crate(&ctx, &mut parser);
        assert!(!lines.contains(&2));
        assert!(!lines.contains(&3));

        let mut ctx = TestContext::default();
        ctx.conf.ignore_tests = true;
        let mut parser = ctx.generate_parser("test.rs", src_string);
        let lines = parse_crate(&ctx, &mut parser);
        assert!(lines.contains(&2));
        assert!(lines.contains(&3));

    }
}
<|endoftext|>


<commit_before>//! Tidy check to verify the validity of long error diagnostic codes.
//!
//! This ensures that error codes are used at most once and also prints out some
//! statistics about the error codes.

use std::collections::HashMap;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

pub fn check(path: &Path, bad: &mut bool) {
    let mut contents = String::new();
    let mut map: HashMap<_, Vec<_>> = HashMap::new();
    super::walk(path,
                &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
                &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if filename != "diagnostics.rs" && filename != "diagnostic_list.rs" {
            return
        }

        contents.truncate(0);
        t!(t!(File::open(file)).read_to_string(&mut contents));

        // In the `register_long_diagnostics!` macro, entries look like this:
        //
        // ```
        // EXXXX: r##"
        // <Long diagnostic message>
        // "##,
        // ```
        //
        // and these long messages often have error codes themselves inside
        // them, but we don't want to report duplicates in these cases. This
        // variable keeps track of whether we're currently inside one of these
        // long diagnostic messages.
        let mut inside_long_diag = false;
        for (num, line) in contents.lines().enumerate() {
            if inside_long_diag {
                inside_long_diag = !line.contains("\"##");
                continue
            }

            let mut search = line;
            while let Some(i) = search.find('E') {
                search = &search[i + 1..];
                let code = if search.len() > 4 {
                    search[..4].parse::<u32>()
                } else {
                    continue
                };
                let code = match code {
                    Ok(n) => n,
                    Err(..) => continue,
                };
                map.entry(code).or_default()
                   .push((file.to_owned(), num + 1, line.to_owned()));
                break
            }

            inside_long_diag = line.contains("r##\"");
        }
    });

    let mut max = 0;
    for (&code, entries) in map.iter() {
        if code > max {
            max = code;
        }
        if entries.len() == 1 {
            continue
        }

        tidy_error!(bad, "duplicate error code: {}", code);
        for &(ref file, line_num, ref line) in entries.iter() {
            tidy_error!(bad, "{}:{}: {}", file.display(), line_num, line);
        }
    }

    if !*bad {
        println!("* {} error codes", map.len());
        println!("* highest error code: E{:04}", max);
    }
}
<commit_msg>Rename filename<commit_after>//! Tidy check to verify the validity of long error diagnostic codes.
//!
//! This ensures that error codes are used at most once and also prints out some
//! statistics about the error codes.

use std::collections::HashMap;
use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

pub fn check(path: &Path, bad: &mut bool) {
    let mut contents = String::new();
    let mut map: HashMap<_, Vec<_>> = HashMap::new();
    super::walk(path,
                &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
                &mut |file| {
        let filename = file.file_name().unwrap().to_string_lossy();
        if filename != "error_codes.rs" && filename != "diagnostic_list.rs" {
            return
        }

        contents.truncate(0);
        t!(t!(File::open(file)).read_to_string(&mut contents));

        // In the `register_long_diagnostics!` macro, entries look like this:
        //
        // ```
        // EXXXX: r##"
        // <Long diagnostic message>
        // "##,
        // ```
        //
        // and these long messages often have error codes themselves inside
        // them, but we don't want to report duplicates in these cases. This
        // variable keeps track of whether we're currently inside one of these
        // long diagnostic messages.
        let mut inside_long_diag = false;
        for (num, line) in contents.lines().enumerate() {
            if inside_long_diag {
                inside_long_diag = !line.contains("\"##");
                continue
            }

            let mut search = line;
            while let Some(i) = search.find('E') {
                search = &search[i + 1..];
                let code = if search.len() > 4 {
                    search[..4].parse::<u32>()
                } else {
                    continue
                };
                let code = match code {
                    Ok(n) => n,
                    Err(..) => continue,
                };
                map.entry(code).or_default()
                   .push((file.to_owned(), num + 1, line.to_owned()));
                break
            }

            inside_long_diag = line.contains("r##\"");
        }
    });

    let mut max = 0;
    for (&code, entries) in map.iter() {
        if code > max {
            max = code;
        }
        if entries.len() == 1 {
            continue
        }

        tidy_error!(bad, "duplicate error code: {}", code);
        for &(ref file, line_num, ref line) in entries.iter() {
            tidy_error!(bad, "{}:{}: {}", file.display(), line_num, line);
        }
    }

    if !*bad {
        println!("* {} error codes", map.len());
        println!("* highest error code: E{:04}", max);
    }
}
<|endoftext|>


<commit_before>// * This file is part of the uutils coreutils package.
// *
// * (c) 2015 Wiktor Kuropatwa <wiktor.kuropatwa@gmail.com>
// * (c) 2020 nicoo            <nicoo@debian.org>
// *
// * For the full copyright and license information, please view the LICENSE file
// * that was distributed with this source code.

use num_traits::{
    identities::{One, Zero},
    int::PrimInt,
    ops::wrapping::{WrappingMul, WrappingNeg, WrappingSub},
};
use std::fmt::{Debug, Display};
use std::mem::swap;

// This is incorrectly reported as dead code,
//  presumably when included in build.rs.
#[allow(dead_code)]
pub(crate) fn gcd(mut a: u64, mut b: u64) -> u64 {
    while b > 0 {
        a %= b;
        swap(&mut a, &mut b);
    }
    a
}

pub(crate) trait Arithmetic: Copy + Sized {
    // The type of integers mod m, in some opaque representation
    type ModInt: Copy + Sized + Eq;

    fn new(m: u64) -> Self;
    fn modulus(&self) -> u64;
    fn from_u64(&self, n: u64) -> Self::ModInt;
    fn to_u64(&self, n: Self::ModInt) -> u64;
    fn add(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt;
    fn mul(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt;

    fn pow(&self, mut a: Self::ModInt, mut b: u64) -> Self::ModInt {
        let (_a, _b) = (a, b);
        let mut result = self.one();
        while b > 0 {
            if b & 1 != 0 {
                result = self.mul(result, a);
            }
            a = self.mul(a, a);
            b >>= 1;
        }

        // Check that r (reduced back to the usual representation) equals
        //  a^b % n, unless the latter computation overflows
        // Temporarily commented-out, as there u64::checked_pow is not available
        //  on the minimum supported Rust version, nor is an appropriate method
        //  for compiling the check conditionally.
        //debug_assert!(self
        //    .to_u64(_a)
        //    .checked_pow(_b as u32)
        //    .map(|r| r % self.modulus() == self.to_u64(result))
        //    .unwrap_or(true));

        result
    }

    fn one(&self) -> Self::ModInt {
        self.from_u64(1)
    }
    fn minus_one(&self) -> Self::ModInt {
        self.from_u64(self.modulus() - 1)
    }
    fn zero(&self) -> Self::ModInt {
        self.from_u64(0)
    }
}

#[derive(Clone, Copy, Debug)]
pub(crate) struct Montgomery<T: DoubleInt> {
    a: T,
    n: T,
}

impl<T: DoubleInt> Montgomery<T> {
    /// computes x/R mod n efficiently
    fn reduce(&self, x: T::Double) -> T {
        let t_bits = T::zero().count_zeros() as usize;

        debug_assert!(x < (self.n.as_double()) << t_bits);
        // TODO: optimiiiiiiise
        let Montgomery { a, n } = self;
        let m = T::from_double(x).wrapping_mul(a);
        let nm = (n.as_double()) * (m.as_double());
        let (xnm, overflow) = x.overflowing_add_(nm); // x + n*m
        debug_assert_eq!(
            xnm % (T::Double::one() << T::zero().count_zeros() as usize),
            T::Double::zero()
        );

        // (x + n*m) / R
        // in case of overflow, this is (2¹²⁸ + xnm)/2⁶⁴ - n = xnm/2⁶⁴ + (2⁶⁴ - n)
        let y = T::from_double(xnm >> t_bits)
            + if !overflow {
                T::zero()
            } else {
                n.wrapping_neg()
            };

        if y >= *n {
            y - *n
        } else {
            y
        }
    }
}

impl<T: DoubleInt> Arithmetic for Montgomery<T> {
    // Montgomery transform, R=2⁶⁴
    // Provides fast arithmetic mod n (n odd, u64)
    type ModInt = T;

    fn new(n: u64) -> Self {
        debug_assert!(T::zero().count_zeros() >= 64 || n < (1 << T::zero().count_zeros() as usize));
        let n = T::from_u64(n);
        let a = modular_inverse(n).wrapping_neg();
        debug_assert_eq!(n.wrapping_mul(&a), T::one().wrapping_neg());
        Montgomery { a, n }
    }

    fn modulus(&self) -> u64 {
        self.n.as_u64()
    }

    fn from_u64(&self, x: u64) -> Self::ModInt {
        // TODO: optimise!
        debug_assert!(x < self.n.as_u64());
        let r = T::from_double(
            ((T::Double::from_u64(x)) << T::zero().count_zeros() as usize) % self.n.as_double(),
        );
        debug_assert_eq!(x, self.to_u64(r));
        r
    }

    fn to_u64(&self, n: Self::ModInt) -> u64 {
        self.reduce(n.as_double()).as_u64()
    }

    fn add(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt {
        let (r, overflow) = a.overflowing_add_(b);

        // In case of overflow, a+b = 2⁶⁴ + r = (2⁶⁴ - n) + r (working mod n)
        let r = if !overflow {
            r
        } else {
            r + self.n.wrapping_neg()
        };

        // Normalise to [0; n[
        let r = if r < self.n { r } else { r - self.n };

        // Check that r (reduced back to the usual representation) equals
        // a+b % n
        #[cfg(debug_assertions)]
        {
            let a_r = self.to_u64(a) as u128;
            let b_r = self.to_u64(b) as u128;
            let r_r = self.to_u64(r);
            let r_2 = ((a_r + b_r) % self.n.as_u128()) as u64;
            debug_assert_eq!(
                r_r, r_2,
                "[{}] = {} ≠ {} = {} + {} = [{}] + [{}] mod {}; a = {}",
                r, r_r, r_2, a_r, b_r, a, b, self.n, self.a
            );
        }
        r
    }

    fn mul(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt {
        let r = self.reduce(a.as_double() * b.as_double());

        // Check that r (reduced back to the usual representation) equals
        // a*b % n
        #[cfg(debug_assertions)]
        {
            let a_r = self.to_u64(a) as u128;
            let b_r = self.to_u64(b) as u128;
            let r_r = self.to_u64(r);
            let r_2: u64 = ((a_r * b_r) % self.n.as_u128()) as u64;
            debug_assert_eq!(
                r_r, r_2,
                "[{}] = {} ≠ {} = {} * {} = [{}] * [{}] mod {}; a = {}",
                r, r_r, r_2, a_r, b_r, a, b, self.n, self.a
            );
        }
        r
    }
}

// NOTE: Trait can be removed once num-traits adds a similar one;
//       see https://github.com/rust-num/num-traits/issues/168
pub(crate) trait OverflowingAdd: Sized {
    fn overflowing_add_(self, n: Self) -> (Self, bool);
}
impl OverflowingAdd for u32 {
    fn overflowing_add_(self, n: Self) -> (Self, bool) {
        self.overflowing_add(n)
    }
}
impl OverflowingAdd for u64 {
    fn overflowing_add_(self, n: Self) -> (Self, bool) {
        self.overflowing_add(n)
    }
}
impl OverflowingAdd for u128 {
    fn overflowing_add_(self, n: Self) -> (Self, bool) {
        self.overflowing_add(n)
    }
}

pub(crate) trait Int:
    Display + Debug + PrimInt + OverflowingAdd + WrappingNeg + WrappingSub + WrappingMul
{
    fn as_u64(&self) -> u64;
    fn from_u64(n: u64) -> Self;
    #[cfg(debug_assertions)]
    fn as_u128(&self) -> u128;
    #[cfg(debug_assertions)]
    fn from_u128(n: u64) -> Self;
}

pub(crate) trait DoubleInt: Int {
    type Double: Int;

    fn as_double(self) -> Self::Double;
    fn from_double(n: Self::Double) -> Self;
}

macro_rules! int {
    ( $x:ty ) => {
        impl Int for $x {
            fn as_u64(&self) -> u64 {
                *self as u64
            }
            fn from_u64(n: u64) -> Self {
                n as _
            }
            #[cfg(debug_assertions)]
            fn as_u128(&self) -> u128 {
                *self as u128
            }
            #[cfg(debug_assertions)]
            fn from_u128(n: u64) -> Self {
                n as _
            }
        }
    };
}
macro_rules! double_int {
    ( $x:ty, $y:ty ) => {
        int!($x);
        impl DoubleInt for $x {
            type Double = $y;

            fn as_double(self) -> $y {
                self as _
            }
            fn from_double(n: $y) -> $x {
                n as _
            }
        }
    };
}

double_int!(u32, u64);
double_int!(u64, u128);
int!(u128);

// extended Euclid algorithm
// precondition: a is odd
pub(crate) fn modular_inverse<T: Int>(a: T) -> T {
    let zero = T::zero();
    let one = T::one();
    debug_assert!(a % (one + one) == one, "{:?} is not odd", a);

    let mut t = zero;
    let mut newt = one;
    let mut r = zero;
    let mut newr = a;

    while newr != zero {
        let quot = if r == zero {
            // special case when we're just starting out
            // This works because we know that
            // a does not divide 2^64, so floor(2^64 / a) == floor((2^64-1) / a);
            T::max_value()
        } else {
            r
        } / newr;

        let newtp = t.wrapping_sub(&quot.wrapping_mul(&newt));
        t = newt;
        newt = newtp;

        let newrp = r.wrapping_sub(&quot.wrapping_mul(&newr));
        r = newr;
        newr = newrp;
    }

    debug_assert_eq!(r, one);
    t
}

#[cfg(test)]
mod tests {
    use super::*;

    fn test_inverter<T: Int>() {
        // All odd integers from 1 to 20 000
        let one = T::from(1).unwrap();
        let two = T::from(2).unwrap();
        let mut test_values = (0..10_000)
            .map(|i| T::from(i).unwrap())
            .map(|i| two * i + one);

        assert!(test_values.all(|x| x.wrapping_mul(&modular_inverse(x)) == one));
    }

    #[test]
    fn test_inverter_u32() {
        test_inverter::<u32>()
    }

    #[test]
    fn test_inverter_u64() {
        test_inverter::<u64>()
    }

    fn test_add<A: Arithmetic>() {
        for n in 0..100 {
            let n = 2 * n + 1;
            let m = A::new(n);
            for x in 0..n {
                let m_x = m.from_u64(x);
                for y in 0..=x {
                    let m_y = m.from_u64(y);
                    println!("{n:?}, {x:?}, {y:?}", n = n, x = x, y = y);
                    assert_eq!((x + y) % n, m.to_u64(m.add(m_x, m_y)));
                }
            }
        }
    }

    #[test]
    fn test_add_m32() {
        test_add::<Montgomery<u32>>()
    }

    #[test]
    fn test_add_m64() {
        test_add::<Montgomery<u64>>()
    }

    fn test_mult<A: Arithmetic>() {
        for n in 0..100 {
            let n = 2 * n + 1;
            let m = A::new(n);
            for x in 0..n {
                let m_x = m.from_u64(x);
                for y in 0..=x {
                    let m_y = m.from_u64(y);
                    assert_eq!((x * y) % n, m.to_u64(m.mul(m_x, m_y)));
                }
            }
        }
    }

    #[test]
    fn test_mult_m32() {
        test_mult::<Montgomery<u32>>()
    }

    #[test]
    fn test_mult_m64() {
        test_mult::<Montgomery<u64>>()
    }

    fn test_roundtrip<A: Arithmetic>() {
        for n in 0..100 {
            let n = 2 * n + 1;
            let m = A::new(n);
            for x in 0..n {
                let x_ = m.from_u64(x);
                assert_eq!(x, m.to_u64(x_));
            }
        }
    }

    #[test]
    fn test_roundtrip_m32() {
        test_roundtrip::<Montgomery<u32>>()
    }

    #[test]
    fn test_roundtrip_m64() {
        test_roundtrip::<Montgomery<u64>>()
    }
}
<commit_msg>factor::numeric::OverflowingAdd: Generate impls with a macro<commit_after>// * This file is part of the uutils coreutils package.
// *
// * (c) 2015 Wiktor Kuropatwa <wiktor.kuropatwa@gmail.com>
// * (c) 2020 nicoo            <nicoo@debian.org>
// *
// * For the full copyright and license information, please view the LICENSE file
// * that was distributed with this source code.

use num_traits::{
    identities::{One, Zero},
    int::PrimInt,
    ops::wrapping::{WrappingMul, WrappingNeg, WrappingSub},
};
use std::fmt::{Debug, Display};
use std::mem::swap;

// This is incorrectly reported as dead code,
//  presumably when included in build.rs.
#[allow(dead_code)]
pub(crate) fn gcd(mut a: u64, mut b: u64) -> u64 {
    while b > 0 {
        a %= b;
        swap(&mut a, &mut b);
    }
    a
}

pub(crate) trait Arithmetic: Copy + Sized {
    // The type of integers mod m, in some opaque representation
    type ModInt: Copy + Sized + Eq;

    fn new(m: u64) -> Self;
    fn modulus(&self) -> u64;
    fn from_u64(&self, n: u64) -> Self::ModInt;
    fn to_u64(&self, n: Self::ModInt) -> u64;
    fn add(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt;
    fn mul(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt;

    fn pow(&self, mut a: Self::ModInt, mut b: u64) -> Self::ModInt {
        let (_a, _b) = (a, b);
        let mut result = self.one();
        while b > 0 {
            if b & 1 != 0 {
                result = self.mul(result, a);
            }
            a = self.mul(a, a);
            b >>= 1;
        }

        // Check that r (reduced back to the usual representation) equals
        //  a^b % n, unless the latter computation overflows
        // Temporarily commented-out, as there u64::checked_pow is not available
        //  on the minimum supported Rust version, nor is an appropriate method
        //  for compiling the check conditionally.
        //debug_assert!(self
        //    .to_u64(_a)
        //    .checked_pow(_b as u32)
        //    .map(|r| r % self.modulus() == self.to_u64(result))
        //    .unwrap_or(true));

        result
    }

    fn one(&self) -> Self::ModInt {
        self.from_u64(1)
    }
    fn minus_one(&self) -> Self::ModInt {
        self.from_u64(self.modulus() - 1)
    }
    fn zero(&self) -> Self::ModInt {
        self.from_u64(0)
    }
}

#[derive(Clone, Copy, Debug)]
pub(crate) struct Montgomery<T: DoubleInt> {
    a: T,
    n: T,
}

impl<T: DoubleInt> Montgomery<T> {
    /// computes x/R mod n efficiently
    fn reduce(&self, x: T::Double) -> T {
        let t_bits = T::zero().count_zeros() as usize;

        debug_assert!(x < (self.n.as_double()) << t_bits);
        // TODO: optimiiiiiiise
        let Montgomery { a, n } = self;
        let m = T::from_double(x).wrapping_mul(a);
        let nm = (n.as_double()) * (m.as_double());
        let (xnm, overflow) = x.overflowing_add_(nm); // x + n*m
        debug_assert_eq!(
            xnm % (T::Double::one() << T::zero().count_zeros() as usize),
            T::Double::zero()
        );

        // (x + n*m) / R
        // in case of overflow, this is (2¹²⁸ + xnm)/2⁶⁴ - n = xnm/2⁶⁴ + (2⁶⁴ - n)
        let y = T::from_double(xnm >> t_bits)
            + if !overflow {
                T::zero()
            } else {
                n.wrapping_neg()
            };

        if y >= *n {
            y - *n
        } else {
            y
        }
    }
}

impl<T: DoubleInt> Arithmetic for Montgomery<T> {
    // Montgomery transform, R=2⁶⁴
    // Provides fast arithmetic mod n (n odd, u64)
    type ModInt = T;

    fn new(n: u64) -> Self {
        debug_assert!(T::zero().count_zeros() >= 64 || n < (1 << T::zero().count_zeros() as usize));
        let n = T::from_u64(n);
        let a = modular_inverse(n).wrapping_neg();
        debug_assert_eq!(n.wrapping_mul(&a), T::one().wrapping_neg());
        Montgomery { a, n }
    }

    fn modulus(&self) -> u64 {
        self.n.as_u64()
    }

    fn from_u64(&self, x: u64) -> Self::ModInt {
        // TODO: optimise!
        debug_assert!(x < self.n.as_u64());
        let r = T::from_double(
            ((T::Double::from_u64(x)) << T::zero().count_zeros() as usize) % self.n.as_double(),
        );
        debug_assert_eq!(x, self.to_u64(r));
        r
    }

    fn to_u64(&self, n: Self::ModInt) -> u64 {
        self.reduce(n.as_double()).as_u64()
    }

    fn add(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt {
        let (r, overflow) = a.overflowing_add_(b);

        // In case of overflow, a+b = 2⁶⁴ + r = (2⁶⁴ - n) + r (working mod n)
        let r = if !overflow {
            r
        } else {
            r + self.n.wrapping_neg()
        };

        // Normalise to [0; n[
        let r = if r < self.n { r } else { r - self.n };

        // Check that r (reduced back to the usual representation) equals
        // a+b % n
        #[cfg(debug_assertions)]
        {
            let a_r = self.to_u64(a) as u128;
            let b_r = self.to_u64(b) as u128;
            let r_r = self.to_u64(r);
            let r_2 = ((a_r + b_r) % self.n.as_u128()) as u64;
            debug_assert_eq!(
                r_r, r_2,
                "[{}] = {} ≠ {} = {} + {} = [{}] + [{}] mod {}; a = {}",
                r, r_r, r_2, a_r, b_r, a, b, self.n, self.a
            );
        }
        r
    }

    fn mul(&self, a: Self::ModInt, b: Self::ModInt) -> Self::ModInt {
        let r = self.reduce(a.as_double() * b.as_double());

        // Check that r (reduced back to the usual representation) equals
        // a*b % n
        #[cfg(debug_assertions)]
        {
            let a_r = self.to_u64(a) as u128;
            let b_r = self.to_u64(b) as u128;
            let r_r = self.to_u64(r);
            let r_2: u64 = ((a_r * b_r) % self.n.as_u128()) as u64;
            debug_assert_eq!(
                r_r, r_2,
                "[{}] = {} ≠ {} = {} * {} = [{}] * [{}] mod {}; a = {}",
                r, r_r, r_2, a_r, b_r, a, b, self.n, self.a
            );
        }
        r
    }
}

// NOTE: Trait can be removed once num-traits adds a similar one;
//       see https://github.com/rust-num/num-traits/issues/168
pub(crate) trait OverflowingAdd: Sized {
    fn overflowing_add_(self, n: Self) -> (Self, bool);
}
macro_rules! overflowing {
    ($x:ty) => {
        impl OverflowingAdd for $x {
            fn overflowing_add_(self, n: Self) -> (Self, bool) {
                self.overflowing_add(n)
            }
        }
    };
}
overflowing!(u32);
overflowing!(u64);
overflowing!(u128);

pub(crate) trait Int:
    Display + Debug + PrimInt + OverflowingAdd + WrappingNeg + WrappingSub + WrappingMul
{
    fn as_u64(&self) -> u64;
    fn from_u64(n: u64) -> Self;
    #[cfg(debug_assertions)]
    fn as_u128(&self) -> u128;
    #[cfg(debug_assertions)]
    fn from_u128(n: u64) -> Self;
}

pub(crate) trait DoubleInt: Int {
    type Double: Int;

    fn as_double(self) -> Self::Double;
    fn from_double(n: Self::Double) -> Self;
}

macro_rules! int {
    ( $x:ty ) => {
        impl Int for $x {
            fn as_u64(&self) -> u64 {
                *self as u64
            }
            fn from_u64(n: u64) -> Self {
                n as _
            }
            #[cfg(debug_assertions)]
            fn as_u128(&self) -> u128 {
                *self as u128
            }
            #[cfg(debug_assertions)]
            fn from_u128(n: u64) -> Self {
                n as _
            }
        }
    };
}
macro_rules! double_int {
    ( $x:ty, $y:ty ) => {
        int!($x);
        impl DoubleInt for $x {
            type Double = $y;

            fn as_double(self) -> $y {
                self as _
            }
            fn from_double(n: $y) -> $x {
                n as _
            }
        }
    };
}

double_int!(u32, u64);
double_int!(u64, u128);
int!(u128);

// extended Euclid algorithm
// precondition: a is odd
pub(crate) fn modular_inverse<T: Int>(a: T) -> T {
    let zero = T::zero();
    let one = T::one();
    debug_assert!(a % (one + one) == one, "{:?} is not odd", a);

    let mut t = zero;
    let mut newt = one;
    let mut r = zero;
    let mut newr = a;

    while newr != zero {
        let quot = if r == zero {
            // special case when we're just starting out
            // This works because we know that
            // a does not divide 2^64, so floor(2^64 / a) == floor((2^64-1) / a);
            T::max_value()
        } else {
            r
        } / newr;

        let newtp = t.wrapping_sub(&quot.wrapping_mul(&newt));
        t = newt;
        newt = newtp;

        let newrp = r.wrapping_sub(&quot.wrapping_mul(&newr));
        r = newr;
        newr = newrp;
    }

    debug_assert_eq!(r, one);
    t
}

#[cfg(test)]
mod tests {
    use super::*;

    fn test_inverter<T: Int>() {
        // All odd integers from 1 to 20 000
        let one = T::from(1).unwrap();
        let two = T::from(2).unwrap();
        let mut test_values = (0..10_000)
            .map(|i| T::from(i).unwrap())
            .map(|i| two * i + one);

        assert!(test_values.all(|x| x.wrapping_mul(&modular_inverse(x)) == one));
    }

    #[test]
    fn test_inverter_u32() {
        test_inverter::<u32>()
    }

    #[test]
    fn test_inverter_u64() {
        test_inverter::<u64>()
    }

    fn test_add<A: Arithmetic>() {
        for n in 0..100 {
            let n = 2 * n + 1;
            let m = A::new(n);
            for x in 0..n {
                let m_x = m.from_u64(x);
                for y in 0..=x {
                    let m_y = m.from_u64(y);
                    println!("{n:?}, {x:?}, {y:?}", n = n, x = x, y = y);
                    assert_eq!((x + y) % n, m.to_u64(m.add(m_x, m_y)));
                }
            }
        }
    }

    #[test]
    fn test_add_m32() {
        test_add::<Montgomery<u32>>()
    }

    #[test]
    fn test_add_m64() {
        test_add::<Montgomery<u64>>()
    }

    fn test_mult<A: Arithmetic>() {
        for n in 0..100 {
            let n = 2 * n + 1;
            let m = A::new(n);
            for x in 0..n {
                let m_x = m.from_u64(x);
                for y in 0..=x {
                    let m_y = m.from_u64(y);
                    assert_eq!((x * y) % n, m.to_u64(m.mul(m_x, m_y)));
                }
            }
        }
    }

    #[test]
    fn test_mult_m32() {
        test_mult::<Montgomery<u32>>()
    }

    #[test]
    fn test_mult_m64() {
        test_mult::<Montgomery<u64>>()
    }

    fn test_roundtrip<A: Arithmetic>() {
        for n in 0..100 {
            let n = 2 * n + 1;
            let m = A::new(n);
            for x in 0..n {
                let x_ = m.from_u64(x);
                assert_eq!(x, m.to_u64(x_));
            }
        }
    }

    #[test]
    fn test_roundtrip_m32() {
        test_roundtrip::<Montgomery<u32>>()
    }

    #[test]
    fn test_roundtrip_m64() {
        test_roundtrip::<Montgomery<u64>>()
    }
}
<|endoftext|>


<commit_before>#![allow(clippy::manual_assert)]

mod progress;

use self::progress::Progress;
use anyhow::Result;
use flate2::read::GzDecoder;
use std::fs;
use std::path::Path;
use tar::Archive;
use walkdir::DirEntry;

const REVISION: &str = "98ad6a5519651af36e246c0335c964dd52c554ba";

#[rustfmt::skip]
static EXCLUDE_FILES: &[&str] = &[
    // TODO
    "compiler/rustc_codegen_gcc/src/intrinsic/archs.rs",
    "src/test/run-make/translation/test.rs",
    "src/test/rustdoc-ui/issue-79467.rs",
    "src/test/ui/dyn-star/const.rs",
    "src/test/ui/dyn-star/drop.rs",
    "src/test/ui/dyn-star/make-dyn-star.rs",
    "src/test/ui/dyn-star/method.rs",
    "src/test/ui/dyn-star/syntax.rs",
    "src/test/ui/generics/issue-94432-garbage-ice.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0012_type_item_where_clause.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0040_crate_keyword_vis.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0102_record_pat_field_list.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0106_lambda_expr.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0131_existential_type.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0138_associated_type_bounds.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0179_use_tree_abs_star.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0188_const_param_default_path.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0200_const_param_default_literal.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0202_typepathfn_with_coloncolon.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0015_use_tree.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0029_range_forms.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0051_parameter_attrs.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0055_dot_dot_dot.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0063_variadic_fun.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0068_item_modifiers.rs",
    "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0031_block_inner_attrs.rs",
    "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0045_ambiguous_trait_object.rs",
    "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0046_mutable_const_item.rs",
    "src/tools/rustfmt/tests/source/closure.rs",
    "src/tools/rustfmt/tests/target/closure.rs",

    // TODO: impl ~const T {}
    // https://github.com/dtolnay/syn/issues/1051
    "src/test/ui/rfc-2632-const-trait-impl/syntax.rs",

    // Compile-fail expr parameter in const generic position: f::<1 + 2>()
    "src/test/ui/const-generics/early/closing-args-token.rs",
    "src/test/ui/const-generics/early/const-expression-parameter.rs",

    // Need at least one trait in impl Trait, no such type as impl 'static
    "src/test/ui/type-alias-impl-trait/generic_type_does_not_live_long_enough.rs",

    // Deprecated anonymous parameter syntax in traits
    "src/test/ui/issues/issue-13105.rs",
    "src/test/ui/issues/issue-13775.rs",
    "src/test/ui/issues/issue-34074.rs",
    "src/test/ui/proc-macro/trait-fn-args-2015.rs",
    "src/tools/rustfmt/tests/source/trait.rs",
    "src/tools/rustfmt/tests/target/trait.rs",

    // Placeholder syntax for "throw expressions"
    "src/test/pretty/yeet-expr.rs",
    "src/test/ui/try-trait/yeet-for-option.rs",
    "src/test/ui/try-trait/yeet-for-result.rs",

    // Excessive nesting
    "src/test/ui/issues/issue-74564-if-expr-stack-overflow.rs",

    // Testing rustfmt on invalid syntax
    "src/tools/rustfmt/tests/coverage/target/comments.rs",
    "src/tools/rustfmt/tests/parser/issue-4126/invalid.rs",
    "src/tools/rustfmt/tests/parser/issue_4418.rs",
    "src/tools/rustfmt/tests/parser/unclosed-delims/issue_4466.rs",
    "src/tools/rustfmt/tests/source/configs/disable_all_formatting/true.rs",
    "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/false.rs",
    "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/true.rs",
    "src/tools/rustfmt/tests/source/type.rs",
    "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/false.rs",
    "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/true.rs",
    "src/tools/rustfmt/tests/target/type.rs",

    // Clippy lint lists represented as expressions
    "src/tools/clippy/clippy_lints/src/lib.deprecated.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_all.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_cargo.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_complexity.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_correctness.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_internal.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_lints.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_nursery.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_pedantic.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_perf.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_restriction.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_style.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_suspicious.rs",

    // Not actually test cases
    "src/test/rustdoc-ui/test-compile-fail2.rs",
    "src/test/rustdoc-ui/test-compile-fail3.rs",
    "src/test/ui/lint/expansion-time-include.rs",
    "src/test/ui/macros/auxiliary/macro-comma-support.rs",
    "src/test/ui/macros/auxiliary/macro-include-items-expr.rs",
    "src/test/ui/macros/include-single-expr-helper.rs",
    "src/test/ui/macros/include-single-expr-helper-1.rs",
    "src/test/ui/parser/issues/auxiliary/issue-21146-inc.rs",
];

#[rustfmt::skip]
static EXCLUDE_DIRS: &[&str] = &[
    // Inputs that intentionally do not parse
    "src/tools/rust-analyzer/crates/parser/test_data/parser/err",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err",

    // Inputs that lex but do not necessarily parse
    "src/tools/rust-analyzer/crates/parser/test_data/lexer",

    // Inputs that used to crash rust-analyzer, but aren't necessarily supposed to parse
    "src/tools/rust-analyzer/crates/syntax/test_data/parser/fuzz-failures",
    "src/tools/rust-analyzer/crates/syntax/test_data/reparse/fuzz-failures",
];

pub fn base_dir_filter(entry: &DirEntry) -> bool {
    let path = entry.path();

    let mut path_string = path.to_string_lossy();
    if cfg!(windows) {
        path_string = path_string.replace('\\', "/").into();
    }
    let path_string = if path_string == "tests/rust" {
        return true;
    } else if let Some(path) = path_string.strip_prefix("tests/rust/") {
        path
    } else {
        panic!("unexpected path in Rust dist: {}", path_string);
    };

    if path.is_dir() {
        return !EXCLUDE_DIRS.contains(&path_string);
    }

    if path.extension().map_or(true, |e| e != "rs") {
        return false;
    }

    if path_string.starts_with("src/test/ui") {
        let stderr_path = path.with_extension("stderr");
        if stderr_path.exists() {
            // Expected to fail in some way
            return false;
        }
    }

    !EXCLUDE_FILES.contains(&path_string)
}

#[allow(dead_code)]
pub fn edition(path: &Path) -> &'static str {
    if path.ends_with("dyn-2015-no-warnings-without-lints.rs") {
        "2015"
    } else {
        "2018"
    }
}

pub fn clone_rust() {
    let needs_clone = match fs::read_to_string("tests/rust/COMMIT") {
        Err(_) => true,
        Ok(contents) => contents.trim() != REVISION,
    };
    if needs_clone {
        download_and_unpack().unwrap();
    }
    let mut missing = String::new();
    let test_src = Path::new("tests/rust");
    for exclude in EXCLUDE_FILES {
        if !test_src.join(exclude).is_file() {
            missing += "\ntests/rust/";
            missing += exclude;
        }
    }
    for exclude in EXCLUDE_DIRS {
        if !test_src.join(exclude).is_dir() {
            missing += "\ntests/rust/";
            missing += exclude;
            missing += "/";
        }
    }
    if !missing.is_empty() {
        panic!("excluded test file does not exist:{}\n", missing);
    }
}

fn download_and_unpack() -> Result<()> {
    let url = format!(
        "https://github.com/rust-lang/rust/archive/{}.tar.gz",
        REVISION
    );
    let response = reqwest::blocking::get(&url)?.error_for_status()?;
    let progress = Progress::new(response);
    let decoder = GzDecoder::new(progress);
    let mut archive = Archive::new(decoder);
    let prefix = format!("rust-{}", REVISION);

    let tests_rust = Path::new("tests/rust");
    if tests_rust.exists() {
        fs::remove_dir_all(tests_rust)?;
    }

    for entry in archive.entries()? {
        let mut entry = entry?;
        let path = entry.path()?;
        if path == Path::new("pax_global_header") {
            continue;
        }
        let relative = path.strip_prefix(&prefix)?;
        let out = tests_rust.join(relative);
        entry.unpack(&out)?;
    }

    fs::write("tests/rust/COMMIT", REVISION)?;
    Ok(())
}
<commit_msg>Categorize all the new parse failures<commit_after>#![allow(clippy::manual_assert)]

mod progress;

use self::progress::Progress;
use anyhow::Result;
use flate2::read::GzDecoder;
use std::fs;
use std::path::Path;
use tar::Archive;
use walkdir::DirEntry;

const REVISION: &str = "98ad6a5519651af36e246c0335c964dd52c554ba";

#[rustfmt::skip]
static EXCLUDE_FILES: &[&str] = &[
    // TODO: for<> closures with empty lifetime list
    "src/tools/rustfmt/tests/source/closure.rs",
    "src/tools/rustfmt/tests/target/closure.rs",

    // TODO: dyn*
    "src/test/ui/dyn-star/const.rs",
    "src/test/ui/dyn-star/drop.rs",
    "src/test/ui/dyn-star/make-dyn-star.rs",
    "src/test/ui/dyn-star/method.rs",
    "src/test/ui/dyn-star/syntax.rs",

    // TODO: Struct { #[attr] .. } in pattern position
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0102_record_pat_field_list.rs",

    // TODO: static async closures, for<> move closures
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0106_lambda_expr.rs",

    // TODO: generic associated type constraint inside trait bound
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0138_associated_type_bounds.rs",

    // TODO: negative literal for const generic parameter default value
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0200_const_param_default_literal.rs",

    // TODO: associated type of a path with parenthesized generic arguments
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0202_typepathfn_with_coloncolon.rs",

    // TODO: trailing comma after variadic in extern fn signature
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0063_variadic_fun.rs",

    // TODO: impl ~const T {}
    // https://github.com/dtolnay/syn/issues/1051
    "src/test/ui/rfc-2632-const-trait-impl/syntax.rs",

    // Compile-fail expr parameter in const generic position: f::<1 + 2>()
    "src/test/ui/const-generics/early/closing-args-token.rs",
    "src/test/ui/const-generics/early/const-expression-parameter.rs",

    // Need at least one trait in impl Trait, no such type as impl 'static
    "src/test/ui/type-alias-impl-trait/generic_type_does_not_live_long_enough.rs",

    // Deprecated anonymous parameter syntax in traits
    "src/test/ui/issues/issue-13105.rs",
    "src/test/ui/issues/issue-13775.rs",
    "src/test/ui/issues/issue-34074.rs",
    "src/test/ui/proc-macro/trait-fn-args-2015.rs",
    "src/tools/rustfmt/tests/source/trait.rs",
    "src/tools/rustfmt/tests/target/trait.rs",

    // Various extensions to Rust syntax made up by rust-analyzer
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0012_type_item_where_clause.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0040_crate_keyword_vis.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0131_existential_type.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0179_use_tree_abs_star.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0188_const_param_default_path.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0015_use_tree.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0029_range_forms.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0051_parameter_attrs.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0055_dot_dot_dot.rs",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0068_item_modifiers.rs",
    "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0031_block_inner_attrs.rs",
    "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0045_ambiguous_trait_object.rs",
    "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0046_mutable_const_item.rs",

    // Placeholder syntax for "throw expressions"
    "src/test/pretty/yeet-expr.rs",
    "src/test/ui/try-trait/yeet-for-option.rs",
    "src/test/ui/try-trait/yeet-for-result.rs",

    // Excessive nesting
    "src/test/ui/issues/issue-74564-if-expr-stack-overflow.rs",

    // Testing tools on invalid syntax
    "src/test/run-make/translation/test.rs",
    "src/test/ui/generics/issue-94432-garbage-ice.rs",
    "src/tools/rustfmt/tests/coverage/target/comments.rs",
    "src/tools/rustfmt/tests/parser/issue-4126/invalid.rs",
    "src/tools/rustfmt/tests/parser/issue_4418.rs",
    "src/tools/rustfmt/tests/parser/unclosed-delims/issue_4466.rs",
    "src/tools/rustfmt/tests/source/configs/disable_all_formatting/true.rs",
    "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/false.rs",
    "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/true.rs",
    "src/tools/rustfmt/tests/source/type.rs",
    "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/false.rs",
    "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/true.rs",
    "src/tools/rustfmt/tests/target/type.rs",

    // Generated file containing a top-level expression, used with `include!`
    "compiler/rustc_codegen_gcc/src/intrinsic/archs.rs",

    // Clippy lint lists represented as expressions
    "src/tools/clippy/clippy_lints/src/lib.deprecated.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_all.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_cargo.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_complexity.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_correctness.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_internal.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_lints.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_nursery.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_pedantic.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_perf.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_restriction.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_style.rs",
    "src/tools/clippy/clippy_lints/src/lib.register_suspicious.rs",

    // Not actually test cases
    "src/test/ui/lint/expansion-time-include.rs",
    "src/test/ui/macros/auxiliary/macro-comma-support.rs",
    "src/test/ui/macros/auxiliary/macro-include-items-expr.rs",
    "src/test/ui/macros/include-single-expr-helper.rs",
    "src/test/ui/macros/include-single-expr-helper-1.rs",
    "src/test/ui/parser/issues/auxiliary/issue-21146-inc.rs",
];

#[rustfmt::skip]
static EXCLUDE_DIRS: &[&str] = &[
    // Inputs that intentionally do not parse
    "src/tools/rust-analyzer/crates/parser/test_data/parser/err",
    "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err",

    // Inputs that lex but do not necessarily parse
    "src/tools/rust-analyzer/crates/parser/test_data/lexer",

    // Inputs that used to crash rust-analyzer, but aren't necessarily supposed to parse
    "src/tools/rust-analyzer/crates/syntax/test_data/parser/fuzz-failures",
    "src/tools/rust-analyzer/crates/syntax/test_data/reparse/fuzz-failures",
];

pub fn base_dir_filter(entry: &DirEntry) -> bool {
    let path = entry.path();

    let mut path_string = path.to_string_lossy();
    if cfg!(windows) {
        path_string = path_string.replace('\\', "/").into();
    }
    let path_string = if path_string == "tests/rust" {
        return true;
    } else if let Some(path) = path_string.strip_prefix("tests/rust/") {
        path
    } else {
        panic!("unexpected path in Rust dist: {}", path_string);
    };

    if path.is_dir() {
        return !EXCLUDE_DIRS.contains(&path_string);
    }

    if path.extension().map_or(true, |e| e != "rs") {
        return false;
    }

    if path_string.starts_with("src/test/ui") || path_string.starts_with("src/test/rustdoc-ui") {
        let stderr_path = path.with_extension("stderr");
        if stderr_path.exists() {
            // Expected to fail in some way
            return false;
        }
    }

    !EXCLUDE_FILES.contains(&path_string)
}

#[allow(dead_code)]
pub fn edition(path: &Path) -> &'static str {
    if path.ends_with("dyn-2015-no-warnings-without-lints.rs") {
        "2015"
    } else {
        "2018"
    }
}

pub fn clone_rust() {
    let needs_clone = match fs::read_to_string("tests/rust/COMMIT") {
        Err(_) => true,
        Ok(contents) => contents.trim() != REVISION,
    };
    if needs_clone {
        download_and_unpack().unwrap();
    }
    let mut missing = String::new();
    let test_src = Path::new("tests/rust");
    for exclude in EXCLUDE_FILES {
        if !test_src.join(exclude).is_file() {
            missing += "\ntests/rust/";
            missing += exclude;
        }
    }
    for exclude in EXCLUDE_DIRS {
        if !test_src.join(exclude).is_dir() {
            missing += "\ntests/rust/";
            missing += exclude;
            missing += "/";
        }
    }
    if !missing.is_empty() {
        panic!("excluded test file does not exist:{}\n", missing);
    }
}

fn download_and_unpack() -> Result<()> {
    let url = format!(
        "https://github.com/rust-lang/rust/archive/{}.tar.gz",
        REVISION
    );
    let response = reqwest::blocking::get(&url)?.error_for_status()?;
    let progress = Progress::new(response);
    let decoder = GzDecoder::new(progress);
    let mut archive = Archive::new(decoder);
    let prefix = format!("rust-{}", REVISION);

    let tests_rust = Path::new("tests/rust");
    if tests_rust.exists() {
        fs::remove_dir_all(tests_rust)?;
    }

    for entry in archive.entries()? {
        let mut entry = entry?;
        let path = entry.path()?;
        if path == Path::new("pax_global_header") {
            continue;
        }
        let relative = path.strip_prefix(&prefix)?;
        let out = tests_rust.join(relative);
        entry.unpack(&out)?;
    }

    fs::write("tests/rust/COMMIT", REVISION)?;
    Ok(())
}
<|endoftext|>


<commit_before>// Copyright 2018 Syn Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate proc_macro2;
extern crate quote;
extern crate syn;

use proc_macro2::{TokenStream, TokenTree};
use quote::ToTokens;
use std::str::FromStr;
use syn::{FloatSuffix, IntSuffix, Lit};

fn lit(s: &str) -> Lit {
    match TokenStream::from_str(s)
        .unwrap()
        .into_iter()
        .next()
        .unwrap()
    {
        TokenTree::Literal(lit) => Lit::new(lit),
        _ => panic!(),
    }
}

#[test]
fn strings() {
    fn test_string(s: &str, value: &str) {
        match lit(s) {
            Lit::Str(lit) => {
                assert_eq!(lit.value(), value);
                let again = lit.into_tokens().to_string();
                if again != s {
                    test_string(&again, value);
                }
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    test_string("\"a\"", "a");
    test_string("\"\\n\"", "\n");
    test_string("\"\\r\"", "\r");
    test_string("\"\\t\"", "\t");
    test_string("\"🐕\"", "🐕"); // NOTE: This is an emoji
    test_string("\"\\\"\"", "\"");
    test_string("\"'\"", "'");
    test_string("\"\"", "");
    test_string("\"\\u{1F415}\"", "\u{1F415}");
    test_string(
        "\"contains\nnewlines\\\nescaped newlines\"",
        "contains\nnewlinesescaped newlines",
    );
    test_string("r\"raw\nstring\\\nhere\"", "raw\nstring\\\nhere");
}

#[test]
fn byte_strings() {
    fn test_byte_string(s: &str, value: &[u8]) {
        match lit(s) {
            Lit::ByteStr(lit) => {
                assert_eq!(lit.value(), value);
                let again = lit.into_tokens().to_string();
                if again != s {
                    test_byte_string(&again, value);
                }
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    test_byte_string("b\"a\"", b"a");
    test_byte_string("b\"\\n\"", b"\n");
    test_byte_string("b\"\\r\"", b"\r");
    test_byte_string("b\"\\t\"", b"\t");
    test_byte_string("b\"\\\"\"", b"\"");
    test_byte_string("b\"'\"", b"'");
    test_byte_string("b\"\"", b"");
    test_byte_string(
        "b\"contains\nnewlines\\\nescaped newlines\"",
        b"contains\nnewlinesescaped newlines",
    );
    test_byte_string("br\"raw\nstring\\\nhere\"", b"raw\nstring\\\nhere");
}

#[test]
fn bytes() {
    fn test_byte(s: &str, value: u8) {
        match lit(s) {
            Lit::Byte(lit) => {
                assert_eq!(lit.value(), value);
                let again = lit.into_tokens().to_string();
                assert_eq!(again, s);
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    test_byte("b'a'", b'a');
    test_byte("b'\\n'", b'\n');
    test_byte("b'\\r'", b'\r');
    test_byte("b'\\t'", b'\t');
    test_byte("b'\\''", b'\'');
    test_byte("b'\"'", b'"');
}

#[test]
fn chars() {
    fn test_char(s: &str, value: char) {
        match lit(s) {
            Lit::Char(lit) => {
                assert_eq!(lit.value(), value);
                let again = lit.into_tokens().to_string();
                if again != s {
                    test_char(&again, value);
                }
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    test_char("'a'", 'a');
    test_char("'\\n'", '\n');
    test_char("'\\r'", '\r');
    test_char("'\\t'", '\t');
    test_char("'🐕'", '🐕'); // NOTE: This is an emoji
    test_char("'\\''", '\'');
    test_char("'\"'", '"');
    test_char("'\\u{1F415}'", '\u{1F415}');
}

#[test]
fn ints() {
    fn test_int(s: &str, value: u64, suffix: IntSuffix) {
        match lit(s) {
            Lit::Int(lit) => {
                assert_eq!(lit.value(), value);
                assert_eq!(lit.suffix(), suffix);
                let again = lit.into_tokens().to_string();
                if again != s {
                    test_int(&again, value, suffix);
                }
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    use syn::IntSuffix::*;
    test_int("5", 5, None);
    test_int("5u32", 5, U32);
    test_int("5_0", 50, None);
    test_int("5_____0_____", 50, None);
    test_int("0x7f", 127, None);
    test_int("0x7F", 127, None);
    test_int("0b1001", 9, None);
    test_int("0o73", 59, None);
    test_int("0x7Fu8", 127, U8);
    test_int("0b1001i8", 9, I8);
    test_int("0o73u32", 59, U32);
    test_int("0x__7___f_", 127, None);
    test_int("0x__7___F_", 127, None);
    test_int("0b_1_0__01", 9, None);
    test_int("0o_7__3", 59, None);
    test_int("0x_7F__u8", 127, U8);
    test_int("0b__10__0_1i8", 9, I8);
    test_int("0o__7__________________3u32", 59, U32);
}

#[test]
fn floats() {
    #[cfg_attr(feature = "cargo-clippy", allow(float_cmp))]
    fn test_float(s: &str, value: f64, suffix: FloatSuffix) {
        match lit(s) {
            Lit::Float(lit) => {
                assert_eq!(lit.value(), value);
                assert_eq!(lit.suffix(), suffix);
                let again = lit.into_tokens().to_string();
                if again != s {
                    test_float(&again, value, suffix);
                }
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    use syn::FloatSuffix::*;
    test_float("5.5", 5.5, None);
    test_float("5.5E12", 5.5e12, None);
    test_float("5.5e12", 5.5e12, None);
    test_float("1.0__3e-12", 1.03e-12, None);
    test_float("1.03e+12", 1.03e12, None);
}
<commit_msg>Fix test_lit<commit_after>// Copyright 2018 Syn Developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

extern crate proc_macro2;
extern crate quote;
extern crate syn;

use proc_macro2::{TokenStream, TokenTree};
use quote::ToTokens;
use std::str::FromStr;
use syn::{FloatSuffix, IntSuffix, Lit};

fn lit(s: &str) -> Lit {
    match TokenStream::from_str(s)
        .unwrap()
        .into_iter()
        .next()
        .unwrap()
    {
        TokenTree::Literal(lit) => Lit::new(lit),
        _ => panic!(),
    }
}

#[test]
fn strings() {
    fn test_string(s: &str, value: &str) {
        match lit(s) {
            Lit::Str(lit) => {
                assert_eq!(lit.value(), value);
                let again = lit.into_token_stream().to_string();
                if again != s {
                    test_string(&again, value);
                }
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    test_string("\"a\"", "a");
    test_string("\"\\n\"", "\n");
    test_string("\"\\r\"", "\r");
    test_string("\"\\t\"", "\t");
    test_string("\"🐕\"", "🐕"); // NOTE: This is an emoji
    test_string("\"\\\"\"", "\"");
    test_string("\"'\"", "'");
    test_string("\"\"", "");
    test_string("\"\\u{1F415}\"", "\u{1F415}");
    test_string(
        "\"contains\nnewlines\\\nescaped newlines\"",
        "contains\nnewlinesescaped newlines",
    );
    test_string("r\"raw\nstring\\\nhere\"", "raw\nstring\\\nhere");
}

#[test]
fn byte_strings() {
    fn test_byte_string(s: &str, value: &[u8]) {
        match lit(s) {
            Lit::ByteStr(lit) => {
                assert_eq!(lit.value(), value);
                let again = lit.into_token_stream().to_string();
                if again != s {
                    test_byte_string(&again, value);
                }
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    test_byte_string("b\"a\"", b"a");
    test_byte_string("b\"\\n\"", b"\n");
    test_byte_string("b\"\\r\"", b"\r");
    test_byte_string("b\"\\t\"", b"\t");
    test_byte_string("b\"\\\"\"", b"\"");
    test_byte_string("b\"'\"", b"'");
    test_byte_string("b\"\"", b"");
    test_byte_string(
        "b\"contains\nnewlines\\\nescaped newlines\"",
        b"contains\nnewlinesescaped newlines",
    );
    test_byte_string("br\"raw\nstring\\\nhere\"", b"raw\nstring\\\nhere");
}

#[test]
fn bytes() {
    fn test_byte(s: &str, value: u8) {
        match lit(s) {
            Lit::Byte(lit) => {
                assert_eq!(lit.value(), value);
                let again = lit.into_token_stream().to_string();
                assert_eq!(again, s);
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    test_byte("b'a'", b'a');
    test_byte("b'\\n'", b'\n');
    test_byte("b'\\r'", b'\r');
    test_byte("b'\\t'", b'\t');
    test_byte("b'\\''", b'\'');
    test_byte("b'\"'", b'"');
}

#[test]
fn chars() {
    fn test_char(s: &str, value: char) {
        match lit(s) {
            Lit::Char(lit) => {
                assert_eq!(lit.value(), value);
                let again = lit.into_token_stream().to_string();
                if again != s {
                    test_char(&again, value);
                }
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    test_char("'a'", 'a');
    test_char("'\\n'", '\n');
    test_char("'\\r'", '\r');
    test_char("'\\t'", '\t');
    test_char("'🐕'", '🐕'); // NOTE: This is an emoji
    test_char("'\\''", '\'');
    test_char("'\"'", '"');
    test_char("'\\u{1F415}'", '\u{1F415}');
}

#[test]
fn ints() {
    fn test_int(s: &str, value: u64, suffix: IntSuffix) {
        match lit(s) {
            Lit::Int(lit) => {
                assert_eq!(lit.value(), value);
                assert_eq!(lit.suffix(), suffix);
                let again = lit.into_token_stream().to_string();
                if again != s {
                    test_int(&again, value, suffix);
                }
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    use syn::IntSuffix::*;
    test_int("5", 5, None);
    test_int("5u32", 5, U32);
    test_int("5_0", 50, None);
    test_int("5_____0_____", 50, None);
    test_int("0x7f", 127, None);
    test_int("0x7F", 127, None);
    test_int("0b1001", 9, None);
    test_int("0o73", 59, None);
    test_int("0x7Fu8", 127, U8);
    test_int("0b1001i8", 9, I8);
    test_int("0o73u32", 59, U32);
    test_int("0x__7___f_", 127, None);
    test_int("0x__7___F_", 127, None);
    test_int("0b_1_0__01", 9, None);
    test_int("0o_7__3", 59, None);
    test_int("0x_7F__u8", 127, U8);
    test_int("0b__10__0_1i8", 9, I8);
    test_int("0o__7__________________3u32", 59, U32);
}

#[test]
fn floats() {
    #[cfg_attr(feature = "cargo-clippy", allow(float_cmp))]
    fn test_float(s: &str, value: f64, suffix: FloatSuffix) {
        match lit(s) {
            Lit::Float(lit) => {
                assert_eq!(lit.value(), value);
                assert_eq!(lit.suffix(), suffix);
                let again = lit.into_token_stream().to_string();
                if again != s {
                    test_float(&again, value, suffix);
                }
            }
            wrong => panic!("{:?}", wrong),
        }
    }

    use syn::FloatSuffix::*;
    test_float("5.5", 5.5, None);
    test_float("5.5E12", 5.5e12, None);
    test_float("5.5e12", 5.5e12, None);
    test_float("1.0__3e-12", 1.03e-12, None);
    test_float("1.03e+12", 1.03e12, None);
}
<|endoftext|>


<commit_before>//!

use std::io::{Read, Write};
use std::net::TcpStream;

use networking::MessageHeader;

use super::bytes_slab::BytesSlab;
use super::bytes_exchange::{MergeQueue, Signal};

use logging_core::Logger;

use ::logging::{CommunicationEvent, CommunicationSetup, MessageEvent, StateEvent};

/// Repeatedly reads from a TcpStream and carves out messages.
///
/// The intended communication pattern is a sequence of (header, message)^* for valid
/// messages, followed by a header for a zero length message indicating the end of stream.
/// If the stream ends without being shut down, the receive thread panics in an attempt to
/// take down the computation and cause the failures to cascade.
pub fn recv_loop(
    mut reader: TcpStream,
    mut targets: Vec<MergeQueue>,
    worker_offset: usize,
    process: usize,
    remote: usize,
    mut logger: Option<Logger<CommunicationEvent, CommunicationSetup>>)
{
    // Log the receive thread's start.
    logger.as_mut().map(|l| l.log(StateEvent { send: false, process, remote, start: true }));

    let mut buffer = BytesSlab::new(20);

    // Where we stash Bytes before handing them off.
    let mut stageds = Vec::with_capacity(targets.len());
    for _ in 0 .. targets.len() {
        stageds.push(Vec::new());
    }

    // Each loop iteration adds to `self.Bytes` and consumes all complete messages.
    // At the start of each iteration, `self.buffer[..self.length]` represents valid
    // data, and the remaining capacity is available for reading from the reader.
    //
    // Once the buffer fills, we need to copy uncomplete messages to a new shared
    // allocation and place the existing Bytes into `self.in_progress`, so that it
    // can be recovered once all readers have read what they need to.
    let mut active = true;
    while active {

        buffer.ensure_capacity(1);

        assert!(!buffer.empty().is_empty());

        // Attempt to read some more bytes into self.buffer.
        let read = match reader.read(&mut buffer.empty()) {
            Ok(n) => n,
            Err(x) => {
                // We don't expect this, as socket closure results in Ok(0) reads.
                println!("Error: {:?}", x);
                0
            },
        };

        assert!(read > 0);
        buffer.make_valid(read);

        // Consume complete messages from the front of self.buffer.
        while let Some(header) = MessageHeader::try_read(buffer.valid()) {

            // TODO: Consolidate message sequences sent to the same worker?
            let peeled_bytes = header.required_bytes();
            let bytes = buffer.extract(peeled_bytes);

            // Record message receipt.
            logger.as_mut().map(|logger| {
                logger.log(MessageEvent { is_send: false, header, });
            });

            if header.length > 0 {
                stageds[header.target - worker_offset].push(bytes);
            }
            else {
                // Shutting down; confirm absence of subsequent data.
                active = false;
                if !buffer.valid().is_empty() {
                    panic!("Clean shutdown followed by data.");
                }
                buffer.ensure_capacity(1);
                if reader.read(&mut buffer.empty()).expect("read failure") > 0 {
                    panic!("Clean shutdown followed by data.");
                }
            }
        }

        // Pass bytes along to targets.
        for (index, staged) in stageds.iter_mut().enumerate() {

            use allocator::zero_copy::bytes_exchange::BytesPush;
            targets[index].extend(staged.drain(..));
        }
    }

    // Log the receive thread's start.
    logger.as_mut().map(|l| l.log(StateEvent { send: false, process, remote, start: false, }));
}

/// Repeatedly sends messages into a TcpStream.
///
/// The intended communication pattern is a sequence of (header, message)^* for valid
/// messages, followed by a header for a zero length message indicating the end of stream.
pub fn send_loop(
    // TODO: Maybe we don't need BufWriter with consolidation in writes.
    writer: TcpStream,
    mut sources: Vec<MergeQueue>,
    signal: Signal,
    process: usize,
    remote: usize,
    mut logger: Option<Logger<CommunicationEvent, CommunicationSetup>>)
{

    // Log the receive thread's start.
    logger.as_mut().map(|l| l.log(StateEvent { send: true, process, remote, start: true, }));

    let mut writer = ::std::io::BufWriter::with_capacity(1 << 16, writer);
    let mut stash = Vec::new();

    while !sources.is_empty() {

        // TODO: Round-robin better, to release resources fairly when overloaded.
        for source in sources.iter_mut() {
            use allocator::zero_copy::bytes_exchange::BytesPull;
            source.drain_into(&mut stash);
        }

        if stash.is_empty() {
            // No evidence of records to read, but sources not yet empty (at start of loop).
            // We are going to flush our writer (to move buffered data) and wait on a signal.
            // We could get awoken by more data, a channel closing, or spuriously perhaps.
            writer.flush().expect("Failed to flush writer.");
            signal.wait();
            sources.retain(|source| !source.is_complete());
        }
        else {
            // TODO: Could do scatter/gather write here.
            for mut bytes in stash.drain(..) {

                // Record message sends.
                logger.as_mut().map(|logger| {
                    let mut offset = 0;
                    while let Some(header) = MessageHeader::try_read(&mut bytes[offset..]) {
                        logger.log(MessageEvent { is_send: true, header, });
                        offset += header.required_bytes();
                    }
                });

                writer.write_all(&bytes[..]).expect("Write failure in send_loop.");
            }
        }
    }

    // Write final zero-length header.
    // Would be better with meaningful metadata, but as this stream merges many
    // workers it isn't clear that there is anything specific to write here.
    let header = MessageHeader {
        channel:    0,
        source:     0,
        target:     0,
        length:     0,
        seqno:      0,
    };
    header.write_to(&mut writer).expect("Failed to write header!");
    writer.flush().expect("Failed to flush writer.");
    writer.get_mut().shutdown(::std::net::Shutdown::Write).expect("Write shutdown failed");
    logger.as_mut().map(|logger| logger.log(MessageEvent { is_send: true, header }));

    // Log the receive thread's start.
    logger.as_mut().map(|l| l.log(StateEvent { send: true, process, remote, start: false, }));
}<commit_msg>wait only if not complete<commit_after>//!

use std::io::{Read, Write};
use std::net::TcpStream;

use networking::MessageHeader;

use super::bytes_slab::BytesSlab;
use super::bytes_exchange::{MergeQueue, Signal};

use logging_core::Logger;

use ::logging::{CommunicationEvent, CommunicationSetup, MessageEvent, StateEvent};

/// Repeatedly reads from a TcpStream and carves out messages.
///
/// The intended communication pattern is a sequence of (header, message)^* for valid
/// messages, followed by a header for a zero length message indicating the end of stream.
/// If the stream ends without being shut down, the receive thread panics in an attempt to
/// take down the computation and cause the failures to cascade.
pub fn recv_loop(
    mut reader: TcpStream,
    mut targets: Vec<MergeQueue>,
    worker_offset: usize,
    process: usize,
    remote: usize,
    mut logger: Option<Logger<CommunicationEvent, CommunicationSetup>>)
{
    // Log the receive thread's start.
    logger.as_mut().map(|l| l.log(StateEvent { send: false, process, remote, start: true }));

    let mut buffer = BytesSlab::new(20);

    // Where we stash Bytes before handing them off.
    let mut stageds = Vec::with_capacity(targets.len());
    for _ in 0 .. targets.len() {
        stageds.push(Vec::new());
    }

    // Each loop iteration adds to `self.Bytes` and consumes all complete messages.
    // At the start of each iteration, `self.buffer[..self.length]` represents valid
    // data, and the remaining capacity is available for reading from the reader.
    //
    // Once the buffer fills, we need to copy uncomplete messages to a new shared
    // allocation and place the existing Bytes into `self.in_progress`, so that it
    // can be recovered once all readers have read what they need to.
    let mut active = true;
    while active {

        buffer.ensure_capacity(1);

        assert!(!buffer.empty().is_empty());

        // Attempt to read some more bytes into self.buffer.
        let read = match reader.read(&mut buffer.empty()) {
            Ok(n) => n,
            Err(x) => {
                // We don't expect this, as socket closure results in Ok(0) reads.
                println!("Error: {:?}", x);
                0
            },
        };

        assert!(read > 0);
        buffer.make_valid(read);

        // Consume complete messages from the front of self.buffer.
        while let Some(header) = MessageHeader::try_read(buffer.valid()) {

            // TODO: Consolidate message sequences sent to the same worker?
            let peeled_bytes = header.required_bytes();
            let bytes = buffer.extract(peeled_bytes);

            // Record message receipt.
            logger.as_mut().map(|logger| {
                logger.log(MessageEvent { is_send: false, header, });
            });

            if header.length > 0 {
                stageds[header.target - worker_offset].push(bytes);
            }
            else {
                // Shutting down; confirm absence of subsequent data.
                active = false;
                if !buffer.valid().is_empty() {
                    panic!("Clean shutdown followed by data.");
                }
                buffer.ensure_capacity(1);
                if reader.read(&mut buffer.empty()).expect("read failure") > 0 {
                    panic!("Clean shutdown followed by data.");
                }
            }
        }

        // Pass bytes along to targets.
        for (index, staged) in stageds.iter_mut().enumerate() {

            use allocator::zero_copy::bytes_exchange::BytesPush;
            targets[index].extend(staged.drain(..));
        }
    }

    // Log the receive thread's start.
    logger.as_mut().map(|l| l.log(StateEvent { send: false, process, remote, start: false, }));
}

/// Repeatedly sends messages into a TcpStream.
///
/// The intended communication pattern is a sequence of (header, message)^* for valid
/// messages, followed by a header for a zero length message indicating the end of stream.
pub fn send_loop(
    // TODO: Maybe we don't need BufWriter with consolidation in writes.
    writer: TcpStream,
    mut sources: Vec<MergeQueue>,
    signal: Signal,
    process: usize,
    remote: usize,
    mut logger: Option<Logger<CommunicationEvent, CommunicationSetup>>)
{

    // Log the receive thread's start.
    logger.as_mut().map(|l| l.log(StateEvent { send: true, process, remote, start: true, }));

    let mut writer = ::std::io::BufWriter::with_capacity(1 << 16, writer);
    let mut stash = Vec::new();

    while !sources.is_empty() {

        // TODO: Round-robin better, to release resources fairly when overloaded.
        for source in sources.iter_mut() {
            use allocator::zero_copy::bytes_exchange::BytesPull;
            source.drain_into(&mut stash);
        }

        if stash.is_empty() {
            // No evidence of records to read, but sources not yet empty (at start of loop).
            // We are going to flush our writer (to move buffered data), double check on the
            // sources for emptiness and wait on a signal only if we are sure that there will
            // still be a signal incoming.
            //
            // We could get awoken by more data, a channel closing, or spuriously perhaps.
            writer.flush().expect("Failed to flush writer.");
            sources.retain(|source| !source.is_complete());
            if !sources.is_empty() {
                signal.wait();
            }
        }
        else {
            // TODO: Could do scatter/gather write here.
            for mut bytes in stash.drain(..) {

                // Record message sends.
                logger.as_mut().map(|logger| {
                    let mut offset = 0;
                    while let Some(header) = MessageHeader::try_read(&mut bytes[offset..]) {
                        logger.log(MessageEvent { is_send: true, header, });
                        offset += header.required_bytes();
                    }
                });

                writer.write_all(&bytes[..]).expect("Write failure in send_loop.");
            }
        }
    }

    // Write final zero-length header.
    // Would be better with meaningful metadata, but as this stream merges many
    // workers it isn't clear that there is anything specific to write here.
    let header = MessageHeader {
        channel:    0,
        source:     0,
        target:     0,
        length:     0,
        seqno:      0,
    };
    header.write_to(&mut writer).expect("Failed to write header!");
    writer.flush().expect("Failed to flush writer.");
    writer.get_mut().shutdown(::std::net::Shutdown::Write).expect("Write shutdown failed");
    logger.as_mut().map(|logger| logger.log(MessageEvent { is_send: true, header }));

    // Log the receive thread's start.
    logger.as_mut().map(|l| l.log(StateEvent { send: true, process, remote, start: false, }));
}<|endoftext|>