<commit_before>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

pub use self::CommentStyle::*;

use ast;
use codemap::{BytePos, CharPos, CodeMap, Pos};
use errors;
use parse::lexer::is_block_doc_comment;
use parse::lexer::{StringReader, TokenAndSpan};
use parse::lexer::{is_whitespace, Reader};
use parse::lexer;
use print::pprust;
use str::char_at;

use std::io::Read;
use std::usize;

#[derive(Clone, Copy, PartialEq)]
pub enum CommentStyle {
    /// No code on either side of each line of the comment
    Isolated,
    /// Code exists to the left of the comment
    Trailing,
    /// Code before /* foo */ and after the comment
    Mixed,
    /// Just a manual blank line "\n\n", for layout
    BlankLine,
}

#[derive(Clone)]
pub struct Comment {
    pub style: CommentStyle,
    pub lines: Vec<String>,
    pub pos: BytePos,
}

pub fn is_doc_comment(s: &str) -> bool {
    (s.starts_with("///") && super::is_doc_comment(s)) || s.starts_with("//!") ||
    (s.starts_with("/**") && is_block_doc_comment(s)) || s.starts_with("/*!")
}

pub fn doc_comment_style(comment: &str) -> ast::AttrStyle {
    assert!(is_doc_comment(comment));
    if comment.starts_with("//!") || comment.starts_with("/*!") {
        ast::AttrStyle::Inner
    } else {
        ast::AttrStyle::Outer
    }
}

pub fn strip_doc_comment_decoration(comment: &str) -> String {
    /// remove whitespace-only lines from the start/end of lines
    fn vertical_trim(lines: Vec<String>) -> Vec<String> {
        let mut i = 0;
        let mut j = lines.len();
        // first line of all-stars should be omitted
        if !lines.is_empty() && lines[0].chars().all(|c| c == '*') {
            i += 1;
        }
        while i < j && lines[i].trim().is_empty() {
            i += 1;
        }
        // like the first, a last line of all stars should be omitted
        if j > i &&
           lines[j - 1]
               .chars()
               .skip(1)
               .all(|c| c == '*') {
            j -= 1;
        }
        while j > i && lines[j - 1].trim().is_empty() {
            j -= 1;
        }
        lines[i..j].iter().cloned().collect()
    }

    /// remove a "[ \t]*\*" block from each line, if possible
    fn horizontal_trim(lines: Vec<String>) -> Vec<String> {
        let mut i = usize::MAX;
        let mut can_trim = true;
        let mut first = true;
        for line in &lines {
            for (j, c) in line.chars().enumerate() {
                if j > i || !"* \t".contains(c) {
                    can_trim = false;
                    break;
                }
                if c == '*' {
                    if first {
                        i = j;
                        first = false;
                    } else if i != j {
                        can_trim = false;
                    }
                    break;
                }
            }
            if i > line.len() {
                can_trim = false;
            }
            if !can_trim {
                break;
            }
        }

        if can_trim {
            lines.iter()
                 .map(|line| (&line[i + 1..line.len()]).to_string())
                 .collect()
        } else {
            lines
        }
    }

    // one-line comments lose their prefix
    const ONELINERS: &'static [&'static str] = &["///!", "///", "//!", "//"];
    for prefix in ONELINERS {
        if comment.starts_with(*prefix) {
            return (&comment[prefix.len()..]).to_string();
        }
    }

    if comment.starts_with("/*") {
        let lines = comment[3..comment.len() - 2]
                        .lines()
                        .map(|s| s.to_string())
                        .collect::<Vec<String>>();

        let lines = vertical_trim(lines);
        let lines = horizontal_trim(lines);

        return lines.join("\n");
    }

    panic!("not a doc-comment: {}", comment);
}

fn push_blank_line_comment(rdr: &StringReader, comments: &mut Vec<Comment>) {
    debug!(">>> blank-line comment");
    comments.push(Comment {
        style: BlankLine,
        lines: Vec::new(),
        pos: rdr.last_pos,
    });
}

fn consume_whitespace_counting_blank_lines(rdr: &mut StringReader, comments: &mut Vec<Comment>) {
    while is_whitespace(rdr.curr) && !rdr.is_eof() {
        if rdr.col == CharPos(0) && rdr.curr_is('\n') {
            push_blank_line_comment(rdr, &mut *comments);
        }
        rdr.bump();
    }
}

fn read_shebang_comment(rdr: &mut StringReader,
                        code_to_the_left: bool,
                        comments: &mut Vec<Comment>) {
    debug!(">>> shebang comment");
    let p = rdr.last_pos;
    debug!("<<< shebang comment");
    comments.push(Comment {
        style: if code_to_the_left {
            Trailing
        } else {
            Isolated
        },
        lines: vec![rdr.read_one_line_comment()],
        pos: p,
    });
}

fn read_line_comments(rdr: &mut StringReader,
                      code_to_the_left: bool,
                      comments: &mut Vec<Comment>) {
    debug!(">>> line comments");
    let p = rdr.last_pos;
    let mut lines: Vec<String> = Vec::new();
    while rdr.curr_is('/') && rdr.nextch_is('/') {
        let line = rdr.read_one_line_comment();
        debug!("{}", line);
        // Doc comments are not put in comments.
        if is_doc_comment(&line[..]) {
            break;
        }
        lines.push(line);
        rdr.consume_non_eol_whitespace();
    }
    debug!("<<< line comments");
    if !lines.is_empty() {
        comments.push(Comment {
            style: if code_to_the_left {
                Trailing
            } else {
                Isolated
            },
            lines: lines,
            pos: p,
        });
    }
}

/// Returns None if the first col chars of s contain a non-whitespace char.
/// Otherwise returns Some(k) where k is first char offset after that leading
/// whitespace.  Note k may be outside bounds of s.
fn all_whitespace(s: &str, col: CharPos) -> Option<usize> {
    let len = s.len();
    let mut col = col.to_usize();
    let mut cursor: usize = 0;
    while col > 0 && cursor < len {
        let ch = char_at(s, cursor);
        if !ch.is_whitespace() {
            return None;
        }
        cursor += ch.len_utf8();
        col -= 1;
    }
    return Some(cursor);
}

fn trim_whitespace_prefix_and_push_line(lines: &mut Vec<String>, s: String, col: CharPos) {
    let len = s.len();
    let s1 = match all_whitespace(&s[..], col) {
        Some(col) => {
            if col < len {
                (&s[col..len]).to_string()
            } else {
                "".to_string()
            }
        }
        None => s,
    };
    debug!("pushing line: {}", s1);
    lines.push(s1);
}

fn read_block_comment(rdr: &mut StringReader,
                      code_to_the_left: bool,
                      comments: &mut Vec<Comment>) {
    debug!(">>> block comment");
    let p = rdr.last_pos;
    let mut lines: Vec<String> = Vec::new();
    let col = rdr.col;
    rdr.bump();
    rdr.bump();

    let mut curr_line = String::from("/*");

    // doc-comments are not really comments, they are attributes
    if (rdr.curr_is('*') && !rdr.nextch_is('*')) || rdr.curr_is('!') {
        while !(rdr.curr_is('*') && rdr.nextch_is('/')) && !rdr.is_eof() {
            curr_line.push(rdr.curr.unwrap());
            rdr.bump();
        }
        if !rdr.is_eof() {
            curr_line.push_str("*/");
            rdr.bump();
            rdr.bump();
        }
        if is_block_doc_comment(&curr_line[..]) {
            return;
        }
        assert!(!curr_line.contains('\n'));
        lines.push(curr_line);
    } else {
        let mut level: isize = 1;
        while level > 0 {
            debug!("=== block comment level {}", level);
            if rdr.is_eof() {
                panic!(rdr.fatal("unterminated block comment"));
            }
            if rdr.curr_is('\n') {
                trim_whitespace_prefix_and_push_line(&mut lines, curr_line, col);
                curr_line = String::new();
                rdr.bump();
            } else {
                curr_line.push(rdr.curr.unwrap());
                if rdr.curr_is('/') && rdr.nextch_is('*') {
                    rdr.bump();
                    rdr.bump();
                    curr_line.push('*');
                    level += 1;
                } else {
                    if rdr.curr_is('*') && rdr.nextch_is('/') {
                        rdr.bump();
                        rdr.bump();
                        curr_line.push('/');
                        level -= 1;
                    } else {
                        rdr.bump();
                    }
                }
            }
        }
        if !curr_line.is_empty() {
            trim_whitespace_prefix_and_push_line(&mut lines, curr_line, col);
        }
    }

    let mut style = if code_to_the_left {
        Trailing
    } else {
        Isolated
    };
    rdr.consume_non_eol_whitespace();
    if !rdr.is_eof() && !rdr.curr_is('\n') && lines.len() == 1 {
        style = Mixed;
    }
    debug!("<<< block comment");
    comments.push(Comment {
        style: style,
        lines: lines,
        pos: p,
    });
}

fn consume_comment(rdr: &mut StringReader, code_to_the_left: bool, comments: &mut Vec<Comment>) {
    debug!(">>> consume comment");
    if rdr.curr_is('/') && rdr.nextch_is('/') {
        read_line_comments(rdr, code_to_the_left, comments);
    } else if rdr.curr_is('/') && rdr.nextch_is('*') {
        read_block_comment(rdr, code_to_the_left, comments);
    } else if rdr.curr_is('#') && rdr.nextch_is('!') {
        read_shebang_comment(rdr, code_to_the_left, comments);
    } else {
        panic!();
    }
    debug!("<<< consume comment");
}

#[derive(Clone)]
pub struct Literal {
    pub lit: String,
    pub pos: BytePos,
}

// it appears this function is called only from pprust... that's
// probably not a good thing.
pub fn gather_comments_and_literals(span_diagnostic: &errors::Handler,
                                    path: String,
                                    srdr: &mut Read)
                                    -> (Vec<Comment>, Vec<Literal>) {
    let mut src = Vec::new();
    srdr.read_to_end(&mut src).unwrap();
    let src = String::from_utf8(src).unwrap();
    let cm = CodeMap::new();
    let filemap = cm.new_filemap(path, src);
    let mut rdr = lexer::StringReader::new_raw(span_diagnostic, filemap);

    let mut comments: Vec<Comment> = Vec::new();
    let mut literals: Vec<Literal> = Vec::new();
    let mut first_read: bool = true;
    while !rdr.is_eof() {
        loop {
            let mut code_to_the_left = !first_read;
            rdr.consume_non_eol_whitespace();
            if rdr.curr_is('\n') {
                code_to_the_left = false;
                consume_whitespace_counting_blank_lines(&mut rdr, &mut comments);
            }
            while rdr.peeking_at_comment() {
                consume_comment(&mut rdr, code_to_the_left, &mut comments);
                consume_whitespace_counting_blank_lines(&mut rdr, &mut comments);
            }
            break;
        }

        let bstart = rdr.last_pos;
        rdr.next_token();
        // discard, and look ahead; we're working with internal state
        let TokenAndSpan { tok, sp } = rdr.peek();
        if tok.is_lit() {
            rdr.with_str_from(bstart, |s| {
                debug!("tok lit: {}", s);
                literals.push(Literal {
                    lit: s.to_string(),
                    pos: sp.lo,
                });
            })
        } else {
            debug!("tok: {}", pprust::token_to_string(&tok));
        }
        first_read = false;
    }

    (comments, literals)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_block_doc_comment_1() {
        let comment = "/**\n * Test \n **  Test\n *   Test\n*/";
        let stripped = strip_doc_comment_decoration(comment);
        assert_eq!(stripped, " Test \n*  Test\n   Test");
    }

    #[test]
    fn test_block_doc_comment_2() {
        let comment = "/**\n * Test\n *  Test\n*/";
        let stripped = strip_doc_comment_decoration(comment);
        assert_eq!(stripped, " Test\n  Test");
    }

    #[test]
    fn test_block_doc_comment_3() {
        let comment = "/**\n let a: *i32;\n *a = 5;\n*/";
        let stripped = strip_doc_comment_decoration(comment);
        assert_eq!(stripped, " let a: *i32;\n *a = 5;");
    }

    #[test]
    fn test_block_doc_comment_4() {
        let comment = "/*******************\n test\n *********************/";
        let stripped = strip_doc_comment_decoration(comment);
        assert_eq!(stripped, " test");
    }

    #[test]
    fn test_line_doc_comment() {
        let stripped = strip_doc_comment_decoration("/// test");
        assert_eq!(stripped, " test");
        let stripped = strip_doc_comment_decoration("///! test");
        assert_eq!(stripped, " test");
        let stripped = strip_doc_comment_decoration("// test");
        assert_eq!(stripped, " test");
        let stripped = strip_doc_comment_decoration("// test");
        assert_eq!(stripped, " test");
        let stripped = strip_doc_comment_decoration("///test");
        assert_eq!(stripped, "test");
        let stripped = strip_doc_comment_decoration("///!test");
        assert_eq!(stripped, "test");
        let stripped = strip_doc_comment_decoration("//test");
        assert_eq!(stripped, "test");
    }
}
<commit_msg>address review comment<commit_after>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

pub use self::CommentStyle::*;

use ast;
use codemap::{BytePos, CharPos, CodeMap, Pos};
use errors;
use parse::lexer::is_block_doc_comment;
use parse::lexer::{StringReader, TokenAndSpan};
use parse::lexer::{is_whitespace, Reader};
use parse::lexer;
use print::pprust;
use str::char_at;

use std::io::Read;
use std::usize;

#[derive(Clone, Copy, PartialEq)]
pub enum CommentStyle {
    /// No code on either side of each line of the comment
    Isolated,
    /// Code exists to the left of the comment
    Trailing,
    /// Code before /* foo */ and after the comment
    Mixed,
    /// Just a manual blank line "\n\n", for layout
    BlankLine,
}

#[derive(Clone)]
pub struct Comment {
    pub style: CommentStyle,
    pub lines: Vec<String>,
    pub pos: BytePos,
}

pub fn is_doc_comment(s: &str) -> bool {
    (s.starts_with("///") && super::is_doc_comment(s)) || s.starts_with("//!") ||
    (s.starts_with("/**") && is_block_doc_comment(s)) || s.starts_with("/*!")
}

pub fn doc_comment_style(comment: &str) -> ast::AttrStyle {
    assert!(is_doc_comment(comment));
    if comment.starts_with("//!") || comment.starts_with("/*!") {
        ast::AttrStyle::Inner
    } else {
        ast::AttrStyle::Outer
    }
}

pub fn strip_doc_comment_decoration(comment: &str) -> String {
    /// remove whitespace-only lines from the start/end of lines
    fn vertical_trim(lines: Vec<String>) -> Vec<String> {
        let mut i = 0;
        let mut j = lines.len();
        // first line of all-stars should be omitted
        if !lines.is_empty() && lines[0].chars().all(|c| c == '*') {
            i += 1;
        }
        while i < j && lines[i].trim().is_empty() {
            i += 1;
        }
        // like the first, a last line of all stars should be omitted
        if j > i &&
           lines[j - 1]
               .chars()
               .skip(1)
               .all(|c| c == '*') {
            j -= 1;
        }
        while j > i && lines[j - 1].trim().is_empty() {
            j -= 1;
        }
        lines[i..j].iter().cloned().collect()
    }

    /// remove a "[ \t]*\*" block from each line, if possible
    fn horizontal_trim(lines: Vec<String>) -> Vec<String> {
        let mut i = usize::MAX;
        let mut can_trim = true;
        let mut first = true;
        for line in &lines {
            for (j, c) in line.chars().enumerate() {
                if j > i || !"* \t".contains(c) {
                    can_trim = false;
                    break;
                }
                if c == '*' {
                    if first {
                        i = j;
                        first = false;
                    } else if i != j {
                        can_trim = false;
                    }
                    break;
                }
            }
            if i > line.len() {
                can_trim = false;
            }
            if !can_trim {
                break;
            }
        }

        if can_trim {
            lines.iter()
                 .map(|line| (&line[i + 1..line.len()]).to_string())
                 .collect()
        } else {
            lines
        }
    }

    // one-line comments lose their prefix
    const ONELINERS: &'static [&'static str] = &["///!", "///", "//!", "//"];
    for prefix in ONELINERS {
        if comment.starts_with(*prefix) {
            return (&comment[prefix.len()..]).to_string();
        }
    }

    if comment.starts_with("/*") {
        let lines = comment[3..comment.len() - 2]
                        .lines()
                        .map(|s| s.to_string())
                        .collect::<Vec<String>>();

        let lines = vertical_trim(lines);
        let lines = horizontal_trim(lines);

        return lines.join("\n");
    }

    panic!("not a doc-comment: {}", comment);
}

fn push_blank_line_comment(rdr: &StringReader, comments: &mut Vec<Comment>) {
    debug!(">>> blank-line comment");
    comments.push(Comment {
        style: BlankLine,
        lines: Vec::new(),
        pos: rdr.last_pos,
    });
}

fn consume_whitespace_counting_blank_lines(rdr: &mut StringReader, comments: &mut Vec<Comment>) {
    while is_whitespace(rdr.curr) && !rdr.is_eof() {
        if rdr.col == CharPos(0) && rdr.curr_is('\n') {
            push_blank_line_comment(rdr, &mut *comments);
        }
        rdr.bump();
    }
}

fn read_shebang_comment(rdr: &mut StringReader,
                        code_to_the_left: bool,
                        comments: &mut Vec<Comment>) {
    debug!(">>> shebang comment");
    let p = rdr.last_pos;
    debug!("<<< shebang comment");
    comments.push(Comment {
        style: if code_to_the_left { Trailing } else { Isolated },
        lines: vec![rdr.read_one_line_comment()],
        pos: p,
    });
}

fn read_line_comments(rdr: &mut StringReader,
                      code_to_the_left: bool,
                      comments: &mut Vec<Comment>) {
    debug!(">>> line comments");
    let p = rdr.last_pos;
    let mut lines: Vec<String> = Vec::new();
    while rdr.curr_is('/') && rdr.nextch_is('/') {
        let line = rdr.read_one_line_comment();
        debug!("{}", line);
        // Doc comments are not put in comments.
        if is_doc_comment(&line[..]) {
            break;
        }
        lines.push(line);
        rdr.consume_non_eol_whitespace();
    }
    debug!("<<< line comments");
    if !lines.is_empty() {
        comments.push(Comment {
            style: if code_to_the_left { Trailing } else { Isolated },
            lines: lines,
            pos: p,
        });
    }
}

/// Returns None if the first col chars of s contain a non-whitespace char.
/// Otherwise returns Some(k) where k is first char offset after that leading
/// whitespace.  Note k may be outside bounds of s.
fn all_whitespace(s: &str, col: CharPos) -> Option<usize> {
    let len = s.len();
    let mut col = col.to_usize();
    let mut cursor: usize = 0;
    while col > 0 && cursor < len {
        let ch = char_at(s, cursor);
        if !ch.is_whitespace() {
            return None;
        }
        cursor += ch.len_utf8();
        col -= 1;
    }
    return Some(cursor);
}

fn trim_whitespace_prefix_and_push_line(lines: &mut Vec<String>, s: String, col: CharPos) {
    let len = s.len();
    let s1 = match all_whitespace(&s[..], col) {
        Some(col) => {
            if col < len {
                (&s[col..len]).to_string()
            } else {
                "".to_string()
            }
        }
        None => s,
    };
    debug!("pushing line: {}", s1);
    lines.push(s1);
}

fn read_block_comment(rdr: &mut StringReader,
                      code_to_the_left: bool,
                      comments: &mut Vec<Comment>) {
    debug!(">>> block comment");
    let p = rdr.last_pos;
    let mut lines: Vec<String> = Vec::new();
    let col = rdr.col;
    rdr.bump();
    rdr.bump();

    let mut curr_line = String::from("/*");

    // doc-comments are not really comments, they are attributes
    if (rdr.curr_is('*') && !rdr.nextch_is('*')) || rdr.curr_is('!') {
        while !(rdr.curr_is('*') && rdr.nextch_is('/')) && !rdr.is_eof() {
            curr_line.push(rdr.curr.unwrap());
            rdr.bump();
        }
        if !rdr.is_eof() {
            curr_line.push_str("*/");
            rdr.bump();
            rdr.bump();
        }
        if is_block_doc_comment(&curr_line[..]) {
            return;
        }
        assert!(!curr_line.contains('\n'));
        lines.push(curr_line);
    } else {
        let mut level: isize = 1;
        while level > 0 {
            debug!("=== block comment level {}", level);
            if rdr.is_eof() {
                panic!(rdr.fatal("unterminated block comment"));
            }
            if rdr.curr_is('\n') {
                trim_whitespace_prefix_and_push_line(&mut lines, curr_line, col);
                curr_line = String::new();
                rdr.bump();
            } else {
                curr_line.push(rdr.curr.unwrap());
                if rdr.curr_is('/') && rdr.nextch_is('*') {
                    rdr.bump();
                    rdr.bump();
                    curr_line.push('*');
                    level += 1;
                } else {
                    if rdr.curr_is('*') && rdr.nextch_is('/') {
                        rdr.bump();
                        rdr.bump();
                        curr_line.push('/');
                        level -= 1;
                    } else {
                        rdr.bump();
                    }
                }
            }
        }
        if !curr_line.is_empty() {
            trim_whitespace_prefix_and_push_line(&mut lines, curr_line, col);
        }
    }

    let mut style = if code_to_the_left {
        Trailing
    } else {
        Isolated
    };
    rdr.consume_non_eol_whitespace();
    if !rdr.is_eof() && !rdr.curr_is('\n') && lines.len() == 1 {
        style = Mixed;
    }
    debug!("<<< block comment");
    comments.push(Comment {
        style: style,
        lines: lines,
        pos: p,
    });
}

fn consume_comment(rdr: &mut StringReader, code_to_the_left: bool, comments: &mut Vec<Comment>) {
    debug!(">>> consume comment");
    if rdr.curr_is('/') && rdr.nextch_is('/') {
        read_line_comments(rdr, code_to_the_left, comments);
    } else if rdr.curr_is('/') && rdr.nextch_is('*') {
        read_block_comment(rdr, code_to_the_left, comments);
    } else if rdr.curr_is('#') && rdr.nextch_is('!') {
        read_shebang_comment(rdr, code_to_the_left, comments);
    } else {
        panic!();
    }
    debug!("<<< consume comment");
}

#[derive(Clone)]
pub struct Literal {
    pub lit: String,
    pub pos: BytePos,
}

// it appears this function is called only from pprust... that's
// probably not a good thing.
pub fn gather_comments_and_literals(span_diagnostic: &errors::Handler,
                                    path: String,
                                    srdr: &mut Read)
                                    -> (Vec<Comment>, Vec<Literal>) {
    let mut src = Vec::new();
    srdr.read_to_end(&mut src).unwrap();
    let src = String::from_utf8(src).unwrap();
    let cm = CodeMap::new();
    let filemap = cm.new_filemap(path, src);
    let mut rdr = lexer::StringReader::new_raw(span_diagnostic, filemap);

    let mut comments: Vec<Comment> = Vec::new();
    let mut literals: Vec<Literal> = Vec::new();
    let mut first_read: bool = true;
    while !rdr.is_eof() {
        loop {
            let mut code_to_the_left = !first_read;
            rdr.consume_non_eol_whitespace();
            if rdr.curr_is('\n') {
                code_to_the_left = false;
                consume_whitespace_counting_blank_lines(&mut rdr, &mut comments);
            }
            while rdr.peeking_at_comment() {
                consume_comment(&mut rdr, code_to_the_left, &mut comments);
                consume_whitespace_counting_blank_lines(&mut rdr, &mut comments);
            }
            break;
        }

        let bstart = rdr.last_pos;
        rdr.next_token();
        // discard, and look ahead; we're working with internal state
        let TokenAndSpan { tok, sp } = rdr.peek();
        if tok.is_lit() {
            rdr.with_str_from(bstart, |s| {
                debug!("tok lit: {}", s);
                literals.push(Literal {
                    lit: s.to_string(),
                    pos: sp.lo,
                });
            })
        } else {
            debug!("tok: {}", pprust::token_to_string(&tok));
        }
        first_read = false;
    }

    (comments, literals)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_block_doc_comment_1() {
        let comment = "/**\n * Test \n **  Test\n *   Test\n*/";
        let stripped = strip_doc_comment_decoration(comment);
        assert_eq!(stripped, " Test \n*  Test\n   Test");
    }

    #[test]
    fn test_block_doc_comment_2() {
        let comment = "/**\n * Test\n *  Test\n*/";
        let stripped = strip_doc_comment_decoration(comment);
        assert_eq!(stripped, " Test\n  Test");
    }

    #[test]
    fn test_block_doc_comment_3() {
        let comment = "/**\n let a: *i32;\n *a = 5;\n*/";
        let stripped = strip_doc_comment_decoration(comment);
        assert_eq!(stripped, " let a: *i32;\n *a = 5;");
    }

    #[test]
    fn test_block_doc_comment_4() {
        let comment = "/*******************\n test\n *********************/";
        let stripped = strip_doc_comment_decoration(comment);
        assert_eq!(stripped, " test");
    }

    #[test]
    fn test_line_doc_comment() {
        let stripped = strip_doc_comment_decoration("/// test");
        assert_eq!(stripped, " test");
        let stripped = strip_doc_comment_decoration("///! test");
        assert_eq!(stripped, " test");
        let stripped = strip_doc_comment_decoration("// test");
        assert_eq!(stripped, " test");
        let stripped = strip_doc_comment_decoration("// test");
        assert_eq!(stripped, " test");
        let stripped = strip_doc_comment_decoration("///test");
        assert_eq!(stripped, "test");
        let stripped = strip_doc_comment_decoration("///!test");
        assert_eq!(stripped, "test");
        let stripped = strip_doc_comment_decoration("//test");
        assert_eq!(stripped, "test");
    }
}
<|endoftext|>


<commit_before>//! This module contains `HashStable` implementations for various data types
//! from `rustc_middle::ty` in no particular order.

use crate::ich::{NodeIdHashingMode, StableHashingContext};
use crate::middle::region;
use crate::mir;
use crate::ty;
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey};
use std::cell::RefCell;
use std::mem;

impl<'a, 'tcx, T> HashStable<StableHashingContext<'a>> for &'tcx ty::List<T>
where
    T: HashStable<StableHashingContext<'a>>,
{
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        thread_local! {
            static CACHE: RefCell<FxHashMap<(usize, usize), Fingerprint>> =
                RefCell::new(Default::default());
        }

        let hash = CACHE.with(|cache| {
            let key = (self.as_ptr() as usize, self.len());
            if let Some(&hash) = cache.borrow().get(&key) {
                return hash;
            }

            let mut hasher = StableHasher::new();
            (&self[..]).hash_stable(hcx, &mut hasher);

            let hash: Fingerprint = hasher.finish();
            cache.borrow_mut().insert(key, hash);
            hash
        });

        hash.hash_stable(hcx, hasher);
    }
}

impl<'a, 'tcx, T> ToStableHashKey<StableHashingContext<'a>> for &'tcx ty::List<T>
where
    T: HashStable<StableHashingContext<'a>>,
{
    type KeyType = Fingerprint;

    #[inline]
    fn to_stable_hash_key(&self, hcx: &StableHashingContext<'a>) -> Fingerprint {
        let mut hasher = StableHasher::new();
        let mut hcx: StableHashingContext<'a> = hcx.clone();
        self.hash_stable(&mut hcx, &mut hasher);
        hasher.finish()
    }
}

impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for ty::subst::GenericArg<'tcx> {
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        self.unpack().hash_stable(hcx, hasher);
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for ty::RegionKind {
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        mem::discriminant(self).hash_stable(hcx, hasher);
        match *self {
            ty::ReErased | ty::ReStatic => {
                // No variant fields to hash for these ...
            }
            ty::ReEmpty(universe) => {
                universe.hash_stable(hcx, hasher);
            }
            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrAnon(i), .. }) => {
                db.hash_stable(hcx, hasher);
                i.hash_stable(hcx, hasher);
            }
            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrNamed(def_id, name), .. }) => {
                db.hash_stable(hcx, hasher);
                def_id.hash_stable(hcx, hasher);
                name.hash_stable(hcx, hasher);
            }
            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrEnv, .. }) => {
                db.hash_stable(hcx, hasher);
            }
            ty::ReEarlyBound(ty::EarlyBoundRegion { def_id, index, name }) => {
                def_id.hash_stable(hcx, hasher);
                index.hash_stable(hcx, hasher);
                name.hash_stable(hcx, hasher);
            }
            ty::ReFree(ref free_region) => {
                free_region.hash_stable(hcx, hasher);
            }
            ty::ReVar(..) | ty::RePlaceholder(..) => {
                bug!("StableHasher: unexpected region {:?}", *self)
            }
        }
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for ty::RegionVid {
    #[inline]
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        self.index().hash_stable(hcx, hasher);
    }
}

impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for ty::ConstVid<'tcx> {
    #[inline]
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        self.index.hash_stable(hcx, hasher);
    }
}

impl<'tcx> HashStable<StableHashingContext<'tcx>> for ty::BoundVar {
    #[inline]
    fn hash_stable(&self, hcx: &mut StableHashingContext<'tcx>, hasher: &mut StableHasher) {
        self.index().hash_stable(hcx, hasher);
    }
}

impl<'a, 'tcx, T> HashStable<StableHashingContext<'a>> for ty::Binder<'tcx, T>
where
    T: HashStable<StableHashingContext<'a>>,
{
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        self.as_ref().skip_binder().hash_stable(hcx, hasher);
        self.bound_vars().hash_stable(hcx, hasher);
    }
}

// AllocIds get resolved to whatever they point to (to be stable)
impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::AllocId {
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        ty::tls::with_opt(|tcx| {
            trace!("hashing {:?}", *self);
            let tcx = tcx.expect("can't hash AllocIds during hir lowering");
            tcx.get_global_alloc(*self).hash_stable(hcx, hasher);
        });
    }
}

// `Relocations` with default type parameters is a sorted map.
impl<'a, Tag> HashStable<StableHashingContext<'a>> for mir::interpret::Relocations<Tag>
where
    Tag: HashStable<StableHashingContext<'a>>,
{
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        self.len().hash_stable(hcx, hasher);
        for reloc in self.iter() {
            reloc.hash_stable(hcx, hasher);
        }
    }
}

impl<'a> ToStableHashKey<StableHashingContext<'a>> for region::Scope {
    type KeyType = region::Scope;

    #[inline]
    fn to_stable_hash_key(&self, _: &StableHashingContext<'a>) -> region::Scope {
        *self
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for ty::TyVid {
    fn hash_stable(&self, _hcx: &mut StableHashingContext<'a>, _hasher: &mut StableHasher) {
        // `TyVid` values are confined to an inference context and hence
        // should not be hashed.
        bug!("ty::TyKind::hash_stable() - can't hash a TyVid {:?}.", *self)
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for ty::IntVid {
    fn hash_stable(&self, _hcx: &mut StableHashingContext<'a>, _hasher: &mut StableHasher) {
        // `IntVid` values are confined to an inference context and hence
        // should not be hashed.
        bug!("ty::TyKind::hash_stable() - can't hash an IntVid {:?}.", *self)
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for ty::FloatVid {
    fn hash_stable(&self, _hcx: &mut StableHashingContext<'a>, _hasher: &mut StableHasher) {
        // `FloatVid` values are confined to an inference context and hence
        // should not be hashed.
        bug!("ty::TyKind::hash_stable() - can't hash a FloatVid {:?}.", *self)
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for crate::middle::privacy::AccessLevels {
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        hcx.with_node_id_hashing_mode(NodeIdHashingMode::HashDefPath, |hcx| {
            let crate::middle::privacy::AccessLevels { ref map } = *self;

            map.hash_stable(hcx, hasher);
        });
    }
}
<commit_msg>Rollup merge of #89142 - nerandell:master, r=jackh726<commit_after>//! This module contains `HashStable` implementations for various data types
//! from `rustc_middle::ty` in no particular order.

use crate::ich::{NodeIdHashingMode, StableHashingContext};
use crate::middle::region;
use crate::mir;
use crate::ty;
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher, ToStableHashKey};
use std::cell::RefCell;
use std::mem;

impl<'a, 'tcx, T> HashStable<StableHashingContext<'a>> for &'tcx ty::List<T>
where
    T: HashStable<StableHashingContext<'a>>,
{
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        thread_local! {
            static CACHE: RefCell<FxHashMap<(usize, usize), Fingerprint>> =
                RefCell::new(Default::default());
        }

        let hash = CACHE.with(|cache| {
            let key = (self.as_ptr() as usize, self.len());
            if let Some(&hash) = cache.borrow().get(&key) {
                return hash;
            }

            let mut hasher = StableHasher::new();
            (&self[..]).hash_stable(hcx, &mut hasher);

            let hash: Fingerprint = hasher.finish();
            cache.borrow_mut().insert(key, hash);
            hash
        });

        hash.hash_stable(hcx, hasher);
    }
}

impl<'a, 'tcx, T> ToStableHashKey<StableHashingContext<'a>> for &'tcx ty::List<T>
where
    T: HashStable<StableHashingContext<'a>>,
{
    type KeyType = Fingerprint;

    #[inline]
    fn to_stable_hash_key(&self, hcx: &StableHashingContext<'a>) -> Fingerprint {
        let mut hasher = StableHasher::new();
        let mut hcx: StableHashingContext<'a> = hcx.clone();
        self.hash_stable(&mut hcx, &mut hasher);
        hasher.finish()
    }
}

impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for ty::subst::GenericArg<'tcx> {
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        self.unpack().hash_stable(hcx, hasher);
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for ty::RegionKind {
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        mem::discriminant(self).hash_stable(hcx, hasher);
        match *self {
            ty::ReErased | ty::ReStatic => {
                // No variant fields to hash for these ...
            }
            ty::ReEmpty(universe) => {
                universe.hash_stable(hcx, hasher);
            }
            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrAnon(i), .. }) => {
                db.hash_stable(hcx, hasher);
                i.hash_stable(hcx, hasher);
            }
            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrNamed(def_id, name), .. }) => {
                db.hash_stable(hcx, hasher);
                def_id.hash_stable(hcx, hasher);
                name.hash_stable(hcx, hasher);
            }
            ty::ReLateBound(db, ty::BoundRegion { kind: ty::BrEnv, .. }) => {
                db.hash_stable(hcx, hasher);
            }
            ty::ReEarlyBound(ty::EarlyBoundRegion { def_id, index, name }) => {
                def_id.hash_stable(hcx, hasher);
                index.hash_stable(hcx, hasher);
                name.hash_stable(hcx, hasher);
            }
            ty::ReFree(ref free_region) => {
                free_region.hash_stable(hcx, hasher);
            }
            ty::RePlaceholder(p) => {
                p.hash_stable(hcx, hasher);
            }
            ty::ReVar(..) => {
                bug!("StableHasher: unexpected region {:?}", *self)
            }
        }
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for ty::RegionVid {
    #[inline]
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        self.index().hash_stable(hcx, hasher);
    }
}

impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for ty::ConstVid<'tcx> {
    #[inline]
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        self.index.hash_stable(hcx, hasher);
    }
}

impl<'tcx> HashStable<StableHashingContext<'tcx>> for ty::BoundVar {
    #[inline]
    fn hash_stable(&self, hcx: &mut StableHashingContext<'tcx>, hasher: &mut StableHasher) {
        self.index().hash_stable(hcx, hasher);
    }
}

impl<'a, 'tcx, T> HashStable<StableHashingContext<'a>> for ty::Binder<'tcx, T>
where
    T: HashStable<StableHashingContext<'a>>,
{
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        self.as_ref().skip_binder().hash_stable(hcx, hasher);
        self.bound_vars().hash_stable(hcx, hasher);
    }
}

// AllocIds get resolved to whatever they point to (to be stable)
impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::AllocId {
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        ty::tls::with_opt(|tcx| {
            trace!("hashing {:?}", *self);
            let tcx = tcx.expect("can't hash AllocIds during hir lowering");
            tcx.get_global_alloc(*self).hash_stable(hcx, hasher);
        });
    }
}

// `Relocations` with default type parameters is a sorted map.
impl<'a, Tag> HashStable<StableHashingContext<'a>> for mir::interpret::Relocations<Tag>
where
    Tag: HashStable<StableHashingContext<'a>>,
{
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        self.len().hash_stable(hcx, hasher);
        for reloc in self.iter() {
            reloc.hash_stable(hcx, hasher);
        }
    }
}

impl<'a> ToStableHashKey<StableHashingContext<'a>> for region::Scope {
    type KeyType = region::Scope;

    #[inline]
    fn to_stable_hash_key(&self, _: &StableHashingContext<'a>) -> region::Scope {
        *self
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for ty::TyVid {
    fn hash_stable(&self, _hcx: &mut StableHashingContext<'a>, _hasher: &mut StableHasher) {
        // `TyVid` values are confined to an inference context and hence
        // should not be hashed.
        bug!("ty::TyKind::hash_stable() - can't hash a TyVid {:?}.", *self)
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for ty::IntVid {
    fn hash_stable(&self, _hcx: &mut StableHashingContext<'a>, _hasher: &mut StableHasher) {
        // `IntVid` values are confined to an inference context and hence
        // should not be hashed.
        bug!("ty::TyKind::hash_stable() - can't hash an IntVid {:?}.", *self)
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for ty::FloatVid {
    fn hash_stable(&self, _hcx: &mut StableHashingContext<'a>, _hasher: &mut StableHasher) {
        // `FloatVid` values are confined to an inference context and hence
        // should not be hashed.
        bug!("ty::TyKind::hash_stable() - can't hash a FloatVid {:?}.", *self)
    }
}

impl<'a> HashStable<StableHashingContext<'a>> for crate::middle::privacy::AccessLevels {
    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
        hcx.with_node_id_hashing_mode(NodeIdHashingMode::HashDefPath, |hcx| {
            let crate::middle::privacy::AccessLevels { ref map } = *self;

            map.hash_stable(hcx, hasher);
        });
    }
}
<|endoftext|>


<commit_before>// Copyright 2017-2019 int08h LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//!
//! Roughtime server
//!
//! # Configuration
//! The server has multiple ways it can be configured, see
//! [`ServerConfig`](config/trait.ServerConfig.html) for details.
//!

#[macro_use]
extern crate log;

use std::env;
use std::process;
use std::sync::atomic::Ordering;

use roughenough::config;
use roughenough::config::ServerConfig;
use roughenough::roughenough_version;
use roughenough::server::Server;

use mio::Events;

macro_rules! check_ctrlc {
    ($keep_running:expr) => {
        if !$keep_running.load(Ordering::Acquire) {
            warn!("Ctrl-C caught, exiting...");
            return;
        }
    };
}

fn polling_loop(config: Box<dyn ServerConfig>) {
    let mut server = Server::new(config);
    let cfg = server.get_config(); // borrow config that was moved above

    info!("Long-term public key       : {}", server.get_public_key());
    info!("Online public key          : {}", server.get_online_key());
    info!("Max response batch size    : {}", cfg.batch_size());
    info!("Status updates every       : {} seconds", cfg.status_interval().as_secs());
    info!("Server listening on        : {}:{}", cfg.interface(), cfg.port());
    if let Some(hc_port) = cfg.health_check_port() {
        info!("TCP health check           : {}:{}", cfg.interface(), hc_port);
    } else {
        info!("TCP health check           : disabled");
    }
    info!("Client req/resp tracking   : {}",
          if cfg.client_stats_enabled() { "per-client" } else { "aggregated" }
    );
    if cfg.fault_percentage() > 0 {
        info!("Deliberate response errors : ~{}%", cfg.fault_percentage());
    } else {
        info!("Deliberate response errors : disabled");
    }

    let kr = server.get_keep_running();
    let kr_new = kr.clone();

    ctrlc::set_handler(move || kr.store(false, Ordering::Release))
        .expect("failed setting Ctrl-C handler");

    let mut events = Events::with_capacity(64);

    loop {
        check_ctrlc!(kr_new);
        if server.process_events(&mut events) {
            return;
        }
    }
}

pub fn main() {
    use log::Level;

    simple_logger::init_with_level(Level::Info).unwrap();

    info!("Roughenough server v{} starting", roughenough_version());

    let mut args = env::args();
    if args.len() != 2 {
        error!("Usage: server <ENV | /path/to/config.yaml>");
        process::exit(1);
    }

    let arg1 = args.nth(1).unwrap();
    let config = match config::make_config(&arg1) {
        Err(e) => {
            error!("{:?}", e);
            process::exit(1)
        }
        Ok(ref cfg) if !config::is_valid_config(cfg.as_ref()) => process::exit(1),
        Ok(cfg) => cfg,
    };

    polling_loop(config);

    info!("Done.");
    process::exit(0);
}
<commit_msg>cleanups: tidy server code; extract configuration display to own function<commit_after>// Copyright 2017-2019 int08h LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//!
//! Roughtime server
//!
//! # Configuration
//! The server has multiple ways it can be configured, see
//! [`ServerConfig`](config/trait.ServerConfig.html) for details.
//!

#[macro_use]
extern crate log;

use std::env;
use std::process;
use std::sync::atomic::Ordering;

use roughenough::config;
use roughenough::config::ServerConfig;
use roughenough::roughenough_version;
use roughenough::server::Server;

use mio::Events;

macro_rules! check_ctrlc {
    ($keep_running:expr) => {
        if !$keep_running.load(Ordering::Acquire) {
            warn!("Ctrl-C caught, exiting...");
            return;
        }
    };
}

fn polling_loop(config: Box<dyn ServerConfig>) {
    let mut server = Server::new(config);
    let cfg = server.get_config();
    display_config(&server, cfg);

    let kr = server.get_keep_running();
    let kr0 = kr.clone();

    ctrlc::set_handler(move || kr.store(false, Ordering::Release))
        .expect("failed setting Ctrl-C handler");

    let mut events = Events::with_capacity(1024);

    loop {
        check_ctrlc!(kr0);
        if server.process_events(&mut events) {
            return;
        }
    }
}

fn display_config(server: &Server, cfg: &dyn ServerConfig) {
    info!("Long-term public key       : {}", server.get_public_key());
    info!("Online public key          : {}", server.get_online_key());
    info!("Max response batch size    : {}", cfg.batch_size());
    info!("Status updates every       : {} seconds", cfg.status_interval().as_secs());
    info!("Server listening on        : {}:{}", cfg.interface(), cfg.port());
    if let Some(hc_port) = cfg.health_check_port() {
        info!("TCP health check           : {}:{}", cfg.interface(), hc_port);
    } else {
        info!("TCP health check           : disabled");
    }
    info!("Client req/resp tracking   : {}",
          if cfg.client_stats_enabled() { "per-client" } else { "aggregated" }
    );
    if cfg.fault_percentage() > 0 {
        info!("Deliberate response errors : ~{}%", cfg.fault_percentage());
    } else {
        info!("Deliberate response errors : disabled");
    }
}

pub fn main() {
    use log::Level;

    simple_logger::init_with_level(Level::Info).unwrap();

    info!("Roughenough server v{} starting", roughenough_version());

    let mut args = env::args();
    if args.len() != 2 {
        error!("Usage: server <ENV | /path/to/config.yaml>");
        process::exit(1);
    }

    let arg1 = args.nth(1).unwrap();
    let config = match config::make_config(&arg1) {
        Err(e) => {
            error!("{:?}", e);
            process::exit(1)
        }
        Ok(ref cfg) if !config::is_valid_config(cfg.as_ref()) => process::exit(1),
        Ok(cfg) => cfg,
    };

    polling_loop(config);

    info!("Done.");
    process::exit(0);
}
<|endoftext|>


<commit_before>use primal_bit::{BitVec};
use std::{cmp};

use wheel;
use primal_smallsieve::Primes;

/// A segmented sieve that yields only a small run of primes at a
/// time.
///
/// This is heavily inspired by this [segmented
/// sieve](http://primesieve.org/segmented_sieve.html) code.
#[derive(Debug)]
pub struct StreamingSieve {
    small: Primes,
    // stores which numbers *aren't* prime, i.e. true == composite.
    sieve: BitVec,
    primes: Vec<wheel::WheelInfo>,
    presieve: BitVec,

    low: usize,
    current: usize,
    limit: usize,
}

const CACHE: usize = (32 << 10);
const SEG_ELEMS: usize = 8 * CACHE;
const SEG_LEN: usize = SEG_ELEMS * wheel::BYTE_MODULO / wheel::BYTE_SIZE;

const PRESIEVE_PROD: usize = 2 * 3 * 5 * 7 * 11;
const PRESIEVE_PRIMES: &'static [usize] = &[2, 3, 5, 7, 11];
const PRESIEVE_NEXT: usize = 13;
const PRESIEVE_ACTIVE: bool = true;

fn bits_for(x: usize) -> usize {
    (x * wheel::BYTE_SIZE + wheel::BYTE_MODULO - 1) / wheel::BYTE_MODULO
}

#[inline(never)]
fn compute_presieve() -> BitVec {
    assert!(PRESIEVE_ACTIVE);
    let len = bits_for(PRESIEVE_PROD);
    let mut bitv = BitVec::from_elem(len, false);

    // this is silly and should be done with a sieve that only uses
    // the primes in PRESIEVE_PRIMES.
    for i in 0..len {
        let true_ = wheel::from_bit_index(i);
        fn gcd(x: usize, y: usize) -> usize {
            if y == 0 { x }
            else { gcd(y, x % y) }
        }
        if gcd(true_, PRESIEVE_PROD) != 1 {
            bitv.set(i, true)
        }
    }
    bitv
}

impl StreamingSieve {
    /// Create a new instance of the streaming sieve that will
    /// correctly progressively filter primes up to `limit`.
    pub fn new(limit: usize) -> StreamingSieve {
        let small = Primes::sieve((limit as f64).sqrt() as usize + 1);
        let mut current = match wheel::MODULO {
            6 => 5,
            30 => 7,
            210 => 11,
            _ => unimplemented!(),
        };
        let low = 0;

        let elems = cmp::min(bits_for(limit), SEG_ELEMS);
        let presieve = if PRESIEVE_ACTIVE {
            current = PRESIEVE_NEXT;
            compute_presieve()
        } else {
            BitVec::new()
        };

        StreamingSieve {
            small: small,
            sieve: BitVec::from_elem(elems, false),
            primes: vec![],
            presieve: presieve,

            low: low,
            current: current,
            limit: limit
        }
    }

    pub fn very_small_sieve(&mut self, low: usize) {
        let offset = (low % PRESIEVE_PROD) * wheel::BYTE_SIZE / wheel::BYTE_MODULO / 8;

        copy_all(self.sieve.as_bytes_mut(),
                 self.presieve.as_bytes(),
                 offset);
        fn copy_all(mut dst: &mut [u8], src: &[u8], init_offset: usize) {
            let mut pos = 0;
            // pre-fill data at the start, as a rotation of `src`.
            pos += memcpy(&mut dst[pos..], &src[init_offset..]);
            if pos >= dst.len() { return }

            pos += memcpy(&mut dst[pos..], &src[..init_offset]);
            if pos >= dst.len() { return }

            // progressively copy the prefix to the rest of the
            // vector, doubling each time.
            while pos < dst.len() {
                let (filled, unfilled) = dst.split_at_mut(pos);
                pos += memcpy(unfilled, filled);
            }
        }
        fn memcpy<'d>(dst: &'d mut [u8], src: &[u8]) -> usize {
            use std::ptr;
            let l = cmp::min(dst.len(), src.len());
            unsafe {
                ptr::copy(src.as_ptr(), dst.as_mut_ptr(), l);
            }
            l
        }
    }

    fn direct_sieve(&mut self) {
        let bytes = self.sieve.as_bytes_mut();
        let top = bytes.len();

        for wi in self.primes.iter_mut() {
            let mut si = wi.sieve_index;
            let mut wi_ = wi.wheel_index;
            let p = wi.prime;
            while si < top {
                wheel::set_bit(bytes, &mut si, &mut wi_, p);
            }
            // if this wraps, we've hit the limit, and so won't be
            // continuing, so whatever, it can be junk.
            wi.sieve_index = si.wrapping_sub(top);
            wi.wheel_index = wi_;
        }
    }

    /// Extract the next chunk of filtered primes, the return value is
    /// `Some((low, v))` or `None` if the sieve has reached the limit.
    ///
    /// The vector stores bits for each odd number starting at `low`.
    /// Bit `n` of `v` is set if and only if `low + 2 * n + 1` is
    /// prime.
    ///
    /// NB. the prime 2 is not included in any of these sieves and so
    /// needs special handling.
    pub fn next(&mut self) -> Option<(usize, &BitVec)> {
        if self.low >= self.limit {
            return None
        }

        let low = self.low;
        self.low += SEG_LEN;
        let high = cmp::min(low + SEG_LEN - 1, self.limit);
        self.sieve.clear();

        let mut s = self.current;

        while s * s <= high {
            if self.small.is_prime(s) {
                self.primes.push(wheel::compute_wheel_elem(s, low));
            }
            s += 1
        }

        self.current = s;
        if PRESIEVE_ACTIVE && self.presieve.len() > 0 {
            self.very_small_sieve(low);
        }
        self.direct_sieve();

        if low == 0 {
            // 1 is not prime.
            self.sieve.set(0, true);
            if PRESIEVE_ACTIVE {
                // but these are
                for &x in PRESIEVE_PRIMES {
                    let (use_, idx) = wheel::bit_index(x);
                    if use_ {
                        self.sieve.set(idx, false)
                    }
                }
            }
        }

        Some((low, &self.sieve))
    }
}

#[cfg(test)]
mod tests {
    use wheel;
    use super::StreamingSieve;
    fn gcd(x: usize, y: usize) -> usize {
        if y == 0 { x }
        else { gcd(y, x % y) }
    }
    fn coprime_to(x: usize) -> Vec<usize> {
        (1..x).filter(|&n| gcd(n, x) == 1).collect()
    }
    #[test]
    fn test() {
        let coprime = coprime_to(wheel::BYTE_MODULO);
        const LIMIT: usize = 2_000_000;
        let mut sieve = StreamingSieve::new(LIMIT);
        let primes = ::primal_smallsieve::Primes::sieve(LIMIT);

        let mut base = 0;
        let mut index = 0;

        while let Some((_low, next)) = sieve.next() {
            for val in next {
                let i = wheel::BYTE_MODULO * base + coprime[index];
                if i >= LIMIT { break }
                assert!(primes.is_prime(i) == !val,
                        "failed for {} (is prime = {})", i, primes.is_prime(i));

                index += 1;
                if index == wheel::BYTE_SIZE {
                    index = 0;
                    base += 1
                }
            }
        }
    }
}

#[cfg(all(test, feature = "unstable"))]
mod benches {
    use test::Bencher;
    use super::StreamingSieve;

    fn run(b: &mut Bencher, n: usize) {
        b.iter(|| {
            let mut sieve = StreamingSieve::new(n);
            while sieve.next().is_some() {}
        })
    }

    #[bench]
    fn sieve_small(b: &mut Bencher) {
        run(b, 100)
    }
    #[bench]
    fn sieve_medium(b: &mut Bencher) {
        run(b, 10_000)
    }
    #[bench]
    fn sieve_large(b: &mut Bencher) {
        run(b, 100_000)
    }
    #[bench]
    fn sieve_larger(b: &mut Bencher) {
        run(b, 1_000_000)
    }
    #[bench]
    fn sieve_huge(b: &mut Bencher) {
        run(b, 10_000_000)
    }
}
<commit_msg>sieve: only presieve as far as necessary.<commit_after>use primal_bit::{BitVec};
use std::{cmp};

use wheel;
use primal_smallsieve::Primes;

/// A segmented sieve that yields only a small run of primes at a
/// time.
///
/// This is heavily inspired by this [segmented
/// sieve](http://primesieve.org/segmented_sieve.html) code.
#[derive(Debug)]
pub struct StreamingSieve {
    small: Primes,
    // stores which numbers *aren't* prime, i.e. true == composite.
    sieve: BitVec,
    primes: Vec<wheel::WheelInfo>,
    presieve: BitVec,

    low: usize,
    current: usize,
    limit: usize,
}

const CACHE: usize = (32 << 10);
const SEG_ELEMS: usize = 8 * CACHE;
const SEG_LEN: usize = SEG_ELEMS * wheel::BYTE_MODULO / wheel::BYTE_SIZE;

const PRESIEVE_PROD: usize = 2 * 3 * 5 * 7 * 11;
const PRESIEVE_PRIMES: &'static [usize] = &[2, 3, 5, 7, 11];
const PRESIEVE_NEXT: usize = 13;
const PRESIEVE_ACTIVE: bool = true;

fn bits_for(x: usize) -> usize {
    (x * wheel::BYTE_SIZE + wheel::BYTE_MODULO - 1) / wheel::BYTE_MODULO
}

#[inline(never)]
fn compute_presieve(limit_bits: usize) -> BitVec {
    assert!(PRESIEVE_ACTIVE);
    let len = cmp::min(bits_for(PRESIEVE_PROD),
                       limit_bits);
    let mut bitv = BitVec::from_elem(len, false);

    // this is silly and should be done with a sieve that only uses
    // the primes in PRESIEVE_PRIMES.
    for i in 0..len {
        let true_ = wheel::from_bit_index(i);
        fn gcd(x: usize, y: usize) -> usize {
            if y == 0 { x }
            else { gcd(y, x % y) }
        }
        if gcd(true_, PRESIEVE_PROD) != 1 {
            bitv.set(i, true)
        }
    }
    bitv
}

impl StreamingSieve {
    /// Create a new instance of the streaming sieve that will
    /// correctly progressively filter primes up to `limit`.
    pub fn new(limit: usize) -> StreamingSieve {
        let small = Primes::sieve((limit as f64).sqrt() as usize + 1);
        let mut current = match wheel::MODULO {
            6 => 5,
            30 => 7,
            210 => 11,
            _ => unimplemented!(),
        };
        let low = 0;

        let elems = cmp::min(bits_for(limit), SEG_ELEMS);
        let presieve = if PRESIEVE_ACTIVE {
            current = PRESIEVE_NEXT;
            compute_presieve(elems)
        } else {
            BitVec::new()
        };

        StreamingSieve {
            small: small,
            sieve: BitVec::from_elem(elems, false),
            primes: vec![],
            presieve: presieve,

            low: low,
            current: current,
            limit: limit
        }
    }

    pub fn very_small_sieve(&mut self, low: usize) {
        let offset = (low % PRESIEVE_PROD) * wheel::BYTE_SIZE / wheel::BYTE_MODULO / 8;

        copy_all(self.sieve.as_bytes_mut(),
                 self.presieve.as_bytes(),
                 offset);
        fn copy_all(mut dst: &mut [u8], src: &[u8], init_offset: usize) {
            let mut pos = 0;
            // pre-fill data at the start, as a rotation of `src`.
            pos += memcpy(&mut dst[pos..], &src[init_offset..]);
            if pos >= dst.len() { return }

            pos += memcpy(&mut dst[pos..], &src[..init_offset]);
            if pos >= dst.len() { return }

            // progressively copy the prefix to the rest of the
            // vector, doubling each time.
            while pos < dst.len() {
                let (filled, unfilled) = dst.split_at_mut(pos);
                pos += memcpy(unfilled, filled);
            }
        }
        fn memcpy<'d>(dst: &'d mut [u8], src: &[u8]) -> usize {
            use std::ptr;
            let l = cmp::min(dst.len(), src.len());
            unsafe {
                ptr::copy(src.as_ptr(), dst.as_mut_ptr(), l);
            }
            l
        }
    }

    fn direct_sieve(&mut self) {
        let bytes = self.sieve.as_bytes_mut();
        let top = bytes.len();

        for wi in self.primes.iter_mut() {
            let mut si = wi.sieve_index;
            let mut wi_ = wi.wheel_index;
            let p = wi.prime;
            while si < top {
                wheel::set_bit(bytes, &mut si, &mut wi_, p);
            }
            // if this wraps, we've hit the limit, and so won't be
            // continuing, so whatever, it can be junk.
            wi.sieve_index = si.wrapping_sub(top);
            wi.wheel_index = wi_;
        }
    }

    /// Extract the next chunk of filtered primes, the return value is
    /// `Some((low, v))` or `None` if the sieve has reached the limit.
    ///
    /// The vector stores bits for each odd number starting at `low`.
    /// Bit `n` of `v` is set if and only if `low + 2 * n + 1` is
    /// prime.
    ///
    /// NB. the prime 2 is not included in any of these sieves and so
    /// needs special handling.
    pub fn next(&mut self) -> Option<(usize, &BitVec)> {
        if self.low >= self.limit {
            return None
        }

        let low = self.low;
        self.low += SEG_LEN;
        let high = cmp::min(low + SEG_LEN - 1, self.limit);
        self.sieve.clear();

        let mut s = self.current;

        while s * s <= high {
            if self.small.is_prime(s) {
                self.primes.push(wheel::compute_wheel_elem(s, low));
            }
            s += 1
        }

        self.current = s;
        if PRESIEVE_ACTIVE && self.presieve.len() > 0 {
            self.very_small_sieve(low);
        }
        self.direct_sieve();

        if low == 0 {
            // 1 is not prime.
            self.sieve.set(0, true);
            if PRESIEVE_ACTIVE {
                // but these are
                for &x in PRESIEVE_PRIMES {
                    let (use_, idx) = wheel::bit_index(x);
                    if use_ {
                        self.sieve.set(idx, false)
                    }
                }
            }
        }

        Some((low, &self.sieve))
    }
}

#[cfg(test)]
mod tests {
    use wheel;
    use super::StreamingSieve;
    fn gcd(x: usize, y: usize) -> usize {
        if y == 0 { x }
        else { gcd(y, x % y) }
    }
    fn coprime_to(x: usize) -> Vec<usize> {
        (1..x).filter(|&n| gcd(n, x) == 1).collect()
    }
    #[test]
    fn test() {
        let coprime = coprime_to(wheel::BYTE_MODULO);
        const LIMIT: usize = 2_000_000;
        let mut sieve = StreamingSieve::new(LIMIT);
        let primes = ::primal_smallsieve::Primes::sieve(LIMIT);

        let mut base = 0;
        let mut index = 0;

        while let Some((_low, next)) = sieve.next() {
            for val in next {
                let i = wheel::BYTE_MODULO * base + coprime[index];
                if i >= LIMIT { break }
                assert!(primes.is_prime(i) == !val,
                        "failed for {} (is prime = {})", i, primes.is_prime(i));

                index += 1;
                if index == wheel::BYTE_SIZE {
                    index = 0;
                    base += 1
                }
            }
        }
    }
}

#[cfg(all(test, feature = "unstable"))]
mod benches {
    use test::Bencher;
    use super::StreamingSieve;

    fn run(b: &mut Bencher, n: usize) {
        b.iter(|| {
            let mut sieve = StreamingSieve::new(n);
            while sieve.next().is_some() {}
        })
    }

    #[bench]
    fn sieve_small(b: &mut Bencher) {
        run(b, 100)
    }
    #[bench]
    fn sieve_medium(b: &mut Bencher) {
        run(b, 10_000)
    }
    #[bench]
    fn sieve_large(b: &mut Bencher) {
        run(b, 100_000)
    }
    #[bench]
    fn sieve_larger(b: &mut Bencher) {
        run(b, 1_000_000)
    }
    #[bench]
    fn sieve_huge(b: &mut Bencher) {
        run(b, 10_000_000)
    }
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io
use super::model;
use regex;
use rs_es;
use rs_es::query::Query as rs_q;
use rs_es::operations::search::SearchResult;
use mimir;
use serde_json;

fn build_rs_client(cnx: &String) -> rs_es::Client {
    let re = regex::Regex::new(r"(?:https?://)?(?P<host>.+?):(?P<port>\d+)")
                 .unwrap();
    let cap = re.captures(&cnx).unwrap();
    let host = cap.name("host").unwrap();
    let port = cap.name("port").unwrap().parse::<u32>().unwrap();

    rs_es::Client::new(&host, port)
}

/// takes a ES json blob and build a Place from it
/// it uses the _type field of ES to know which type of the Place enum to fill
pub fn make_place(doc_type: String, value: Option<Box<serde_json::Value>>) -> Option<mimir::Place> {
    value.and_then(|v| {
        match doc_type.as_ref() {
            "addr" => {
                serde_json::from_value::<mimir::Addr>(*v)
                    .ok()
                    .and_then(|o| Some(mimir::Place::Addr(o)))
            }
            "street" => {
                serde_json::from_value::<mimir::Street>(*v)
                    .ok()
                    .and_then(|o| Some(mimir::Place::Street(o)))
            }
            "admin" => {
                serde_json::from_value::<mimir::Admin>(*v)
                    .ok()
                    .and_then(|o| Some(mimir::Place::Admin(o)))
            }
            _ => {
                warn!("unknown ES return value, _type field = {}", doc_type);
                None
            }
        }
    })
}

fn query(q: &str,
         cnx: &str,
         match_type: &str,
	 coord: &Option<model::Coord>,
         shape: Option<Vec<rs_es::units::Location>>)
         -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
    let sub_query = rs_q::build_bool()
                        .with_should(vec![
                       rs_q::build_term("_type","addr").with_boost(1000).build(),
                       rs_q::build_match(match_type, q.to_string())
                              .with_boost(100)
                              .build(),
                       rs_q::build_function_score()
                              .with_boost_mode(rs_es::query::compound::BoostMode::Multiply)
                              .with_boost(300)
                              .with_query(rs_q::build_match_all().build())
			                  .with_function(
                          rs_es::query::functions::Function::build_field_value_factor("weight")
			                                  .with_factor(1)
                                      .with_modifier(rs_es::query::functions::Modifier::Log1p)
			                                                          .build())
                              .build()])
                        .build();

    let mut must = vec![rs_q::build_match(match_type, q.to_string())
             .with_minimum_should_match(rs_es::query::MinimumShouldMatch::from(100f64)).build()];

    if let Some(s) = shape {
        must.push(rs_q::build_geo_polygon("coord", s).build());
    }

    let filter = rs_q::build_bool()
        .with_should(vec![rs_q::build_bool()
                              .with_must_not(rs_q::build_exists("house_number").build())
                              .build(),
                          rs_q::build_match("house_number", q.to_string()).build()])
        .with_must(must)
        .build();

    let final_query = rs_q::build_bool()
                          .with_must(vec![sub_query])
                          .with_filter(filter)
                          .build();

    let mut client = build_rs_client(cnx);

    let result: SearchResult<serde_json::Value> = try!(client.search_query()
                                                       .with_indexes(&["munin"])
                                                       .with_query(&final_query)
                                                       .send());

    debug!("{} documents found", result.hits.total);

    // for the moment rs-es does not handle enum Document,
    // so we need to convert the ES glob to a Place
    Ok(result.hits
             .hits
             .into_iter()
             .filter_map(|hit| make_place(hit.doc_type, hit.source))
             .collect())
}

fn query_prefix(q: &String, cnx: &String) -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
	query(&q, cnx, "name.prefix")
}

fn query_ngram(q: &String, cnx: &String) -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
	query(&q, cnx, "name.ngram")
}

pub fn autocomplete(q: String,
                    coord: Option<model::Coord>,
                    cnx: &String,
                    shape: Option<Vec<(f64, f64)>>)
                    -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
	// First search with match = "name.prefix".
	// If no result then another search with match = "name.ngram"
	fn make_shape(shape: &Option<Vec<(f64, f64)>>) -> Option<Vec<rs_es::units::Location>> {
	    shape.as_ref().map(|v| v.iter().map(|&l| l.into()).collect())
	}

    let results = try!(query_prefix(&q, cnx, &coord, make_shape(&shape)));
    if results.is_empty() {
    	query_ngram(&q, cnx, &coord, make_shape(&shape))
    } else {
    	Ok(results)
    }
}
<commit_msg>query handle coords<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io
use super::model;
use regex;
use rs_es;
use rs_es::query::Query as rs_q;
use rs_es::operations::search::SearchResult;
use rs_es::units as rs_u;
use mimir;
use serde_json;

fn build_rs_client(cnx: &String) -> rs_es::Client {
    let re = regex::Regex::new(r"(?:https?://)?(?P<host>.+?):(?P<port>\d+)")
                 .unwrap();
    let cap = re.captures(&cnx).unwrap();
    let host = cap.name("host").unwrap();
    let port = cap.name("port").unwrap().parse::<u32>().unwrap();

    rs_es::Client::new(&host, port)
}

/// takes a ES json blob and build a Place from it
/// it uses the _type field of ES to know which type of the Place enum to fill
pub fn make_place(doc_type: String, value: Option<Box<serde_json::Value>>) -> Option<mimir::Place> {
    value.and_then(|v| {
        match doc_type.as_ref() {
            "addr" => {
                serde_json::from_value::<mimir::Addr>(*v)
                    .ok()
                    .and_then(|o| Some(mimir::Place::Addr(o)))
            }
            "street" => {
                serde_json::from_value::<mimir::Street>(*v)
                    .ok()
                    .and_then(|o| Some(mimir::Place::Street(o)))
            }
            "admin" => {
                serde_json::from_value::<mimir::Admin>(*v)
                    .ok()
                    .and_then(|o| Some(mimir::Place::Admin(o)))
            }
            _ => {
                warn!("unknown ES return value, _type field = {}", doc_type);
                None
            }
        }
    })
}

fn build_query(q: &String, match_type: &str, coord: &Option<model::Coord>) -> rs_es::query::Query {
    let boost_addr = rs_q::build_term("_type","addr").with_boost(1000).build();
    let boost_match_query = rs_q::build_match(match_type, q.to_string()).with_boost(100).build();

    let mut should_query = vec![
   boost_addr,
   boost_match_query];
   if let &Some(ref c) = coord {
       // if we have coordinate, we boost we result near this coordinate
       let boost_on_proximity = rs_q::build_function_score()
              .with_boost_mode(rs_es::query::compound::BoostMode::Multiply)
              .with_boost(300)
              .with_function(
          rs_es::query::functions::Function::build_decay("coord", rs_u::Location::LatLon(c.lat, c.lon), rs_u::Distance::new(3f64, rs_u::DistanceUnit::Kilometer)).build_gauss()).build();
        should_query.push(boost_on_proximity);
   } else {
       // if we don't have coords, we take the field `weight` into account
       let boost_on_weight = rs_q::build_function_score()
              .with_boost_mode(rs_es::query::compound::BoostMode::Multiply)
              .with_boost(300)
              .with_query(rs_q::build_match_all().build())
              .with_function(
          rs_es::query::functions::Function::build_field_value_factor("weight")
                              .with_factor(1)
                      .with_modifier(rs_es::query::functions::Modifier::Log1p)
                                                      .build())
              .build();
        should_query.push(boost_on_weight);
   }
}

fn query(q: &str,
         cnx: &str,
         match_type: &str,
	 coord: &Option<model::Coord>,
         shape: Option<Vec<rs_es::units::Location>>)
         -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
    let sub_query = rs_q::build_bool()
                        .with_should(should_query)
                        .build();

    let mut must = vec![rs_q::build_match(match_type, q.to_string())
             .with_minimum_should_match(rs_es::query::MinimumShouldMatch::from(100f64)).build()];

    if let Some(s) = shape {
        must.push(rs_q::build_geo_polygon("coord", s).build());
    }

    let filter = rs_q::build_bool()
        .with_should(vec![rs_q::build_bool()
                              .with_must_not(rs_q::build_exists("house_number").build())
                              .build(),
                          rs_q::build_match("house_number", q.to_string()).build()])
        .with_must(must)
        .build();

    rs_q::build_bool()
                          .with_must(vec![sub_query])
                          .with_filter(filter)
                          .build()
}

fn query(q: &String, cnx: &String, match_type: &str, coord: &Option<model::Coord>,
         shape: Option<Vec<rs_es::units::Location>>) -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
    let query = build_query(q, match_type, coord, shape);

    let mut client = build_rs_client(cnx);

    let result: SearchResult<serde_json::Value> = try!(client.search_query()
                                                       .with_indexes(&["munin"])
                                                       .with_query(&query)
                                                       .send());

    debug!("{} documents found", result.hits.total);

    // for the moment rs-es does not handle enum Document,
    // so we need to convert the ES glob to a Place
    Ok(result.hits
             .hits
             .into_iter()
             .filter_map(|hit| make_place(hit.doc_type, hit.source))
             .collect())
}

fn query_prefix(q: &String, cnx: &String, coord: &Option<model::Coord>,
         shape: Option<Vec<rs_es::units::Location>>) -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
	query(&q, cnx, "name.prefix", coord, shape)
}

fn query_ngram(q: &String, cnx: &String, coord: &Option<model::Coord>,
         shape: Option<Vec<rs_es::units::Location>>) -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
	query(&q, cnx, "name.ngram", coord, shape)
}

pub fn autocomplete(q: String,
                    coord: Option<model::Coord>,
                    cnx: &String,
                    shape: Option<Vec<(f64, f64)>>)
                    -> Result<Vec<mimir::Place>, rs_es::error::EsError> {
	// First search with match = "name.prefix".
	// If no result then another search with match = "name.ngram"
	fn make_shape(shape: &Option<Vec<(f64, f64)>>) -> Option<Vec<rs_es::units::Location>> {
	    shape.as_ref().map(|v| v.iter().map(|&l| l.into()).collect())
	}

    let results = try!(query_prefix(&q, cnx, &coord, make_shape(&shape)));
    if results.is_empty() {
    	query_ngram(&q, cnx, &coord, make_shape(&shape))
    } else {
    	Ok(results)
    }
}
<|endoftext|>


<commit_before>use clap::{value_t, App, Arg, ArgMatches};
use core::*;

use std::fs::File;
use std::io::prelude::*;
use std::io::Error as IoError;
use std::time::Instant;

use env_logger;
use log::{error, info, trace};

#[derive(Debug)]
pub enum NativeError {
    GeneralError,
    NoneError,
    IoError(IoError),
    SenError(core::error::Error),
}

impl From<IoError> for NativeError {
    fn from(e: IoError) -> NativeError {
        NativeError::IoError(e)
    }
}

impl From<core::error::Error> for NativeError {
    fn from(e: core::error::Error) -> NativeError {
        NativeError::SenError(e)
    }
}

pub type Result<T> = ::std::result::Result<T, NativeError>;

fn main() {
    let start = Instant::now();
    let matches = App::new("seni-cli")
        .version("0.1.0")
        .author("Inderjit Gill <email@indy.io>")
        .about("native cli build of seni")
        .arg(
            Arg::with_name("SCRIPT")
                .help("Sets the input seni script to use")
                .required(true)
                .index(1),
        )
        .arg(
            Arg::with_name("seed")
                .short("s")
                .long("seed")
                .help("The seed to use")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("debug")
                .short("d")
                .long("debug")
                .help("debug mode")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("packed_trait_list")
                .short("t")
                .long("packed_trait_list")
                .help("print the packed trait list")
                .takes_value(false),
        )
        .get_matches();

    env_logger::init();
    if let Err(e) = run(&matches) {
        error!("Application error: {:?}", e);
    }
    let duration = start.elapsed();
    info!("Complete time elapsed: {:?}", duration);
}

fn run(matches: &ArgMatches) -> Result<()> {
    trace!("run");

    if let Some(script) = matches.value_of("SCRIPT") {
        // this should always pass as SCRIPT is required
        info!("Using script file: {}", script);

        if matches.is_present("packed_trait_list") {
            print_packed_trait_list(script)?;
        } else if matches.is_present("debug") {
            if let Ok(seed) = value_t!(matches.value_of("seed"), u32) {
                run_debug_script_with_seed(script, seed)?;
            } else {
                run_debug_script(script)?;
            }
        } else if let Ok(seed) = value_t!(matches.value_of("seed"), u32) {
            run_script_with_seed(script, seed)?;
        } else {
            run_script(script)?;
        }
    }

    Ok(())
}

fn read_script_file(filename: &str) -> Result<String> {
    trace!("read_script_file");

    let mut f = File::open(filename)?;
    let mut contents = String::new();
    f.read_to_string(&mut contents)?;

    Ok(contents)
}

fn run_debug_script(script: &str) -> Result<()> {
    trace!("run_debug_script");

    let source = read_script_file(script)?;
    let program = compile_str(&source)?;

    println!("{}", &source);
    println!("{}", &program);

    Ok(())
}

fn run_debug_script_with_seed(_script: &str, _seed: u32) -> Result<()> {
    trace!("run_debug_script_with_seed");

    Ok(())
}

fn run_script(script: &str) -> Result<()> {
    trace!("run_script");

    // --------------------------------------------------------------------------------

    let time_read_script_file = Instant::now();
    let source = read_script_file(script)?;
    info!("read_script_file: {:?}", time_read_script_file.elapsed());

    let mut vm = Vm::new();

    // --------------------------------------------------------------------------------

    let time_parse = Instant::now();
    let (ast, _word_lut) = parse(&source)?;
    info!("parse: {:?}", time_parse.elapsed());

    // --------------------------------------------------------------------------------

    let time_compile_program = Instant::now();
    let program = compile_program(&ast)?;
    info!("compile_program: {:?}", time_compile_program.elapsed());

    // --------------------------------------------------------------------------------

    let time_run_program = Instant::now();

    vm.reset();

    // setup the env with the global variables in preamble
    let time_preamble = Instant::now();
    let preamble = compile_preamble()?;
    vm.interpret(&preamble)?;
    info!("preamble: {:?}", time_preamble.elapsed());

    vm.ip = 0;
    let time_interpret = Instant::now();
    vm.interpret(&program)?;
    let res = vm.top_stack_value()?;
    info!("interpret {:?}", time_interpret.elapsed());

    // vm.opcode_profiler_report();

    info!("run_program: {:?}", time_run_program.elapsed());

    println!("res = {}", res);

    Ok(())
}

fn run_script_with_seed(_script: &str, _seed: u32) -> Result<()> {
    trace!("run_script_with_seed");

    Ok(())
}

fn print_packed_trait_list(script: &str) -> Result<()> {
    trace!("print_packed_trait_list");

    let source = read_script_file(script)?;
    let trait_list = build_traits(&source)?;
    let mut packed: String = "".to_string();

    trait_list.pack(&mut packed)?;
    println!("{}", packed);

    Ok(())
}
<commit_msg>cli: reinstated debug argument<commit_after>use clap::{value_t, App, Arg, ArgMatches};
use core::*;

use std::fs::File;
use std::io::prelude::*;
use std::io::Error as IoError;
use std::time::Instant;

use env_logger;
use log::{error, info, trace};

#[derive(Debug)]
pub enum NativeError {
    GeneralError,
    NoneError,
    IoError(IoError),
    SenError(core::error::Error),
}

impl From<IoError> for NativeError {
    fn from(e: IoError) -> NativeError {
        NativeError::IoError(e)
    }
}

impl From<core::error::Error> for NativeError {
    fn from(e: core::error::Error) -> NativeError {
        NativeError::SenError(e)
    }
}

pub type Result<T> = ::std::result::Result<T, NativeError>;

fn main() {
    let start = Instant::now();
    let matches = App::new("seni-cli")
        .version("0.1.0")
        .author("Inderjit Gill <email@indy.io>")
        .about("native cli build of seni")
        .arg(
            Arg::with_name("SCRIPT")
                .help("Sets the input seni script to use")
                .required(true)
                .index(1),
        )
        .arg(
            Arg::with_name("seed")
                .short("s")
                .long("seed")
                .help("The seed to use")
                .takes_value(true),
        )
        .arg(
            Arg::with_name("debug")
                .short("d")
                .long("debug")
                .help("print bytecode")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("profiling")
                .short("p")
                .long("profiling")
                .help("Show opcode count")
                .takes_value(false),
        )
        .arg(
            Arg::with_name("packed_trait_list")
                .short("t")
                .long("packed_trait_list")
                .help("print the packed trait list")
                .takes_value(false),
        )
        .get_matches();

    env_logger::init();
    if let Err(e) = run(&matches) {
        error!("Application error: {:?}", e);
    }
    let duration = start.elapsed();
    info!("Complete time elapsed: {:?}", duration);
}

fn run(matches: &ArgMatches) -> Result<()> {
    trace!("run");

    if let Some(script) = matches.value_of("SCRIPT") {
        // this should always pass as SCRIPT is required
        info!("Using script file: {}", script);

        let profiling = if matches.is_present("profiling") {
            VMProfiling::On
        } else {
            VMProfiling::Off
        };

        let debug = matches.is_present("debug");

        if matches.is_present("packed_trait_list") {
            print_packed_trait_list(script)?;
        } else if let Ok(seed) = value_t!(matches.value_of("seed"), u32) {
            run_script_with_seed(script, seed, profiling, debug)?;
        } else {
            run_script(script, profiling, debug)?;
        }
    }

    Ok(())
}

fn read_script_file(filename: &str) -> Result<String> {
    trace!("read_script_file");

    let mut f = File::open(filename)?;
    let mut contents = String::new();
    f.read_to_string(&mut contents)?;

    Ok(contents)
}

fn run_script(script: &str, profiling: VMProfiling, debug: bool) -> Result<()> {
    trace!("run_script");

    // --------------------------------------------------------------------------------

    let time_read_script_file = Instant::now();
    let source = read_script_file(script)?;

    info!("read_script_file: {:?}", time_read_script_file.elapsed());

    let mut vm = Vm::new();

    // --------------------------------------------------------------------------------

    let time_parse = Instant::now();
    let (ast, _word_lut) = parse(&source)?;
    info!("parse: {:?}", time_parse.elapsed());

    // --------------------------------------------------------------------------------

    let time_compile_program = Instant::now();
    let program = compile_program(&ast)?;
    info!("compile_program: {:?}", time_compile_program.elapsed());

    // --------------------------------------------------------------------------------

    let time_run_program = Instant::now();

    vm.reset();

    // setup the env with the global variables in preamble
    let time_preamble = Instant::now();
    let preamble = compile_preamble()?;
    vm.interpret(&preamble)?;
    info!("preamble: {:?}", time_preamble.elapsed());

    // reset the ip and setup any profiling of the main program
    vm.init_for_main_program(&program, profiling)?;

    let time_interpret = Instant::now();
    vm.interpret(&program)?;
    let res = vm.top_stack_value()?;
    info!("interpret {:?}", time_interpret.elapsed());

    // vm.opcode_profiler_report();

    info!("run_program: {:?}", time_run_program.elapsed());

    if profiling == VMProfiling::On {
        vm.println_profiling(&program)?;
    } else if debug {
        println!("{}", source);
        println!("{}", program);
    }

    println!("res = {}", res);

    Ok(())
}

fn run_script_with_seed(_script: &str, _seed: u32, _profiling: VMProfiling, _debug: bool) -> Result<()> {
    trace!("run_script_with_seed");

    Ok(())
}

fn print_packed_trait_list(script: &str) -> Result<()> {
    trace!("print_packed_trait_list");

    let source = read_script_file(script)?;
    let trait_list = build_traits(&source)?;
    let mut packed: String = "".to_string();

    trait_list.pack(&mut packed)?;
    println!("{}", packed);

    Ok(())
}
<|endoftext|>