<commit_before>use regex::Regex;
use chrono::NaiveDate;
use reqwest;
use std::io::Read;
use select::document::Document;
use select::predicate::{Name, Attr};
use rustc_serialize::json::Json;
use errors::*;

// Some formats borrowed from https://github.com/amir/article-date-extractor
static FMTS: &'static [&str] = &["%A, %B %e, %Y", "%Y-%m-%dT%H:%M:%S%:z", "/%Y/%m/%d/", "/%Y/%d/%m/", "%Y-%m-%d", "%B %e, %Y", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%SZ", "%B %k, %Y, %H:%M %p", "%Y-%m-%d %H:%M:%S.000000"];

// Use lazy_static to ensure we only compile the regex once
lazy_static! {
    // Regex by Newspaper3k  - https://github.com/codelucas/newspaper/blob/master/newspaper/urls.py
    static ref RE: Regex = Regex::new(r"([\./\-_]{0,1}(19|20)\d{2})[\./\-_]{0,1}(([0-3]{0,1}[0-9][\./\-_])|(\w{3,5}[\./\-_]))([0-3]{0,1}[0-9][\./\-]{0,1})").unwrap();
}

// Parse the date, trying out each format
fn parse_date(input: &str) -> Result<NaiveDate> {
    let mut result: Result<NaiveDate> = Err("None of the formats matched the date".into());

    'outer: for fmt in FMTS {
        if let Ok(v) = NaiveDate::parse_from_str(input, fmt) {
            { result = Ok(v);
              break 'outer; }
        }
    }

    result
}

// Extract date from a URL
fn extract_from_url(url: &str) -> Option<String> {
    if let Some(val) = RE.find(url) {
        return Some(val.as_str().to_string())
    } else {
        return None
    }
}

// Extract date from JSON-LD
fn extract_from_ldjson<'a>(html: &'a Document) -> Option<String> {
    let mut json_date: Option<String> = None;
    let mut ldjson: String = String::new();
    if let Some(ldj) = html.find(Attr("type", "application/ld+json")).next() {
        ldjson = ldj.text();
    }

    let mut _decoded_ldjson: Json = Json::from_str("{}").unwrap();

    match Json::from_str(ldjson.as_str()) {
        Ok(v) => _decoded_ldjson = v,
        _ => return None,
    }

    if let Some(date_published) = _decoded_ldjson.search("datePublished") {
        if let Some(date) = date_published.as_string() {
            json_date = Some(date.to_string())
        }
    } else if let Some(date_created) = _decoded_ldjson.search("dateCreated") {
        if let Some(date) = date_created.as_string() {
            json_date = Some(date.to_string())
        }
    }

    json_date
}

// Extract date from meta tags
fn extract_from_meta<'a>(html: &'a Document) -> Option<String> {
    let mut meta_date: Option<String> = None;

    'outer: for meta in html.find(Name("meta")) {
        let meta_name: String     = meta.attr("name").unwrap_or("").to_lowercase();
        let item_prop: String     = meta.attr("itemprop").unwrap_or("").to_lowercase();
        let http_equiv: String    = meta.attr("http-equiv").unwrap_or("").to_lowercase();
        let meta_property: String = meta.attr("property").unwrap_or("").to_lowercase();

        match meta_name.as_ref() {
            "pubdate"               | "publishdate"                  | "timestamp"       |
            "dc.date.issued"        | "date"                         | "sailthru.date"   |
            "article.published"     | "published-date"               | "article.created" |
            "article_date_original" | "cxenseparse:recs:publishtime" | "date_published"  => { if let Some(ct) = meta.attr("content") {
                                                                                meta_date = Some(ct.trim().to_string())
                                                                                }
                                                                                break 'outer; },
            _ => {},
        }

        match item_prop.as_ref() {
            "datepublished" | "datecreated" => { if let Some(ct) = meta.attr("content") {
                                                   meta_date = Some(ct.trim().to_string())
                                                 }
                                                 break 'outer; },
            _ => {},
        }

        match http_equiv.as_ref() {
            "date" =>  { if let Some(ct) = meta.attr("content") {
                           meta_date = Some(ct.trim().to_string())
                         }
                         break 'outer; },
            _ => {},
        }

        match meta_property.as_ref() {
            "article:published_time" | "bt:pubdate" => { if let Some(ct) = meta.attr("content") {
                                                           meta_date = Some(ct.trim().to_string())
                                                         }
                                                         break 'outer; },
            "og:image"                              => { if let Some(url) = meta.attr("content") {
                                                           meta_date = extract_from_url(url.trim())
                                                         }
                                                         break 'outer; },

            _ => {},
        }

    }

    meta_date
}

// Extract from html tags
fn extract_from_html_tag<'a>(html: &'a Document) -> Option<String> {
    lazy_static! {
        static ref TAG_RE: Regex = Regex::new(r"(?i)publishdate|pubdate|timestamp|article_date|articledate|date").unwrap();
    }

    let mut date: Option<String> = None;

    'initial: for time in html.find(Name("time")) {
        if let Some(dt) = time.attr("datetime") {
            { date = Some(dt.to_string());
              break 'initial; }
        } else if let Some("timestamp") = time.attr("class") {
            { date = Some(time.text().trim_matches('\n').to_string());
              break 'initial; }
        }
    }

    if date.is_none() {
        'outer: for tag in html.find(Name("span")) {
            if let Some("datePublished") = tag.attr("itemprop") {
                if let Some(v) = tag.attr("content") {
                    { date = Some(v.to_string());
                      break 'outer; }
                } else if tag.text() != "" {
                    { date = Some(tag.text().trim_matches('\n').to_string());
                      break 'outer; }
                }
            }
        }
    }

    // These next three loops are due to the lack of `find_all` method for select.rs library
    // See also: https://github.com/Webhose/article-date-extractor/blob/master/articleDateExtractor/__init__.py#L191
    if date.is_none() {
        'outer_first: for tag in html.find(Name("span")) {
            if TAG_RE.is_match(tag.attr("class").unwrap_or("")) {
                { date = Some(tag.text().trim_matches('\n').to_string());
                  break 'outer_first; }
            }
        }
    }

    if date.is_none() {
        'outer_second: for tag in html.find(Name("p")) {
            if TAG_RE.is_match(tag.attr("class").unwrap_or("")) {
                { date = Some(tag.text().trim_matches('\n').to_string());
                  break 'outer_second; }
            }
        }
    }

    if date.is_none() {
        'outer_third: for tag in html.find(Name("div")) {
            if TAG_RE.is_match(tag.attr("class").unwrap_or("")) {
                { date = Some(tag.text().trim_matches('\n').to_string());
                  break 'outer_third; }
            }
        }
    }

    date
}

// Try to extract the date by using each function one by one
pub fn extract_article_published_date(link: &str, html: Option<String>) -> Result<NaiveDate> {
    let mut body: String = String::new();
    let mut _parsed_body: Option<Document> = None;

    if let Some(v) = extract_from_url(link) {
        return parse_date(v.as_str())
    }

    if html.is_none() {
        if let Ok(mut response) = reqwest::get(link) {
            response.read_to_string(&mut body).unwrap();
            let doc = Document::from(body.as_str());
            _parsed_body = Some(doc);
        } else {
            return Err("Couldn't open the link".into())
        }
    } else {
        _parsed_body = Some(Document::from(html.unwrap().as_str()))
    }

    if let Some(v) = extract_from_url(link) {
        return parse_date(v.as_str())
    } else if let Some(v) = extract_from_ldjson(_parsed_body.as_ref().unwrap()) {
        return parse_date(v.as_str())
    } else if let Some(v) = extract_from_meta(_parsed_body.as_ref().unwrap()) {
        return parse_date(v.as_str())
    } else if let Some(v) = extract_from_html_tag(_parsed_body.as_ref().unwrap()) {
        return parse_date(v.as_str())
    } else {
        return Err("Couldn't find the date to parse".into())
    }
}

// Unit tests
#[cfg(test)]
mod test {
    use super::extract_from_url;
    use super::parse_date;
    use super::extract_from_meta;
    use super::extract_from_ldjson;
    use super::extract_from_html_tag;
    use chrono::NaiveDate;
    use reqwest;
    use reqwest::Response;
    use std::io::Read;
    use select::document::Document;

    #[test]
    fn parsing_date() {
        assert_eq!(NaiveDate::from_ymd(2015,11,30), parse_date("/2015/11/30/").unwrap());
        assert_eq!(NaiveDate::from_ymd(2015,11,30), parse_date("/2015/30/11/").unwrap());

        assert!(parse_date("bad_format").is_err());
    }

    #[test]
    fn extracting_from_url() {
        let link: &str = "http://edition.cnn.com/2015/11/28/opinions/sutter-cop21-paris-preview-two-degrees/index.html";
        assert_eq!(Some("/2015/11/28/".to_string()), extract_from_url(link));

        let link: &str = "";
        assert_eq!(None, extract_from_url(link));
    }

    #[test]
    fn extracting_from_ldjson() {
        let mut response: Response = reqwest::get("https://techcrunch.com/2015/11/30/atlassian-share-price/").unwrap();
        let mut body: String = String::new();
        response.read_to_string(&mut body).unwrap();
        let document: Document = Document::from(body.as_str());

        assert_eq!(Some("2015-12-01T07:50:48Z".to_string()), extract_from_ldjson(&document));
    }

    #[test]
    fn extracting_from_meta() {
        let mut response: Response = reqwest::get("https://techcrunch.com/2015/11/30/atlassian-share-price/").unwrap();
        let mut body: String = String::new();
        response.read_to_string(&mut body).unwrap();
        let document: Document = Document::from(body.as_str());

        assert_eq!(Some(("2015-11-30 23:50:48".to_string())), extract_from_meta(&document));
    }

    #[test]
    fn extracting_from_html_tag() {
        let mut response: Response = reqwest::get("https://research.googleblog.com/2017/03/announcing-guetzli-new-open-source-jpeg.html").unwrap();
        let mut body: String = String::new();
        response.read_to_string(&mut body).unwrap();
        let document: Document = Document::from(body.as_str());

        assert_eq!(Some("Thursday, March 16, 2017".to_string()), extract_from_html_tag(&document));
    }
}
<commit_msg>Add early returns<commit_after>use regex::Regex;
use chrono::NaiveDate;
use reqwest;
use std::io::Read;
use select::document::Document;
use select::predicate::{Name, Attr};
use rustc_serialize::json::Json;
use errors::*;

// Some formats borrowed from https://github.com/amir/article-date-extractor
static FMTS: &'static [&str] = &["%A, %B %e, %Y",
                                 "%Y-%m-%dT%H:%M:%S%:z",
                                 "/%Y/%m/%d/",
                                 "/%Y/%d/%m/",
                                 "%Y-%m-%d",
                                 "%B %e, %Y",
                                 "%Y-%m-%d %H:%M:%S",
                                 "%Y-%m-%dT%H:%M:%SZ",
                                 "%B %k, %Y, %H:%M %p",
                                 "%Y-%m-%d %H:%M:%S.000000"];

// Use lazy_static to ensure we only compile the regex once
lazy_static! {
    // Regex by Newspaper3k  - https://github.com/codelucas/newspaper/blob/master/newspaper/urls.py
    static ref RE: Regex =
        Regex::new(r"([\./\-_]{0,1}(19|20)\d{2})[\./\-_]{0,1}(([0-3]{0,1}[0-9][\./\-_])|(\w{3,5}[\./\-_]))([0-3]{0,1}[0-9][\./\-]{0,1})").unwrap();
}

// Parse the date, trying out each format
fn parse_date(input: &str) -> Result<NaiveDate> {
    let mut result: Result<NaiveDate> = Err("None of the formats matched the date".into());

    'outer: for fmt in FMTS {
        if let Ok(v) = NaiveDate::parse_from_str(input, fmt) {
            {
                result = Ok(v);
                break 'outer;
            }
        }
    }

    result
}

// Extract date from a URL
fn extract_from_url(url: &str) -> Option<String> {
    if let Some(val) = RE.find(url) {
        return Some(val.as_str().to_string());
    } else {
        return None;
    }
}

// Extract date from JSON-LD
fn extract_from_ldjson<'a>(html: &'a Document) -> Option<String> {
    let mut json_date: Option<String> = None;
    let mut _ldjson: String = String::new();
    if let Some(ldj) = html.find(Attr("type", "application/ld+json")).next() {
        _ldjson = ldj.text();
    } else {
        return None;
    }

    let mut _decoded_ldjson: Json = Json::from_str("{}").unwrap();

    match Json::from_str(&_ldjson) {
        Ok(v) => _decoded_ldjson = v,
        _ => return None,
    }

    if let Some(date_published) = _decoded_ldjson.search("datePublished") {
        if let Some(date) = date_published.as_string() {
            json_date = Some(date.to_string())
        }
    } else if let Some(date_created) = _decoded_ldjson.search("dateCreated") {
        if let Some(date) = date_created.as_string() {
            json_date = Some(date.to_string())
        }
    }

    json_date
}

// Extract date from meta tags
fn extract_from_meta<'a>(html: &'a Document) -> Option<String> {
    let mut meta_date: Option<String> = None;

    'outer: for meta in html.find(Name("meta")) {
        let meta_name: Option<&str> = meta.attr("name");
        let item_prop: Option<&str> = meta.attr("itemprop");
        let http_equiv: Option<&str> = meta.attr("http-equiv");
        let meta_property: Option<&str> = meta.attr("property");

        if let Some(v) = meta_name {
            match v.to_lowercase().as_ref() {
                "pubdate" |
                "publishdate" |
                "timestamp" |
                "dc.date.issued" |
                "date" |
                "sailthru.date" |
                "article.published" |
                "published-date" |
                "article.created" |
                "article_date_original" |
                "cxenseparse:recs:publishtime" |
                "date_published" => {
                    if let Some(ct) = meta.attr("content") {
                        {
                            meta_date = Some(ct.trim().to_string());
                            break 'outer;
                        }
                    }
                }
                _ => {}
            }
        }

        if let Some(v) = item_prop {
            match v.to_lowercase().as_ref() {
                "datepublished" | "datecreated" => {
                    if let Some(ct) = meta.attr("content") {
                        {
                            meta_date = Some(ct.trim().to_string());
                            break 'outer;
                        }
                    }
                }
                _ => {}
            }
        }

        if let Some(v) = http_equiv {
            match v.to_lowercase().as_ref() {
                "date" => {
                    if let Some(ct) = meta.attr("content") {
                        {
                            meta_date = Some(ct.trim().to_string());
                            break 'outer;
                        }
                    }
                }
                _ => {}
            }
        }

        if let Some(v) = meta_property {
            match v.as_ref() {
                "article:published_time" |
                "bt:pubdate" => {
                    if let Some(ct) = meta.attr("content") {
                        {
                            meta_date = Some(ct.trim().to_string());
                            break 'outer;
                        }
                    }
                }
                "og:image" => {
                    if let Some(url) = meta.attr("content") {
                        {
                            meta_date = extract_from_url(url.trim());
                            break 'outer;
                        }
                    }
                }

                _ => {}
            }
        }

    }

    meta_date
}

// Extract from html tags
fn extract_from_html_tag<'a>(html: &'a Document) -> Option<String> {
    lazy_static! {
        static ref TAG_RE: Regex =
            Regex::new(r"(?i)publishdate|pubdate|timestamp|article_date|articledate|date").unwrap();
    }

    let mut date: Option<String> = None;

    'initial: for time in html.find(Name("time")) {
        if let Some(dt) = time.attr("datetime") {
            {
                date = Some(dt.to_string());
                break 'initial;
            }
        } else if let Some("timestamp") = time.attr("class") {
            {
                date = Some(time.text().trim_matches('\n').to_string());
                break 'initial;
            }
        }
    }

    if date.is_none() {
        'outer: for tag in html.find(Name("span")) {
            if let Some("datePublished") = tag.attr("itemprop") {
                if let Some(v) = tag.attr("content") {
                    {
                        date = Some(v.to_string());
                        break 'outer;
                    }
                } else if !tag.text().is_empty() {
                    {
                        date = Some(tag.text().trim_matches('\n').to_string());
                        break 'outer;
                    }
                }
            }
        }
    }

    // These next three loops are due to the lack of `find_all` method for select.rs library
    if date.is_none() {
        'outer_first: for tag in html.find(Name("span")) {
            if TAG_RE.is_match(tag.attr("class").unwrap_or("")) {
                {
                    date = Some(tag.text().trim_matches('\n').to_string());
                    break 'outer_first;
                }
            }
        }
    }

    if date.is_none() {
        'outer_second: for tag in html.find(Name("p")) {
            if TAG_RE.is_match(tag.attr("class").unwrap_or("")) {
                {
                    date = Some(tag.text().trim_matches('\n').to_string());
                    break 'outer_second;
                }
            }
        }
    }

    if date.is_none() {
        'outer_third: for tag in html.find(Name("div")) {
            if TAG_RE.is_match(tag.attr("class").unwrap_or("")) {
                {
                    date = Some(tag.text().trim_matches('\n').to_string());
                    break 'outer_third;
                }
            }
        }
    }

    date
}

// Try to extract the date by using each function one by one
pub fn extract_article_published_date(link: &str, html: Option<String>) -> Result<NaiveDate> {
    let mut body: String = String::new();
    let mut _parsed_body: Option<Document> = None;

    if let Some(v) = extract_from_url(link) {
        return parse_date(&v);
    }

    if html.is_none() {
        if let Ok(mut response) = reqwest::get(link) {
            response.read_to_string(&mut body).unwrap();
            let doc = Document::from(body.as_str());
            _parsed_body = Some(doc);
        } else {
            return Err("Couldn't open the link".into());
        }
    } else {
        _parsed_body = Some(Document::from(html.unwrap().as_str()))
    }

    if let Some(v) = extract_from_url(link) {
        return parse_date(&v);
    } else if let Some(v) = extract_from_ldjson(_parsed_body.as_ref().unwrap()) {
        return parse_date(&v);
    } else if let Some(v) = extract_from_meta(_parsed_body.as_ref().unwrap()) {
        return parse_date(&v);
    } else if let Some(v) = extract_from_html_tag(_parsed_body.as_ref().unwrap()) {
        return parse_date(&v);
    } else {
        return Err("Couldn't find the date to parse".into());
    }
}

// Unit tests
#[cfg(test)]
mod test {
    use super::extract_from_url;
    use super::parse_date;
    use super::extract_from_meta;
    use super::extract_from_ldjson;
    use super::extract_from_html_tag;
    use chrono::NaiveDate;
    use reqwest;
    use reqwest::Response;
    use std::io::Read;
    use select::document::Document;

    #[test]
    fn parsing_date() {
        assert_eq!(NaiveDate::from_ymd(2015, 11, 30),
                   parse_date("/2015/11/30/").unwrap());
        assert_eq!(NaiveDate::from_ymd(2015, 11, 30),
                   parse_date("/2015/30/11/").unwrap());

        assert!(parse_date("bad_format").is_err());
    }

    #[test]
    fn extracting_from_url() {
        let link: &str = "http://edition.cnn.\
                          com/2015/11/28/opinions/sutter-cop21-paris-preview-two-degrees/index.\
                          html";
        assert_eq!(Some("/2015/11/28/".to_string()), extract_from_url(link));

        let link: &str = "";
        assert_eq!(None, extract_from_url(link));
    }

    #[test]
    fn extracting_from_ldjson() {
        let mut response: Response =
            reqwest::get("https://techcrunch.com/2015/11/30/atlassian-share-price/").unwrap();
        let mut body: String = String::new();
        response.read_to_string(&mut body).unwrap();
        let document: Document = Document::from(body.as_str());

        assert_eq!(Some("2015-12-01T07:50:48Z".to_string()),
                   extract_from_ldjson(&document));
    }

    #[test]
    fn extracting_from_meta() {
        let mut response: Response =
            reqwest::get("https://techcrunch.com/2015/11/30/atlassian-share-price/").unwrap();
        let mut body: String = String::new();
        response.read_to_string(&mut body).unwrap();
        let document: Document = Document::from(body.as_str());

        assert_eq!(Some(("2015-11-30 23:50:48".to_string())),
                   extract_from_meta(&document));
    }

    #[test]
    fn extracting_from_html_tag() {
        let mut response: Response =
            reqwest::get("https://research.googleblog.\
                          com/2017/03/announcing-guetzli-new-open-source-jpeg.html")
                .unwrap();
        let mut body: String = String::new();
        response.read_to_string(&mut body).unwrap();
        let document: Document = Document::from(body.as_str());

        assert_eq!(Some("Thursday, March 16, 2017".to_string()),
                   extract_from_html_tag(&document));
    }
}
<|endoftext|>


<commit_before>//use core::cmp::expect;
use core::intrinsics::offset;

use util::ptr::mut_offset;

use rust_core::c_types::c_int;

mod stack;

// TODO: use SSE

#[inline]
fn stosb(s: *mut u8, c: u8, n: uint) {
    unsafe {
        asm!("rep stosb" :: "{al}"(c), "{edi}"(s), "{ecx}"(n))
    }
}

#[inline]
fn stosd(s: *mut u8, c: u32, n: uint) {
    unsafe {
        asm!("rep stosl" :: "A"(c), "{edi}"(s), "{ecx}"(n))
    }
}

#[inline]
fn stosd8(s: *mut u8, c: u8, n: uint) {
    unsafe {
        let mut dword: u32 = c as u32;
        dword |= (dword << 24) | (dword << 16) | (dword << 8);
        asm!("rep stosl" :: "A"(dword), "{edi}"(s), "{ecx}"(n))
    }
}

#[inline]
fn stosd16(s: *mut u8, c: u16, n: uint) {
    unsafe {
        let mut dword: u32 = c as u32;
        dword |= dword << 16;
        asm!("rep stosl" :: "A"(dword), "{edi}"(s), "{ecx}"(n))
    }
}

#[inline]
fn memset_nonzero(mut s: *mut u8, c: u8, mut n: uint) {
    if unlikely!(n == 0) {
        return
    }
    if unlikely!(n == 1) {
        unsafe { *s = c; }
        return
    }

    while n > 0 {
        match n % 4 {
            0 => {
                stosd8(s, c, n / 4);
                n = 0;
            }
            /*2 => unsafe {
                let mut word: u16 = c as u16;
                word = (word << 8) | word;
                asm!("rep stosw" :: "A"(word), "{edi}"(s), "{ecx}"(n / 2))
                n = 0;
            },*/
            q => {
                stosb(s, c, q);
                s = unsafe { mut_offset(s, q as int) };
                n -= q;
            }
        }
    }
}

pub fn wmemset(mut dest: *mut u8, c: u16, n: uint) {
    if unlikely!(n == 0) {
        return;
    }

    if (n % 2) == 1 {
        unsafe {
            *(dest as *mut u16) = c;
            dest = mut_offset(dest, 2);
        }
    }

    stosd16(dest, c, n >> 1);
}

fn dmemset(s: *mut u8, c: u32, n: uint) {
    if unlikely!(n == 0) {
        return;
    }

    stosd(s, c, n);
}

#[no_mangle]
pub fn memset(s: *mut u8, c: c_int, n: int) {
    memset_nonzero(s, (c & 0xFF) as u8, n as uint);
}

#[allow(dead_assignment)]
#[no_mangle]
pub fn memcpy(dest: *mut u8, src: *u8, mut n: uint) {
    if unlikely!(n == 0) {
        return;
    }
    unsafe {
        if n < 12 {
            asm!("rep movsb" :: "{edi}"(dest), "{esi}"(src), "{ecx}"(n))
            return;
        }

        let offset = (4 - (dest as uint % 4)) % 4;
        n -= offset;

        let mut pd: *mut u8;
        let mut ps: *u8;
        asm!("rep movsb" : "={edi}"(pd), "={esi}"(ps) : "{edi}"(dest), "{esi}"(src), "{ecx}"(offset))
        asm!("rep movsl" : "={edi}"(pd), "={esi}"(ps) : "{edi}"(pd), "{esi}"(ps), "{ecx}"(n >> 2))
        asm!("rep movsb" :: "{edi}"(pd), "{esi}"(ps), "{ecx}"(n % 4))
    }
}

#[no_mangle]
pub fn memmove(dest: *mut u8, src: *u8, n: uint) {
    unsafe {
        if src < dest as *u8 {
            asm!("std")
            memcpy(mut_offset(dest, n as int), offset(src, n as int), n);
            asm!("cld")
        }
        else {
            asm!("cld")
            memcpy(dest, src, n);
        }
    }
}
<commit_msg>Add memcmp<commit_after>//use core::cmp::expect;
use core::intrinsics::offset;

use util::ptr::mut_offset;

use rust_core::c_types::c_int;

mod stack;

// TODO: use SSE

#[inline]
fn stosb(s: *mut u8, c: u8, n: uint) {
    unsafe {
        asm!("rep stosb" :: "{al}"(c), "{edi}"(s), "{ecx}"(n))
    }
}

#[inline]
fn stosd(s: *mut u8, c: u32, n: uint) {
    unsafe {
        asm!("rep stosl" :: "A"(c), "{edi}"(s), "{ecx}"(n))
    }
}

#[inline]
fn stosd8(s: *mut u8, c: u8, n: uint) {
    unsafe {
        let mut dword: u32 = c as u32;
        dword |= (dword << 24) | (dword << 16) | (dword << 8);
        asm!("rep stosl" :: "A"(dword), "{edi}"(s), "{ecx}"(n))
    }
}

#[inline]
fn stosd16(s: *mut u8, c: u16, n: uint) {
    unsafe {
        let mut dword: u32 = c as u32;
        dword |= dword << 16;
        asm!("rep stosl" :: "A"(dword), "{edi}"(s), "{ecx}"(n))
    }
}

#[inline]
fn memset_nonzero(mut s: *mut u8, c: u8, mut n: uint) {
    if unlikely!(n == 0) {
        return
    }
    if unlikely!(n == 1) {
        unsafe { *s = c; }
        return
    }

    while n > 0 {
        match n % 4 {
            0 => {
                stosd8(s, c, n / 4);
                n = 0;
            }
            /*2 => unsafe {
                let mut word: u16 = c as u16;
                word = (word << 8) | word;
                asm!("rep stosw" :: "A"(word), "{edi}"(s), "{ecx}"(n / 2))
                n = 0;
            },*/
            q => {
                stosb(s, c, q);
                s = unsafe { mut_offset(s, q as int) };
                n -= q;
            }
        }
    }
}

pub fn wmemset(mut dest: *mut u8, c: u16, n: uint) {
    if unlikely!(n == 0) {
        return;
    }

    if (n % 2) == 1 {
        unsafe {
            *(dest as *mut u16) = c;
            dest = mut_offset(dest, 2);
        }
    }

    stosd16(dest, c, n >> 1);
}

fn dmemset(s: *mut u8, c: u32, n: uint) {
    if unlikely!(n == 0) {
        return;
    }

    stosd(s, c, n);
}

#[no_mangle]
pub fn memset(s: *mut u8, c: c_int, n: int) {
    memset_nonzero(s, (c & 0xFF) as u8, n as uint);
}

#[allow(dead_assignment)]
#[no_mangle]
pub fn memcpy(dest: *mut u8, src: *u8, mut n: uint) {
    if unlikely!(n == 0) {
        return;
    }
    unsafe {
        if n < 12 {
            asm!("rep movsb" :: "{edi}"(dest), "{esi}"(src), "{ecx}"(n))
            return;
        }

        let offset = (4 - (dest as uint % 4)) % 4;
        n -= offset;

        let mut pd: *mut u8;
        let mut ps: *u8;
        asm!("rep movsb" : "={edi}"(pd), "={esi}"(ps) : "{edi}"(dest), "{esi}"(src), "{ecx}"(offset))
        asm!("rep movsl" : "={edi}"(pd), "={esi}"(ps) : "{edi}"(pd), "{esi}"(ps), "{ecx}"(n >> 2))
        asm!("rep movsb" :: "{edi}"(pd), "{esi}"(ps), "{ecx}"(n % 4))
    }
}

#[no_mangle]
pub fn memmove(dest: *mut u8, src: *u8, n: uint) {
    unsafe {
        if src < dest as *u8 {
            asm!("std")
            memcpy(mut_offset(dest, n as int), offset(src, n as int), n);
            asm!("cld")
        }
        else {
            asm!("cld")
            memcpy(dest, src, n);
        }
    }
}

#[no_mangle]
pub unsafe fn memcmp(s1: *u8, s2: *u8, n: uint) -> i32 {
    let mut i = 0;
    while i < n {
        let a = *offset(s1, i as int);
        let b = *offset(s2, i as int);
        if a != b {
            return (a - b) as i32
        }
        i += 1;
    }
    return 0;
}
<|endoftext|>


<commit_before>#![deny(elided_lifetimes_in_paths)]

use actix_web::body::BoxBody;
use actix_web::http::StatusCode;
use actix_web::{HttpResponse, HttpResponseBuilder, ResponseError};
use askama::mime::extension_to_mime_type;
pub use askama::*;

pub trait TemplateToResponse {
    fn to_response(&self) -> HttpResponse<BoxBody>;
}

impl<T: askama::Template> TemplateToResponse for T {
    fn to_response(&self) -> HttpResponse<BoxBody> {
        match self.render() {
            Ok(buffer) => {
                let ctype = extension_to_mime_type(T::EXTENSION.unwrap_or("txt"));
                HttpResponseBuilder::new(StatusCode::OK)
                    .content_type(ctype)
                    .body(buffer)
            }
            Err(err) => {
                HttpResponse::from_error(Box::new(err) as Box<dyn std::error::Error + 'static>)
            }
        }
    }
}
<commit_msg>No needless boxing of the error<commit_after>#![deny(elided_lifetimes_in_paths)]

use std::fmt;

use actix_web::body::BoxBody;
use actix_web::http::StatusCode;
use actix_web::{HttpResponse, HttpResponseBuilder, ResponseError};
use askama::mime::extension_to_mime_type;
pub use askama::*;

/// Newtype to let askama::Error implement actix_web::ResponseError.
struct ActixError(Error);

impl fmt::Debug for ActixError {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        <Error as fmt::Debug>::fmt(&self.0, f)
    }
}

impl fmt::Display for ActixError {
    #[inline]
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        <Error as fmt::Display>::fmt(&self.0, f)
    }
}

impl ResponseError for ActixError {}

pub trait TemplateToResponse {
    fn to_response(&self) -> HttpResponse<BoxBody>;
}

impl<T: askama::Template> TemplateToResponse for T {
    fn to_response(&self) -> HttpResponse<BoxBody> {
        match self.render() {
            Ok(buffer) => {
                let ctype = extension_to_mime_type(T::EXTENSION.unwrap_or("txt"));
                HttpResponseBuilder::new(StatusCode::OK)
                    .content_type(ctype)
                    .body(buffer)
            }
            Err(err) => HttpResponse::from_error(ActixError(err)),
        }
    }
}
<|endoftext|>


<commit_before>extern crate pocketsphinx;
extern crate sphinxad;

enum Language{
    English,
    Russian,
}

impl Language{
    fn print(&self) -> &'static str {
        match *self{
            Language::English => "english",
            Language::Russian => "russian",
        }
    }
}

//select your language
const language:Language = Language::English;

fn main(){
    println!("{}",match language{
        Language::English => "sphinx understands only phrases from english/grammar.jsgf. Say `exit` to exit",
        Language::Russian => "sphinx понимает только фразы из russian/grammar.jsgf. Cкажите `выход` чтобы выйти",
    });

    let (hiddenMarkovModel,grammar,vocabulary)=match language{
        Language::English => ("english/cmusphinx-en-us-ptm-5.2", "english/grammar.jsgf", "english/vocabulary.dict" ),
        Language::Russian => ("russian/zero_ru.cd_cont_4000", "russian/grammar.jsgf", "russian/vocabulary.dict" ),
    };

    let psConfig = pocketsphinx::CmdLn::init(true, &[
        "-hmm", hiddenMarkovModel,
        "-jsgf", grammar,
        "-dict", vocabulary,
        //"-samprate", "16000",
        "-logfn", "log.txt" // 2>log.txt
    ]).unwrap();

    let decoder = pocketsphinx::PsDecoder::init(psConfig);

    //open Microphone and start recording
    let mut device=sphinxad::AudioDevice::default_with_sps(16000).unwrap();
    device.start_recording().unwrap();

    let mut isSpeechStarted=false;
    let mut buffer = [0i16; 2048];

    decoder.start_utt(None).unwrap();

    loop{
        let microphoneData=match device.read(&mut buffer[..]).unwrap(){
            Some(d) => d,
            None => continue,
        };

        decoder.process_raw(microphoneData, false, false).unwrap();

        if decoder.get_in_speech() {
            if !isSpeechStarted {//begining of utterance
                isSpeechStarted=true;
                println!("listening speech");
            }
        }else{
            if isSpeechStarted {//end of utterance
                isSpeechStarted=false;
                println!("end of speech");

                decoder.end_utt().unwrap();

                match decoder.get_hyp() {
                    None => println!("Not recognized"),
                    Some((hypothesis, _utt_id, _score)) => {
                        println!("Recognized: {}", hypothesis);

                        for seg in decoder.seg_iter() {
                            println!("{} : {}, {}",seg.word(), seg.prob().ascr, seg.prob().lscr);
                        }

                        if hypothesis==match language {
                            Language::English => "EXIT",
                            Language::Russian => "выход",
                        }{
                            break;
                        }
                    },
                }

                decoder.start_utt(None).unwrap();
            }
        }
    }
}
<commit_msg>come changes<commit_after>extern crate pocketsphinx;
extern crate sphinxad;

enum Language{
    English,
    Russian,
}

impl Language{
    fn print(&self) -> &'static str {
        match *self{
            Language::English => "english",
            Language::Russian => "russian",
        }
    }
}

//select your language
const language:Language = Language::English;

fn main(){
    println!("{}",match language{
        Language::English => "sphinx understands only phrases from english/grammar.jsgf. Say `exit` to exit",
        Language::Russian => "sphinx понимает только фразы из russian/grammar.jsgf. Cкажите `выход` чтобы выйти",
    });

    let (hiddenMarkovModel,grammar,vocabulary)=match language{
        Language::English => ("english/cmusphinx-en-us-ptm-5.2", "english/grammar.jsgf", "english/vocabulary.dict" ),
        Language::Russian => ("russian/zero_ru.cd_cont_4000", "russian/grammar.jsgf", "russian/vocabulary.dict" ),
    };

    let psConfig = pocketsphinx::CmdLn::init(true, &[
        "-hmm", hiddenMarkovModel,
        "-jsgf", grammar,
        "-dict", vocabulary,
        //"-samprate", "16000",
        "-logfn", "log.txt" // 2>log.txt If you will get panics, remove this line to see log in console
    ]).unwrap();

    let decoder = pocketsphinx::PsDecoder::init(psConfig);

    //open Microphone and start recording
    let mut device=sphinxad::AudioDevice::default_with_sps(16000).unwrap();
    device.start_recording().unwrap();

    let mut isSpeechStarted=false;
    let mut buffer = [0i16; 2048];

    decoder.start_utt(None).unwrap();

    loop{
        let microphoneData=match device.read(&mut buffer[..]).unwrap(){
            Some(d) => d,
            None => continue,
        };

        decoder.process_raw(microphoneData, false, false).unwrap();

        if decoder.get_in_speech() {
            if !isSpeechStarted {//begining of utterance
                isSpeechStarted=true;
                println!("listening speech");
            }
        }else{
            if isSpeechStarted {//end of utterance
                isSpeechStarted=false;
                println!("end of speech");

                decoder.end_utt().unwrap();

                match decoder.get_hyp() {
                    None => println!("Not recognized"),
                    Some((hypothesis, _utt_id, _score)) => {
                        println!("Recognized: {}", hypothesis);

                        for seg in decoder.seg_iter() {
                            println!("{} : {}, {}",seg.word(), seg.prob().ascr, seg.prob().lscr);
                        }

                        if hypothesis==match language {
                            Language::English => "EXIT",
                            Language::Russian => "выход",
                        }{
                            break;
                        }
                    },
                }

                decoder.start_utt(None).unwrap();
            }
        }
    }
}
<|endoftext|>


<commit_before>use gmp::mpq::Mpq;
use gmp::mpz::Mpz;
use std::borrow::Borrow;
use std::collections::hash_map::{Entry, HashMap};
use std::error::Error;
use std::fs::File;
use std::io::{BufRead, BufReader, Read, stdin};
use std::result::Result;
use std::str::FromStr;

pub struct BallotParser {
    pub candidates: Vec<String>,
    pub candidate_index: HashMap<String, usize>,
    pub ballots: Vec<(Box<[Box<[usize]>]>, Mpq)>,
}

impl BallotParser {
    fn new() -> BallotParser {
        return BallotParser {
            candidates: Vec::new(),
            candidate_index: HashMap::new(),
            ballots: Vec::new(),
        };
    }

    fn parse_candidate(&mut self, name: &str) -> usize {
        let name = name.trim();
        match self.candidate_index.entry(name.to_string()) {
            Entry::Occupied(e) => *e.get(),
            Entry::Vacant(e) => {
                let n = self.candidates.len();
                e.insert(n);
                self.candidates.push(name.to_string());
                n
            }
        }
    }

    fn parse_group(&mut self, group: &str) -> Box<[usize]> {
        group.split('=')
            .map(|name| self.parse_candidate(name))
            .collect::<Vec<_>>()
            .into_boxed_slice()
    }

    fn parse_groups(&mut self, groups: &str) -> Box<[Box<[usize]>]> {
        groups.split('>')
            .map(|group| self.parse_group(group))
            .collect::<Vec<_>>()
            .into_boxed_slice()
    }

    fn add_ballot(&mut self, line: &str) -> Result<(), String> {
        let line = line.trim();
        if line.is_empty() {
            return Ok(());
        }

        let (w, groups) = match line.find(':') {
            Some(i) => {
                (Mpq::ratio(&Mpz::from_str(&line[..i]).map_err(|()|
                                                               "cannot parse ballot weight"
                )?, &Mpz::one()), &line[i + 1..])
            }
            None => (Mpq::one(), &line[..]),
        };

        let ballot = (self.parse_groups(groups), w);
        self.ballots.push(ballot);
        Ok(())
    }

    fn add_ballots<R: Read>(&mut self, buf: BufReader<R>) -> Result<(), (usize, String)> {
        for (lineno, line) in buf.lines().enumerate() {
            self.add_ballot(&line.map_err(|e| (lineno, String::from(e.description())))?)
                .map_err(|e| (lineno, e))?;
        }
        Ok(())
    }

    fn add_ballot_file(&mut self, filename: &str) -> Result<(), String> {
        let file: Box<Read> = if filename == "-" {
            Box::new(stdin())
        } else {
            Box::new(File::open(filename).map_err(|e| format!("error: {}: {}", filename, e))?)
        };
        self.add_ballots(BufReader::new(file))
            .map_err(|(lineno, e)| format!("{}:{}: error: {}", filename, lineno + 1, e))
    }
}

pub fn parse_ballot_files<Str>(filenames: &[Str]) -> Result<BallotParser, String>
    where Str: Borrow<str>
{
    let mut bp = BallotParser::new();
    for filename in filenames.iter() {
        bp.add_ballot_file(filename.borrow())?;
    }
    Ok(bp)
}
<commit_msg>Support rational ballot weights<commit_after>use gmp::mpq::Mpq;
use gmp::mpz::Mpz;
use std::borrow::Borrow;
use std::collections::hash_map::{Entry, HashMap};
use std::error::Error;
use std::fs::File;
use std::io::{BufRead, BufReader, Read, stdin};
use std::result::Result;
use std::str::FromStr;

pub struct BallotParser {
    pub candidates: Vec<String>,
    pub candidate_index: HashMap<String, usize>,
    pub ballots: Vec<(Box<[Box<[usize]>]>, Mpq)>,
}

fn parse_rational(s: &str) -> Result<Mpq, ()> {
    match s.find('/') {
        Some(i) => Ok(Mpq::ratio(&Mpz::from_str(&s[..i])?, &Mpz::from_str(&s[i + 1..])?)),
        None => Ok(Mpq::ratio(&Mpz::from_str(s)?, &Mpz::one())),
    }
}

impl BallotParser {
    fn new() -> BallotParser {
        return BallotParser {
            candidates: Vec::new(),
            candidate_index: HashMap::new(),
            ballots: Vec::new(),
        };
    }

    fn parse_candidate(&mut self, name: &str) -> usize {
        let name = name.trim();
        match self.candidate_index.entry(name.to_string()) {
            Entry::Occupied(e) => *e.get(),
            Entry::Vacant(e) => {
                let n = self.candidates.len();
                e.insert(n);
                self.candidates.push(name.to_string());
                n
            }
        }
    }

    fn parse_group(&mut self, group: &str) -> Box<[usize]> {
        group.split('=')
            .map(|name| self.parse_candidate(name))
            .collect::<Vec<_>>()
            .into_boxed_slice()
    }

    fn parse_groups(&mut self, groups: &str) -> Box<[Box<[usize]>]> {
        groups.split('>')
            .map(|group| self.parse_group(group))
            .collect::<Vec<_>>()
            .into_boxed_slice()
    }

    fn add_ballot(&mut self, line: &str) -> Result<(), String> {
        let line = line.trim();
        if line.is_empty() {
            return Ok(());
        }

        let (w, groups) = match line.find(':') {
            Some(i) =>
                (parse_rational(&line[..i]).map_err(|()| "cannot parse ballot weight")?,
                 &line[i + 1..]),
            None => (Mpq::one(), &line[..]),
        };

        let ballot = (self.parse_groups(groups), w);
        self.ballots.push(ballot);
        Ok(())
    }

    fn add_ballots<R: Read>(&mut self, buf: BufReader<R>) -> Result<(), (usize, String)> {
        for (lineno, line) in buf.lines().enumerate() {
            self.add_ballot(&line.map_err(|e| (lineno, String::from(e.description())))?)
                .map_err(|e| (lineno, e))?;
        }
        Ok(())
    }

    fn add_ballot_file(&mut self, filename: &str) -> Result<(), String> {
        let file: Box<Read> = if filename == "-" {
            Box::new(stdin())
        } else {
            Box::new(File::open(filename).map_err(|e| format!("error: {}: {}", filename, e))?)
        };
        self.add_ballots(BufReader::new(file))
            .map_err(|(lineno, e)| format!("{}:{}: error: {}", filename, lineno + 1, e))
    }
}

pub fn parse_ballot_files<Str>(filenames: &[Str]) -> Result<BallotParser, String>
    where Str: Borrow<str>
{
    let mut bp = BallotParser::new();
    for filename in filenames.iter() {
        bp.add_ballot_file(filename.borrow())?;
    }
    Ok(bp)
}
<|endoftext|>


<commit_before>// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.

use engine_rocks::RocksSnapshot;
use kvproto::raft_cmdpb::{RaftCmdRequest, RaftCmdResponse};
use raft::eraftpb::MessageType;
use raftstore::store::msg::*;
use test_raftstore::*;
use tikv_util::HandyRwLock;

use std::sync::mpsc::{self, TryRecvError};
use std::time::Duration;

struct CbReceivers {
    proposed: mpsc::Receiver<()>,
    committed: mpsc::Receiver<()>,
    applied: mpsc::Receiver<RaftCmdResponse>,
}

impl CbReceivers {
    fn assert_not_ready(&self) {
        sleep_ms(100);
        assert_eq!(self.proposed.try_recv().unwrap_err(), TryRecvError::Empty);
        assert_eq!(self.committed.try_recv().unwrap_err(), TryRecvError::Empty);
        assert_eq!(self.applied.try_recv().unwrap_err(), TryRecvError::Empty);
    }

    fn assert_ok(&self) {
        let resp = self.applied.recv_timeout(Duration::from_secs(1)).unwrap();
        assert!(
            !resp.get_header().has_error(),
            "{:?}",
            resp.get_header().get_error()
        );
        // proposed and committed should be invoked before applied
        self.proposed.try_recv().unwrap();
        self.committed.try_recv().unwrap();
    }

    // When fails to propose, only applied callback will be invoked.
    fn assert_err(&self) {
        let resp = self.applied.recv_timeout(Duration::from_secs(1)).unwrap();
        assert!(resp.get_header().has_error(), "{:?}", resp);
        self.proposed.try_recv().unwrap_err();
        self.committed.try_recv().unwrap_err();
    }
}

fn make_cb(cmd: &RaftCmdRequest) -> (Callback<RocksSnapshot>, CbReceivers) {
    let (proposed_tx, proposed_rx) = mpsc::channel();
    let (committed_tx, committed_rx) = mpsc::channel();
    let (cb, applied_rx) = make_cb_ext(
        cmd,
        Some(Box::new(move || proposed_tx.send(()).unwrap())),
        Some(Box::new(move || committed_tx.send(()).unwrap())),
    );
    (
        cb,
        CbReceivers {
            proposed: proposed_rx,
            committed: committed_rx,
            applied: applied_rx,
        },
    )
}

fn make_write_req(cluster: &mut Cluster<NodeCluster>, k: &[u8]) -> RaftCmdRequest {
    let r = cluster.get_region(k);
    let mut req = new_request(
        r.get_id(),
        r.get_region_epoch().clone(),
        vec![new_put_cmd(k, b"v")],
        false,
    );
    let leader = cluster.leader_of_region(r.get_id()).unwrap();
    req.mut_header().set_peer(leader);
    req
}

#[test]
fn test_reject_proposal_during_region_split() {
    let mut cluster = new_node_cluster(0, 3);
    let pd_client = cluster.pd_client.clone();
    pd_client.disable_default_operator();
    cluster.run();
    cluster.must_transfer_leader(1, new_peer(1, 1));
    cluster.must_put(b"k", b"v");

    // Pause on applying so that region split is not finished.
    let fp = "apply_before_split";
    fail::cfg(fp, "pause").unwrap();

    // Try to split region.
    let (split_tx, split_rx) = mpsc::channel();
    let cb = Callback::Read(Box::new(move |resp: ReadResponse<RocksSnapshot>| {
        split_tx.send(resp.response).unwrap()
    }));
    let r = cluster.get_region(b"");
    cluster.split_region(&r, b"k", cb);
    split_rx
        .recv_timeout(Duration::from_millis(100))
        .unwrap_err();

    // Try to put a key.
    let write_req = make_write_req(&mut cluster, b"k1");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    // The write request should be blocked until split is finished.
    cb_receivers.assert_not_ready();

    fail::remove(fp);
    // Split is finished.
    assert!(
        !split_rx
            .recv_timeout(Duration::from_secs(1))
            .unwrap()
            .get_header()
            .has_error()
    );

    // The write request fails due to epoch not match.
    cb_receivers.assert_err();

    // New write request can succeed.
    let write_req = make_write_req(&mut cluster, b"k1");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_ok();
}

#[test]
fn test_reject_proposal_during_region_merge() {
    let mut cluster = new_node_cluster(0, 3);
    configure_for_merge(&mut cluster);
    let pd_client = cluster.pd_client.clone();
    pd_client.disable_default_operator();
    cluster.run();
    cluster.must_transfer_leader(1, new_peer(1, 1));
    cluster.must_put(b"k", b"v");

    let r = cluster.get_region(b"");
    cluster.must_split(&r, b"k");
    // Let the new region catch up.
    cluster.must_put(b"a", b"v");
    cluster.must_put(b"k", b"v");

    let prepare_merge_fp = "apply_before_prepare_merge";
    let commit_merge_fp = "apply_before_commit_merge";

    // Pause on applying so that prepare-merge is not finished.
    fail::cfg(prepare_merge_fp, "pause").unwrap();
    // Try to merge region.
    let (merge_tx, merge_rx) = mpsc::channel();
    let cb = Callback::Read(Box::new(move |resp: ReadResponse<RocksSnapshot>| {
        merge_tx.send(resp.response).unwrap()
    }));
    let source = cluster.get_region(b"");
    let target = cluster.get_region(b"k");
    cluster.merge_region(source.get_id(), target.get_id(), cb);
    merge_rx
        .recv_timeout(Duration::from_millis(100))
        .unwrap_err();

    // Try to put a key on the source region.
    let write_req = make_write_req(&mut cluster, b"a");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    // The write request should be blocked until prepare-merge is finished.
    cb_receivers.assert_not_ready();

    // Pause on the second phase of region merge.
    fail::cfg(commit_merge_fp, "pause").unwrap();

    // prepare-merge is finished.
    fail::remove(prepare_merge_fp);
    assert!(
        !merge_rx
            .recv_timeout(Duration::from_secs(1))
            .unwrap()
            .get_header()
            .has_error()
    );
    // The write request fails due to epoch not match.
    cb_receivers.assert_err();

    // Write request is rejected because the source region is merging.
    // It's not handled by epoch checker now.
    let write_req = make_write_req(&mut cluster, b"a");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_err();

    // Try to put a key on the target region.
    let write_req = make_write_req(&mut cluster, b"k");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    // The write request should be blocked until commit-merge is finished.
    cb_receivers.assert_not_ready();

    // Wait for region merge done.
    fail::remove(commit_merge_fp);
    pd_client.check_merged_timeout(source.get_id(), Duration::from_secs(1));
    // The write request fails due to epoch not match.
    cb_receivers.assert_err();

    // New write request can succeed.
    let write_req = make_write_req(&mut cluster, b"k");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_ok();
}

#[test]
fn test_reject_proposal_during_rollback_region_merge() {
    let mut cluster = new_node_cluster(0, 2);
    configure_for_merge(&mut cluster);
    let pd_client = cluster.pd_client.clone();
    pd_client.disable_default_operator();
    cluster.run_conf_change();

    let r = cluster.get_region(b"");
    cluster.must_split(&r, b"k");

    // Don't enter the second phase of region merge.
    let schedule_merge_fp = "on_schedule_merge";
    fail::cfg(schedule_merge_fp, "return()").unwrap();

    let source = cluster.get_region(b"");
    let target = cluster.get_region(b"k");
    // The call is finished when prepare_merge is applied.
    cluster.must_try_merge(source.get_id(), target.get_id());

    // Add a peer to trigger rollback.
    pd_client.must_add_peer(target.get_id(), new_peer(2, 4));
    cluster.must_put(b"k", b"v");
    must_get_equal(&cluster.get_engine(1), b"k", b"v");

    // Pause on applying so that rolling back merge is not finished.
    let rollback_merge_fp = "apply_before_rollback_merge";
    fail::cfg(rollback_merge_fp, "pause").unwrap();
    fail::remove(schedule_merge_fp);
    sleep_ms(200);

    // Write request is rejected because the source region is merging.
    // It's not handled by epoch checker now.
    let write_req = make_write_req(&mut cluster, b"a");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_err();

    fail::remove(rollback_merge_fp);
    // Make sure the rollback is done.
    cluster.must_put(b"a", b"v");

    // New write request can succeed.
    let write_req = make_write_req(&mut cluster, b"a");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_ok();
}

#[test]
fn test_reject_proposal_during_leader_transfer() {
    let mut cluster = new_node_cluster(0, 2);
    let pd_client = cluster.pd_client.clone();
    pd_client.disable_default_operator();
    let r = cluster.run_conf_change();
    pd_client.must_add_peer(r, new_peer(2, 2));

    // Don't allow leader transfer succeed if it is actually triggered.
    cluster.add_send_filter(CloneFilterFactory(
        RegionPacketFilter::new(r, 2)
            .msg_type(MessageType::MsgTimeoutNow)
            .direction(Direction::Recv),
    ));

    cluster.must_put(b"k", b"v");
    cluster.transfer_leader(r, new_peer(2, 2));
    // The leader can't change to transferring state immediately due to pre-transfer-leader
    // feature, so wait for a while.
    sleep_ms(100);
    assert_ne!(cluster.leader_of_region(r).unwrap(), new_peer(2, 2));

    let write_req = make_write_req(&mut cluster, b"k");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_err();
    cluster.clear_send_filters();
}

#[test]
fn test_accept_proposal_during_conf_change() {
    let mut cluster = new_node_cluster(0, 2);
    cluster.pd_client.disable_default_operator();
    let r = cluster.run_conf_change();
    cluster.must_put(b"a", b"v");

    let conf_change_fp = "apply_on_conf_change_all_1";
    fail::cfg(conf_change_fp, "pause").unwrap();
    let add_peer_rx = cluster.async_add_peer(r, new_peer(2, 2)).unwrap();
    add_peer_rx
        .recv_timeout(Duration::from_millis(100))
        .unwrap_err();

    // Conf change doesn't affect proposals.
    let write_req = make_write_req(&mut cluster, b"k");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers
        .committed
        .recv_timeout(Duration::from_millis(300))
        .unwrap();
    cb_receivers.proposed.try_recv().unwrap();

    fail::remove(conf_change_fp);
    assert!(
        !add_peer_rx
            .recv_timeout(Duration::from_secs(1))
            .unwrap()
            .get_header()
            .has_error()
    );
    assert!(
        !cb_receivers
            .applied
            .recv_timeout(Duration::from_secs(1))
            .unwrap()
            .get_header()
            .has_error()
    );
    must_get_equal(&cluster.get_engine(2), b"k", b"v");
}

#[test]
fn test_not_invoke_committed_cb_when_fail_to_commit() {
    let mut cluster = new_node_cluster(0, 3);
    cluster.pd_client.disable_default_operator();
    cluster.run();
    cluster.must_transfer_leader(1, new_peer(1, 1));
    cluster.must_put(b"k", b"v");

    // Partiton the leader and followers to let the leader fails to commit the proposal.
    cluster.partition(vec![1], vec![2, 3]);
    let write_req = make_write_req(&mut cluster, b"k1");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    // Check the request is proposed but not committed.
    cb_receivers
        .committed
        .recv_timeout(Duration::from_millis(200))
        .unwrap_err();
    cb_receivers.proposed.try_recv().unwrap();

    // The election timeout is 250ms by default.
    let election_timeout = cluster.cfg.raft_store.raft_base_tick_interval.0
        * cluster.cfg.raft_store.raft_election_timeout_ticks as u32;
    std::thread::sleep(2 * election_timeout);

    // Make sure a new leader is elected and will discard the previous proposal when partition is
    // recovered.
    cluster.must_put(b"k2", b"v");
    cluster.clear_send_filters();

    let resp = cb_receivers
        .applied
        .recv_timeout(Duration::from_secs(1))
        .unwrap();
    assert!(resp.get_header().has_error(), "{:?}", resp);
    // The committed callback shouldn't be invoked.
    cb_receivers.committed.try_recv().unwrap_err();
}
<commit_msg>test: fix test_reject_proposal_during_region_merge (#10627)<commit_after>// Copyright 2020 TiKV Project Authors. Licensed under Apache-2.0.

use engine_rocks::RocksSnapshot;
use kvproto::raft_cmdpb::{RaftCmdRequest, RaftCmdResponse};
use raft::eraftpb::MessageType;
use raftstore::store::msg::*;
use test_raftstore::*;
use tikv_util::HandyRwLock;

use std::sync::mpsc::{self, TryRecvError};
use std::time::Duration;

struct CbReceivers {
    proposed: mpsc::Receiver<()>,
    committed: mpsc::Receiver<()>,
    applied: mpsc::Receiver<RaftCmdResponse>,
}

impl CbReceivers {
    fn assert_not_ready(&self) {
        sleep_ms(100);
        assert_eq!(self.proposed.try_recv().unwrap_err(), TryRecvError::Empty);
        assert_eq!(self.committed.try_recv().unwrap_err(), TryRecvError::Empty);
        assert_eq!(self.applied.try_recv().unwrap_err(), TryRecvError::Empty);
    }

    fn assert_ok(&self) {
        let resp = self.applied.recv_timeout(Duration::from_secs(1)).unwrap();
        assert!(
            !resp.get_header().has_error(),
            "{:?}",
            resp.get_header().get_error()
        );
        // proposed and committed should be invoked before applied
        self.proposed.try_recv().unwrap();
        self.committed.try_recv().unwrap();
    }

    // When fails to propose, only applied callback will be invoked.
    fn assert_err(&self) {
        let resp = self.applied.recv_timeout(Duration::from_secs(1)).unwrap();
        assert!(resp.get_header().has_error(), "{:?}", resp);
        self.proposed.try_recv().unwrap_err();
        self.committed.try_recv().unwrap_err();
    }
}

fn make_cb(cmd: &RaftCmdRequest) -> (Callback<RocksSnapshot>, CbReceivers) {
    let (proposed_tx, proposed_rx) = mpsc::channel();
    let (committed_tx, committed_rx) = mpsc::channel();
    let (cb, applied_rx) = make_cb_ext(
        cmd,
        Some(Box::new(move || proposed_tx.send(()).unwrap())),
        Some(Box::new(move || committed_tx.send(()).unwrap())),
    );
    (
        cb,
        CbReceivers {
            proposed: proposed_rx,
            committed: committed_rx,
            applied: applied_rx,
        },
    )
}

fn make_write_req(cluster: &mut Cluster<NodeCluster>, k: &[u8]) -> RaftCmdRequest {
    let r = cluster.get_region(k);
    let mut req = new_request(
        r.get_id(),
        r.get_region_epoch().clone(),
        vec![new_put_cmd(k, b"v")],
        false,
    );
    let leader = cluster.leader_of_region(r.get_id()).unwrap();
    req.mut_header().set_peer(leader);
    req
}

#[test]
fn test_reject_proposal_during_region_split() {
    let mut cluster = new_node_cluster(0, 3);
    let pd_client = cluster.pd_client.clone();
    pd_client.disable_default_operator();
    cluster.run();
    cluster.must_transfer_leader(1, new_peer(1, 1));
    cluster.must_put(b"k", b"v");

    // Pause on applying so that region split is not finished.
    let fp = "apply_before_split";
    fail::cfg(fp, "pause").unwrap();

    // Try to split region.
    let (split_tx, split_rx) = mpsc::channel();
    let cb = Callback::Read(Box::new(move |resp: ReadResponse<RocksSnapshot>| {
        split_tx.send(resp.response).unwrap()
    }));
    let r = cluster.get_region(b"");
    cluster.split_region(&r, b"k", cb);
    split_rx
        .recv_timeout(Duration::from_millis(100))
        .unwrap_err();

    // Try to put a key.
    let write_req = make_write_req(&mut cluster, b"k1");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    // The write request should be blocked until split is finished.
    cb_receivers.assert_not_ready();

    fail::remove(fp);
    // Split is finished.
    assert!(
        !split_rx
            .recv_timeout(Duration::from_secs(1))
            .unwrap()
            .get_header()
            .has_error()
    );

    // The write request fails due to epoch not match.
    cb_receivers.assert_err();

    // New write request can succeed.
    let write_req = make_write_req(&mut cluster, b"k1");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_ok();
}

#[test]
fn test_reject_proposal_during_region_merge() {
    let mut cluster = new_node_cluster(0, 3);
    configure_for_merge(&mut cluster);
    let pd_client = cluster.pd_client.clone();
    pd_client.disable_default_operator();
    cluster.run();
    cluster.must_transfer_leader(1, new_peer(1, 1));
    cluster.must_put(b"k", b"v");

    let r = cluster.get_region(b"");
    cluster.must_split(&r, b"k");
    // Let the new region catch up.
    cluster.must_put(b"a", b"v");
    cluster.must_put(b"k", b"v");

    let prepare_merge_fp = "apply_before_prepare_merge";
    let commit_merge_fp = "apply_before_commit_merge";

    // Pause on applying so that prepare-merge is not finished.
    fail::cfg(prepare_merge_fp, "pause").unwrap();
    // Try to merge region.
    let (merge_tx, merge_rx) = mpsc::channel();
    let cb = Callback::Read(Box::new(move |resp: ReadResponse<RocksSnapshot>| {
        merge_tx.send(resp.response).unwrap()
    }));
    let source = cluster.get_region(b"");
    let target = cluster.get_region(b"k");
    cluster.merge_region(source.get_id(), target.get_id(), cb);
    merge_rx
        .recv_timeout(Duration::from_millis(100))
        .unwrap_err();

    // Try to put a key on the source region.
    let write_req = make_write_req(&mut cluster, b"a");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    // The write request should be blocked until prepare-merge is finished.
    cb_receivers.assert_not_ready();

    // Pause on the second phase of region merge.
    fail::cfg(commit_merge_fp, "pause").unwrap();

    // prepare-merge is finished.
    fail::remove(prepare_merge_fp);
    assert!(
        !merge_rx
            .recv_timeout(Duration::from_secs(5))
            .unwrap()
            .get_header()
            .has_error()
    );
    // The write request fails due to epoch not match.
    cb_receivers.assert_err();

    // Write request is rejected because the source region is merging.
    // It's not handled by epoch checker now.
    let write_req = make_write_req(&mut cluster, b"a");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_err();

    // Try to put a key on the target region.
    let write_req = make_write_req(&mut cluster, b"k");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    // The write request should be blocked until commit-merge is finished.
    cb_receivers.assert_not_ready();

    // Wait for region merge done.
    fail::remove(commit_merge_fp);
    pd_client.check_merged_timeout(source.get_id(), Duration::from_secs(5));
    // The write request fails due to epoch not match.
    cb_receivers.assert_err();

    // New write request can succeed.
    let write_req = make_write_req(&mut cluster, b"k");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_ok();
}

#[test]
fn test_reject_proposal_during_rollback_region_merge() {
    let mut cluster = new_node_cluster(0, 2);
    configure_for_merge(&mut cluster);
    let pd_client = cluster.pd_client.clone();
    pd_client.disable_default_operator();
    cluster.run_conf_change();

    let r = cluster.get_region(b"");
    cluster.must_split(&r, b"k");

    // Don't enter the second phase of region merge.
    let schedule_merge_fp = "on_schedule_merge";
    fail::cfg(schedule_merge_fp, "return()").unwrap();

    let source = cluster.get_region(b"");
    let target = cluster.get_region(b"k");
    // The call is finished when prepare_merge is applied.
    cluster.must_try_merge(source.get_id(), target.get_id());

    // Add a peer to trigger rollback.
    pd_client.must_add_peer(target.get_id(), new_peer(2, 4));
    cluster.must_put(b"k", b"v");
    must_get_equal(&cluster.get_engine(1), b"k", b"v");

    // Pause on applying so that rolling back merge is not finished.
    let rollback_merge_fp = "apply_before_rollback_merge";
    fail::cfg(rollback_merge_fp, "pause").unwrap();
    fail::remove(schedule_merge_fp);
    sleep_ms(200);

    // Write request is rejected because the source region is merging.
    // It's not handled by epoch checker now.
    let write_req = make_write_req(&mut cluster, b"a");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_err();

    fail::remove(rollback_merge_fp);
    // Make sure the rollback is done.
    cluster.must_put(b"a", b"v");

    // New write request can succeed.
    let write_req = make_write_req(&mut cluster, b"a");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_ok();
}

#[test]
fn test_reject_proposal_during_leader_transfer() {
    let mut cluster = new_node_cluster(0, 2);
    let pd_client = cluster.pd_client.clone();
    pd_client.disable_default_operator();
    let r = cluster.run_conf_change();
    pd_client.must_add_peer(r, new_peer(2, 2));

    // Don't allow leader transfer succeed if it is actually triggered.
    cluster.add_send_filter(CloneFilterFactory(
        RegionPacketFilter::new(r, 2)
            .msg_type(MessageType::MsgTimeoutNow)
            .direction(Direction::Recv),
    ));

    cluster.must_put(b"k", b"v");
    cluster.transfer_leader(r, new_peer(2, 2));
    // The leader can't change to transferring state immediately due to pre-transfer-leader
    // feature, so wait for a while.
    sleep_ms(100);
    assert_ne!(cluster.leader_of_region(r).unwrap(), new_peer(2, 2));

    let write_req = make_write_req(&mut cluster, b"k");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers.assert_err();
    cluster.clear_send_filters();
}

#[test]
fn test_accept_proposal_during_conf_change() {
    let mut cluster = new_node_cluster(0, 2);
    cluster.pd_client.disable_default_operator();
    let r = cluster.run_conf_change();
    cluster.must_put(b"a", b"v");

    let conf_change_fp = "apply_on_conf_change_all_1";
    fail::cfg(conf_change_fp, "pause").unwrap();
    let add_peer_rx = cluster.async_add_peer(r, new_peer(2, 2)).unwrap();
    add_peer_rx
        .recv_timeout(Duration::from_millis(100))
        .unwrap_err();

    // Conf change doesn't affect proposals.
    let write_req = make_write_req(&mut cluster, b"k");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    cb_receivers
        .committed
        .recv_timeout(Duration::from_millis(300))
        .unwrap();
    cb_receivers.proposed.try_recv().unwrap();

    fail::remove(conf_change_fp);
    assert!(
        !add_peer_rx
            .recv_timeout(Duration::from_secs(1))
            .unwrap()
            .get_header()
            .has_error()
    );
    assert!(
        !cb_receivers
            .applied
            .recv_timeout(Duration::from_secs(1))
            .unwrap()
            .get_header()
            .has_error()
    );
    must_get_equal(&cluster.get_engine(2), b"k", b"v");
}

#[test]
fn test_not_invoke_committed_cb_when_fail_to_commit() {
    let mut cluster = new_node_cluster(0, 3);
    cluster.pd_client.disable_default_operator();
    cluster.run();
    cluster.must_transfer_leader(1, new_peer(1, 1));
    cluster.must_put(b"k", b"v");

    // Partiton the leader and followers to let the leader fails to commit the proposal.
    cluster.partition(vec![1], vec![2, 3]);
    let write_req = make_write_req(&mut cluster, b"k1");
    let (cb, cb_receivers) = make_cb(&write_req);
    cluster
        .sim
        .rl()
        .async_command_on_node(1, write_req, cb)
        .unwrap();
    // Check the request is proposed but not committed.
    cb_receivers
        .committed
        .recv_timeout(Duration::from_millis(200))
        .unwrap_err();
    cb_receivers.proposed.try_recv().unwrap();

    // The election timeout is 250ms by default.
    let election_timeout = cluster.cfg.raft_store.raft_base_tick_interval.0
        * cluster.cfg.raft_store.raft_election_timeout_ticks as u32;
    std::thread::sleep(2 * election_timeout);

    // Make sure a new leader is elected and will discard the previous proposal when partition is
    // recovered.
    cluster.must_put(b"k2", b"v");
    cluster.clear_send_filters();

    let resp = cb_receivers
        .applied
        .recv_timeout(Duration::from_secs(1))
        .unwrap();
    assert!(resp.get_header().has_error(), "{:?}", resp);
    // The committed callback shouldn't be invoked.
    cb_receivers.committed.try_recv().unwrap_err();
}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate log;
extern crate osmpbfreader;
extern crate rustc_serialize;
extern crate docopt;
extern crate mimir;
extern crate rs_es;

use std::collections::HashSet;
use std::collections::HashMap;
use mimir::rubber::Rubber;
use mimir::objects::{Polygon, MultiPolygon};

pub type AdminsVec = Vec<mimir::Admin>;
pub type StreetsVec = Vec< mimir::Street>;
pub type ParsedPbf = osmpbfreader::OsmPbfReader<std::fs::File>;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_connection_string: String,
}

static USAGE: &'static str = "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> [--connection-string=<connection-string>] --level=<level>...

Options:
    -h, --help            Show this message.
    -i, --input=<file>    OSM PBF file.
    -l, --level=<level>   Admin levels to keep.
    -c, --connection-string=<connection-string>
                          Elasticsearch parameters, [default: http://localhost:9200/munin]
";

#[derive(Debug)]
struct AdminMatcher {
    admin_levels: HashSet<u32>,
}
impl AdminMatcher {
    pub fn new(levels: HashSet<u32>) -> AdminMatcher {
        AdminMatcher { admin_levels: levels }
    }

    pub fn is_admin(&self, obj: &osmpbfreader::OsmObj) -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("boundary").map_or(false, |v| v == "administrative") &&
                rel.tags.get("admin_level").map_or(false, |lvl| {
                    self.admin_levels.contains(&lvl.parse::<u32>().unwrap_or(0))
                })
            }
            _ => false,
        }
    }
}

struct BoundaryPart {
    nodes: Vec<osmpbfreader::Node>,
}

impl BoundaryPart {
    pub fn new(nodes: Vec<osmpbfreader::Node>) -> BoundaryPart {
        BoundaryPart { nodes: nodes }
    }
    pub fn first(&self) -> i64 {
        self.nodes.first().unwrap().id
    }
    pub fn last(&self) -> i64 {
        self.nodes.last().unwrap().id
    }
}

fn get_nodes(way: &osmpbfreader::Way,
             objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
             -> Vec<osmpbfreader::Node> {
    way.nodes
       .iter()
       .filter_map(|node_id| objects.get(&osmpbfreader::OsmId::Node(*node_id)))
       .filter_map(|node_obj| if let &osmpbfreader::OsmObj::Node(ref node) = node_obj {
           Some(node.clone())
       } else {
           None
       })
       .collect()
}

fn build_boundary(relation: &osmpbfreader::Relation,
                  objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
                  -> Option<MultiPolygon> {
    let mut boundary_parts: Vec<BoundaryPart> =
        relation.refs
                .iter()
                .filter(|rf| rf.role == "outer" || rf.role == "" || rf.role == "enclave")
                .filter_map(|refe| {
                    objects.get(&refe.member).or_else(|| {
                        warn!("missing element for relation {}", relation.id);
                        None
                    })
                })
                .filter_map(|way_obj| if let &osmpbfreader::OsmObj::Way(ref way) = way_obj {
                    Some(way)
                } else {
                    None
                })
                .map(|way| get_nodes(&way, objects))
                .filter(|nodes| nodes.len() > 1)
                .map(|nodes| BoundaryPart::new(nodes))
                .collect();
    let mut multipoly = MultiPolygon { polygons: Vec::new() };
    // we want to try build a polygon for a least each way
    while !boundary_parts.is_empty() {
        let mut current = -1;
        let mut first = 0;
        let mut nb_try = 0;

        let mut outer: Vec<osmpbfreader::Node> = Vec::new();
        // we try to close the polygon, if we can't we want to at least have tried one time per
        // way. We could improve that latter by trying to attach the way to both side of the
        // polygon
        let max_try = boundary_parts.len();
        while current != first && nb_try < max_try {
            let mut i = 0;
            while i < boundary_parts.len() {
                if outer.is_empty() {
                    // our polygon is empty, we initialise it with the current way
                    first = boundary_parts[i].first();
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    // this way has been used, we remove it from the pool
                    boundary_parts.remove(i);
                    continue;
                }
                if current == boundary_parts[i].first() {
                    // the start of current way touch the polygon, we add it and remove it from the
                    // pool
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else if current == boundary_parts[i].last() {
                    // the end of the current way touch the polygon, we reverse the way and add it
                    current = boundary_parts[i].first();
                    boundary_parts[i].nodes.reverse();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else {
                    i += 1;
                    // didnt do anything, we want to explore the next way, if we had do something we
                    // will have removed the current way and there will be no need to increment
                }
                if current == first {
                    // our polygon is closed, we create it and add it to the multipolygon
                    let polygon = Polygon::new(outer.iter()
                                                    .map(|n| {
                                                        mimir::Coord {
                                                            lat: n.lat,
                                                            lon: n.lon,
                                                        }
                                                    })
                                                    .collect());
                    multipoly.polygons.push(polygon);
                    break;
                }
            }
            nb_try += 1;
        }
    }
    debug!("polygon for relation {};{}", relation.id, multipoly.to_wkt());
    if multipoly.polygons.is_empty() {
        None
    } else {
        Some(multipoly)
    }
}

fn parse_osm_pbf(path: &String) -> ParsedPbf {
    let path = std::path::Path::new(&path);
    match std::fs::File::open(&path) {
        Err(e) => panic!("failed to parse file {:?} because: {}", &path, e),
        Ok(r) => osmpbfreader::OsmPbfReader::new(r)
    }
}

fn administrative_regions(pbf: &mut ParsedPbf, levels: HashSet<u32>) -> AdminsVec {
    let mut administrative_regions = AdminsVec::new();
    let matcher = AdminMatcher::new(levels);
    let objects = osmpbfreader::get_objs_and_deps(pbf, |o| matcher.is_admin(o)).unwrap();
    // load administratives regions
    for (_, obj) in &objects {
        if !matcher.is_admin(&obj) {
            continue;
        }
        if let &osmpbfreader::OsmObj::Relation(ref relation) = obj {
            let level = relation.tags
                                .get("admin_level")
                                .and_then(|s| s.parse().ok());
            let level = match level {
                None => {
                    info!("invalid admin_level for relation {}: admin_level {:?}",
                          relation.id,
                          relation.tags.get("admin_level"));
                    continue;
                }
                Some(l) => l,
            };
            // administrative region with name ?
            let name = match relation.tags.get("name") {
                Some(val) => val,
                None => {
                    warn!("adminstrative region without name for relation {}:  admin_level {} \
                           ignored.",
                          relation.id,
                          level);
                    continue;
                }
            };
            // admininstrative region without coordinates
            let coord_centre = relation.refs
                                       .iter()
                                       .find(|rf| rf.role == "admin_centre")
                                       .and_then(|r| {
                                           objects.get(&r.member).and_then(|value| {
                                               match value {
                                                   &osmpbfreader::OsmObj::Node(ref node) => {
                                                       Some(mimir::Coord {
                                                           lat: node.lat,
                                                           lon: node.lon,
                                                       })
                                                   }
                                                   _ => None,
                                               }
                                           })
                                       });

            let admin_id = match relation.tags.get("ref:INSEE") {
                Some(val) => format!("admin:fr:{}", val.trim_left_matches('0')),
                None => format!("admin:osm:{}", relation.id),
            };
            let zip_code = match relation.tags.get("addr:postcode") {
                Some(val) => &val[..],
                None => "",
            };
            let boundary = build_boundary(&relation, &objects);
            let admin = mimir::Admin {
                id: admin_id,
                level: level,
                name: name.to_string(),
                zip_code: zip_code.to_string(),
                // TODO weight value ?
                weight: 1,
                coord: coord_centre,
                boundary: boundary,
            };
            administrative_regions.push(admin);
        }
    }
    return administrative_regions;
}

fn streets(pbf: &mut ParsedPbf) -> StreetsVec {

    let is_valid_street = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                way.tags.get("highway").map_or(false, |x: &String| !x.is_empty()) &&
                way.tags.get("name").map_or(false, |x: &String| !x.is_empty())
            }
            _ => false,
        }
    };

    let mut streets = StreetsVec::new();

    if let Result::Ok(ref objects) = osmpbfreader::get_objs_and_deps(pbf, is_valid_street) {
        for (osm_id, obj) in objects {
            if let &osmpbfreader::OsmObj::Way(ref way) = obj {
                if let &osmpbfreader::OsmId::Way(ref way_id) = osm_id {
                    if let Some(ref way_name) = way.tags.get("name") {
                        let street = mimirsbrunn::Street {
                            id: way_id.to_string(),
                            street_name: way_name.to_string(),
                            name: way_name.to_string(),
                            weight: 1,
                            administrative_region: None,
                        };
                        streets.push(street);
                    };
                };
            };
        };
    };

    streets
}

fn index_osm(es_cnx_string: &str, admins: &AdminsVec, streets: &StreetsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    rubber.create_index();
    match rubber.clean_db_by_doc_type(&["admin", "street"]) {
        Err(e) => panic!("failed to clean data by document type: {}", e),
        Ok(nb) => info!("clean data by document type : {}", nb),
    }
    info!("Add data in elasticsearch db.");
    match rubber.bulk_index(admins.iter()) {
       Err(e) => panic!("failed to index admins of osm because: {}", e),
       Ok(nb) => info!("Nb of indexed adminstrative regions: {}", nb),
    }

    match rubber.bulk_index(streets.iter()) {
       Err(e) => panic!("failed to index streets of osm because: {}", e),
       Ok(nb) => info!("Nb of indexed streets: {}", nb),
    }
}

fn main() {
    mimir::logger_init().unwrap();
    debug!("importing adminstrative region into Mimir");
    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);
    let admins = administrative_regions(&mut parsed_pbf, levels);
    let streets = streets(&mut parsed_pbf);
    index_osm(&args.flag_connection_string, &admins, &streets);
}
<commit_msg>rust format<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate log;
extern crate osmpbfreader;
extern crate rustc_serialize;
extern crate docopt;
extern crate mimir;
extern crate rs_es;

use std::collections::HashSet;
use std::collections::HashMap;
use mimirsbrunn::rubber::Rubber;
use mimirsbrunn::objects::{Polygon, MultiPolygon};

pub type AdminsVec = Vec<mimir::Admin>;
pub type StreetsVec = Vec<mimir::Street>;
pub type ParsedPbf = osmpbfreader::OsmPbfReader<std::fs::File>;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_connection_string: String,
}

static USAGE: &'static str = "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> [--connection-string=<connection-string>] --level=<level>...

Options:
    -h, --help            Show this message.
    -i, --input=<file>    OSM PBF file.
    -l, --level=<level>   Admin levels to keep.
    -c, --connection-string=<connection-string>
                          Elasticsearch parameters, [default: http://localhost:9200/munin]
";

#[derive(Debug)]
struct AdminMatcher {
    admin_levels: HashSet<u32>,
}
impl AdminMatcher {
    pub fn new(levels: HashSet<u32>) -> AdminMatcher {
        AdminMatcher { admin_levels: levels }
    }

    pub fn is_admin(&self, obj: &osmpbfreader::OsmObj) -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("boundary").map_or(false, |v| v == "administrative") &&
                rel.tags.get("admin_level").map_or(false, |lvl| {
                    self.admin_levels.contains(&lvl.parse::<u32>().unwrap_or(0))
                })
            }
            _ => false,
        }
    }
}

struct BoundaryPart {
    nodes: Vec<osmpbfreader::Node>,
}

impl BoundaryPart {
    pub fn new(nodes: Vec<osmpbfreader::Node>) -> BoundaryPart {
        BoundaryPart { nodes: nodes }
    }
    pub fn first(&self) -> i64 {
        self.nodes.first().unwrap().id
    }
    pub fn last(&self) -> i64 {
        self.nodes.last().unwrap().id
    }
}

fn get_nodes(way: &osmpbfreader::Way,
             objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
             -> Vec<osmpbfreader::Node> {
    way.nodes
       .iter()
       .filter_map(|node_id| objects.get(&osmpbfreader::OsmId::Node(*node_id)))
       .filter_map(|node_obj| if let &osmpbfreader::OsmObj::Node(ref node) = node_obj {
           Some(node.clone())
       } else {
           None
       })
       .collect()
}

fn build_boundary(relation: &osmpbfreader::Relation,
                  objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
                  -> Option<MultiPolygon> {
    let mut boundary_parts: Vec<BoundaryPart> =
        relation.refs
                .iter()
                .filter(|rf| rf.role == "outer" || rf.role == "" || rf.role == "enclave")
                .filter_map(|refe| {
                    objects.get(&refe.member).or_else(|| {
                        warn!("missing element for relation {}", relation.id);
                        None
                    })
                })
                .filter_map(|way_obj| if let &osmpbfreader::OsmObj::Way(ref way) = way_obj {
                    Some(way)
                } else {
                    None
                })
                .map(|way| get_nodes(&way, objects))
                .filter(|nodes| nodes.len() > 1)
                .map(|nodes| BoundaryPart::new(nodes))
                .collect();
    let mut multipoly = MultiPolygon { polygons: Vec::new() };
    // we want to try build a polygon for a least each way
    while !boundary_parts.is_empty() {
        let mut current = -1;
        let mut first = 0;
        let mut nb_try = 0;

        let mut outer: Vec<osmpbfreader::Node> = Vec::new();
        // we try to close the polygon, if we can't we want to at least have tried one time per
        // way. We could improve that latter by trying to attach the way to both side of the
        // polygon
        let max_try = boundary_parts.len();
        while current != first && nb_try < max_try {
            let mut i = 0;
            while i < boundary_parts.len() {
                if outer.is_empty() {
                    // our polygon is empty, we initialise it with the current way
                    first = boundary_parts[i].first();
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    // this way has been used, we remove it from the pool
                    boundary_parts.remove(i);
                    continue;
                }
                if current == boundary_parts[i].first() {
                    // the start of current way touch the polygon, we add it and remove it from the
                    // pool
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else if current == boundary_parts[i].last() {
                    // the end of the current way touch the polygon, we reverse the way and add it
                    current = boundary_parts[i].first();
                    boundary_parts[i].nodes.reverse();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else {
                    i += 1;
                    // didnt do anything, we want to explore the next way, if we had do something we
                    // will have removed the current way and there will be no need to increment
                }
                if current == first {
                    // our polygon is closed, we create it and add it to the multipolygon
                    let polygon = Polygon::new(outer.iter()
                                                    .map(|n| {
                                                        mimir::Coord {
                                                            lat: n.lat,
                                                            lon: n.lon,
                                                        }
                                                    })
                                                    .collect());
                    multipoly.polygons.push(polygon);
                    break;
                }
            }
            nb_try += 1;
        }
    }
    debug!("polygon for relation {};{}", relation.id, multipoly.to_wkt());
    if multipoly.polygons.is_empty() {
        None
    } else {
        Some(multipoly)
    }
}

fn parse_osm_pbf(path: &String) -> ParsedPbf {
    let path = std::path::Path::new(&path);
    match std::fs::File::open(&path) {
        Err(e) => panic!("failed to parse file {:?} because: {}", &path, e),
        Ok(r) => osmpbfreader::OsmPbfReader::new(r)
    }
}

fn administrative_regions(pbf: &mut ParsedPbf, levels: HashSet<u32>) -> AdminsVec {
    let mut administrative_regions = AdminsVec::new();
    let matcher = AdminMatcher::new(levels);
    let objects = osmpbfreader::get_objs_and_deps(pbf, |o| matcher.is_admin(o)).unwrap();
    // load administratives regions
    for (_, obj) in &objects {
        if !matcher.is_admin(&obj) {
            continue;
        }
        if let &osmpbfreader::OsmObj::Relation(ref relation) = obj {
            let level = relation.tags
                                .get("admin_level")
                                .and_then(|s| s.parse().ok());
            let level = match level {
                None => {
                    info!("invalid admin_level for relation {}: admin_level {:?}",
                          relation.id,
                          relation.tags.get("admin_level"));
                    continue;
                }
                Some(l) => l,
            };
            // administrative region with name ?
            let name = match relation.tags.get("name") {
                Some(val) => val,
                None => {
                    warn!("adminstrative region without name for relation {}:  admin_level {} \
                           ignored.",
                          relation.id,
                          level);
                    continue;
                }
            };
            // admininstrative region without coordinates
            let coord_centre = relation.refs
                                       .iter()
                                       .find(|rf| rf.role == "admin_centre")
                                       .and_then(|r| {
                                           objects.get(&r.member).and_then(|value| {
                                               match value {
                                                   &osmpbfreader::OsmObj::Node(ref node) => {
                                                       Some(mimir::Coord {
                                                           lat: node.lat,
                                                           lon: node.lon,
                                                       })
                                                   }
                                                   _ => None,
                                               }
                                           })
                                       });

            let admin_id = match relation.tags.get("ref:INSEE") {
                Some(val) => format!("admin:fr:{}", val.trim_left_matches('0')),
                None => format!("admin:osm:{}", relation.id),
            };
            let zip_code = match relation.tags.get("addr:postcode") {
                Some(val) => &val[..],
                None => "",
            };
            let boundary = build_boundary(&relation, &objects);
            let admin = mimir::Admin {
                id: admin_id,
                level: level,
                name: name.to_string(),
                zip_code: zip_code.to_string(),
                // TODO weight value ?
                weight: 1,
                coord: coord_centre,
                boundary: boundary,
            };
            administrative_regions.push(admin);
        }
    }
    return administrative_regions;
}

fn streets(pbf: &mut ParsedPbf) -> StreetsVec {

    let is_valid_street = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                way.tags.get("highway").map_or(false, |x: &String| !x.is_empty()) &&
                way.tags.get("name").map_or(false, |x: &String| !x.is_empty())
            }
            _ => false,
        }
    };

    let mut streets = StreetsVec::new();

    if let Result::Ok(ref objects) = osmpbfreader::get_objs_and_deps(pbf, is_valid_street) {
        for (osm_id, obj) in objects {
            if let &osmpbfreader::OsmObj::Way(ref way) = obj {
                if let &osmpbfreader::OsmId::Way(ref way_id) = osm_id {
                    if let Some(ref way_name) = way.tags.get("name") {
                        let street = mimirsbrunn::Street {
                            id: way_id.to_string(),
                            street_name: way_name.to_string(),
                            name: way_name.to_string(),
                            weight: 1,
                            administrative_region: None,
                        };
                        streets.push(street);
                    };
                };
            };
        }
    };

    streets
}

fn index_osm(es_cnx_string: &str, admins: &AdminsVec, streets: &StreetsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    rubber.create_index();
    match rubber.clean_db_by_doc_type(&["admin", "street"]) {
        Err(e) => panic!("failed to clean data by document type: {}", e),
        Ok(nb) => info!("clean data by document type : {}", nb),
    }
    info!("Add data in elasticsearch db.");
    match rubber.bulk_index(admins.iter()) {
       Err(e) => panic!("failed to index admins of osm because: {}", e),
       Ok(nb) => info!("Nb of indexed adminstrative regions: {}", nb),
    }

    match rubber.bulk_index(streets.iter()) {
       Err(e) => panic!("failed to index streets of osm because: {}", e),
       Ok(nb) => info!("Nb of indexed streets: {}", nb),
    }
}

fn main() {
    mimir::logger_init().unwrap();
    debug!("importing adminstrative region into Mimir");
    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);
    let admins = administrative_regions(&mut parsed_pbf, levels);
    let streets = streets(&mut parsed_pbf);
    index_osm(&args.flag_connection_string, &admins, &streets);
}
<|endoftext|>


<commit_before>#[derive(Copy, Clone, Debug)]
pub struct Score {
    /// The computed score value
    pub value: f32
}

impl Score {
    pub fn new(value: f32) -> Score {
        Score {
            value: value
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn new() {
        let score = Score::new(1.0);
        assert_eq!(score.value, 1.0);
    }
}
<commit_msg>Improve documentation of Score::new<commit_after>#[derive(Copy, Clone, Debug)]
pub struct Score {
    /// The computed score value
    pub value: f32
}

impl Score {
    /// Creates a new Score with the provided value
    ///
    /// # Examples
    ///
    /// ```
    /// let score = rff::fuzzy::Score::new(1.0);
    /// assert_eq!(score.value, 1.0);
    /// ```
    pub fn new(value: f32) -> Score {
        Score {
            value: value
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
}
<|endoftext|>