<commit_before>pub use self::tokens::{Doctype, Attributes, TagKind, StartTag, EndTag, Tag, Token};
pub use self::tokens::{DoctypeToken, TagToken, CommentToken, CharacterToken};

use std::util::replace;

mod tokens;
mod states;

pub trait TokenSink {
    fn process_token(&mut self, token: Token);
}

fn ascii_letter(c: char) -> Option<char> {
    c.to_ascii_opt()
        .filtered(|a| a.is_alpha())
        .map(|a| a.to_lower().to_char())
}

pub struct Tokenizer<'sink, Sink> {
    priv sink: &'sink mut Sink,
    priv state: states::State,

    // FIXME: The state machine guarantees the tag exists when
    // we need it, so we could eliminate the Option overhead.
    // Leaving it as Option for now, to find bugs.
    priv current_tag: Option<Tag>,

    /// Last start tag name, for use in checking "appropriate end tag".
    priv last_start_tag_name: Option<~str>,

    /// The "temporary buffer" mentioned in the spec.
    priv temp_buf: ~str
}

#[deriving(Eq)]
enum ConsumeCharResult {
    Reconsume,
    Finished,
}

impl<'sink, Sink: TokenSink> Tokenizer<'sink, Sink> {
    pub fn new(sink: &'sink mut Sink) -> Tokenizer<'sink, Sink> {
        Tokenizer {
            sink: sink,
            state: states::Data,
            current_tag: None,
            last_start_tag_name: None,
            temp_buf: ~"",
        }
    }

    pub fn feed(&mut self, input: &str) {
        debug!("feeding {:s}", input);
        let mut it = input.chars(); //.peekable();
        loop {
            match self.state {
                // These states do something other than consume a single character.
                states::CharacterReferenceInData | states::CharacterReferenceInRcdata
                | states::CharacterReferenceInAttributeValue | states::BogusComment
                | states::MarkupDeclarationOpen | states::CdataSection => {
                    fail!("FIXME: state {:?} not implemented", self.state);
                }

                _ => match it.next() {
                    None => return,
                    Some(c) => {
                        while self.process_char(c) == Reconsume {
                            // reconsume
                        }
                    }
                }
            }
        }
    }

    fn parse_error(&self, c: char) {
        error!("Parse error: saw {:?} in state {:?}", c, self.state);
    }

    fn emit_char(&mut self, c: char) {
        self.sink.process_token(CharacterToken(c));
    }

    fn emit_current_tag(&mut self) {
        let tag = self.current_tag.take().unwrap();
        match tag.kind {
            StartTag => self.last_start_tag_name = Some(tag.name.clone()),
            _ => ()
        }
        self.sink.process_token(TagToken(tag));
    }

    fn emit_temp_buf(&mut self) {
        let buf = replace(&mut self.temp_buf, ~""); // FIXME

        // FIXME: add a multiple-character token
        for c in buf.chars() {
            self.emit_char(c);
        }
    }

    fn create_tag(&mut self, kind: TagKind, c: char) {
        assert!(self.current_tag.is_none());
        let mut t = Tag::new(kind);
        t.name.push_char(c);
        self.current_tag = Some(t);
    }

    fn append_to_tag_name(&mut self, c: char) {
        self.current_tag.get_mut_ref().name.push_char(c);
    }

    fn have_appropriate_end_tag(&self) -> bool {
        match (self.last_start_tag_name.as_ref(), self.current_tag.as_ref()) {
            (Some(last), Some(tag)) =>
                (tag.kind == EndTag) && (tag.name.as_slice() == last.as_slice()),
            _ => false
        }
    }
}

// A little DSL for common state machine behaviors.
macro_rules! go (
    ( to $state:ident $($rest:tt)* ) => ({
        self.state = states::$state;
        go!($($rest)*);
    });

    ( emit $c:expr $($rest:tt)* ) => ({
        self.emit_char($c);
        go!($($rest)*);
    });

    ( error $($rest:tt)* ) => ({
        self.parse_error(c); // CAPTURE
        go!($($rest)*);
    });

    ( create_tag $kind:expr $c:expr $($rest:tt)* ) => ({
        self.create_tag($kind, $c);
        go!($($rest)*);
    });

    ( append_tag $c:expr $($rest:tt)* ) => ({
        self.append_to_tag_name($c);
        go!($($rest)*);
    });

    ( emit_tag $($rest:tt)* ) => ({
        self.emit_current_tag();
        go!($($rest)*);
    });

    ( clear_temp $($rest:tt)* ) => ({
        self.temp_buf = ~""; // FIXME: don't allocate if already empty
        go!($($rest)*);
    });

    ( append_temp $c:expr $($rest:tt)* ) => ({
        self.temp_buf.push_char($c);
        go!($($rest)*);
    });

    ( emit_temp $($rest:tt)* ) => ({
        self.emit_temp_buf();
        go!($($rest)*);
    });

    ( reconsume ) => ({
        return Reconsume;
    });

    // FIXME: better name
    ( finish ) => ({
        return Finished;
    });

    () => ({});
)

impl<'sink, Sink: TokenSink> Tokenizer<'sink, Sink> {
    // FIXME: explicitly represent the EOF character?
    // For now the plan is to handle EOF in a separate match.
    fn process_char(&mut self, c: char) -> ConsumeCharResult {
        debug!("Processing {:?} in state {:?}", c, self.state);
        match self.state {
            states::Data => match c {
                '&'  => go!(to CharacterReferenceInData),
                '<'  => go!(to TagOpen),
                '\0' => go!(error emit '\0'),
                _    => go!(emit c),
            },

            states::Rcdata => match c {
                '&'  => go!(to CharacterReferenceInRcdata),
                '<'  => go!(to RcdataLessThanSign),
                '\0' => go!(error emit '\ufffd'),
                _    => go!(emit c),
            },

            states::Rawtext => match c {
                '<'  => go!(to RawtextLessThanSign),
                '\0' => go!(error emit '\ufffd'),
                _    => go!(emit c),
            },

            states::ScriptData => match c {
                '<'  => go!(to ScriptDataLessThanSign),
                '\0' => go!(error emit '\ufffd'),
                _    => go!(emit c),
            },

            states::Plaintext => match c {
                '\0' => go!(error emit '\ufffd'),
                _    => go!(emit c),
            },

            states::TagOpen => match c {
                '!' => go!(to MarkupDeclarationOpen),
                '/' => go!(to EndTagOpen),
                '?' => go!(error to BogusComment),
                _ => match ascii_letter(c) {
                    Some(cl) => go!(create_tag StartTag cl to TagName),
                    None     => go!(error emit '<' to Data reconsume)
                }
            },

            states::EndTagOpen => match c {
                '>' => go!(error to Data),
                _ => match ascii_letter(c) {
                    Some(cl) => go!(create_tag EndTag cl to TagName),
                    None     => go!(error to BogusComment),
                }
            },

            states::TagName => match c {
                '\t' | '\n' | '\x0C' | ' '
                     => go!(to BeforeAttributeName),
                '/'  => go!(to SelfClosingStartTag),
                '>'  => go!(emit_tag to Data),
                '\0' => go!(error append_tag '\ufffd'),
                _    => go!(append_tag (ascii_letter(c).unwrap_or(c))),
            },

            states::RcdataLessThanSign => match c {
                '/' => go!(clear_temp to RcdataEndTagOpen),
                _   => go!(to Rcdata emit '<' reconsume),
            },

            states::RcdataEndTagOpen => match ascii_letter(c) {
                Some(cl) => go!(create_tag EndTag cl append_temp c to RcdataEndTagName),
                None     => go!(to Rcdata emit '<' emit '/' reconsume)
            },

            states::RcdataEndTagName => {
                if self.have_appropriate_end_tag() {
                    match c {
                        '\t' | '\n' | '\x0C' | ' '
                            => go!(to BeforeAttributeName finish),
                        '/' => go!(to SelfClosingStartTag finish),
                        '>' => go!(emit_tag to Data finish),
                        _ => (),
                    }
                }

                match ascii_letter(c) {
                    Some(cl) => go!(append_tag cl append_temp c),
                    None     => go!(emit '<' emit '/' emit_temp to Rcdata reconsume),
                }
            },

            states::RawtextLessThanSign => match c {
                '/' => go!(clear_temp to RawtextEndTagOpen),
                _   => go!(to Rawtext emit '<' reconsume),
            },

            states::RawtextEndTagOpen => match ascii_letter(c) {
                Some(cl) => go!(create_tag EndTag cl append_temp c to RawtextEndTagName),
                None     => go!(to Rawtext emit '<' emit '/' reconsume),
            },

            s => fail!("FIXME: state {:?} not implemented", s),
        }

        Finished

    }
}
<commit_msg>Clean up macro implementation and introduce semicolons for readability<commit_after>pub use self::tokens::{Doctype, Attributes, TagKind, StartTag, EndTag, Tag, Token};
pub use self::tokens::{DoctypeToken, TagToken, CommentToken, CharacterToken};

use std::util::replace;

mod tokens;
mod states;

pub trait TokenSink {
    fn process_token(&mut self, token: Token);
}

fn ascii_letter(c: char) -> Option<char> {
    c.to_ascii_opt()
        .filtered(|a| a.is_alpha())
        .map(|a| a.to_lower().to_char())
}

pub struct Tokenizer<'sink, Sink> {
    priv sink: &'sink mut Sink,
    priv state: states::State,

    // FIXME: The state machine guarantees the tag exists when
    // we need it, so we could eliminate the Option overhead.
    // Leaving it as Option for now, to find bugs.
    priv current_tag: Option<Tag>,

    /// Last start tag name, for use in checking "appropriate end tag".
    priv last_start_tag_name: Option<~str>,

    /// The "temporary buffer" mentioned in the spec.
    priv temp_buf: ~str
}

#[deriving(Eq)]
enum ConsumeCharResult {
    Reconsume,
    Finished,
}

impl<'sink, Sink: TokenSink> Tokenizer<'sink, Sink> {
    pub fn new(sink: &'sink mut Sink) -> Tokenizer<'sink, Sink> {
        Tokenizer {
            sink: sink,
            state: states::Data,
            current_tag: None,
            last_start_tag_name: None,
            temp_buf: ~"",
        }
    }

    pub fn feed(&mut self, input: &str) {
        debug!("feeding {:s}", input);
        let mut it = input.chars(); //.peekable();
        loop {
            match self.state {
                // These states do something other than consume a single character.
                states::CharacterReferenceInData | states::CharacterReferenceInRcdata
                | states::CharacterReferenceInAttributeValue | states::BogusComment
                | states::MarkupDeclarationOpen | states::CdataSection => {
                    fail!("FIXME: state {:?} not implemented", self.state);
                }

                _ => match it.next() {
                    None => return,
                    Some(c) => {
                        while self.process_char(c) == Reconsume {
                            // reconsume
                        }
                    }
                }
            }
        }
    }

    fn parse_error(&self, c: char) {
        error!("Parse error: saw {:?} in state {:?}", c, self.state);
    }

    fn emit_char(&mut self, c: char) {
        self.sink.process_token(CharacterToken(c));
    }

    fn emit_current_tag(&mut self) {
        let tag = self.current_tag.take().unwrap();
        match tag.kind {
            StartTag => self.last_start_tag_name = Some(tag.name.clone()),
            _ => ()
        }
        self.sink.process_token(TagToken(tag));
    }

    fn emit_temp_buf(&mut self) {
        let buf = replace(&mut self.temp_buf, ~""); // FIXME

        // FIXME: add a multiple-character token
        for c in buf.chars() {
            self.emit_char(c);
        }
    }

    fn create_tag(&mut self, kind: TagKind, c: char) {
        assert!(self.current_tag.is_none());
        let mut t = Tag::new(kind);
        t.name.push_char(c);
        self.current_tag = Some(t);
    }

    fn append_to_tag_name(&mut self, c: char) {
        self.current_tag.get_mut_ref().name.push_char(c);
    }

    fn have_appropriate_end_tag(&self) -> bool {
        match (self.last_start_tag_name.as_ref(), self.current_tag.as_ref()) {
            (Some(last), Some(tag)) =>
                (tag.kind == EndTag) && (tag.name.as_slice() == last.as_slice()),
            _ => false
        }
    }
}

// Shorthand for common state machine behaviors.
macro_rules! shorthand (
    ( to          $state:ident         ) => ( self.state = states::$state; );
    ( emit        $c:expr              ) => ( self.emit_char($c);          );
    ( create_tag  $kind:expr   $c:expr ) => ( self.create_tag($kind, $c);  );
    ( append_tag  $c:expr              ) => ( self.append_to_tag_name($c); );
    ( emit_tag                         ) => ( self.emit_current_tag();     );
    ( append_temp $c:expr              ) => ( self.temp_buf.push_char($c); );
    ( emit_temp                        ) => ( self.emit_temp_buf();        );

    // FIXME: don't allocate if already empty
    ( clear_temp                       ) => ( self.temp_buf = ~"";         );

    // NB: Deliberate capture of 'c'
    ( error                            ) => ( self.parse_error(c);         );
)

// A little DSL for sequencing shorthand actions.
macro_rules! go (
    // A pattern like $($cmd:tt)* ; $($rest:tt)* causes parse ambiguity.
    // We have to tell the parser how much lookahead we need.
    ( $a:tt             ; $($rest:tt)* ) => ({ shorthand!($a);       go!($($rest)*); });
    ( $a:tt $b:tt       ; $($rest:tt)* ) => ({ shorthand!($a $b);    go!($($rest)*); });
    ( $a:tt $b:tt $c:tt ; $($rest:tt)* ) => ({ shorthand!($a $b $c); go!($($rest)*); });

    // These can only come at the end.
    // FIXME: Come up with a better name for 'finish'.
    ( reconsume ) => ( return Reconsume; );
    ( finish    ) => ( return Finished;  );

    // If nothing else matched, it's a single command
    ( $($cmd:tt)+ ) => ( shorthand!($($cmd)+); );

    // or nothing.
    () => (());
)

impl<'sink, Sink: TokenSink> Tokenizer<'sink, Sink> {
    // FIXME: explicitly represent the EOF character?
    // For now the plan is to handle EOF in a separate match.
    fn process_char(&mut self, c: char) -> ConsumeCharResult {
        debug!("Processing {:?} in state {:?}", c, self.state);
        match self.state {
            states::Data => match c {
                '&'  => go!(to CharacterReferenceInData),
                '<'  => go!(to TagOpen),
                '\0' => go!(error; emit '\0'),
                _    => go!(emit c),
            },

            states::Rcdata => match c {
                '&'  => go!(to CharacterReferenceInRcdata),
                '<'  => go!(to RcdataLessThanSign),
                '\0' => go!(error; emit '\ufffd'),
                _    => go!(emit c),
            },

            states::Rawtext => match c {
                '<'  => go!(to RawtextLessThanSign),
                '\0' => go!(error; emit '\ufffd'),
                _    => go!(emit c),
            },

            states::ScriptData => match c {
                '<'  => go!(to ScriptDataLessThanSign),
                '\0' => go!(error; emit '\ufffd'),
                _    => go!(emit c),
            },

            states::Plaintext => match c {
                '\0' => go!(error; emit '\ufffd'),
                _    => go!(emit c),
            },

            states::TagOpen => match c {
                '!' => go!(to MarkupDeclarationOpen),
                '/' => go!(to EndTagOpen),
                '?' => go!(error; to BogusComment),
                _ => match ascii_letter(c) {
                    Some(cl) => go!(create_tag StartTag cl; to TagName),
                    None     => go!(error; emit '<'; to Data; reconsume),
                }
            },

            states::EndTagOpen => match c {
                '>' => go!(error; to Data),
                _ => match ascii_letter(c) {
                    Some(cl) => go!(create_tag EndTag cl; to TagName),
                    None     => go!(error; to BogusComment),
                }
            },

            states::TagName => match c {
                '\t' | '\n' | '\x0C' | ' '
                     => go!(to BeforeAttributeName),
                '/'  => go!(to SelfClosingStartTag),
                '>'  => go!(emit_tag; to Data),
                '\0' => go!(error; append_tag '\ufffd'),
                _    => go!(append_tag (ascii_letter(c).unwrap_or(c))),
            },

            states::RcdataLessThanSign => match c {
                '/' => go!(clear_temp; to RcdataEndTagOpen),
                _   => go!(to Rcdata; emit '<'; reconsume),
            },

            states::RcdataEndTagOpen => match ascii_letter(c) {
                Some(cl) => go!(create_tag EndTag cl; append_temp c; to RcdataEndTagName),
                None     => go!(to Rcdata; emit '<'; emit '/'; reconsume),
            },

            states::RcdataEndTagName => {
                if self.have_appropriate_end_tag() {
                    match c {
                        '\t' | '\n' | '\x0C' | ' '
                            => go!(to BeforeAttributeName; finish),
                        '/' => go!(to SelfClosingStartTag; finish),
                        '>' => go!(emit_tag; to Data; finish),
                        _ => (),
                    }
                }

                match ascii_letter(c) {
                    Some(cl) => go!(append_tag cl; append_temp c),
                    None     => go!(emit '<'; emit '/'; emit_temp; to Rcdata; reconsume),
                }
            },

            states::RawtextLessThanSign => match c {
                '/' => go!(clear_temp; to RawtextEndTagOpen),
                _   => go!(to Rawtext; emit '<'; reconsume),
            },

            states::RawtextEndTagOpen => match ascii_letter(c) {
                Some(cl) => go!(create_tag EndTag cl; append_temp c; to RawtextEndTagName),
                None     => go!(to Rawtext; emit '<'; emit '/'; reconsume),
            },

            s => fail!("FIXME: state {:?} not implemented", s),
        }

        Finished

    }
}
<|endoftext|>


<commit_before>/*
  This Source Code Form is subject to the terms of the Mozilla Public
  License, v. 2.0. If a copy of the MPL was not distributed with this
  file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

extern crate plist;

use std::path::Path;
use aplib::plutils::*;
use aplib::{AplibObject,AplibType};
use self::plist::Plist;
use std::collections::BTreeMap;

pub struct Keyword {
    uuid: String,
    model_id: i64,
    parent_uuid: String,

    pub name: String,
    pub children: Vec<Keyword>,
}

impl AplibObject for Keyword {
    fn from_path(plist_path: &Path) -> Keyword {
        assert!(false, "must not be called");
        Keyword { uuid: "".to_string(),
                  model_id: 0,
                  parent_uuid: "".to_string(),
                  name: "".to_string(),
                  children: Vec::new() }
    }
    fn obj_type(&self) -> AplibType {
        return AplibType::KEYWORD;
    }
    fn uuid(&self) -> &String {
        return &self.uuid;
    }
    fn parent(&self) -> &String {
        return &self.parent_uuid;
    }
    fn model_id(&self) -> i64 {
        return self.model_id;
    }
    fn is_valid(&self) -> bool {
        return !self.uuid.is_empty();
    }

}

/// Parse keywords from the .plist file
pub fn parse_keywords(path: &Path) -> Vec<Keyword>
{
    let plist = parse_plist(path);

    return match plist {
        Plist::Dictionary(ref dict) => {
            let version = get_int_value(dict, "keywords_version");
            // XXX deal with proper errors here.
            if version != 6 {
                println!("Wrong keyword version !");
            }

            Keyword::from_array(get_array_value(dict, "keywords"))
        },
        _ => Vec::new()
    }
}

impl Keyword {

    /// convert a Plist array to a vec of keyword.
    fn from_array(a: Vec<Plist>) -> Vec<Keyword>
    {
        let mut keywords = Vec::new();
        for item in a {
            match item {
                Plist::Dictionary(ref kw) => {
                    keywords.push(Keyword::from(kw));
                },
                _ => ()
            }
        }
        return keywords;
    }

    /// create a new keyword from a plist dictionary
    /// will recursively create the children
    pub fn from(d: &BTreeMap<String, Plist>) -> Keyword
    {
        return Keyword {
            uuid: get_str_value(d, "uuid"),
            model_id: get_int_value(d, "modelId"),
            parent_uuid: get_str_value(d, "parentUuid"),
            name: get_str_value(d, "name"),
            children: Keyword::from_array(get_array_value(d, "zChildren")),
        };
    }
}
<commit_msg>Suppress a warning<commit_after>/*
  This Source Code Form is subject to the terms of the Mozilla Public
  License, v. 2.0. If a copy of the MPL was not distributed with this
  file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

extern crate plist;

use std::path::Path;
use aplib::plutils::*;
use aplib::{AplibObject,AplibType};
use self::plist::Plist;
use std::collections::BTreeMap;

pub struct Keyword {
    uuid: String,
    model_id: i64,
    parent_uuid: String,

    pub name: String,
    pub children: Vec<Keyword>,
}

impl AplibObject for Keyword {
    #[warn(unused_variables)]
    fn from_path(plist_path: &Path) -> Keyword {
        assert!(false, "must not be called");
        Keyword { uuid: "".to_string(),
                  model_id: 0,
                  parent_uuid: "".to_string(),
                  name: "".to_string(),
                  children: Vec::new() }
    }
    fn obj_type(&self) -> AplibType {
        return AplibType::KEYWORD;
    }
    fn uuid(&self) -> &String {
        return &self.uuid;
    }
    fn parent(&self) -> &String {
        return &self.parent_uuid;
    }
    fn model_id(&self) -> i64 {
        return self.model_id;
    }
    fn is_valid(&self) -> bool {
        return !self.uuid.is_empty();
    }

}

/// Parse keywords from the .plist file
pub fn parse_keywords(path: &Path) -> Vec<Keyword>
{
    let plist = parse_plist(path);

    return match plist {
        Plist::Dictionary(ref dict) => {
            let version = get_int_value(dict, "keywords_version");
            // XXX deal with proper errors here.
            if version != 6 {
                println!("Wrong keyword version !");
            }

            Keyword::from_array(get_array_value(dict, "keywords"))
        },
        _ => Vec::new()
    }
}

impl Keyword {

    /// convert a Plist array to a vec of keyword.
    fn from_array(a: Vec<Plist>) -> Vec<Keyword>
    {
        let mut keywords = Vec::new();
        for item in a {
            match item {
                Plist::Dictionary(ref kw) => {
                    keywords.push(Keyword::from(kw));
                },
                _ => ()
            }
        }
        return keywords;
    }

    /// create a new keyword from a plist dictionary
    /// will recursively create the children
    pub fn from(d: &BTreeMap<String, Plist>) -> Keyword
    {
        return Keyword {
            uuid: get_str_value(d, "uuid"),
            model_id: get_int_value(d, "modelId"),
            parent_uuid: get_str_value(d, "parentUuid"),
            name: get_str_value(d, "name"),
            children: Keyword::from_array(get_array_value(d, "zChildren")),
        };
    }
}
<|endoftext|>


<commit_before>// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Script to check the validity of `href` links in our HTML documentation.
//!
//! In the past we've been quite error prone to writing in broken links as most
//! of them are manually rather than automatically added. As files move over
//! time or apis change old links become stale or broken. The purpose of this
//! script is to check all relative links in our documentation to make sure they
//! actually point to a valid place.
//!
//! Currently this doesn't actually do any HTML parsing or anything fancy like
//! that, it just has a simple "regex" to search for `href` and `id` tags.
//! These values are then translated to file URLs if possible and then the
//! destination is asserted to exist.
//!
//! A few whitelisted exceptions are allowed as there's known bugs in rustdoc,
//! but this should catch the majority of "broken link" cases.

extern crate url;

use std::env;
use std::fs::File;
use std::io::prelude::*;
use std::path::{Path, PathBuf};
use std::collections::{HashMap, HashSet};
use std::collections::hash_map::Entry;

use url::{Url, UrlParser};

use Redirect::*;

macro_rules! t {
    ($e:expr) => (match $e {
        Ok(e) => e,
        Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
    })
}

fn main() {
    let docs = env::args().nth(1).unwrap();
    let docs = env::current_dir().unwrap().join(docs);
    let mut url = Url::from_file_path(&docs).unwrap();
    let mut errors = false;
    walk(&mut HashMap::new(), &docs, &docs, &mut url, &mut errors);
    if errors {
        panic!("found some broken links");
    }
}

#[derive(Debug)]
pub enum LoadError {
    IOError(std::io::Error),
    BrokenRedirect(PathBuf, std::io::Error),
    IsRedirect,
}

enum Redirect {
    SkipRedirect,
    FromRedirect(bool),
}

struct FileEntry {
    source: String,
    ids: HashSet<String>,
}

type Cache = HashMap<PathBuf, FileEntry>;

impl FileEntry {
    fn parse_ids(&mut self,
                file: &Path,
                contents: &str,
                errors: &mut bool)
{
        if self.ids.is_empty() {
            with_attrs_in_source(contents, " id", |fragment, i| {
                let frag = fragment.trim_left_matches("#").to_owned();
                if !self.ids.insert(frag) {
                    *errors = true;
                    println!("{}:{}: id is not unique: `{}`",
                             file.display(), i, fragment);
                }
            });
        }
    }
}

fn walk(cache: &mut Cache,
        root: &Path,
        dir: &Path,
        url: &mut Url,
        errors: &mut bool)
{
    for entry in t!(dir.read_dir()).map(|e| t!(e)) {
        let path = entry.path();
        let kind = t!(entry.file_type());
        url.path_mut().unwrap().push(entry.file_name().into_string().unwrap());
        if kind.is_dir() {
            walk(cache, root, &path, url, errors);
        } else {
            let pretty_path = check(cache, root, &path, url, errors);
            if let Some(pretty_path) = pretty_path {
                let entry = cache.get_mut(&pretty_path).unwrap();
                // we don't need the source anymore,
                // so drop to reduce memory-usage
                entry.source = String::new();
            }
        }
        url.path_mut().unwrap().pop();
    }
}

fn check(cache: &mut Cache,
         root: &Path,
         file: &Path,
         base: &Url,
         errors: &mut bool) -> Option<PathBuf>
{
    // ignore js files as they are not prone to errors as the rest of the
    // documentation is and they otherwise bring up false positives.
    if file.extension().and_then(|s| s.to_str()) == Some("js") {
        return None;
    }

    // Unfortunately we're not 100% full of valid links today to we need a few
    // whitelists to get this past `make check` today.
    // FIXME(#32129)
    if file.ends_with("std/string/struct.String.html") {
        return None;
    }
    // FIXME(#32553)
    if file.ends_with("collections/string/struct.String.html") {
        return None;
    }
    // FIXME(#32130)
    if file.ends_with("btree_set/struct.BTreeSet.html") ||
       file.ends_with("collections/struct.BTreeSet.html") ||
       file.ends_with("collections/btree_map/struct.BTreeMap.html") ||
       file.ends_with("collections/hash_map/struct.HashMap.html") {
        return None;
    }

    if file.ends_with("std/sys/ext/index.html") {
        return None;
    }

    if let Some(file) = file.to_str() {
        // FIXME(#31948)
        if file.contains("ParseFloatError") {
            return None;
        }
        // weird reexports, but this module is on its way out, so chalk it up to
        // "rustdoc weirdness" and move on from there
        if file.contains("scoped_tls") {
            return None;
        }
    }

    let mut parser = UrlParser::new();
    parser.base_url(base);

    let res = load_file(cache, root, PathBuf::from(file), SkipRedirect);
    let (pretty_file, contents) = match res {
        Ok(res) => res,
        Err(_) => return None,
    };
    {
        cache.get_mut(&pretty_file).unwrap()
                                   .parse_ids(&pretty_file, &contents, errors);
    }

    // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
    with_attrs_in_source(&contents, " href", |url, i| {
        // Once we've plucked out the URL, parse it using our base url and
        // then try to extract a file path. If either of these fail then we
        // just keep going.
        let (parsed_url, path) = match url_to_file_path(&parser, url) {
            Some((url, path)) => (url, PathBuf::from(path)),
            None => return,
        };

        // Alright, if we've found a file name then this file had better
        // exist! If it doesn't then we register and print an error.
        if path.exists() {
            if path.is_dir() {
                // Links to directories show as directory listings when viewing
                // the docs offline so it's best to avoid them.
                *errors = true;
                let pretty_path = path.strip_prefix(root).unwrap_or(&path);
                println!("{}:{}: directory link - {}", pretty_file.display(),
                         i + 1, pretty_path.display());
                return;
            }
            let res = load_file(cache, root, path.clone(), FromRedirect(false));
            let (pretty_path, contents) = match res {
                Ok(res) => res,
                Err(LoadError::IOError(err)) => panic!(format!("{}", err)),
                Err(LoadError::BrokenRedirect(target, _)) => {
                    print!("{}:{}: broken redirect to {}",
                           pretty_file.display(), i + 1, target.display());
                    return;
                }
                Err(LoadError::IsRedirect) => unreachable!(),
            };

            if let Some(ref fragment) = parsed_url.fragment {
                // Fragments like `#1-6` are most likely line numbers to be
                // interpreted by javascript, so we're ignoring these
                if fragment.splitn(2, '-')
                           .all(|f| f.chars().all(|c| c.is_numeric())) {
                    return;
                }

                let entry = &mut cache.get_mut(&pretty_path).unwrap();
                entry.parse_ids(&pretty_path, &contents, errors);

                if !entry.ids.contains(fragment) {
                    *errors = true;
                    print!("{}:{}: broken link fragment  ",
                           pretty_file.display(), i + 1);
                    println!("`#{}` pointing to `{}`",
                             fragment, pretty_path.display());
                };
            }
        } else {
            *errors = true;
            print!("{}:{}: broken link - ", pretty_file.display(), i + 1);
            let pretty_path = path.strip_prefix(root).unwrap_or(&path);
            println!("{}", pretty_path.display());
        }
    });
    Some(pretty_file)
}

fn load_file(cache: &mut Cache,
             root: &Path,
             file: PathBuf,
             redirect: Redirect) -> Result<(PathBuf, String), LoadError> {
    let mut contents = String::new();
    let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file));

    let maybe_redirect = match cache.entry(pretty_file.clone()) {
        Entry::Occupied(entry) => {
            contents = entry.get().source.clone();
            None
        },
        Entry::Vacant(entry) => {
            let mut fp = try!(File::open(file.clone()).map_err(|err| {
                if let FromRedirect(true) = redirect {
                    LoadError::BrokenRedirect(file.clone(), err)
                } else {
                    LoadError::IOError(err)
                }
            }));
            try!(fp.read_to_string(&mut contents)
                   .map_err(|err| LoadError::IOError(err)));

            let maybe = maybe_redirect(&contents);
            if maybe.is_some() {
                if let SkipRedirect = redirect {
                    return Err(LoadError::IsRedirect);
                }
            } else {
                entry.insert(FileEntry {
                    source: contents.clone(),
                    ids: HashSet::new(),
                });
            }
            maybe
        },
    };
    let base = Url::from_file_path(&file).unwrap();
    let mut parser = UrlParser::new();
    parser.base_url(&base);

    match maybe_redirect.and_then(|url| url_to_file_path(&parser, &url)) {
        Some((_, redirect_file)) => {
            let path = PathBuf::from(redirect_file);
            load_file(cache, root, path, FromRedirect(true))
        }
        None => Ok((pretty_file, contents))
    }
}

fn maybe_redirect(source: &str) -> Option<String> {
    const REDIRECT: &'static str = "
Redirecting to  l,
        None => return None,
    };

    redirect_line.find(REDIRECT).map(|i| {
        let rest = &redirect_line[(i + REDIRECT.len() + 1)..];
        let pos_quote = rest.find('"').unwrap();
        rest[..pos_quote].to_owned()
    })
}

fn url_to_file_path(parser: &UrlParser, url: &str) -> Option<(Url, PathBuf)> {
    parser.parse(url).ok().and_then(|parsed_url| {
        parsed_url.to_file_path().ok().map(|f| (parsed_url, f))
    })
}

fn with_attrs_in_source<F: FnMut(&str, usize)>(contents: &str,
                                               attr: &str,
                                               mut f: F)
{
    for (i, mut line) in contents.lines().enumerate() {
        while let Some(j) = line.find(attr) {
            let rest = &line[j + attr.len() ..];
            line = rest;
            let pos_equals = match rest.find("=") {
                Some(i) => i,
                None => continue,
            };
            if rest[..pos_equals].trim_left_matches(" ") != "" {
                continue
            }

            let rest = &rest[pos_equals + 1..];

            let pos_quote = match rest.find(&['"', '\''][..]) {
                Some(i) => i,
                None => continue,
            };
            let quote_delim = rest.as_bytes()[pos_quote] as char;

            if rest[..pos_quote].trim_left_matches(" ") != "" {
                continue
            }
            let rest = &rest[pos_quote + 1..];
            let url = match rest.find(quote_delim) {
                Some(i) => &rest[..i],
                None => continue,
            };
            f(url, i)
        }
    }
}
<commit_msg>run rustfmt on linkchecker folderin src/tools/linkchecker<commit_after>// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Script to check the validity of `href` links in our HTML documentation.
//!
//! In the past we've been quite error prone to writing in broken links as most
//! of them are manually rather than automatically added. As files move over
//! time or apis change old links become stale or broken. The purpose of this
//! script is to check all relative links in our documentation to make sure they
//! actually point to a valid place.
//!
//! Currently this doesn't actually do any HTML parsing or anything fancy like
//! that, it just has a simple "regex" to search for `href` and `id` tags.
//! These values are then translated to file URLs if possible and then the
//! destination is asserted to exist.
//!
//! A few whitelisted exceptions are allowed as there's known bugs in rustdoc,
//! but this should catch the majority of "broken link" cases.

extern crate url;

use std::env;
use std::fs::File;
use std::io::prelude::*;
use std::path::{Path, PathBuf};
use std::collections::{HashMap, HashSet};
use std::collections::hash_map::Entry;

use url::{Url, UrlParser};

use Redirect::*;

macro_rules! t {
    ($e:expr) => (match $e {
        Ok(e) => e,
        Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
    })
}

fn main() {
    let docs = env::args().nth(1).unwrap();
    let docs = env::current_dir().unwrap().join(docs);
    let mut url = Url::from_file_path(&docs).unwrap();
    let mut errors = false;
    walk(&mut HashMap::new(), &docs, &docs, &mut url, &mut errors);
    if errors {
        panic!("found some broken links");
    }
}

#[derive(Debug)]
pub enum LoadError {
    IOError(std::io::Error),
    BrokenRedirect(PathBuf, std::io::Error),
    IsRedirect,
}

enum Redirect {
    SkipRedirect,
    FromRedirect(bool),
}

struct FileEntry {
    source: String,
    ids: HashSet<String>,
}

type Cache = HashMap<PathBuf, FileEntry>;

impl FileEntry {
    fn parse_ids(&mut self, file: &Path, contents: &str, errors: &mut bool) {
        if self.ids.is_empty() {
            with_attrs_in_source(contents, " id", |fragment, i| {
                let frag = fragment.trim_left_matches("#").to_owned();
                if !self.ids.insert(frag) {
                    *errors = true;
                    println!("{}:{}: id is not unique: `{}`", file.display(), i, fragment);
                }
            });
        }
    }
}

fn walk(cache: &mut Cache, root: &Path, dir: &Path, url: &mut Url, errors: &mut bool) {
    for entry in t!(dir.read_dir()).map(|e| t!(e)) {
        let path = entry.path();
        let kind = t!(entry.file_type());
        url.path_mut().unwrap().push(entry.file_name().into_string().unwrap());
        if kind.is_dir() {
            walk(cache, root, &path, url, errors);
        } else {
            let pretty_path = check(cache, root, &path, url, errors);
            if let Some(pretty_path) = pretty_path {
                let entry = cache.get_mut(&pretty_path).unwrap();
                // we don't need the source anymore,
                // so drop to reduce memory-usage
                entry.source = String::new();
            }
        }
        url.path_mut().unwrap().pop();
    }
}

fn check(cache: &mut Cache,
         root: &Path,
         file: &Path,
         base: &Url,
         errors: &mut bool)
         -> Option<PathBuf> {
    // ignore js files as they are not prone to errors as the rest of the
    // documentation is and they otherwise bring up false positives.
    if file.extension().and_then(|s| s.to_str()) == Some("js") {
        return None;
    }

    // Unfortunately we're not 100% full of valid links today to we need a few
    // whitelists to get this past `make check` today.
    // FIXME(#32129)
    if file.ends_with("std/string/struct.String.html") {
        return None;
    }
    // FIXME(#32553)
    if file.ends_with("collections/string/struct.String.html") {
        return None;
    }
    // FIXME(#32130)
    if file.ends_with("btree_set/struct.BTreeSet.html") ||
       file.ends_with("collections/struct.BTreeSet.html") ||
       file.ends_with("collections/btree_map/struct.BTreeMap.html") ||
       file.ends_with("collections/hash_map/struct.HashMap.html") {
        return None;
    }

    if file.ends_with("std/sys/ext/index.html") {
        return None;
    }

    if let Some(file) = file.to_str() {
        // FIXME(#31948)
        if file.contains("ParseFloatError") {
            return None;
        }
        // weird reexports, but this module is on its way out, so chalk it up to
        // "rustdoc weirdness" and move on from there
        if file.contains("scoped_tls") {
            return None;
        }
    }

    let mut parser = UrlParser::new();
    parser.base_url(base);

    let res = load_file(cache, root, PathBuf::from(file), SkipRedirect);
    let (pretty_file, contents) = match res {
        Ok(res) => res,
        Err(_) => return None,
    };
    {
        cache.get_mut(&pretty_file)
             .unwrap()
             .parse_ids(&pretty_file, &contents, errors);
    }

    // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
    with_attrs_in_source(&contents, " href", |url, i| {
        // Once we've plucked out the URL, parse it using our base url and
        // then try to extract a file path. If either of these fail then we
        // just keep going.
        let (parsed_url, path) = match url_to_file_path(&parser, url) {
            Some((url, path)) => (url, PathBuf::from(path)),
            None => return,
        };

        // Alright, if we've found a file name then this file had better
        // exist! If it doesn't then we register and print an error.
        if path.exists() {
            if path.is_dir() {
                // Links to directories show as directory listings when viewing
                // the docs offline so it's best to avoid them.
                *errors = true;
                let pretty_path = path.strip_prefix(root).unwrap_or(&path);
                println!("{}:{}: directory link - {}",
                         pretty_file.display(),
                         i + 1,
                         pretty_path.display());
                return;
            }
            let res = load_file(cache, root, path.clone(), FromRedirect(false));
            let (pretty_path, contents) = match res {
                Ok(res) => res,
                Err(LoadError::IOError(err)) => panic!(format!("{}", err)),
                Err(LoadError::BrokenRedirect(target, _)) => {
                    print!("{}:{}: broken redirect to {}",
                           pretty_file.display(),
                           i + 1,
                           target.display());
                    return;
                }
                Err(LoadError::IsRedirect) => unreachable!(),
            };

            if let Some(ref fragment) = parsed_url.fragment {
                // Fragments like `#1-6` are most likely line numbers to be
                // interpreted by javascript, so we're ignoring these
                if fragment.splitn(2, '-')
                           .all(|f| f.chars().all(|c| c.is_numeric())) {
                    return;
                }

                let entry = &mut cache.get_mut(&pretty_path).unwrap();
                entry.parse_ids(&pretty_path, &contents, errors);

                if !entry.ids.contains(fragment) {
                    *errors = true;
                    print!("{}:{}: broken link fragment  ",
                           pretty_file.display(),
                           i + 1);
                    println!("`#{}` pointing to `{}`", fragment, pretty_path.display());
                };
            }
        } else {
            *errors = true;
            print!("{}:{}: broken link - ", pretty_file.display(), i + 1);
            let pretty_path = path.strip_prefix(root).unwrap_or(&path);
            println!("{}", pretty_path.display());
        }
    });
    Some(pretty_file)
}

fn load_file(cache: &mut Cache,
             root: &Path,
             file: PathBuf,
             redirect: Redirect)
             -> Result<(PathBuf, String), LoadError> {
    let mut contents = String::new();
    let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file));

    let maybe_redirect = match cache.entry(pretty_file.clone()) {
        Entry::Occupied(entry) => {
            contents = entry.get().source.clone();
            None
        }
        Entry::Vacant(entry) => {
            let mut fp = try!(File::open(file.clone()).map_err(|err| {
                if let FromRedirect(true) = redirect {
                    LoadError::BrokenRedirect(file.clone(), err)
                } else {
                    LoadError::IOError(err)
                }
            }));
            try!(fp.read_to_string(&mut contents)
                   .map_err(|err| LoadError::IOError(err)));

            let maybe = maybe_redirect(&contents);
            if maybe.is_some() {
                if let SkipRedirect = redirect {
                    return Err(LoadError::IsRedirect);
                }
            } else {
                entry.insert(FileEntry {
                    source: contents.clone(),
                    ids: HashSet::new(),
                });
            }
            maybe
        }
    };
    let base = Url::from_file_path(&file).unwrap();
    let mut parser = UrlParser::new();
    parser.base_url(&base);

    match maybe_redirect.and_then(|url| url_to_file_path(&parser, &url)) {
        Some((_, redirect_file)) => {
            let path = PathBuf::from(redirect_file);
            load_file(cache, root, path, FromRedirect(true))
        }
        None => Ok((pretty_file, contents)),
    }
}

fn maybe_redirect(source: &str) -> Option<String> {
    const REDIRECT: &'static str = "
Redirecting to  l,
        None => return None,
    };

    redirect_line.find(REDIRECT).map(|i| {
        let rest = &redirect_line[(i + REDIRECT.len() + 1)..];
        let pos_quote = rest.find('"').unwrap();
        rest[..pos_quote].to_owned()
    })
}

fn url_to_file_path(parser: &UrlParser, url: &str) -> Option<(Url, PathBuf)> {
    parser.parse(url)
          .ok()
          .and_then(|parsed_url| parsed_url.to_file_path().ok().map(|f| (parsed_url, f)))
}

fn with_attrs_in_source<F: FnMut(&str, usize)>(contents: &str, attr: &str, mut f: F) {
    for (i, mut line) in contents.lines().enumerate() {
        while let Some(j) = line.find(attr) {
            let rest = &line[j + attr.len()..];
            line = rest;
            let pos_equals = match rest.find("=") {
                Some(i) => i,
                None => continue,
            };
            if rest[..pos_equals].trim_left_matches(" ") != "" {
                continue;
            }

            let rest = &rest[pos_equals + 1..];

            let pos_quote = match rest.find(&['"', '\''][..]) {
                Some(i) => i,
                None => continue,
            };
            let quote_delim = rest.as_bytes()[pos_quote] as char;

            if rest[..pos_quote].trim_left_matches(" ") != "" {
                continue;
            }
            let rest = &rest[pos_quote + 1..];
            let url = match rest.find(quote_delim) {
                Some(i) => &rest[..i],
                None => continue,
            };
            f(url, i)
        }
    }
}
<|endoftext|>


<commit_before>extern crate websocket;
extern crate futures;
extern crate futures_cpupool;
extern crate tokio_core;
extern crate serde;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;

extern crate shootr;

use websocket::message::OwnedMessage;
use websocket::server::InvalidConnection;
use websocket::async::Server;

use tokio_core::reactor::{Handle, Remote, Core};

use futures::{Future, BoxFuture, Sink, Stream};
use futures::future::{self, Loop};
use futures::sync::mpsc;
use futures_cpupool::{CpuPool, CpuFuture};

use std::sync::{RwLock, Arc};
use std::thread;
use std::rc::Rc;
use std::fmt::Debug;
use std::time::Duration;
use std::ops::Deref;
use std::collections::HashMap;
use std::cell::RefCell;

use shootr::model::ClientState;
use shootr::ecs::res;
use shootr::engine;

fn main() {
    shootr::engine::execute(main_loop);
}

fn main_loop(engine: engine::Engine) -> BoxFuture<(), ()> {
    future::loop_fn(engine, |conn_info| {
        thread::sleep(Duration::from_millis(100));

        let should_continue = update(
            conn_info.connections.clone(),
            conn_info.channel.clone(),
            &conn_info.remote,
        );
        match should_continue {
            Ok(true) => Ok(Loop::Continue(conn_info)),
            Ok(false) => Ok(Loop::Break(())),
            Err(()) => Err(()),
        }
    }).boxed()
}

type Id = u32;

type SinkContent = websocket::client::async::Framed<
    tokio_core::net::TcpStream,
    websocket::async::MessageCodec<OwnedMessage>,
>;
type SplitSink = futures::stream::SplitSink<SinkContent>;
// Represents one tick in the main loop
fn update(
    connections: Arc<RwLock<HashMap<Id, SplitSink>>>,
    channel: mpsc::UnboundedSender<(Id, Arc<RwLock<ClientState>>)>,
    remote: &Remote,
) -> Result<bool, ()> {
    remote.spawn(move |handle| {
        let state = Arc::new(RwLock::new(ClientState {}));
        for (id, _) in connections.read().unwrap().iter() {
            let f = channel.clone().send((*id, state.clone()));
            shootr::engine::spawn_future(f, "Send message to write handler", handle);
        }
        Ok(())
    });
    Ok(true)
}
<commit_msg>Core: Run clippy<commit_after>extern crate websocket;
extern crate futures;
extern crate tokio_core;
extern crate serde;
extern crate serde_json;

extern crate shootr;

use websocket::message::OwnedMessage;

use tokio_core::reactor::Remote;

use futures::{Future, BoxFuture, Sink};
use futures::future::{self, Loop};
use futures::sync::mpsc;

use std::sync::{RwLock, Arc};
use std::thread;
use std::time::Duration;
use std::collections::HashMap;

use shootr::model::ClientState;
use shootr::engine;

fn main() {
    shootr::engine::execute(main_loop);
}

fn main_loop(engine: engine::Engine) -> BoxFuture<(), ()> {
    future::loop_fn(engine, |conn_info| {
        thread::sleep(Duration::from_millis(100));

        let should_continue = update(
            conn_info.connections.clone(),
            conn_info.channel.clone(),
            &conn_info.remote,
        );
        match should_continue {
            Ok(true) => Ok(Loop::Continue(conn_info)),
            Ok(false) => Ok(Loop::Break(())),
            Err(()) => Err(()),
        }
    }).boxed()
}

type Id = u32;

type SinkContent = websocket::client::async::Framed<
    tokio_core::net::TcpStream,
    websocket::async::MessageCodec<OwnedMessage>,
>;
type SplitSink = futures::stream::SplitSink<SinkContent>;
// Represents one tick in the main loop
fn update(
    connections: Arc<RwLock<HashMap<Id, SplitSink>>>,
    channel: mpsc::UnboundedSender<(Id, Arc<RwLock<ClientState>>)>,
    remote: &Remote,
) -> Result<bool, ()> {
    remote.spawn(move |handle| {
        let state = Arc::new(RwLock::new(ClientState {}));
        for (id, _) in connections.read().unwrap().iter() {
            let f = channel.clone().send((*id, state.clone()));
            shootr::engine::spawn_future(f, "Send message to write handler", handle);
        }
        Ok(())
    });
    Ok(true)
}
<|endoftext|>


<commit_before>//! TODO Documentation

use std::panic;
use std::rc::{Rc, Weak};
use std::sync::atomic::{AtomicBool, Ordering};

use errors::{UpgradeHandleErr, UpgradeHandleResult};
use wlroots_sys::{wlr_input_device, wlr_pointer};

use InputDevice;

#[derive(Debug)]
pub struct Pointer {
    /// The structure that ensures weak handles to this structure are still alive.
    ///
    /// They contain weak handles, and will safely not use dead memory when this
    /// is freed by wlroots.
    ///
    /// If this is `None`, then this is from an upgraded `PointerHandle`, and
    /// the operations are **unchecked**.
    /// This is means safe operations might fail, but only if you use the unsafe
    /// marked function `upgrade` on a `PointerHandle`.
    liveliness: Option<Rc<AtomicBool>>,
    /// The device that refers to this pointer.
    device: InputDevice,
    /// The underlying pointer data.
    pointer: *mut wlr_pointer
}

/// A wlr_input_device that is guaranteed to be a pointer.
#[derive(Debug)]
pub struct PointerHandle {
    /// The Rc that ensures that this handle is still alive.
    ///
    /// When wlroots deallocates the pointer associated with this handle,
    /// this can no longer be used.
    handle: Weak<AtomicBool>,
    /// The device that refers to this pointer.
    device: InputDevice,
    /// The underlying pointer data.
    pointer: *mut wlr_pointer
}

impl Pointer {
    /// Tries to convert an input device to a Pointer
    ///
    /// Returns none if it is of a different input variant.
    ///
    /// # Safety
    /// This creates a totally new Pointer (e.g with its own reference count)
    /// so only do this once per `wlr_input_device`!
    pub(crate) unsafe fn new_from_input_device(device: *mut wlr_input_device) -> Option<Self> {
        use wlroots_sys::wlr_input_device_type::*;
        match (*device).type_ {
            WLR_INPUT_DEVICE_POINTER => {
                let pointer = (*device).__bindgen_anon_1.pointer;
                Some(Pointer { liveliness: Some(Rc::new(AtomicBool::new(false))),
                               device: InputDevice::from_ptr(device),
                               pointer })
            }
            _ => None
        }
    }

    /// Creates an unbound Pointer from a `PointerHandle`
    unsafe fn from_handle(handle: &PointerHandle) -> Self {
        Pointer { liveliness: None,
                  device: handle.input_device().clone(),
                  pointer: handle.as_ptr() }
    }

    /// Gets the wlr_input_device associated with this Pointer.
    pub fn input_device(&self) -> &InputDevice {
        &self.device
    }

    /// Gets the wlr_pointer associated with this Pointer.
    #[allow(dead_code)]
    pub(crate) unsafe fn as_ptr(&self) -> *mut wlr_pointer {
        self.pointer
    }

    /// Creates a weak reference to a `Pointer`.
    ///
    /// # Panics
    /// If this `Pointer` is a previously upgraded `PointerHandle`,
    /// then this function will panic.
    pub fn weak_reference(&self) -> PointerHandle {
        let arc = self.liveliness.as_ref()
                      .expect("Cannot downgrade a previously upgraded PointerHandle!");
        PointerHandle { handle: Rc::downgrade(arc),
                        // NOTE Rationale for cloning:
                        // We can't use the keyboard handle unless the keyboard is alive,
                        // which means the device pointer is still alive.
                        device: unsafe { self.device.clone() },
                        pointer: self.pointer }
    }

    /// Manually set the lock used to determine if a double-borrow is
    /// occuring on this structure.
    ///
    /// # Panics
    /// Panics when trying to set the lock on an upgraded handle.
    pub(crate) unsafe fn set_lock(&self, val: bool) {
        self.liveliness.as_ref()
            .expect("Tried to set lock on borrowed Pointer")
            .store(val, Ordering::Release);
    }
}

impl Drop for Pointer {
    fn drop(&mut self) {
        match self.liveliness {
            None => {}
            Some(ref liveliness) => {
                if Rc::strong_count(liveliness) == 1 {
                    wlr_log!(L_DEBUG, "Dropped Pointer {:p}", self.pointer);
                    let weak_count = Rc::weak_count(liveliness);
                    if weak_count > 0 {
                        wlr_log!(L_DEBUG,
                                 "Still {} weak pointers to Pointer {:p}",
                                 weak_count,
                                 self.pointer);
                    }
                }
            }
        }
    }
}

impl PointerHandle {
    /// Upgrades the pointer handle to a reference to the backing `Pointer`.
    ///
    /// # Unsafety
    /// This function is unsafe, because it creates an unbound `Pointer`
    /// which may live forever..
    /// But no pointer lives forever and might be disconnected at any time.
    pub unsafe fn upgrade(&self) -> UpgradeHandleResult<Pointer> {
        self.handle.upgrade()
            .ok_or(UpgradeHandleErr::AlreadyDropped)
            // NOTE
            // We drop the Rc here because having two would allow a dangling
            // pointer to exist!
            .and_then(|check| {
                let pointer = Pointer::from_handle(self);
                if check.load(Ordering::Acquire) {
                    wlr_log!(L_ERROR, "Double mutable borrows on {:?}", pointer);
                    panic!("Double mutable borrows detected");
                }
                check.store(true, Ordering::Release);
                Ok(pointer)
            })
    }

    /// Run a function on the referenced Pointer, if it still exists
    ///
    /// Returns the result of the function, if successful
    ///
    /// # Safety
    /// By enforcing a rather harsh limit on the lifetime of the output
    /// to a short lived scope of an anonymous function,
    /// this function ensures the Pointer does not live longer
    /// than it exists.
    ///
    /// # Panics
    /// This function will panic if multiple mutable borrows are detected.
    /// This will happen if you call `upgrade` directly within this callback,
    /// or if you run this function within the another run to the same `Output`.
    ///
    /// So don't nest `run` calls and everything will be ok :).
    pub fn run<F, R>(&mut self, runner: F) -> UpgradeHandleResult<R>
        where F: FnOnce(&Pointer) -> R
    {
        let mut pointer = unsafe { self.upgrade()? };
        let res = panic::catch_unwind(panic::AssertUnwindSafe(|| runner(&mut pointer)));
        self.handle.upgrade().map(|check| {
                                      // Sanity check that it hasn't been tampered with.
                                      if !check.load(Ordering::Acquire) {
                                          wlr_log!(L_ERROR,
                                                   "After running pointer callback, mutable lock \
                                                    was false for: {:?}",
                                                   pointer);
                                          panic!("Lock in incorrect state!");
                                      }
                                      check.store(false, Ordering::Release);
                                  });
        match res {
            Ok(res) => Ok(res),
            Err(err) => panic::resume_unwind(err)
        }
    }

    /// Gets the wlr_input_device associated with this PointerHandle.
    pub fn input_device(&self) -> &InputDevice {
        &self.device
    }

    /// Gets the wlr_pointer associated with this PointerHandle.
    pub(crate) unsafe fn as_ptr(&self) -> *mut wlr_pointer {
        self.pointer
    }
}

impl Clone for PointerHandle {
    fn clone(&self) -> Self {
        PointerHandle { pointer: self.pointer,
                        handle: self.handle.clone(),
                        /// NOTE Rationale for unsafe clone:
                        ///
                        /// You can only access it after a call to `upgrade`,
                        /// and that implicitly checks that it is valid.
                        device: unsafe { self.device.clone() } }
    }
}

impl PartialEq for PointerHandle {
    fn eq(&self, other: &PointerHandle) -> bool {
        self.pointer == other.pointer
    }
}

impl Eq for PointerHandle {}
<commit_msg>Remove bad doc comment on PointerHandle<commit_after>//! TODO Documentation

use std::panic;
use std::rc::{Rc, Weak};
use std::sync::atomic::{AtomicBool, Ordering};

use errors::{UpgradeHandleErr, UpgradeHandleResult};
use wlroots_sys::{wlr_input_device, wlr_pointer};

use InputDevice;

#[derive(Debug)]
pub struct Pointer {
    /// The structure that ensures weak handles to this structure are still alive.
    ///
    /// They contain weak handles, and will safely not use dead memory when this
    /// is freed by wlroots.
    ///
    /// If this is `None`, then this is from an upgraded `PointerHandle`, and
    /// the operations are **unchecked**.
    /// This is means safe operations might fail, but only if you use the unsafe
    /// marked function `upgrade` on a `PointerHandle`.
    liveliness: Option<Rc<AtomicBool>>,
    /// The device that refers to this pointer.
    device: InputDevice,
    /// The underlying pointer data.
    pointer: *mut wlr_pointer
}

#[derive(Debug)]
pub struct PointerHandle {
    /// The Rc that ensures that this handle is still alive.
    ///
    /// When wlroots deallocates the pointer associated with this handle,
    /// this can no longer be used.
    handle: Weak<AtomicBool>,
    /// The device that refers to this pointer.
    device: InputDevice,
    /// The underlying pointer data.
    pointer: *mut wlr_pointer
}

impl Pointer {
    /// Tries to convert an input device to a Pointer
    ///
    /// Returns none if it is of a different input variant.
    ///
    /// # Safety
    /// This creates a totally new Pointer (e.g with its own reference count)
    /// so only do this once per `wlr_input_device`!
    pub(crate) unsafe fn new_from_input_device(device: *mut wlr_input_device) -> Option<Self> {
        use wlroots_sys::wlr_input_device_type::*;
        match (*device).type_ {
            WLR_INPUT_DEVICE_POINTER => {
                let pointer = (*device).__bindgen_anon_1.pointer;
                Some(Pointer { liveliness: Some(Rc::new(AtomicBool::new(false))),
                               device: InputDevice::from_ptr(device),
                               pointer })
            }
            _ => None
        }
    }

    /// Creates an unbound Pointer from a `PointerHandle`
    unsafe fn from_handle(handle: &PointerHandle) -> Self {
        Pointer { liveliness: None,
                  device: handle.input_device().clone(),
                  pointer: handle.as_ptr() }
    }

    /// Gets the wlr_input_device associated with this Pointer.
    pub fn input_device(&self) -> &InputDevice {
        &self.device
    }

    /// Gets the wlr_pointer associated with this Pointer.
    #[allow(dead_code)]
    pub(crate) unsafe fn as_ptr(&self) -> *mut wlr_pointer {
        self.pointer
    }

    /// Creates a weak reference to a `Pointer`.
    ///
    /// # Panics
    /// If this `Pointer` is a previously upgraded `PointerHandle`,
    /// then this function will panic.
    pub fn weak_reference(&self) -> PointerHandle {
        let arc = self.liveliness.as_ref()
                      .expect("Cannot downgrade a previously upgraded PointerHandle!");
        PointerHandle { handle: Rc::downgrade(arc),
                        // NOTE Rationale for cloning:
                        // We can't use the keyboard handle unless the keyboard is alive,
                        // which means the device pointer is still alive.
                        device: unsafe { self.device.clone() },
                        pointer: self.pointer }
    }

    /// Manually set the lock used to determine if a double-borrow is
    /// occuring on this structure.
    ///
    /// # Panics
    /// Panics when trying to set the lock on an upgraded handle.
    pub(crate) unsafe fn set_lock(&self, val: bool) {
        self.liveliness.as_ref()
            .expect("Tried to set lock on borrowed Pointer")
            .store(val, Ordering::Release);
    }
}

impl Drop for Pointer {
    fn drop(&mut self) {
        match self.liveliness {
            None => {}
            Some(ref liveliness) => {
                if Rc::strong_count(liveliness) == 1 {
                    wlr_log!(L_DEBUG, "Dropped Pointer {:p}", self.pointer);
                    let weak_count = Rc::weak_count(liveliness);
                    if weak_count > 0 {
                        wlr_log!(L_DEBUG,
                                 "Still {} weak pointers to Pointer {:p}",
                                 weak_count,
                                 self.pointer);
                    }
                }
            }
        }
    }
}

impl PointerHandle {
    /// Upgrades the pointer handle to a reference to the backing `Pointer`.
    ///
    /// # Unsafety
    /// This function is unsafe, because it creates an unbound `Pointer`
    /// which may live forever..
    /// But no pointer lives forever and might be disconnected at any time.
    pub unsafe fn upgrade(&self) -> UpgradeHandleResult<Pointer> {
        self.handle.upgrade()
            .ok_or(UpgradeHandleErr::AlreadyDropped)
            // NOTE
            // We drop the Rc here because having two would allow a dangling
            // pointer to exist!
            .and_then(|check| {
                let pointer = Pointer::from_handle(self);
                if check.load(Ordering::Acquire) {
                    wlr_log!(L_ERROR, "Double mutable borrows on {:?}", pointer);
                    panic!("Double mutable borrows detected");
                }
                check.store(true, Ordering::Release);
                Ok(pointer)
            })
    }

    /// Run a function on the referenced Pointer, if it still exists
    ///
    /// Returns the result of the function, if successful
    ///
    /// # Safety
    /// By enforcing a rather harsh limit on the lifetime of the output
    /// to a short lived scope of an anonymous function,
    /// this function ensures the Pointer does not live longer
    /// than it exists.
    ///
    /// # Panics
    /// This function will panic if multiple mutable borrows are detected.
    /// This will happen if you call `upgrade` directly within this callback,
    /// or if you run this function within the another run to the same `Output`.
    ///
    /// So don't nest `run` calls and everything will be ok :).
    pub fn run<F, R>(&mut self, runner: F) -> UpgradeHandleResult<R>
        where F: FnOnce(&Pointer) -> R
    {
        let mut pointer = unsafe { self.upgrade()? };
        let res = panic::catch_unwind(panic::AssertUnwindSafe(|| runner(&mut pointer)));
        self.handle.upgrade().map(|check| {
                                      // Sanity check that it hasn't been tampered with.
                                      if !check.load(Ordering::Acquire) {
                                          wlr_log!(L_ERROR,
                                                   "After running pointer callback, mutable lock \
                                                    was false for: {:?}",
                                                   pointer);
                                          panic!("Lock in incorrect state!");
                                      }
                                      check.store(false, Ordering::Release);
                                  });
        match res {
            Ok(res) => Ok(res),
            Err(err) => panic::resume_unwind(err)
        }
    }

    /// Gets the wlr_input_device associated with this PointerHandle.
    pub fn input_device(&self) -> &InputDevice {
        &self.device
    }

    /// Gets the wlr_pointer associated with this PointerHandle.
    pub(crate) unsafe fn as_ptr(&self) -> *mut wlr_pointer {
        self.pointer
    }
}

impl Clone for PointerHandle {
    fn clone(&self) -> Self {
        PointerHandle { pointer: self.pointer,
                        handle: self.handle.clone(),
                        /// NOTE Rationale for unsafe clone:
                        ///
                        /// You can only access it after a call to `upgrade`,
                        /// and that implicitly checks that it is valid.
                        device: unsafe { self.device.clone() } }
    }
}

impl PartialEq for PointerHandle {
    fn eq(&self, other: &PointerHandle) -> bool {
        self.pointer == other.pointer
    }
}

impl Eq for PointerHandle {}
<|endoftext|>


<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate log;
extern crate osmpbfreader;
extern crate rustc_serialize;
extern crate docopt;
extern crate mimir;
extern crate rs_es;

use std::collections::HashSet;
use std::collections::HashMap;
use mimirsbrunn::rubber::Rubber;
use mimirsbrunn::objects::{Polygon, MultiPolygon};

pub type AdminsVec = Vec<mimir::Admin>;
pub type StreetsVec = Vec<mimir::Street>;
pub type ParsedPbf = osmpbfreader::OsmPbfReader<std::fs::File>;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_connection_string: String,
}

static USAGE: &'static str = "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> [--connection-string=<connection-string>] --level=<level>...

Options:
    -h, --help            Show this message.
    -i, --input=<file>    OSM PBF file.
    -l, --level=<level>   Admin levels to keep.
    -c, --connection-string=<connection-string>
                          Elasticsearch parameters, [default: http://localhost:9200/munin]
";

#[derive(Debug)]
struct AdminMatcher {
    admin_levels: HashSet<u32>,
}
impl AdminMatcher {
    pub fn new(levels: HashSet<u32>) -> AdminMatcher {
        AdminMatcher { admin_levels: levels }
    }

    pub fn is_admin(&self, obj: &osmpbfreader::OsmObj) -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("boundary").map_or(false, |v| v == "administrative") &&
                rel.tags.get("admin_level").map_or(false, |lvl| {
                    self.admin_levels.contains(&lvl.parse::<u32>().unwrap_or(0))
                })
            }
            _ => false,
        }
    }
}

struct BoundaryPart {
    nodes: Vec<osmpbfreader::Node>,
}

impl BoundaryPart {
    pub fn new(nodes: Vec<osmpbfreader::Node>) -> BoundaryPart {
        BoundaryPart { nodes: nodes }
    }
    pub fn first(&self) -> i64 {
        self.nodes.first().unwrap().id
    }
    pub fn last(&self) -> i64 {
        self.nodes.last().unwrap().id
    }
}

fn get_nodes(way: &osmpbfreader::Way,
             objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
             -> Vec<osmpbfreader::Node> {
    way.nodes
       .iter()
       .filter_map(|node_id| objects.get(&osmpbfreader::OsmId::Node(*node_id)))
       .filter_map(|node_obj| if let &osmpbfreader::OsmObj::Node(ref node) = node_obj {
           Some(node.clone())
       } else {
           None
       })
       .collect()
}

fn build_boundary(relation: &osmpbfreader::Relation,
                  objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
                  -> Option<MultiPolygon> {
    let mut boundary_parts: Vec<BoundaryPart> =
        relation.refs
                .iter()
                .filter(|rf| rf.role == "outer" || rf.role == "" || rf.role == "enclave")
                .filter_map(|refe| {
                    objects.get(&refe.member).or_else(|| {
                        warn!("missing element for relation {}", relation.id);
                        None
                    })
                })
                .filter_map(|way_obj| if let &osmpbfreader::OsmObj::Way(ref way) = way_obj {
                    Some(way)
                } else {
                    None
                })
                .map(|way| get_nodes(&way, objects))
                .filter(|nodes| nodes.len() > 1)
                .map(|nodes| BoundaryPart::new(nodes))
                .collect();
    let mut multipoly = MultiPolygon { polygons: Vec::new() };
    // we want to try build a polygon for a least each way
    while !boundary_parts.is_empty() {
        let mut current = -1;
        let mut first = 0;
        let mut nb_try = 0;

        let mut outer: Vec<osmpbfreader::Node> = Vec::new();
        // we try to close the polygon, if we can't we want to at least have tried one time per
        // way. We could improve that latter by trying to attach the way to both side of the
        // polygon
        let max_try = boundary_parts.len();
        while current != first && nb_try < max_try {
            let mut i = 0;
            while i < boundary_parts.len() {
                if outer.is_empty() {
                    // our polygon is empty, we initialise it with the current way
                    first = boundary_parts[i].first();
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    // this way has been used, we remove it from the pool
                    boundary_parts.remove(i);
                    continue;
                }
                if current == boundary_parts[i].first() {
                    // the start of current way touch the polygon, we add it and remove it from the
                    // pool
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else if current == boundary_parts[i].last() {
                    // the end of the current way touch the polygon, we reverse the way and add it
                    current = boundary_parts[i].first();
                    boundary_parts[i].nodes.reverse();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else {
                    i += 1;
                    // didnt do anything, we want to explore the next way, if we had do something we
                    // will have removed the current way and there will be no need to increment
                }
                if current == first {
                    // our polygon is closed, we create it and add it to the multipolygon
                    let polygon = Polygon::new(outer.iter()
                                                    .map(|n| {
                                                        mimir::Coord {
                                                            lat: n.lat,
                                                            lon: n.lon,
                                                        }
                                                    })
                                                    .collect());
                    multipoly.polygons.push(polygon);
                    break;
                }
            }
            nb_try += 1;
        }
    }
    debug!("polygon for relation {};{}", relation.id, multipoly.to_wkt());
    if multipoly.polygons.is_empty() {
        None
    } else {
        Some(multipoly)
    }
}

fn parse_osm_pbf(path: &String) -> ParsedPbf {
    let path = std::path::Path::new(&path);
    osmpbfreader::OsmPbfReader::new(std::fs::File::open(&path).unwrap())
}

fn administrative_regions(pbf: &mut ParsedPbf, levels: HashSet<u32>) -> AdminsVec {
    let mut administrative_regions = AdminsVec::new();
    let matcher = AdminMatcher::new(levels);
    let objects = osmpbfreader::get_objs_and_deps(pbf, |o| matcher.is_admin(o)).unwrap();
    // load administratives regions
    for (_, obj) in &objects {
        if !matcher.is_admin(&obj) {
            continue;
        }
        if let &osmpbfreader::OsmObj::Relation(ref relation) = obj {
            let level = relation.tags
                                .get("admin_level")
                                .and_then(|s| s.parse().ok());
            let level = match level {
                None => {
                    info!("invalid admin_level for relation {}: admin_level {:?}",
                          relation.id,
                          relation.tags.get("admin_level"));
                    continue;
                }
                Some(l) => l,
            };
            // administrative region with name ?
            let name = match relation.tags.get("name") {
                Some(val) => val,
                None => {
                    warn!("adminstrative region without name for relation {}:  admin_level {} \
                           ignored.",
                          relation.id,
                          level);
                    continue;
                }
            };
            // admininstrative region without coordinates
            let coord_centre = relation.refs
                                       .iter()
                                       .find(|rf| rf.role == "admin_centre")
                                       .and_then(|r| {
                                           objects.get(&r.member).and_then(|value| {
                                               match value {
                                                   &osmpbfreader::OsmObj::Node(ref node) => {
                                                       Some(mimir::Coord {
                                                           lat: node.lat,
                                                           lon: node.lon,
                                                       })
                                                   }
                                                   _ => None,
                                               }
                                           })
                                       });

            let admin_id = match relation.tags.get("ref:INSEE") {
                Some(val) => format!("admin:fr:{}", val.trim_left_matches('0')),
                None => format!("admin:osm:{}", relation.id),
            };
            let zip_code = match relation.tags.get("addr:postcode") {
                Some(val) => &val[..],
                None => "",
            };
            let boundary = build_boundary(&relation, &objects);
            let admin = mimir::Admin {
                id: admin_id,
                level: level,
                name: name.to_string(),
                zip_code: zip_code.to_string(),
                // TODO weight value ?
                weight: 1,
                coord: coord_centre,
                boundary: boundary,
            };
            administrative_regions.push(admin);
        }
    }
    return administrative_regions;
}

fn streets(pbf: &mut ParsedPbf) -> StreetsVec {

    let is_valid_street = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                way.tags.get("highway").map_or(false, |x: &String| !x.is_empty()) &&
                way.tags.get("name").map_or(false, |x: &String| !x.is_empty())
            }
            _ => false,
        }
    };

    osmpbfreader::get_objs_and_deps(pbf, is_valid_street)
        .unwrap()
        .iter()
        .filter_map(|(ref osm_id, ref obj)| {
            if let &&osmpbfreader::OsmObj::Way(ref way) = obj {
                if let &&osmpbfreader::OsmId::Way(ref way_id) = osm_id {
                    way.tags.get("name").and_then(|way_name| {
                        Some(mimirsbrunn::Street {
                            id: way_id.to_string(),
                            street_name: way_name.to_string(),
                            name: way_name.to_string(),
                            weight: 1,
                            administrative_region: None,
                        })
                    })
                } else {
                    None
                }
            } else {
                None
            }
        })
        .collect()
}

fn index_osm(es_cnx_string: &str, admins: &AdminsVec, streets: &StreetsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    rubber.create_index();
    match rubber.clean_db_by_doc_type(&["admin", "street"]) {
        Err(e) => panic!("failed to clean data by document type: {}", e),
        Ok(nb) => info!("clean data by document type : {}", nb),
    }
    info!("Add data in elasticsearch db.");
    match rubber.bulk_index(admins.iter()) {
        Err(e) => panic!("failed to index admins of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed adminstrative regions: {}", nb),
    }

    match rubber.bulk_index(streets.iter()) {
        Err(e) => panic!("failed to index streets of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed streets: {}", nb),
    }
}

fn main() {
    mimir::logger_init().unwrap();
    debug!("importing adminstrative region into Mimir");
    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);
    let admins = administrative_regions(&mut parsed_pbf, levels);
    let streets = streets(&mut parsed_pbf);
    index_osm(&args.flag_connection_string, &admins, &streets);
}
<commit_msg>manage the relation for street<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

#[macro_use]
extern crate log;
extern crate osmpbfreader;
extern crate rustc_serialize;
extern crate docopt;
extern crate mimir;
extern crate rs_es;

use std::collections::HashSet;
use std::collections::HashMap;
use mimirsbrunn::rubber::Rubber;
use mimirsbrunn::objects::{Polygon, MultiPolygon};

pub type AdminsVec = Vec<mimir::Admin>;
pub type StreetsVec = Vec<mimir::Street>;
pub type ParsedPbf = osmpbfreader::OsmPbfReader<std::fs::File>;

#[derive(RustcDecodable, Debug)]
struct Args {
    flag_input: String,
    flag_level: Vec<u32>,
    flag_connection_string: String,
}

static USAGE: &'static str = "
Usage:
    osm2mimir --help
    osm2mimir --input=<file> [--connection-string=<connection-string>] --level=<level>...

Options:
    -h, --help            Show this message.
    -i, --input=<file>    OSM PBF file.
    -l, --level=<level>   Admin levels to keep.
    -c, --connection-string=<connection-string>
                          Elasticsearch parameters, [default: http://localhost:9200/munin]
";

#[derive(Debug)]
struct AdminMatcher {
    admin_levels: HashSet<u32>,
}
impl AdminMatcher {
    pub fn new(levels: HashSet<u32>) -> AdminMatcher {
        AdminMatcher { admin_levels: levels }
    }

    pub fn is_admin(&self, obj: &osmpbfreader::OsmObj) -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("boundary").map_or(false, |v| v == "administrative") &&
                rel.tags.get("admin_level").map_or(false, |lvl| {
                    self.admin_levels.contains(&lvl.parse::<u32>().unwrap_or(0))
                })
            }
            _ => false,
        }
    }
}

struct BoundaryPart {
    nodes: Vec<osmpbfreader::Node>,
}

impl BoundaryPart {
    pub fn new(nodes: Vec<osmpbfreader::Node>) -> BoundaryPart {
        BoundaryPart { nodes: nodes }
    }
    pub fn first(&self) -> i64 {
        self.nodes.first().unwrap().id
    }
    pub fn last(&self) -> i64 {
        self.nodes.last().unwrap().id
    }
}

fn get_nodes(way: &osmpbfreader::Way,
             objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
             -> Vec<osmpbfreader::Node> {
    way.nodes
       .iter()
       .filter_map(|node_id| objects.get(&osmpbfreader::OsmId::Node(*node_id)))
       .filter_map(|node_obj| if let &osmpbfreader::OsmObj::Node(ref node) = node_obj {
           Some(node.clone())
       } else {
           None
       })
       .collect()
}

fn build_boundary(relation: &osmpbfreader::Relation,
                  objects: &HashMap<osmpbfreader::OsmId, osmpbfreader::OsmObj>)
                  -> Option<MultiPolygon> {
    let mut boundary_parts: Vec<BoundaryPart> =
        relation.refs
                .iter()
                .filter(|rf| rf.role == "outer" || rf.role == "" || rf.role == "enclave")
                .filter_map(|refe| {
                    objects.get(&refe.member).or_else(|| {
                        warn!("missing element for relation {}", relation.id);
                        None
                    })
                })
                .filter_map(|way_obj| if let &osmpbfreader::OsmObj::Way(ref way) = way_obj {
                    Some(way)
                } else {
                    None
                })
                .map(|way| get_nodes(&way, objects))
                .filter(|nodes| nodes.len() > 1)
                .map(|nodes| BoundaryPart::new(nodes))
                .collect();
    let mut multipoly = MultiPolygon { polygons: Vec::new() };
    // we want to try build a polygon for a least each way
    while !boundary_parts.is_empty() {
        let mut current = -1;
        let mut first = 0;
        let mut nb_try = 0;

        let mut outer: Vec<osmpbfreader::Node> = Vec::new();
        // we try to close the polygon, if we can't we want to at least have tried one time per
        // way. We could improve that latter by trying to attach the way to both side of the
        // polygon
        let max_try = boundary_parts.len();
        while current != first && nb_try < max_try {
            let mut i = 0;
            while i < boundary_parts.len() {
                if outer.is_empty() {
                    // our polygon is empty, we initialise it with the current way
                    first = boundary_parts[i].first();
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    // this way has been used, we remove it from the pool
                    boundary_parts.remove(i);
                    continue;
                }
                if current == boundary_parts[i].first() {
                    // the start of current way touch the polygon, we add it and remove it from the
                    // pool
                    current = boundary_parts[i].last();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else if current == boundary_parts[i].last() {
                    // the end of the current way touch the polygon, we reverse the way and add it
                    current = boundary_parts[i].first();
                    boundary_parts[i].nodes.reverse();
                    outer.append(&mut boundary_parts[i].nodes);
                    boundary_parts.remove(i);
                } else {
                    i += 1;
                    // didnt do anything, we want to explore the next way, if we had do something we
                    // will have removed the current way and there will be no need to increment
                }
                if current == first {
                    // our polygon is closed, we create it and add it to the multipolygon
                    let polygon = Polygon::new(outer.iter()
                                                    .map(|n| {
                                                        mimir::Coord {
                                                            lat: n.lat,
                                                            lon: n.lon,
                                                        }
                                                    })
                                                    .collect());
                    multipoly.polygons.push(polygon);
                    break;
                }
            }
            nb_try += 1;
        }
    }
    debug!("polygon for relation {};{}", relation.id, multipoly.to_wkt());
    if multipoly.polygons.is_empty() {
        None
    } else {
        Some(multipoly)
    }
}

fn parse_osm_pbf(path: &String) -> ParsedPbf {
    let path = std::path::Path::new(&path);
    osmpbfreader::OsmPbfReader::new(std::fs::File::open(&path).unwrap())
}

fn administrative_regions(pbf: &mut ParsedPbf, levels: HashSet<u32>) -> AdminsVec {
    let mut administrative_regions = AdminsVec::new();
    let matcher = AdminMatcher::new(levels);
    let objects = osmpbfreader::get_objs_and_deps(pbf, |o| matcher.is_admin(o)).unwrap();
    // load administratives regions
    for (_, obj) in &objects {
        if !matcher.is_admin(&obj) {
            continue;
        }
        if let &osmpbfreader::OsmObj::Relation(ref relation) = obj {
            let level = relation.tags
                                .get("admin_level")
                                .and_then(|s| s.parse().ok());
            let level = match level {
                None => {
                    info!("invalid admin_level for relation {}: admin_level {:?}",
                          relation.id,
                          relation.tags.get("admin_level"));
                    continue;
                }
                Some(l) => l,
            };
            // administrative region with name ?
            let name = match relation.tags.get("name") {
                Some(val) => val,
                None => {
                    warn!("adminstrative region without name for relation {}:  admin_level {} \
                           ignored.",
                          relation.id,
                          level);
                    continue;
                }
            };
            // admininstrative region without coordinates
            let coord_centre = relation.refs
                                       .iter()
                                       .find(|rf| rf.role == "admin_centre")
                                       .and_then(|r| {
                                           objects.get(&r.member).and_then(|value| {
                                               match value {
                                                   &osmpbfreader::OsmObj::Node(ref node) => {
                                                       Some(mimir::Coord {
                                                           lat: node.lat,
                                                           lon: node.lon,
                                                       })
                                                   }
                                                   _ => None,
                                               }
                                           })
                                       });

            let admin_id = match relation.tags.get("ref:INSEE") {
                Some(val) => format!("admin:fr:{}", val.trim_left_matches('0')),
                None => format!("admin:osm:{}", relation.id),
            };
            let zip_code = match relation.tags.get("addr:postcode") {
                Some(val) => &val[..],
                None => "",
            };
            let boundary = build_boundary(&relation, &objects);
            let admin = mimir::Admin {
                id: admin_id,
                level: level,
                name: name.to_string(),
                zip_code: zip_code.to_string(),
                // TODO weight value ?
                weight: 1,
                coord: coord_centre,
                boundary: boundary,
            };
            administrative_regions.push(admin);
        }
    }
    return administrative_regions;
}

fn streets(pbf: &mut ParsedPbf) -> StreetsVec {

    let is_valid_street = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Way(ref way) => {
                way.tags.get("highway").map_or(false, |x: &String| !x.is_empty()) &&
                way.tags.get("name").map_or(false, |x: &String| !x.is_empty())
            }
            _ => false,
        }
    };

    let is_associated_street_relation = |obj: &osmpbfreader::OsmObj| -> bool {
        match *obj {
            osmpbfreader::OsmObj::Relation(ref rel) => {
                rel.tags.get("type").map_or(false, |v| v == "associatedStreet")
            }
            _ => false,
        }
    };

    let mut street_map = osmpbfreader::get_objs_and_deps(pbf, is_valid_street).unwrap();
    let relation_map = osmpbfreader::get_objs_and_deps(pbf, is_associated_street_relation).unwrap();

    for (_, rel_obj) in &relation_map {
        if let &osmpbfreader::OsmObj::Relation(ref rel) = rel_obj {
            let mut found = false;
            for ref_obj in &rel.refs {
                if let &osmpbfreader::OsmId::Way(_) = &ref_obj.member {
                    if !found {
                        found = true;
                        continue;
                    };
                    street_map.remove(&ref_obj.member);
                };
            }
        };
    }

    street_map.iter()
              .filter_map(|(osm_id, obj)| {
                  if let &osmpbfreader::OsmObj::Way(ref way) = obj {
                      if let &osmpbfreader::OsmId::Way(ref way_id) = osm_id {
                          way.tags.get("name").and_then(|way_name| {
                              Some(mimirsbrunn::Street {
                                  id: way_id.to_string(),
                                  street_name: way_name.to_string(),
                                  name: way_name.to_string(),
                                  weight: 1,
                                  administrative_region: None,
                              })
                          })
                      } else {
                          None
                      }
                  } else {
                      None
                  }
              })
              .collect()

}

fn index_osm(es_cnx_string: &str, admins: &AdminsVec, streets: &StreetsVec) {
    let mut rubber = Rubber::new(es_cnx_string);
    rubber.create_index();
    match rubber.clean_db_by_doc_type(&["admin", "street"]) {
        Err(e) => panic!("failed to clean data by document type: {}", e),
        Ok(nb) => info!("clean data by document type : {}", nb),
    }
    info!("Add data in elasticsearch db.");
    match rubber.bulk_index(admins.iter()) {
        Err(e) => panic!("failed to index admins of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed adminstrative regions: {}", nb),
    }

    match rubber.bulk_index(streets.iter()) {
        Err(e) => panic!("failed to index streets of osm because: {}", e),
        Ok(nb) => info!("Nb of indexed streets: {}", nb),
    }
}

fn main() {
    mimir::logger_init().unwrap();
    debug!("importing adminstrative region into Mimir");
    let args: Args = docopt::Docopt::new(USAGE)
                         .and_then(|d| d.decode())
                         .unwrap_or_else(|e| e.exit());

    let levels = args.flag_level.iter().cloned().collect();
    let mut parsed_pbf = parse_osm_pbf(&args.flag_input);
    let admins = administrative_regions(&mut parsed_pbf, levels);
    let streets = streets(&mut parsed_pbf);
    index_osm(&args.flag_connection_string, &admins, &streets);
}
<|endoftext|>