<commit_before>// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Check license of third-party deps by inspecting src/vendor

use std::collections::{BTreeSet, HashSet, HashMap};
use std::fs::File;
use std::io::Read;
use std::path::Path;
use std::process::Command;

use serde_json;

static LICENSES: &'static [&'static str] = &[
    "MIT/Apache-2.0",
    "MIT / Apache-2.0",
    "Apache-2.0/MIT",
    "Apache-2.0 / MIT",
    "MIT OR Apache-2.0",
    "MIT",
    "Unlicense/MIT",
    "Unlicense OR MIT",
];

/// These are exceptions to Rust's permissive licensing policy, and
/// should be considered bugs. Exceptions are only allowed in Rust
/// tooling. It is _crucial_ that no exception crates be dependencies
/// of the Rust runtime (std / test).
static EXCEPTIONS: &'static [&'static str] = &[
    "mdbook",             // MPL2, mdbook
    "openssl",            // BSD+advertising clause, cargo, mdbook
    "pest",               // MPL2, mdbook via handlebars
    "thread-id",          // Apache-2.0, mdbook
    "toml-query",         // MPL-2.0, mdbook
    "is-match",           // MPL-2.0, mdbook
    "cssparser",          // MPL-2.0, rustdoc
    "smallvec",           // MPL-2.0, rustdoc
    "fuchsia-zircon-sys", // BSD-3-Clause, rustdoc, rustc, cargo
    "fuchsia-zircon",     // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir)
    "cssparser-macros",   // MPL-2.0, rustdoc
    "selectors",          // MPL-2.0, rustdoc
    "clippy_lints",       // MPL-2.0, rls
    "colored",            // MPL-2.0, rustfmt
    "ordslice",           // Apache-2.0, rls
    "cloudabi",           // BSD-2-Clause, (rls -> crossbeam-channel 0.2 -> rand 0.5)
];

/// Which crates to check against the whitelist?
static WHITELIST_CRATES: &'static [CrateVersion] = &[
    CrateVersion("rustc", "0.0.0"),
    CrateVersion("rustc_codegen_llvm", "0.0.0"),
];

/// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible.
static WHITELIST: &'static [Crate] = &[
    Crate("aho-corasick"),
    Crate("arrayvec"),
    Crate("atty"),
    Crate("backtrace"),
    Crate("backtrace-sys"),
    Crate("bitflags"),
    Crate("byteorder"),
    Crate("cc"),
    Crate("chalk-engine"),
    Crate("chalk-macros"),
    Crate("cfg-if"),
    Crate("cmake"),
    Crate("crossbeam-deque"),
    Crate("crossbeam-epoch"),
    Crate("crossbeam-utils"),
    Crate("datafrog"),
    Crate("either"),
    Crate("ena"),
    Crate("env_logger"),
    Crate("filetime"),
    Crate("flate2"),
    Crate("fuchsia-zircon"),
    Crate("fuchsia-zircon-sys"),
    Crate("getopts"),
    Crate("humantime"),
    Crate("jobserver"),
    Crate("kernel32-sys"),
    Crate("lazy_static"),
    Crate("libc"),
    Crate("log"),
    Crate("log_settings"),
    Crate("memchr"),
    Crate("memoffset"),
    Crate("miniz-sys"),
    Crate("nodrop"),
    Crate("num_cpus"),
    Crate("owning_ref"),
    Crate("parking_lot"),
    Crate("parking_lot_core"),
    Crate("polonius-engine"),
    Crate("pkg-config"),
    Crate("quick-error"),
    Crate("rand"),
    Crate("redox_syscall"),
    Crate("redox_termios"),
    Crate("regex"),
    Crate("regex-syntax"),
    Crate("remove_dir_all"),
    Crate("rustc-demangle"),
    Crate("rustc-hash"),
    Crate("rustc-rayon"),
    Crate("rustc-rayon-core"),
    Crate("scoped-tls"),
    Crate("scopeguard"),
    Crate("smallvec"),
    Crate("stable_deref_trait"),
    Crate("tempfile"),
    Crate("termcolor"),
    Crate("terminon"),
    Crate("termion"),
    Crate("thread_local"),
    Crate("ucd-util"),
    Crate("unicode-width"),
    Crate("unreachable"),
    Crate("utf8-ranges"),
    Crate("void"),
    Crate("winapi"),
    Crate("winapi-build"),
    Crate("winapi-i686-pc-windows-gnu"),
    Crate("winapi-x86_64-pc-windows-gnu"),
    Crate("wincolor"),
];

// Some types for Serde to deserialize the output of `cargo metadata` to...

#[derive(Deserialize)]
struct Output {
    resolve: Resolve,
}

#[derive(Deserialize)]
struct Resolve {
    nodes: Vec<ResolveNode>,
}

#[derive(Deserialize)]
struct ResolveNode {
    id: String,
    dependencies: Vec<String>,
}

/// A unique identifier for a crate
#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
struct Crate<'a>(&'a str); // (name,)

#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
struct CrateVersion<'a>(&'a str, &'a str); // (name, version)

impl<'a> Crate<'a> {
    pub fn id_str(&self) -> String {
        format!("{} ", self.0)
    }
}

impl<'a> CrateVersion<'a> {
    /// Returns the struct and whether or not the dep is in-tree
    pub fn from_str(s: &'a str) -> (Self, bool) {
        let mut parts = s.split(' ');
        let name = parts.next().unwrap();
        let version = parts.next().unwrap();
        let path = parts.next().unwrap();

        let is_path_dep = path.starts_with("(path+");

        (CrateVersion(name, version), is_path_dep)
    }

    pub fn id_str(&self) -> String {
        format!("{} {}", self.0, self.1)
    }
}

impl<'a> From<CrateVersion<'a>> for Crate<'a> {
    fn from(cv: CrateVersion<'a>) -> Crate<'a> {
        Crate(cv.0)
    }
}

/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed.
///
/// Specifically, this checks that the license is correct.
pub fn check(path: &Path, bad: &mut bool) {
    // Check licences
    let path = path.join("vendor");
    assert!(path.exists(), "vendor directory missing");
    let mut saw_dir = false;
    for dir in t!(path.read_dir()) {
        saw_dir = true;
        let dir = t!(dir);

        // skip our exceptions
        if EXCEPTIONS.iter().any(|exception| {
            dir.path()
                .to_str()
                .unwrap()
                .contains(&format!("src/vendor/{}", exception))
        }) {
            continue;
        }

        let toml = dir.path().join("Cargo.toml");
        *bad = *bad || !check_license(&toml);
    }
    assert!(saw_dir, "no vendored source");
}

/// Checks the dependency of WHITELIST_CRATES at the given path. Changes `bad` to `true` if a check
/// failed.
///
/// Specifically, this checks that the dependencies are on the WHITELIST.
pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) {
    // Get dependencies from cargo metadata
    let resolve = get_deps(path, cargo);

    // Get the whitelist into a convenient form
    let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect();

    // Check dependencies
    let mut visited = BTreeSet::new();
    let mut unapproved = BTreeSet::new();
    for &krate in WHITELIST_CRATES.iter() {
        let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate, false);
        unapproved.append(&mut bad);
    }

    if unapproved.len() > 0 {
        println!("Dependencies not on the whitelist:");
        for dep in unapproved {
            println!("* {}", dep.id_str());
        }
        *bad = true;
    }

    check_crate_duplicate(&resolve, bad);
}

fn check_license(path: &Path) -> bool {
    if !path.exists() {
        panic!("{} does not exist", path.display());
    }
    let mut contents = String::new();
    t!(t!(File::open(path)).read_to_string(&mut contents));

    let mut found_license = false;
    for line in contents.lines() {
        if !line.starts_with("license") {
            continue;
        }
        let license = extract_license(line);
        if !LICENSES.contains(&&*license) {
            println!("invalid license {} in {}", license, path.display());
            return false;
        }
        found_license = true;
        break;
    }
    if !found_license {
        println!("no license in {}", path.display());
        return false;
    }

    true
}

fn extract_license(line: &str) -> String {
    let first_quote = line.find('"');
    let last_quote = line.rfind('"');
    if let (Some(f), Some(l)) = (first_quote, last_quote) {
        let license = &line[f + 1..l];
        license.into()
    } else {
        "bad-license-parse".into()
    }
}

/// Get the dependencies of the crate at the given path using `cargo metadata`.
fn get_deps(path: &Path, cargo: &Path) -> Resolve {
    // Run `cargo metadata` to get the set of dependencies
    let output = Command::new(cargo)
        .arg("metadata")
        .arg("--format-version")
        .arg("1")
        .arg("--manifest-path")
        .arg(path.join("Cargo.toml"))
        .output()
        .expect("Unable to run `cargo metadata`")
        .stdout;
    let output = String::from_utf8_lossy(&output);
    let output: Output = serde_json::from_str(&output).unwrap();

    output.resolve
}

/// Checks the dependencies of the given crate from the given cargo metadata to see if they are on
/// the whitelist. Returns a list of illegal dependencies.
fn check_crate_whitelist<'a, 'b>(
    whitelist: &'a HashSet<Crate>,
    resolve: &'a Resolve,
    visited: &'b mut BTreeSet<CrateVersion<'a>>,
    krate: CrateVersion<'a>,
    must_be_on_whitelist: bool,
) -> BTreeSet<Crate<'a>> {
    // Will contain bad deps
    let mut unapproved = BTreeSet::new();

    // Check if we have already visited this crate
    if visited.contains(&krate) {
        return unapproved;
    }

    visited.insert(krate);

    // If this path is in-tree, we don't require it to be on the whitelist
    if must_be_on_whitelist {
        // If this dependency is not on the WHITELIST, add to bad set
        if !whitelist.contains(&krate.into()) {
            unapproved.insert(krate.into());
        }
    }

    // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!)
    let to_check = resolve
        .nodes
        .iter()
        .find(|n| n.id.starts_with(&krate.id_str()))
        .expect("crate does not exist");

    for dep in to_check.dependencies.iter() {
        let (krate, is_path_dep) = CrateVersion::from_str(dep);

        let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate, !is_path_dep);
        unapproved.append(&mut bad);
    }

    unapproved
}

fn check_crate_duplicate(resolve: &Resolve, bad: &mut bool) {
    const FORBIDDEN_TO_HAVE_DUPLICATES: &[&str] = &[
        // These two crates take quite a long time to build, let's not let two
        // versions of them accidentally sneak into our dependency graph to
        // ensure we keep our CI times under control
        // "cargo", // FIXME(#53005)
        // "rustc-ap-syntax", // FIXME(#53006)
    ];
    let mut name_to_id = HashMap::new();
    for node in resolve.nodes.iter() {
        name_to_id.entry(node.id.split_whitespace().next().unwrap())
            .or_insert(Vec::new())
            .push(&node.id);
    }

    for name in FORBIDDEN_TO_HAVE_DUPLICATES {
        if name_to_id[name].len() <= 1 {
            continue
        }
        println!("crate `{}` is duplicated in `Cargo.lock`", name);
        for id in name_to_id[name].iter() {
            println!("  * {}", id);
        }
        *bad = true;
    }
}
<commit_msg>Deny future duplication of rustc-ap-syntax<commit_after>// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! Check license of third-party deps by inspecting src/vendor

use std::collections::{BTreeSet, HashSet, HashMap};
use std::fs::File;
use std::io::Read;
use std::path::Path;
use std::process::Command;

use serde_json;

static LICENSES: &'static [&'static str] = &[
    "MIT/Apache-2.0",
    "MIT / Apache-2.0",
    "Apache-2.0/MIT",
    "Apache-2.0 / MIT",
    "MIT OR Apache-2.0",
    "MIT",
    "Unlicense/MIT",
    "Unlicense OR MIT",
];

/// These are exceptions to Rust's permissive licensing policy, and
/// should be considered bugs. Exceptions are only allowed in Rust
/// tooling. It is _crucial_ that no exception crates be dependencies
/// of the Rust runtime (std / test).
static EXCEPTIONS: &'static [&'static str] = &[
    "mdbook",             // MPL2, mdbook
    "openssl",            // BSD+advertising clause, cargo, mdbook
    "pest",               // MPL2, mdbook via handlebars
    "thread-id",          // Apache-2.0, mdbook
    "toml-query",         // MPL-2.0, mdbook
    "is-match",           // MPL-2.0, mdbook
    "cssparser",          // MPL-2.0, rustdoc
    "smallvec",           // MPL-2.0, rustdoc
    "fuchsia-zircon-sys", // BSD-3-Clause, rustdoc, rustc, cargo
    "fuchsia-zircon",     // BSD-3-Clause, rustdoc, rustc, cargo (jobserver & tempdir)
    "cssparser-macros",   // MPL-2.0, rustdoc
    "selectors",          // MPL-2.0, rustdoc
    "clippy_lints",       // MPL-2.0, rls
    "colored",            // MPL-2.0, rustfmt
    "ordslice",           // Apache-2.0, rls
    "cloudabi",           // BSD-2-Clause, (rls -> crossbeam-channel 0.2 -> rand 0.5)
];

/// Which crates to check against the whitelist?
static WHITELIST_CRATES: &'static [CrateVersion] = &[
    CrateVersion("rustc", "0.0.0"),
    CrateVersion("rustc_codegen_llvm", "0.0.0"),
];

/// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible.
static WHITELIST: &'static [Crate] = &[
    Crate("aho-corasick"),
    Crate("arrayvec"),
    Crate("atty"),
    Crate("backtrace"),
    Crate("backtrace-sys"),
    Crate("bitflags"),
    Crate("byteorder"),
    Crate("cc"),
    Crate("chalk-engine"),
    Crate("chalk-macros"),
    Crate("cfg-if"),
    Crate("cmake"),
    Crate("crossbeam-deque"),
    Crate("crossbeam-epoch"),
    Crate("crossbeam-utils"),
    Crate("datafrog"),
    Crate("either"),
    Crate("ena"),
    Crate("env_logger"),
    Crate("filetime"),
    Crate("flate2"),
    Crate("fuchsia-zircon"),
    Crate("fuchsia-zircon-sys"),
    Crate("getopts"),
    Crate("humantime"),
    Crate("jobserver"),
    Crate("kernel32-sys"),
    Crate("lazy_static"),
    Crate("libc"),
    Crate("log"),
    Crate("log_settings"),
    Crate("memchr"),
    Crate("memoffset"),
    Crate("miniz-sys"),
    Crate("nodrop"),
    Crate("num_cpus"),
    Crate("owning_ref"),
    Crate("parking_lot"),
    Crate("parking_lot_core"),
    Crate("polonius-engine"),
    Crate("pkg-config"),
    Crate("quick-error"),
    Crate("rand"),
    Crate("redox_syscall"),
    Crate("redox_termios"),
    Crate("regex"),
    Crate("regex-syntax"),
    Crate("remove_dir_all"),
    Crate("rustc-demangle"),
    Crate("rustc-hash"),
    Crate("rustc-rayon"),
    Crate("rustc-rayon-core"),
    Crate("scoped-tls"),
    Crate("scopeguard"),
    Crate("smallvec"),
    Crate("stable_deref_trait"),
    Crate("tempfile"),
    Crate("termcolor"),
    Crate("terminon"),
    Crate("termion"),
    Crate("thread_local"),
    Crate("ucd-util"),
    Crate("unicode-width"),
    Crate("unreachable"),
    Crate("utf8-ranges"),
    Crate("void"),
    Crate("winapi"),
    Crate("winapi-build"),
    Crate("winapi-i686-pc-windows-gnu"),
    Crate("winapi-x86_64-pc-windows-gnu"),
    Crate("wincolor"),
];

// Some types for Serde to deserialize the output of `cargo metadata` to...

#[derive(Deserialize)]
struct Output {
    resolve: Resolve,
}

#[derive(Deserialize)]
struct Resolve {
    nodes: Vec<ResolveNode>,
}

#[derive(Deserialize)]
struct ResolveNode {
    id: String,
    dependencies: Vec<String>,
}

/// A unique identifier for a crate
#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
struct Crate<'a>(&'a str); // (name,)

#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)]
struct CrateVersion<'a>(&'a str, &'a str); // (name, version)

impl<'a> Crate<'a> {
    pub fn id_str(&self) -> String {
        format!("{} ", self.0)
    }
}

impl<'a> CrateVersion<'a> {
    /// Returns the struct and whether or not the dep is in-tree
    pub fn from_str(s: &'a str) -> (Self, bool) {
        let mut parts = s.split(' ');
        let name = parts.next().unwrap();
        let version = parts.next().unwrap();
        let path = parts.next().unwrap();

        let is_path_dep = path.starts_with("(path+");

        (CrateVersion(name, version), is_path_dep)
    }

    pub fn id_str(&self) -> String {
        format!("{} {}", self.0, self.1)
    }
}

impl<'a> From<CrateVersion<'a>> for Crate<'a> {
    fn from(cv: CrateVersion<'a>) -> Crate<'a> {
        Crate(cv.0)
    }
}

/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed.
///
/// Specifically, this checks that the license is correct.
pub fn check(path: &Path, bad: &mut bool) {
    // Check licences
    let path = path.join("vendor");
    assert!(path.exists(), "vendor directory missing");
    let mut saw_dir = false;
    for dir in t!(path.read_dir()) {
        saw_dir = true;
        let dir = t!(dir);

        // skip our exceptions
        if EXCEPTIONS.iter().any(|exception| {
            dir.path()
                .to_str()
                .unwrap()
                .contains(&format!("src/vendor/{}", exception))
        }) {
            continue;
        }

        let toml = dir.path().join("Cargo.toml");
        *bad = *bad || !check_license(&toml);
    }
    assert!(saw_dir, "no vendored source");
}

/// Checks the dependency of WHITELIST_CRATES at the given path. Changes `bad` to `true` if a check
/// failed.
///
/// Specifically, this checks that the dependencies are on the WHITELIST.
pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) {
    // Get dependencies from cargo metadata
    let resolve = get_deps(path, cargo);

    // Get the whitelist into a convenient form
    let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect();

    // Check dependencies
    let mut visited = BTreeSet::new();
    let mut unapproved = BTreeSet::new();
    for &krate in WHITELIST_CRATES.iter() {
        let mut bad = check_crate_whitelist(&whitelist, &resolve, &mut visited, krate, false);
        unapproved.append(&mut bad);
    }

    if unapproved.len() > 0 {
        println!("Dependencies not on the whitelist:");
        for dep in unapproved {
            println!("* {}", dep.id_str());
        }
        *bad = true;
    }

    check_crate_duplicate(&resolve, bad);
}

fn check_license(path: &Path) -> bool {
    if !path.exists() {
        panic!("{} does not exist", path.display());
    }
    let mut contents = String::new();
    t!(t!(File::open(path)).read_to_string(&mut contents));

    let mut found_license = false;
    for line in contents.lines() {
        if !line.starts_with("license") {
            continue;
        }
        let license = extract_license(line);
        if !LICENSES.contains(&&*license) {
            println!("invalid license {} in {}", license, path.display());
            return false;
        }
        found_license = true;
        break;
    }
    if !found_license {
        println!("no license in {}", path.display());
        return false;
    }

    true
}

fn extract_license(line: &str) -> String {
    let first_quote = line.find('"');
    let last_quote = line.rfind('"');
    if let (Some(f), Some(l)) = (first_quote, last_quote) {
        let license = &line[f + 1..l];
        license.into()
    } else {
        "bad-license-parse".into()
    }
}

/// Get the dependencies of the crate at the given path using `cargo metadata`.
fn get_deps(path: &Path, cargo: &Path) -> Resolve {
    // Run `cargo metadata` to get the set of dependencies
    let output = Command::new(cargo)
        .arg("metadata")
        .arg("--format-version")
        .arg("1")
        .arg("--manifest-path")
        .arg(path.join("Cargo.toml"))
        .output()
        .expect("Unable to run `cargo metadata`")
        .stdout;
    let output = String::from_utf8_lossy(&output);
    let output: Output = serde_json::from_str(&output).unwrap();

    output.resolve
}

/// Checks the dependencies of the given crate from the given cargo metadata to see if they are on
/// the whitelist. Returns a list of illegal dependencies.
fn check_crate_whitelist<'a, 'b>(
    whitelist: &'a HashSet<Crate>,
    resolve: &'a Resolve,
    visited: &'b mut BTreeSet<CrateVersion<'a>>,
    krate: CrateVersion<'a>,
    must_be_on_whitelist: bool,
) -> BTreeSet<Crate<'a>> {
    // Will contain bad deps
    let mut unapproved = BTreeSet::new();

    // Check if we have already visited this crate
    if visited.contains(&krate) {
        return unapproved;
    }

    visited.insert(krate);

    // If this path is in-tree, we don't require it to be on the whitelist
    if must_be_on_whitelist {
        // If this dependency is not on the WHITELIST, add to bad set
        if !whitelist.contains(&krate.into()) {
            unapproved.insert(krate.into());
        }
    }

    // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!)
    let to_check = resolve
        .nodes
        .iter()
        .find(|n| n.id.starts_with(&krate.id_str()))
        .expect("crate does not exist");

    for dep in to_check.dependencies.iter() {
        let (krate, is_path_dep) = CrateVersion::from_str(dep);

        let mut bad = check_crate_whitelist(whitelist, resolve, visited, krate, !is_path_dep);
        unapproved.append(&mut bad);
    }

    unapproved
}

fn check_crate_duplicate(resolve: &Resolve, bad: &mut bool) {
    const FORBIDDEN_TO_HAVE_DUPLICATES: &[&str] = &[
        // These two crates take quite a long time to build, let's not let two
        // versions of them accidentally sneak into our dependency graph to
        // ensure we keep our CI times under control
        // "cargo", // FIXME(#53005)
        "rustc-ap-syntax",
    ];
    let mut name_to_id = HashMap::new();
    for node in resolve.nodes.iter() {
        name_to_id.entry(node.id.split_whitespace().next().unwrap())
            .or_insert(Vec::new())
            .push(&node.id);
    }

    for name in FORBIDDEN_TO_HAVE_DUPLICATES {
        if name_to_id[name].len() <= 1 {
            continue
        }
        println!("crate `{}` is duplicated in `Cargo.lock`", name);
        for id in name_to_id[name].iter() {
            println!("  * {}", id);
        }
        *bad = true;
    }
}
<|endoftext|>


<commit_before>//! Script to check the validity of `href` links in our HTML documentation.
//!
//! In the past we've been quite error prone to writing in broken links as most
//! of them are manually rather than automatically added. As files move over
//! time or apis change old links become stale or broken. The purpose of this
//! script is to check all relative links in our documentation to make sure they
//! actually point to a valid place.
//!
//! Currently this doesn't actually do any HTML parsing or anything fancy like
//! that, it just has a simple "regex" to search for `href` and `id` tags.
//! These values are then translated to file URLs if possible and then the
//! destination is asserted to exist.
//!
//! A few whitelisted exceptions are allowed as there's known bugs in rustdoc,
//! but this should catch the majority of "broken link" cases.

#![deny(warnings)]

use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
use std::env;
use std::fs;
use std::path::{Component, Path, PathBuf};
use std::rc::Rc;

use crate::Redirect::*;

macro_rules! t {
    ($e:expr) => {
        match $e {
            Ok(e) => e,
            Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
        }
    };
}

fn main() {
    let docs = env::args_os().nth(1).unwrap();
    let docs = env::current_dir().unwrap().join(docs);
    let mut errors = false;
    walk(&mut HashMap::new(), &docs, &docs, &mut errors);
    if errors {
        panic!("found some broken links");
    }
}

#[derive(Debug)]
pub enum LoadError {
    IOError(std::io::Error),
    BrokenRedirect(PathBuf, std::io::Error),
    IsRedirect,
}

enum Redirect {
    SkipRedirect,
    FromRedirect(bool),
}

struct FileEntry {
    source: Rc<String>,
    ids: HashSet<String>,
}

type Cache = HashMap<PathBuf, FileEntry>;

fn small_url_encode(s: &str) -> String {
    s.replace("<", "%3C")
        .replace(">", "%3E")
        .replace(" ", "%20")
        .replace("?", "%3F")
        .replace("'", "%27")
        .replace("&", "%26")
        .replace(",", "%2C")
        .replace(":", "%3A")
        .replace(";", "%3B")
        .replace("[", "%5B")
        .replace("]", "%5D")
        .replace("\"", "%22")
}

impl FileEntry {
    fn parse_ids(&mut self, file: &Path, contents: &str, errors: &mut bool) {
        if self.ids.is_empty() {
            with_attrs_in_source(contents, " id", |fragment, i, _| {
                let frag = fragment.trim_start_matches("#").to_owned();
                let encoded = small_url_encode(&frag);
                if !self.ids.insert(frag) {
                    *errors = true;
                    println!("{}:{}: id is not unique: `{}`", file.display(), i, fragment);
                }
                // Just in case, we also add the encoded id.
                self.ids.insert(encoded);
            });
        }
    }
}

fn walk(cache: &mut Cache, root: &Path, dir: &Path, errors: &mut bool) {
    for entry in t!(dir.read_dir()).map(|e| t!(e)) {
        let path = entry.path();
        let kind = t!(entry.file_type());
        if kind.is_dir() {
            walk(cache, root, &path, errors);
        } else {
            let pretty_path = check(cache, root, &path, errors);
            if let Some(pretty_path) = pretty_path {
                let entry = cache.get_mut(&pretty_path).unwrap();
                // we don't need the source anymore,
                // so drop to reduce memory-usage
                entry.source = Rc::new(String::new());
            }
        }
    }
}

fn check(cache: &mut Cache, root: &Path, file: &Path, errors: &mut bool) -> Option<PathBuf> {
    // Ignore none HTML files.
    if file.extension().and_then(|s| s.to_str()) != Some("html") {
        return None;
    }

    // Unfortunately we're not 100% full of valid links today to we need a few
    // whitelists to get this past `make check` today.
    // FIXME(#32129)
    if file.ends_with("std/string/struct.String.html")
        || file.ends_with("interpret/struct.ImmTy.html")
        || file.ends_with("ast/struct.ThinVec.html")
        || file.ends_with("util/struct.ThinVec.html")
        || file.ends_with("layout/struct.TyLayout.html")
        || file.ends_with("humantime/struct.Timestamp.html")
        || file.ends_with("log/index.html")
        || file.ends_with("ty/struct.Slice.html")
        || file.ends_with("ty/enum.Attributes.html")
        || file.ends_with("ty/struct.SymbolName.html")
        || file.ends_with("io/struct.IoSlice.html")
        || file.ends_with("io/struct.IoSliceMut.html")
    {
        return None;
    }
    // FIXME(#32553)
    if file.ends_with("string/struct.String.html") {
        return None;
    }
    // FIXME(#32130)
    if file.ends_with("btree_set/struct.BTreeSet.html")
        || file.ends_with("struct.BTreeSet.html")
        || file.ends_with("btree_map/struct.BTreeMap.html")
        || file.ends_with("hash_map/struct.HashMap.html")
        || file.ends_with("hash_set/struct.HashSet.html")
        || file.ends_with("sync/struct.Lrc.html")
        || file.ends_with("sync/struct.RwLock.html")
    {
        return None;
    }

    let res = load_file(cache, root, file, SkipRedirect);
    let (pretty_file, contents) = match res {
        Ok(res) => res,
        Err(_) => return None,
    };
    {
        cache.get_mut(&pretty_file).unwrap().parse_ids(&pretty_file, &contents, errors);
    }

    // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
    with_attrs_in_source(&contents, " href", |url, i, base| {
        // Ignore external URLs
        if url.starts_with("http:")
            || url.starts_with("https:")
            || url.starts_with("javascript:")
            || url.starts_with("ftp:")
            || url.starts_with("irc:")
            || url.starts_with("data:")
        {
            return;
        }
        let mut parts = url.splitn(2, "#");
        let url = parts.next().unwrap();
        let fragment = parts.next();
        let mut parts = url.splitn(2, "?");
        let url = parts.next().unwrap();

        // Once we've plucked out the URL, parse it using our base url and
        // then try to extract a file path.
        let mut path = file.to_path_buf();
        if !base.is_empty() || !url.is_empty() {
            path.pop();
            for part in Path::new(base).join(url).components() {
                match part {
                    Component::Prefix(_) | Component::RootDir => {
                        // Avoid absolute paths as they make the docs not
                        // relocatable by making assumptions on where the docs
                        // are hosted relative to the site root.
                        *errors = true;
                        println!(
                            "{}:{}: absolute path - {}",
                            pretty_file.display(),
                            i + 1,
                            Path::new(base).join(url).display()
                        );
                        return;
                    }
                    Component::CurDir => {}
                    Component::ParentDir => {
                        path.pop();
                    }
                    Component::Normal(s) => {
                        path.push(s);
                    }
                }
            }
        }

        // Alright, if we've found a file name then this file had better
        // exist! If it doesn't then we register and print an error.
        if path.exists() {
            if path.is_dir() {
                // Links to directories show as directory listings when viewing
                // the docs offline so it's best to avoid them.
                *errors = true;
                let pretty_path = path.strip_prefix(root).unwrap_or(&path);
                println!(
                    "{}:{}: directory link - {}",
                    pretty_file.display(),
                    i + 1,
                    pretty_path.display()
                );
                return;
            }
            if let Some(extension) = path.extension() {
                // Ignore none HTML files.
                if extension != "html" {
                    return;
                }
            }
            let res = load_file(cache, root, &path, FromRedirect(false));
            let (pretty_path, contents) = match res {
                Ok(res) => res,
                Err(LoadError::IOError(err)) => {
                    panic!("error loading {}: {}", path.display(), err);
                }
                Err(LoadError::BrokenRedirect(target, _)) => {
                    *errors = true;
                    println!(
                        "{}:{}: broken redirect to {}",
                        pretty_file.display(),
                        i + 1,
                        target.display()
                    );
                    return;
                }
                Err(LoadError::IsRedirect) => unreachable!(),
            };

            if let Some(ref fragment) = fragment {
                // Fragments like `#1-6` are most likely line numbers to be
                // interpreted by javascript, so we're ignoring these
                if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
                    return;
                }

                // These appear to be broken in mdbook right now?
                if fragment.starts_with("-") {
                    return;
                }

                let entry = &mut cache.get_mut(&pretty_path).unwrap();
                entry.parse_ids(&pretty_path, &contents, errors);

                if !entry.ids.contains(*fragment) {
                    *errors = true;
                    print!("{}:{}: broken link fragment ", pretty_file.display(), i + 1);
                    println!("`#{}` pointing to `{}`", fragment, pretty_path.display());
                };
            }
        } else {
            *errors = true;
            print!("{}:{}: broken link - ", pretty_file.display(), i + 1);
            let pretty_path = path.strip_prefix(root).unwrap_or(&path);
            println!("{}", pretty_path.display());
        }
    });
    Some(pretty_file)
}

fn load_file(
    cache: &mut Cache,
    root: &Path,
    file: &Path,
    redirect: Redirect,
) -> Result<(PathBuf, Rc<String>), LoadError> {
    let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file));

    let (maybe_redirect, contents) = match cache.entry(pretty_file.clone()) {
        Entry::Occupied(entry) => (None, entry.get().source.clone()),
        Entry::Vacant(entry) => {
            let contents = match fs::read_to_string(file) {
                Ok(s) => Rc::new(s),
                Err(err) => {
                    return Err(if let FromRedirect(true) = redirect {
                        LoadError::BrokenRedirect(file.to_path_buf(), err)
                    } else {
                        LoadError::IOError(err)
                    });
                }
            };

            let maybe = maybe_redirect(&contents);
            if maybe.is_some() {
                if let SkipRedirect = redirect {
                    return Err(LoadError::IsRedirect);
                }
            } else {
                entry.insert(FileEntry { source: contents.clone(), ids: HashSet::new() });
            }
            (maybe, contents)
        }
    };
    match maybe_redirect.map(|url| file.parent().unwrap().join(url)) {
        Some(redirect_file) => load_file(cache, root, &redirect_file, FromRedirect(true)),
        None => Ok((pretty_file, contents)),
    }
}

fn maybe_redirect(source: &str) -> Option<String> {
    const REDIRECT: &'static str = "
Redirecting to <a href=";

    let mut lines = source.lines();
    let redirect_line = lines.nth(6)?;

    redirect_line.find(REDIRECT).map(|i| {
        let rest = &redirect_line[(i + REDIRECT.len() + 1)..];
        let pos_quote = rest.find('"').unwrap();
        rest[..pos_quote].to_owned()
    })
}

fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(contents: &str, attr: &str, mut f: F) {
    let mut base = "";
    for (i, mut line) in contents.lines().enumerate() {
        while let Some(j) = line.find(attr) {
            let rest = &line[j + attr.len()..];
            // The base tag should always be the first link in the document so
            // we can get away with using one pass.
            let is_base = line[..j].ends_with("<base");
            line = rest;
            let pos_equals = match rest.find("=") {
                Some(i) => i,
                None => continue,
            };
            if rest[..pos_equals].trim_start_matches(" ") != "" {
                continue;
            }

            let rest = &rest[pos_equals + 1..];

            let pos_quote = match rest.find(&['"', '\''][..]) {
                Some(i) => i,
                None => continue,
            };
            let quote_delim = rest.as_bytes()[pos_quote] as char;

            if rest[..pos_quote].trim_start_matches(" ") != "" {
                continue;
            }
            let rest = &rest[pos_quote + 1..];
            let url = match rest.find(quote_delim) {
                Some(i) => &rest[..i],
                None => continue,
            };
            if is_base {
                base = url;
                continue;
            }
            f(url, i, base)
        }
    }
}
<commit_msg>Cleanup linkchecker whitelist<commit_after>//! Script to check the validity of `href` links in our HTML documentation.
//!
//! In the past we've been quite error prone to writing in broken links as most
//! of them are manually rather than automatically added. As files move over
//! time or apis change old links become stale or broken. The purpose of this
//! script is to check all relative links in our documentation to make sure they
//! actually point to a valid place.
//!
//! Currently this doesn't actually do any HTML parsing or anything fancy like
//! that, it just has a simple "regex" to search for `href` and `id` tags.
//! These values are then translated to file URLs if possible and then the
//! destination is asserted to exist.
//!
//! A few whitelisted exceptions are allowed as there's known bugs in rustdoc,
//! but this should catch the majority of "broken link" cases.

#![deny(warnings)]

use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
use std::env;
use std::fs;
use std::path::{Component, Path, PathBuf};
use std::rc::Rc;

use crate::Redirect::*;

macro_rules! t {
    ($e:expr) => {
        match $e {
            Ok(e) => e,
            Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
        }
    };
}

fn main() {
    let docs = env::args_os().nth(1).unwrap();
    let docs = env::current_dir().unwrap().join(docs);
    let mut errors = false;
    walk(&mut HashMap::new(), &docs, &docs, &mut errors);
    if errors {
        panic!("found some broken links");
    }
}

#[derive(Debug)]
pub enum LoadError {
    IOError(std::io::Error),
    BrokenRedirect(PathBuf, std::io::Error),
    IsRedirect,
}

enum Redirect {
    SkipRedirect,
    FromRedirect(bool),
}

struct FileEntry {
    source: Rc<String>,
    ids: HashSet<String>,
}

type Cache = HashMap<PathBuf, FileEntry>;

fn small_url_encode(s: &str) -> String {
    s.replace("<", "%3C")
        .replace(">", "%3E")
        .replace(" ", "%20")
        .replace("?", "%3F")
        .replace("'", "%27")
        .replace("&", "%26")
        .replace(",", "%2C")
        .replace(":", "%3A")
        .replace(";", "%3B")
        .replace("[", "%5B")
        .replace("]", "%5D")
        .replace("\"", "%22")
}

impl FileEntry {
    fn parse_ids(&mut self, file: &Path, contents: &str, errors: &mut bool) {
        if self.ids.is_empty() {
            with_attrs_in_source(contents, " id", |fragment, i, _| {
                let frag = fragment.trim_start_matches("#").to_owned();
                let encoded = small_url_encode(&frag);
                if !self.ids.insert(frag) {
                    *errors = true;
                    println!("{}:{}: id is not unique: `{}`", file.display(), i, fragment);
                }
                // Just in case, we also add the encoded id.
                self.ids.insert(encoded);
            });
        }
    }
}

fn walk(cache: &mut Cache, root: &Path, dir: &Path, errors: &mut bool) {
    for entry in t!(dir.read_dir()).map(|e| t!(e)) {
        let path = entry.path();
        let kind = t!(entry.file_type());
        if kind.is_dir() {
            walk(cache, root, &path, errors);
        } else {
            let pretty_path = check(cache, root, &path, errors);
            if let Some(pretty_path) = pretty_path {
                let entry = cache.get_mut(&pretty_path).unwrap();
                // we don't need the source anymore,
                // so drop to reduce memory-usage
                entry.source = Rc::new(String::new());
            }
        }
    }
}

fn check(cache: &mut Cache, root: &Path, file: &Path, errors: &mut bool) -> Option<PathBuf> {
    // Ignore none HTML files.
    if file.extension().and_then(|s| s.to_str()) != Some("html") {
        return None;
    }

    // Unfortunately we're not 100% full of valid links today to we need a few
    // whitelists to get this past `make check` today.
    // FIXME(#32129)
    if file.ends_with("std/io/struct.IoSlice.html")
        || file.ends_with("std/string/struct.String.html")
    {
        return None;
    }
    // FIXME(#32553)
    if file.ends_with("alloc/string/struct.String.html") {
        return None;
    }
    // FIXME(#32130)
    if file.ends_with("alloc/collections/btree_map/struct.BTreeMap.html")
        || file.ends_with("alloc/collections/btree_set/struct.BTreeSet.html")
        || file.ends_with("std/collections/btree_map/struct.BTreeMap.html")
        || file.ends_with("std/collections/btree_set/struct.BTreeSet.html")
        || file.ends_with("std/collections/hash_map/struct.HashMap.html")
        || file.ends_with("std/collections/hash_set/struct.HashSet.html")
    {
        return None;
    }

    let res = load_file(cache, root, file, SkipRedirect);
    let (pretty_file, contents) = match res {
        Ok(res) => res,
        Err(_) => return None,
    };
    {
        cache.get_mut(&pretty_file).unwrap().parse_ids(&pretty_file, &contents, errors);
    }

    // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
    with_attrs_in_source(&contents, " href", |url, i, base| {
        // Ignore external URLs
        if url.starts_with("http:")
            || url.starts_with("https:")
            || url.starts_with("javascript:")
            || url.starts_with("ftp:")
            || url.starts_with("irc:")
            || url.starts_with("data:")
        {
            return;
        }
        let mut parts = url.splitn(2, "#");
        let url = parts.next().unwrap();
        let fragment = parts.next();
        let mut parts = url.splitn(2, "?");
        let url = parts.next().unwrap();

        // Once we've plucked out the URL, parse it using our base url and
        // then try to extract a file path.
        let mut path = file.to_path_buf();
        if !base.is_empty() || !url.is_empty() {
            path.pop();
            for part in Path::new(base).join(url).components() {
                match part {
                    Component::Prefix(_) | Component::RootDir => {
                        // Avoid absolute paths as they make the docs not
                        // relocatable by making assumptions on where the docs
                        // are hosted relative to the site root.
                        *errors = true;
                        println!(
                            "{}:{}: absolute path - {}",
                            pretty_file.display(),
                            i + 1,
                            Path::new(base).join(url).display()
                        );
                        return;
                    }
                    Component::CurDir => {}
                    Component::ParentDir => {
                        path.pop();
                    }
                    Component::Normal(s) => {
                        path.push(s);
                    }
                }
            }
        }

        // Alright, if we've found a file name then this file had better
        // exist! If it doesn't then we register and print an error.
        if path.exists() {
            if path.is_dir() {
                // Links to directories show as directory listings when viewing
                // the docs offline so it's best to avoid them.
                *errors = true;
                let pretty_path = path.strip_prefix(root).unwrap_or(&path);
                println!(
                    "{}:{}: directory link - {}",
                    pretty_file.display(),
                    i + 1,
                    pretty_path.display()
                );
                return;
            }
            if let Some(extension) = path.extension() {
                // Ignore none HTML files.
                if extension != "html" {
                    return;
                }
            }
            let res = load_file(cache, root, &path, FromRedirect(false));
            let (pretty_path, contents) = match res {
                Ok(res) => res,
                Err(LoadError::IOError(err)) => {
                    panic!("error loading {}: {}", path.display(), err);
                }
                Err(LoadError::BrokenRedirect(target, _)) => {
                    *errors = true;
                    println!(
                        "{}:{}: broken redirect to {}",
                        pretty_file.display(),
                        i + 1,
                        target.display()
                    );
                    return;
                }
                Err(LoadError::IsRedirect) => unreachable!(),
            };

            if let Some(ref fragment) = fragment {
                // Fragments like `#1-6` are most likely line numbers to be
                // interpreted by javascript, so we're ignoring these
                if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
                    return;
                }

                // These appear to be broken in mdbook right now?
                if fragment.starts_with("-") {
                    return;
                }

                let entry = &mut cache.get_mut(&pretty_path).unwrap();
                entry.parse_ids(&pretty_path, &contents, errors);

                if !entry.ids.contains(*fragment) {
                    *errors = true;
                    print!("{}:{}: broken link fragment ", pretty_file.display(), i + 1);
                    println!("`#{}` pointing to `{}`", fragment, pretty_path.display());
                };
            }
        } else {
            *errors = true;
            print!("{}:{}: broken link - ", pretty_file.display(), i + 1);
            let pretty_path = path.strip_prefix(root).unwrap_or(&path);
            println!("{}", pretty_path.display());
        }
    });
    Some(pretty_file)
}

fn load_file(
    cache: &mut Cache,
    root: &Path,
    file: &Path,
    redirect: Redirect,
) -> Result<(PathBuf, Rc<String>), LoadError> {
    let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file));

    let (maybe_redirect, contents) = match cache.entry(pretty_file.clone()) {
        Entry::Occupied(entry) => (None, entry.get().source.clone()),
        Entry::Vacant(entry) => {
            let contents = match fs::read_to_string(file) {
                Ok(s) => Rc::new(s),
                Err(err) => {
                    return Err(if let FromRedirect(true) = redirect {
                        LoadError::BrokenRedirect(file.to_path_buf(), err)
                    } else {
                        LoadError::IOError(err)
                    });
                }
            };

            let maybe = maybe_redirect(&contents);
            if maybe.is_some() {
                if let SkipRedirect = redirect {
                    return Err(LoadError::IsRedirect);
                }
            } else {
                entry.insert(FileEntry { source: contents.clone(), ids: HashSet::new() });
            }
            (maybe, contents)
        }
    };
    match maybe_redirect.map(|url| file.parent().unwrap().join(url)) {
        Some(redirect_file) => load_file(cache, root, &redirect_file, FromRedirect(true)),
        None => Ok((pretty_file, contents)),
    }
}

fn maybe_redirect(source: &str) -> Option<String> {
    const REDIRECT: &'static str = "
Redirecting to <a href=";

    let mut lines = source.lines();
    let redirect_line = lines.nth(6)?;

    redirect_line.find(REDIRECT).map(|i| {
        let rest = &redirect_line[(i + REDIRECT.len() + 1)..];
        let pos_quote = rest.find('"').unwrap();
        rest[..pos_quote].to_owned()
    })
}

fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(contents: &str, attr: &str, mut f: F) {
    let mut base = "";
    for (i, mut line) in contents.lines().enumerate() {
        while let Some(j) = line.find(attr) {
            let rest = &line[j + attr.len()..];
            // The base tag should always be the first link in the document so
            // we can get away with using one pass.
            let is_base = line[..j].ends_with("<base");
            line = rest;
            let pos_equals = match rest.find("=") {
                Some(i) => i,
                None => continue,
            };
            if rest[..pos_equals].trim_start_matches(" ") != "" {
                continue;
            }

            let rest = &rest[pos_equals + 1..];

            let pos_quote = match rest.find(&['"', '\''][..]) {
                Some(i) => i,
                None => continue,
            };
            let quote_delim = rest.as_bytes()[pos_quote] as char;

            if rest[..pos_quote].trim_start_matches(" ") != "" {
                continue;
            }
            let rest = &rest[pos_quote + 1..];
            let url = match rest.find(quote_delim) {
                Some(i) => &rest[..i],
                None => continue,
            };
            if is_base {
                base = url;
                continue;
            }
            f(url, i, base)
        }
    }
}
<|endoftext|>


<commit_before><commit_msg>Add first hack<commit_after>use std::io::buffered::BufferedStream;
use std::io::net::ip::SocketAddr;
use std::io::net::tcp::TcpStream;

static PROTO_VERSION: i32 = 196608; // 3 << 16

#[deriving(ToStr)]
enum AuthType {
    AuthUnknown,
    AuthOk,
    AuthKerberosV4,
    AuthKerberosV5,
    AuthClearTextPassword,
    AuthCryptPassword(u16),
    AuthMD5Password(u32),
    AuthSCMCredential
}

fn write_cstring(io: &mut BufferedStream<TcpStream>, str: &str) {
      io.write_str(str);
      io.write_u8(0);
}

fn write_startup_message(io: &mut BufferedStream<TcpStream>, params: &[(&str, &str)]) {
    let mut sz: uint = 8 + 1;

    for &(k, v) in params.iter() {
      sz += k.len() + 1 + v.len() + 1;
    }

    io.write_be_i32(sz as i32);
    io.write_be_i32(PROTO_VERSION);

    for &(k, v) in params.iter() {
        write_cstring(io, k);
        write_cstring(io, v);
    }
    io.write_u8(0);
    io.flush();
}

// body_size excludes the typ byte and the size i32
fn write_message_header(io: &mut BufferedStream<TcpStream>, typ: u8, payload: uint) {
    io.write_u8(typ);
    io.write_be_i32((payload + 4) as i32);
}

fn write_query_message(io: &mut BufferedStream<TcpStream>, query: &str) {
    write_message_header(io, 'Q' as u8, query.len() + 1);
    write_cstring(io, query);
    io.flush();
}

fn parse_auth_message(io: &mut BufferedStream<TcpStream>, rem_len: i32) -> AuthType {
    assert!(rem_len >= 4);
    let authtype = io.read_be_i32();
    match authtype {
        0 => AuthOk,
        1 => AuthKerberosV4,
        2 => AuthKerberosV5,
        3 => AuthClearTextPassword,
        4 => {
            assert!(rem_len >= 4 + 2);
            let salt = io.read_be_u16();
            AuthCryptPassword(salt)
        }
        5 => {
            assert!(rem_len >= 4 + 4);
            let salt = io.read_be_u32();
            AuthMD5Password(salt)
        }
        6 => AuthSCMCredential,
        _ => AuthUnknown
    }
}

fn parse_message_header(io: &mut BufferedStream<TcpStream>) -> (u8, i32) {
    let typ = io.read_u8();
    let len = io.read_be_i32();
    return (typ, len);
}

fn main() {
    let addr = from_str::<SocketAddr>("127.0.0.1:5432").unwrap();
    let tcp_stream = TcpStream::connect(addr).unwrap();
    let mut io = BufferedStream::new(tcp_stream);
    write_startup_message(&mut io, [("user", "mneumann"), ("database", "test")]);

    let (typ, len) = parse_message_header(&mut io);
    println!("Typ: {:u} / len: {:d}", typ, len);

    if typ == 82 /* 'R' */ {
        let authmsg = parse_auth_message(&mut io, len-4);
        println!("{:?}", authmsg);
    }
    else {
        fail!();
    }

    write_query_message(&mut io, "select * from articles;");

    let (typ, len) = parse_message_header(&mut io);
    println!("Typ: {:u} / len: {:d}", typ, len);
}
<|endoftext|>


<commit_before><commit_msg>aggregation added<commit_after>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);
    let mut start = time::precise_time_s();
    let mut going = start;

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let graph = GraphMMap::<u32>::new(&filename);

        let nodes = graph.nodes();

        let mut src = vec![1.0; 1 + (nodes / peers as usize)];  // local rank accumulation
        let mut dst = vec![0.0; nodes];                         // local rank accumulation

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    if iter.inner == 10 {
                        going = time::precise_time_s();
                    }

                    if iter.inner == 20 {
                        if index == 0 {
                            println!("average over 10 iters: {}", (time::precise_time_s() - going) / 10.0);
                        }
                    }

                    // /---- should look familiar! ----\
                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node];
                    }

                    for node in 0..src.len() {
                        let edges = graph.edges(index + peers * node);
                        let value = src[node] / edges.len() as f32;
                        for &b in edges {
                            dst[b as usize] += value;
                        }
                    }
                    // \------ end familiar part ------/

                    output.give_at(&iter, dst.drain_temp()
                                             .enumerate()
                                             .filter(|&(_,f)| f != 0.0)
                                             .map(|(u,f)| (u as u32, f)));

                    for _ in 0..(graph.nodes() + 1) { dst.push(0.0); }

                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        let local_index = index as usize % workers;

        let mut acc = vec![0.0; 1 + (nodes / workers)];

        ranks
        .unary_notify(
            Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
            format!("Aggregation"),
            vec![],
            move |input, output, iterator| {
                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        acc[node as usize / workers] += rank;
                    }
                }

                while let Some((item, _)) = iterator.next() {

                    output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
                                             .map(|(u,f)| (((u * workers + local_index) as u32), f)));

                    for _ in 0..(1 + (nodes/workers)) { acc.push(0.0); }
                    assert!(acc.len() == (1 + (nodes/workers)));
                }
            }
        )
        .connect_loop(helper);
    }

    while root.step() { }

    if index == 0 {
        println!("elapsed: {}", time::precise_time_s() - start);
    }

}
<|endoftext|>


<commit_before><commit_msg>Add iron-intercept example<commit_after>extern crate iron;
extern crate multipart;

use iron::prelude::*;

use multipart::server::Entries;
use multipart::server::iron::Intercept;

fn main() {
    // We start with a basic request handler chain.
    let mut chain = Chain::new(|req: &mut Request|
        if let Some(entries) = req.extensions.get::<Entries>() {
            Ok(Response::with(format!("{:?}", entries)))
        } else {
            Ok(Response::with("Not a multipart request"))
        }
    );

    // `Intercept` will read out the entries and place them as an extension in the request.
    // It has various builder-style methods for changing how it will behave, but has sane settings
    // by default.
    chain.link_before(Intercept::default());

    Iron::new(chain).http("localhost:80").unwrap();
}<|endoftext|>


<commit_before>use std::ops::DerefMut;

use runtime::Runtime;
use error::RuntimeError;
use error::RuntimeErrorKind;

use libimagstore::store::FileLockEntry;
use libimagstore::store::Entry;

pub type EditResult<T> = Result<T, RuntimeError>;

pub trait Edit {
    fn edit_content(&mut self, rt: &Runtime) -> EditResult<()>;
}

impl Edit for Entry {

    fn edit_content(&mut self, rt: &Runtime) -> EditResult<()> {
        edit_in_tmpfile(rt, self.get_content_mut())
            .map(|_| ())
    }

}

impl<'a> Edit for FileLockEntry<'a> {

    fn edit_content(&mut self, rt: &Runtime) -> EditResult<()> {
        self.deref_mut().edit_content(rt)
    }

}

pub fn edit_in_tmpfile(rt: &Runtime, s: &mut String) -> EditResult<()> {
    use tempfile::NamedTempFile;
    use std::io::Seek;
    use std::io::Read;
    use std::io::SeekFrom;
    use std::io::Write;

    let file      = try!(NamedTempFile::new());
    let file_path = file.path();
    let mut file  = try!(file.reopen());

    try!(file.write_all(&s.clone().into_bytes()[..]));
    try!(file.sync_data());

    if let Some(mut editor) = rt.editor() {
        let exit_status = editor.arg(file_path).status();

        match exit_status.map(|s| s.success()) {
            Ok(true)  => {
                file.sync_data()
                    .and_then(|_| file.seek(SeekFrom::Start(0)))
                    .and_then(|_| file.read_to_string(s))
                    .map(|_| ())
                    .map_err(|e| RuntimeError::new(RuntimeErrorKind::IOError, Some(Box::new(e))))
            },
            Ok(false) => Err(RuntimeError::new(RuntimeErrorKind::ProcessExitFailure, None)),
            Err(e)    => Err(RuntimeError::new(RuntimeErrorKind::IOError, Some(Box::new(e)))),
        }
    } else {
        Err(RuntimeError::new(RuntimeErrorKind::Instantiate, None))
    }
}
<commit_msg>Implement Edit for String<commit_after>use std::ops::DerefMut;

use runtime::Runtime;
use error::RuntimeError;
use error::RuntimeErrorKind;

use libimagstore::store::FileLockEntry;
use libimagstore::store::Entry;

pub type EditResult<T> = Result<T, RuntimeError>;

pub trait Edit {
    fn edit_content(&mut self, rt: &Runtime) -> EditResult<()>;
}

impl Edit for String {

    fn edit_content(&mut self, rt: &Runtime) -> EditResult<()> {
        edit_in_tmpfile(rt, self).map(|_| ())
    }

}

impl Edit for Entry {

    fn edit_content(&mut self, rt: &Runtime) -> EditResult<()> {
        edit_in_tmpfile(rt, self.get_content_mut())
            .map(|_| ())
    }

}

impl<'a> Edit for FileLockEntry<'a> {

    fn edit_content(&mut self, rt: &Runtime) -> EditResult<()> {
        self.deref_mut().edit_content(rt)
    }

}

pub fn edit_in_tmpfile(rt: &Runtime, s: &mut String) -> EditResult<()> {
    use tempfile::NamedTempFile;
    use std::io::Seek;
    use std::io::Read;
    use std::io::SeekFrom;
    use std::io::Write;

    let file      = try!(NamedTempFile::new());
    let file_path = file.path();
    let mut file  = try!(file.reopen());

    try!(file.write_all(&s.clone().into_bytes()[..]));
    try!(file.sync_data());

    if let Some(mut editor) = rt.editor() {
        let exit_status = editor.arg(file_path).status();

        match exit_status.map(|s| s.success()) {
            Ok(true)  => {
                file.sync_data()
                    .and_then(|_| file.seek(SeekFrom::Start(0)))
                    .and_then(|_| file.read_to_string(s))
                    .map(|_| ())
                    .map_err(|e| RuntimeError::new(RuntimeErrorKind::IOError, Some(Box::new(e))))
            },
            Ok(false) => Err(RuntimeError::new(RuntimeErrorKind::ProcessExitFailure, None)),
            Err(e)    => Err(RuntimeError::new(RuntimeErrorKind::IOError, Some(Box::new(e)))),
        }
    } else {
        Err(RuntimeError::new(RuntimeErrorKind::Instantiate, None))
    }
}
<|endoftext|>