<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use prometheus::{Counter, CounterVec, HistogramVec, exponential_buckets};

lazy_static! {
    pub static ref ASYNC_REQUESTS_COUNTER_VEC: CounterVec =
        register_counter_vec!(
            "tikv_storage_engine_async_request_total",
            "Total number of engine asynchronous requests",
            &["type", "status"]
        ).unwrap();

    pub static ref ASYNC_REQUESTS_DURATIONS_VEC: HistogramVec =
        register_histogram_vec!(
            histogram_opts!(
                "tikv_storage_engine_async_request_duration_seconds",
                "Bucketed histogram of processing successful asynchronous requests.",
                [exponential_buckets(0.0005, 2.0, 13).unwrap()]),
            &["type"]
        ).unwrap();

    pub static ref CURSOR_OVER_SEEK_BOUND_COUNTER: Counter =
        register_counter!(
            "tikv_storage_engine_cursor_over_seek_bound_total",
            "Total number of cursor next over seek bound number"
        ).unwrap();
}
<commit_msg>engine: enlarge buckets (#1341)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use prometheus::{Counter, CounterVec, HistogramVec, exponential_buckets};

lazy_static! {
    pub static ref ASYNC_REQUESTS_COUNTER_VEC: CounterVec =
        register_counter_vec!(
            "tikv_storage_engine_async_request_total",
            "Total number of engine asynchronous requests",
            &["type", "status"]
        ).unwrap();

    pub static ref ASYNC_REQUESTS_DURATIONS_VEC: HistogramVec =
        register_histogram_vec!(
            histogram_opts!(
                "tikv_storage_engine_async_request_duration_seconds",
                "Bucketed histogram of processing successful asynchronous requests.",
                [exponential_buckets(0.0005, 2.0, 20).unwrap()]),
            &["type"]
        ).unwrap();

    pub static ref CURSOR_OVER_SEEK_BOUND_COUNTER: Counter =
        register_counter!(
            "tikv_storage_engine_cursor_over_seek_bound_total",
            "Total number of cursor next over seek bound number"
        ).unwrap();
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::sync::Arc;
use std::time::Duration;
use std::thread;

use rocksdb::DB;
use uuid::Uuid;
use protobuf;
use time::Duration as TimeDuration;

use kvproto::metapb::{self, RegionEpoch};
use kvproto::raft_cmdpb::{Request, StatusRequest, AdminRequest, RaftCmdRequest, RaftCmdResponse};
use kvproto::raft_cmdpb::{CmdType, StatusCmdType, AdminCmdType};
use kvproto::pdpb::{ChangePeer, RegionHeartbeatResponse, TransferLeader};
use kvproto::eraftpb::ConfChangeType;

use tikv::raftstore::store::*;
use tikv::server::Config as ServerConfig;
use tikv::storage::Config as StorageConfig;
use tikv::util::escape;

pub use tikv::raftstore::store::util::find_peer;

pub fn must_get(engine: &Arc<DB>, cf: &str, key: &[u8], value: Option<&[u8]>) {
    for _ in 1..300 {
        let res = engine.get_value_cf(cf, &keys::data_key(key)).unwrap();
        if value.is_some() && res.is_some() {
            assert_eq!(value.unwrap(), &*res.unwrap());
            return;
        }
        if value.is_none() && res.is_none() {
            return;
        }
        thread::sleep(Duration::from_millis(20));
    }
    debug!("last try to get {}", escape(key));
    let res = engine.get_value_cf(cf, &keys::data_key(key)).unwrap();
    if value.is_none() && res.is_none() ||
       value.is_some() && res.is_some() && value.unwrap() == &*res.unwrap() {
        return;
    }
    panic!("can't get value {:?} for key {:?}",
           value.map(escape),
           escape(key))
}

pub fn must_get_equal(engine: &Arc<DB>, key: &[u8], value: &[u8]) {
    must_get(engine, "default", key, Some(value));
}

pub fn must_get_none(engine: &Arc<DB>, key: &[u8]) {
    must_get(engine, "default", key, None);
}

pub fn must_get_cf_equal(engine: &Arc<DB>, cf: &str, key: &[u8], value: &[u8]) {
    must_get(engine, cf, key, Some(value));
}

pub fn must_get_cf_none(engine: &Arc<DB>, cf: &str, key: &[u8]) {
    must_get(engine, cf, key, None);
}

pub fn new_store_cfg() -> Config {
    Config {
        sync_log: false,
        raft_base_tick_interval: 10,
        raft_heartbeat_ticks: 2,
        raft_election_timeout_ticks: 25,
        raft_log_gc_tick_interval: 100,
        raft_log_gc_threshold: 1,
        pd_heartbeat_tick_interval: 20,
        region_check_size_diff: 10000,
        // Use a value of 3 seconds as max_leader_missing_duration just for test.
        // In production environment, the value of max_leader_missing_duration
        // should be configured far beyond the election timeout.
        max_leader_missing_duration: Duration::from_secs(3),
        report_region_flow_interval: 100, // 100ms
        raft_store_max_leader_lease: TimeDuration::milliseconds(25 * 10),
        use_sst_file_snapshot: true,
        ..Config::default()
    }
}

pub fn new_server_config(cluster_id: u64) -> ServerConfig {
    let store_cfg = new_store_cfg();

    ServerConfig {
        cluster_id: cluster_id,
        addr: "127.0.0.1:0".to_owned(),
        raft_store: store_cfg,
        storage: StorageConfig::default(),
        ..ServerConfig::default()
    }
}

// Create a base request.
pub fn new_base_request(region_id: u64, epoch: RegionEpoch, read_quorum: bool) -> RaftCmdRequest {
    let mut req = RaftCmdRequest::new();
    req.mut_header().set_region_id(region_id);
    req.mut_header().set_region_epoch(epoch);
    req.mut_header().set_uuid(Uuid::new_v4().as_bytes().to_vec());
    req.mut_header().set_read_quorum(read_quorum);
    req
}

pub fn new_request(region_id: u64,
                   epoch: RegionEpoch,
                   requests: Vec<Request>,
                   read_quorum: bool)
                   -> RaftCmdRequest {
    let mut req = new_base_request(region_id, epoch, read_quorum);
    req.set_requests(protobuf::RepeatedField::from_vec(requests));
    req
}

pub fn new_put_cmd(key: &[u8], value: &[u8]) -> Request {
    let mut cmd = Request::new();
    cmd.set_cmd_type(CmdType::Put);
    cmd.mut_put().set_key(key.to_vec());
    cmd.mut_put().set_value(value.to_vec());
    cmd
}

pub fn new_put_cf_cmd(cf: &str, key: &[u8], value: &[u8]) -> Request {
    let mut cmd = Request::new();
    cmd.set_cmd_type(CmdType::Put);
    cmd.mut_put().set_key(key.to_vec());
    cmd.mut_put().set_value(value.to_vec());
    cmd.mut_put().set_cf(cf.to_string());
    cmd
}

pub fn new_get_cmd(key: &[u8]) -> Request {
    let mut cmd = Request::new();
    cmd.set_cmd_type(CmdType::Get);
    cmd.mut_get().set_key(key.to_vec());
    cmd
}

pub fn new_delete_cmd(cf: &str, key: &[u8]) -> Request {
    let mut cmd = Request::new();
    cmd.set_cmd_type(CmdType::Delete);
    cmd.mut_delete().set_key(key.to_vec());
    cmd.mut_delete().set_cf(cf.to_string());
    cmd
}

pub fn new_status_request(region_id: u64,
                          peer: metapb::Peer,
                          request: StatusRequest)
                          -> RaftCmdRequest {
    let mut req = new_base_request(region_id, RegionEpoch::new(), false);
    req.mut_header().set_peer(peer);
    req.set_status_request(request);
    req
}

pub fn new_region_detail_cmd() -> StatusRequest {
    let mut cmd = StatusRequest::new();
    cmd.set_cmd_type(StatusCmdType::RegionDetail);
    cmd
}

pub fn new_region_leader_cmd() -> StatusRequest {
    let mut cmd = StatusRequest::new();
    cmd.set_cmd_type(StatusCmdType::RegionLeader);
    cmd
}

pub fn new_admin_request(region_id: u64,
                         epoch: &RegionEpoch,
                         request: AdminRequest)
                         -> RaftCmdRequest {
    let mut req = new_base_request(region_id, epoch.clone(), false);
    req.set_admin_request(request);
    req
}

pub fn new_change_peer_request(change_type: ConfChangeType, peer: metapb::Peer) -> AdminRequest {
    let mut req = AdminRequest::new();
    req.set_cmd_type(AdminCmdType::ChangePeer);
    req.mut_change_peer().set_change_type(change_type);
    req.mut_change_peer().set_peer(peer);
    req
}

pub fn new_transfer_leader_cmd(peer: metapb::Peer) -> AdminRequest {
    let mut cmd = AdminRequest::new();
    cmd.set_cmd_type(AdminCmdType::TransferLeader);
    cmd.mut_transfer_leader().set_peer(peer);
    cmd
}

pub fn new_peer(store_id: u64, peer_id: u64) -> metapb::Peer {
    let mut peer = metapb::Peer::new();
    peer.set_store_id(store_id);
    peer.set_id(peer_id);
    peer
}

pub fn new_store(store_id: u64, addr: String) -> metapb::Store {
    let mut store = metapb::Store::new();
    store.set_id(store_id);
    store.set_address(addr);

    store
}

pub fn sleep_ms(ms: u64) {
    thread::sleep(Duration::from_millis(ms));
}

pub fn is_error_response(resp: &RaftCmdResponse) -> bool {
    resp.get_header().has_error()
}

pub fn new_pd_change_peer(change_type: ConfChangeType,
                          peer: metapb::Peer)
                          -> RegionHeartbeatResponse {
    let mut change_peer = ChangePeer::new();
    change_peer.set_change_type(change_type.into());
    change_peer.set_peer(peer);

    let mut resp = RegionHeartbeatResponse::new();
    resp.set_change_peer(change_peer);
    resp
}

pub fn new_pd_add_change_peer(region: &metapb::Region,
                              peer: metapb::Peer)
                              -> Option<RegionHeartbeatResponse> {
    if let Some(p) = find_peer(region, peer.get_store_id()) {
        assert_eq!(p.get_id(), peer.get_id());
        return None;
    }

    Some(new_pd_change_peer(ConfChangeType::AddNode, peer))
}

pub fn new_pd_remove_change_peer(region: &metapb::Region,
                                 peer: metapb::Peer)
                                 -> Option<RegionHeartbeatResponse> {
    if find_peer(region, peer.get_store_id()).is_none() {
        return None;
    }

    Some(new_pd_change_peer(ConfChangeType::RemoveNode, peer))
}

pub fn new_pd_transfer_leader(peer: metapb::Peer) -> Option<RegionHeartbeatResponse> {
    let mut transfer_leader = TransferLeader::new();
    transfer_leader.set_peer(peer);

    let mut resp = RegionHeartbeatResponse::new();
    resp.set_transfer_leader(transfer_leader);
    Some(resp)
}
<commit_msg>tests: limit server case resources (#1943)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

use std::sync::Arc;
use std::time::Duration;
use std::thread;

use rocksdb::DB;
use uuid::Uuid;
use protobuf;
use time::Duration as TimeDuration;

use kvproto::metapb::{self, RegionEpoch};
use kvproto::raft_cmdpb::{Request, StatusRequest, AdminRequest, RaftCmdRequest, RaftCmdResponse};
use kvproto::raft_cmdpb::{CmdType, StatusCmdType, AdminCmdType};
use kvproto::pdpb::{ChangePeer, RegionHeartbeatResponse, TransferLeader};
use kvproto::eraftpb::ConfChangeType;

use tikv::raftstore::store::*;
use tikv::server::Config as ServerConfig;
use tikv::storage::Config as StorageConfig;
use tikv::util::escape;

pub use tikv::raftstore::store::util::find_peer;

pub fn must_get(engine: &Arc<DB>, cf: &str, key: &[u8], value: Option<&[u8]>) {
    for _ in 1..300 {
        let res = engine.get_value_cf(cf, &keys::data_key(key)).unwrap();
        if value.is_some() && res.is_some() {
            assert_eq!(value.unwrap(), &*res.unwrap());
            return;
        }
        if value.is_none() && res.is_none() {
            return;
        }
        thread::sleep(Duration::from_millis(20));
    }
    debug!("last try to get {}", escape(key));
    let res = engine.get_value_cf(cf, &keys::data_key(key)).unwrap();
    if value.is_none() && res.is_none() ||
       value.is_some() && res.is_some() && value.unwrap() == &*res.unwrap() {
        return;
    }
    panic!("can't get value {:?} for key {:?}",
           value.map(escape),
           escape(key))
}

pub fn must_get_equal(engine: &Arc<DB>, key: &[u8], value: &[u8]) {
    must_get(engine, "default", key, Some(value));
}

pub fn must_get_none(engine: &Arc<DB>, key: &[u8]) {
    must_get(engine, "default", key, None);
}

pub fn must_get_cf_equal(engine: &Arc<DB>, cf: &str, key: &[u8], value: &[u8]) {
    must_get(engine, cf, key, Some(value));
}

pub fn must_get_cf_none(engine: &Arc<DB>, cf: &str, key: &[u8]) {
    must_get(engine, cf, key, None);
}

pub fn new_store_cfg() -> Config {
    Config {
        sync_log: false,
        raft_base_tick_interval: 10,
        raft_heartbeat_ticks: 2,
        raft_election_timeout_ticks: 25,
        raft_log_gc_tick_interval: 100,
        raft_log_gc_threshold: 1,
        pd_heartbeat_tick_interval: 20,
        region_check_size_diff: 10000,
        // Use a value of 3 seconds as max_leader_missing_duration just for test.
        // In production environment, the value of max_leader_missing_duration
        // should be configured far beyond the election timeout.
        max_leader_missing_duration: Duration::from_secs(3),
        report_region_flow_interval: 100, // 100ms
        raft_store_max_leader_lease: TimeDuration::milliseconds(25 * 10),
        use_sst_file_snapshot: true,
        ..Config::default()
    }
}

pub fn new_server_config(cluster_id: u64) -> ServerConfig {
    let store_cfg = new_store_cfg();

    ServerConfig {
        cluster_id: cluster_id,
        addr: "127.0.0.1:0".to_owned(),
        raft_store: store_cfg,
        storage: StorageConfig { sched_worker_pool_size: 1, ..StorageConfig::default() },
        grpc_concurrency: 1,
        // Considering connection selection algo is involved, maybe
        // use 2 or larger value here?
        grpc_raft_conn_num: 1,
        end_point_concurrency: 1,
        ..ServerConfig::default()
    }
}

// Create a base request.
pub fn new_base_request(region_id: u64, epoch: RegionEpoch, read_quorum: bool) -> RaftCmdRequest {
    let mut req = RaftCmdRequest::new();
    req.mut_header().set_region_id(region_id);
    req.mut_header().set_region_epoch(epoch);
    req.mut_header().set_uuid(Uuid::new_v4().as_bytes().to_vec());
    req.mut_header().set_read_quorum(read_quorum);
    req
}

pub fn new_request(region_id: u64,
                   epoch: RegionEpoch,
                   requests: Vec<Request>,
                   read_quorum: bool)
                   -> RaftCmdRequest {
    let mut req = new_base_request(region_id, epoch, read_quorum);
    req.set_requests(protobuf::RepeatedField::from_vec(requests));
    req
}

pub fn new_put_cmd(key: &[u8], value: &[u8]) -> Request {
    let mut cmd = Request::new();
    cmd.set_cmd_type(CmdType::Put);
    cmd.mut_put().set_key(key.to_vec());
    cmd.mut_put().set_value(value.to_vec());
    cmd
}

pub fn new_put_cf_cmd(cf: &str, key: &[u8], value: &[u8]) -> Request {
    let mut cmd = Request::new();
    cmd.set_cmd_type(CmdType::Put);
    cmd.mut_put().set_key(key.to_vec());
    cmd.mut_put().set_value(value.to_vec());
    cmd.mut_put().set_cf(cf.to_string());
    cmd
}

pub fn new_get_cmd(key: &[u8]) -> Request {
    let mut cmd = Request::new();
    cmd.set_cmd_type(CmdType::Get);
    cmd.mut_get().set_key(key.to_vec());
    cmd
}

pub fn new_delete_cmd(cf: &str, key: &[u8]) -> Request {
    let mut cmd = Request::new();
    cmd.set_cmd_type(CmdType::Delete);
    cmd.mut_delete().set_key(key.to_vec());
    cmd.mut_delete().set_cf(cf.to_string());
    cmd
}

pub fn new_status_request(region_id: u64,
                          peer: metapb::Peer,
                          request: StatusRequest)
                          -> RaftCmdRequest {
    let mut req = new_base_request(region_id, RegionEpoch::new(), false);
    req.mut_header().set_peer(peer);
    req.set_status_request(request);
    req
}

pub fn new_region_detail_cmd() -> StatusRequest {
    let mut cmd = StatusRequest::new();
    cmd.set_cmd_type(StatusCmdType::RegionDetail);
    cmd
}

pub fn new_region_leader_cmd() -> StatusRequest {
    let mut cmd = StatusRequest::new();
    cmd.set_cmd_type(StatusCmdType::RegionLeader);
    cmd
}

pub fn new_admin_request(region_id: u64,
                         epoch: &RegionEpoch,
                         request: AdminRequest)
                         -> RaftCmdRequest {
    let mut req = new_base_request(region_id, epoch.clone(), false);
    req.set_admin_request(request);
    req
}

pub fn new_change_peer_request(change_type: ConfChangeType, peer: metapb::Peer) -> AdminRequest {
    let mut req = AdminRequest::new();
    req.set_cmd_type(AdminCmdType::ChangePeer);
    req.mut_change_peer().set_change_type(change_type);
    req.mut_change_peer().set_peer(peer);
    req
}

pub fn new_transfer_leader_cmd(peer: metapb::Peer) -> AdminRequest {
    let mut cmd = AdminRequest::new();
    cmd.set_cmd_type(AdminCmdType::TransferLeader);
    cmd.mut_transfer_leader().set_peer(peer);
    cmd
}

pub fn new_peer(store_id: u64, peer_id: u64) -> metapb::Peer {
    let mut peer = metapb::Peer::new();
    peer.set_store_id(store_id);
    peer.set_id(peer_id);
    peer
}

pub fn new_store(store_id: u64, addr: String) -> metapb::Store {
    let mut store = metapb::Store::new();
    store.set_id(store_id);
    store.set_address(addr);

    store
}

pub fn sleep_ms(ms: u64) {
    thread::sleep(Duration::from_millis(ms));
}

pub fn is_error_response(resp: &RaftCmdResponse) -> bool {
    resp.get_header().has_error()
}

pub fn new_pd_change_peer(change_type: ConfChangeType,
                          peer: metapb::Peer)
                          -> RegionHeartbeatResponse {
    let mut change_peer = ChangePeer::new();
    change_peer.set_change_type(change_type.into());
    change_peer.set_peer(peer);

    let mut resp = RegionHeartbeatResponse::new();
    resp.set_change_peer(change_peer);
    resp
}

pub fn new_pd_add_change_peer(region: &metapb::Region,
                              peer: metapb::Peer)
                              -> Option<RegionHeartbeatResponse> {
    if let Some(p) = find_peer(region, peer.get_store_id()) {
        assert_eq!(p.get_id(), peer.get_id());
        return None;
    }

    Some(new_pd_change_peer(ConfChangeType::AddNode, peer))
}

pub fn new_pd_remove_change_peer(region: &metapb::Region,
                                 peer: metapb::Peer)
                                 -> Option<RegionHeartbeatResponse> {
    if find_peer(region, peer.get_store_id()).is_none() {
        return None;
    }

    Some(new_pd_change_peer(ConfChangeType::RemoveNode, peer))
}

pub fn new_pd_transfer_leader(peer: metapb::Peer) -> Option<RegionHeartbeatResponse> {
    let mut transfer_leader = TransferLeader::new();
    transfer_leader.set_peer(peer);

    let mut resp = RegionHeartbeatResponse::new();
    resp.set_transfer_leader(transfer_leader);
    Some(resp)
}
<|endoftext|>


<commit_before>use std::fs::{self, File};
use std::io::prelude::*;
use std::env;
use tempdir::TempDir;

use support::{execs, paths};
use support::paths::CargoPathExt;
use hamcrest::{assert_that, existing_file, existing_dir, is_not};

use cargo::util::ProcessBuilder;

fn setup() {
}

fn cargo_process(s: &str) -> ProcessBuilder {
    let mut p = ::cargo_process();
    p.arg(s);
    return p;
}

test!(simple_lib {
    assert_that(cargo_process("new").arg("foo").arg("--vcs").arg("none")
                                    .env("USER", "foo"),
                execs().with_status(0));

    assert_that(&paths::root().join("foo"), existing_dir());
    assert_that(&paths::root().join("foo/Cargo.toml"), existing_file());
    assert_that(&paths::root().join("foo/src/lib.rs"), existing_file());
    assert_that(&paths::root().join("foo/.gitignore"), is_not(existing_file()));

    assert_that(cargo_process("build").cwd(&paths::root().join("foo")),
                execs().with_status(0));
});

test!(simple_bin {
    assert_that(cargo_process("new").arg("foo").arg("--bin")
                                    .env("USER", "foo"),
                execs().with_status(0));

    assert_that(&paths::root().join("foo"), existing_dir());
    assert_that(&paths::root().join("foo/Cargo.toml"), existing_file());
    assert_that(&paths::root().join("foo/src/main.rs"), existing_file());

    assert_that(cargo_process("build").cwd(&paths::root().join("foo")),
                execs().with_status(0));
    assert_that(&paths::root().join(&format!("foo/target/debug/foo{}",
                                             env::consts::EXE_SUFFIX)),
                existing_file());
});

test!(simple_git {
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo").cwd(td.path().clone())
                                    .env("USER", "foo"),
                execs().with_status(0));

    assert_that(td.path(), existing_dir());
    assert_that(&td.path().join("foo/Cargo.toml"), existing_file());
    assert_that(&td.path().join("foo/src/lib.rs"), existing_file());
    assert_that(&td.path().join("foo/.git"), existing_dir());
    assert_that(&td.path().join("foo/.gitignore"), existing_file());

    assert_that(cargo_process("build").cwd(&td.path().clone().join("foo")),
                execs().with_status(0));
});

test!(no_argument {
    assert_that(cargo_process("new"),
                execs().with_status(1)
                       .with_stderr("\
[ERROR] Invalid arguments.

Usage:
    cargo new [options] <path>
    cargo new -h | --help
"));
});

test!(existing {
    let dst = paths::root().join("foo");
    fs::create_dir(&dst).unwrap();
    assert_that(cargo_process("new").arg("foo"),
                execs().with_status(101)
                       .with_stderr(format!("[ERROR] destination `{}` already exists\n",
                                            dst.display())));
});

test!(invalid_characters {
    assert_that(cargo_process("new").arg("foo.rs"),
                execs().with_status(101)
                       .with_stderr("\
[ERROR] Invalid character `.` in crate name: `foo.rs`
use --name to override crate name"));
});

test!(reserved_name {
    assert_that(cargo_process("new").arg("test"),
                execs().with_status(101)
                       .with_stderr("\
[ERROR] The name `test` cannot be used as a crate name\n\
use --name to override crate name"));
});

test!(keyword_name {
    assert_that(cargo_process("new").arg("pub"),
                execs().with_status(101)
                       .with_stderr("\
[ERROR] The name `pub` cannot be used as a crate name\n\
use --name to override crate name"));
});

test!(rust_prefix_stripped {
    assert_that(cargo_process("new").arg("rust-foo").env("USER", "foo"),
                execs().with_status(0)
                       .with_stdout("note: package will be named `foo`; use --name to override"));
    let toml = paths::root().join("rust-foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"name = "foo""#));
});

test!(bin_disables_stripping {
    assert_that(cargo_process("new").arg("rust-foo").arg("--bin").env("USER", "foo"),
                execs().with_status(0));
    let toml = paths::root().join("rust-foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"name = "rust-foo""#));
});

test!(explicit_name_not_stripped {
    assert_that(cargo_process("new").arg("foo").arg("--name").arg("rust-bar").env("USER", "foo"),
                execs().with_status(0));
    let toml = paths::root().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"name = "rust-bar""#));
});

test!(finds_author_user {
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo").env("USER", "foo")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["foo"]"#));
});

test!(finds_author_user_escaped {
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo").env("USER", "foo \"bar\"")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["foo \"bar\""]"#));
});

test!(finds_author_username {
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo")
                                    .env_remove("USER")
                                    .env("USERNAME", "foo")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["foo"]"#));
});

test!(finds_author_priority {
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo")
                                    .env("USER", "bar2")
                                    .env("EMAIL", "baz2")
                                    .env("CARGO_NAME", "bar")
                                    .env("CARGO_EMAIL", "baz")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["bar <baz>"]"#));
});

test!(finds_author_email {
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo")
                                    .env("USER", "bar")
                                    .env("EMAIL", "baz")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["bar <baz>"]"#));
});

test!(finds_author_git {
    ::process("git").args(&["config", "--global", "user.name", "bar"])
                    .exec().unwrap();
    ::process("git").args(&["config", "--global", "user.email", "baz"])
                    .exec().unwrap();
    assert_that(cargo_process("new").arg("foo").env("USER", "foo"),
                execs().with_status(0));

    let toml = paths::root().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["bar <baz>"]"#));
});

test!(finds_git_email{
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo")
                                    .env("GIT_AUTHOR_NAME", "foo")
                                    .env("GIT_AUTHOR_EMAIL", "gitfoo")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["foo <gitfoo>"]"#), contents);
});

test!(finds_git_author{
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo")
                                    .env_remove("USER")
                                    .env("GIT_COMMITTER_NAME", "gitfoo")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["gitfoo"]"#));
});

test!(author_prefers_cargo {
    ::process("git").args(&["config", "--global", "user.name", "foo"])
                    .exec().unwrap();
    ::process("git").args(&["config", "--global", "user.email", "bar"])
                    .exec().unwrap();
    let root = paths::root();
    fs::create_dir(&root.join(".cargo")).unwrap();
    File::create(&root.join(".cargo/config")).unwrap().write_all(br#"
        [cargo-new]
        name = "new-foo"
        email = "new-bar"
        git = false
    "#).unwrap();

    assert_that(cargo_process("new").arg("foo").env("USER", "foo"),
                execs().with_status(0));

    let toml = paths::root().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["new-foo <new-bar>"]"#));
    assert!(!root.join("foo/.gitignore").c_exists());
});

test!(git_prefers_command_line {
    let root = paths::root();
    let td = TempDir::new("cargo").unwrap();
    fs::create_dir(&root.join(".cargo")).unwrap();
    File::create(&root.join(".cargo/config")).unwrap().write_all(br#"
        [cargo-new]
        vcs = "none"
        name = "foo"
        email = "bar"
    "#).unwrap();

    assert_that(cargo_process("new").arg("foo").arg("--vcs").arg("git")
                                    .cwd(td.path())
                                    .env("USER", "foo"),
                execs().with_status(0));
    assert!(td.path().join("foo/.gitignore").c_exists());
});

test!(subpackage_no_git {
    assert_that(cargo_process("new").arg("foo").env("USER", "foo"),
                execs().with_status(0));

    let subpackage = paths::root().join("foo").join("components");
    fs::create_dir(&subpackage).unwrap();
    assert_that(cargo_process("new").arg("foo/components/subcomponent")
                                    .env("USER", "foo"),
                execs().with_status(0));

    assert_that(&paths::root().join("foo/components/subcomponent/.git"),
                 is_not(existing_file()));
    assert_that(&paths::root().join("foo/components/subcomponent/.gitignore"),
                 is_not(existing_file()));
});

test!(subpackage_git_with_vcs_arg {
    assert_that(cargo_process("new").arg("foo").env("USER", "foo"),
                execs().with_status(0));

    let subpackage = paths::root().join("foo").join("components");
    fs::create_dir(&subpackage).unwrap();
    assert_that(cargo_process("new").arg("foo/components/subcomponent")
                                    .arg("--vcs").arg("git")
                                    .env("USER", "foo"),
                execs().with_status(0));

    assert_that(&paths::root().join("foo/components/subcomponent/.git"),
                 existing_dir());
    assert_that(&paths::root().join("foo/components/subcomponent/.gitignore"),
                 existing_file());
});

test!(unknown_flags {
    assert_that(cargo_process("new").arg("foo").arg("--flag"),
                execs().with_status(1)
                       .with_stderr("\
[ERROR] Unknown flag: '--flag'

Usage:
    cargo new [..]
    cargo new [..]
"));
});
<commit_msg>PR #1124 have replaced `git` with `vcs`<commit_after>use std::fs::{self, File};
use std::io::prelude::*;
use std::env;
use tempdir::TempDir;

use support::{execs, paths};
use support::paths::CargoPathExt;
use hamcrest::{assert_that, existing_file, existing_dir, is_not};

use cargo::util::ProcessBuilder;

fn setup() {
}

fn cargo_process(s: &str) -> ProcessBuilder {
    let mut p = ::cargo_process();
    p.arg(s);
    return p;
}

test!(simple_lib {
    assert_that(cargo_process("new").arg("foo").arg("--vcs").arg("none")
                                    .env("USER", "foo"),
                execs().with_status(0));

    assert_that(&paths::root().join("foo"), existing_dir());
    assert_that(&paths::root().join("foo/Cargo.toml"), existing_file());
    assert_that(&paths::root().join("foo/src/lib.rs"), existing_file());
    assert_that(&paths::root().join("foo/.gitignore"), is_not(existing_file()));

    assert_that(cargo_process("build").cwd(&paths::root().join("foo")),
                execs().with_status(0));
});

test!(simple_bin {
    assert_that(cargo_process("new").arg("foo").arg("--bin")
                                    .env("USER", "foo"),
                execs().with_status(0));

    assert_that(&paths::root().join("foo"), existing_dir());
    assert_that(&paths::root().join("foo/Cargo.toml"), existing_file());
    assert_that(&paths::root().join("foo/src/main.rs"), existing_file());

    assert_that(cargo_process("build").cwd(&paths::root().join("foo")),
                execs().with_status(0));
    assert_that(&paths::root().join(&format!("foo/target/debug/foo{}",
                                             env::consts::EXE_SUFFIX)),
                existing_file());
});

test!(simple_git {
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo").cwd(td.path().clone())
                                    .env("USER", "foo"),
                execs().with_status(0));

    assert_that(td.path(), existing_dir());
    assert_that(&td.path().join("foo/Cargo.toml"), existing_file());
    assert_that(&td.path().join("foo/src/lib.rs"), existing_file());
    assert_that(&td.path().join("foo/.git"), existing_dir());
    assert_that(&td.path().join("foo/.gitignore"), existing_file());

    assert_that(cargo_process("build").cwd(&td.path().clone().join("foo")),
                execs().with_status(0));
});

test!(no_argument {
    assert_that(cargo_process("new"),
                execs().with_status(1)
                       .with_stderr("\
[ERROR] Invalid arguments.

Usage:
    cargo new [options] <path>
    cargo new -h | --help
"));
});

test!(existing {
    let dst = paths::root().join("foo");
    fs::create_dir(&dst).unwrap();
    assert_that(cargo_process("new").arg("foo"),
                execs().with_status(101)
                       .with_stderr(format!("[ERROR] destination `{}` already exists\n",
                                            dst.display())));
});

test!(invalid_characters {
    assert_that(cargo_process("new").arg("foo.rs"),
                execs().with_status(101)
                       .with_stderr("\
[ERROR] Invalid character `.` in crate name: `foo.rs`
use --name to override crate name"));
});

test!(reserved_name {
    assert_that(cargo_process("new").arg("test"),
                execs().with_status(101)
                       .with_stderr("\
[ERROR] The name `test` cannot be used as a crate name\n\
use --name to override crate name"));
});

test!(keyword_name {
    assert_that(cargo_process("new").arg("pub"),
                execs().with_status(101)
                       .with_stderr("\
[ERROR] The name `pub` cannot be used as a crate name\n\
use --name to override crate name"));
});

test!(rust_prefix_stripped {
    assert_that(cargo_process("new").arg("rust-foo").env("USER", "foo"),
                execs().with_status(0)
                       .with_stdout("note: package will be named `foo`; use --name to override"));
    let toml = paths::root().join("rust-foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"name = "foo""#));
});

test!(bin_disables_stripping {
    assert_that(cargo_process("new").arg("rust-foo").arg("--bin").env("USER", "foo"),
                execs().with_status(0));
    let toml = paths::root().join("rust-foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"name = "rust-foo""#));
});

test!(explicit_name_not_stripped {
    assert_that(cargo_process("new").arg("foo").arg("--name").arg("rust-bar").env("USER", "foo"),
                execs().with_status(0));
    let toml = paths::root().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"name = "rust-bar""#));
});

test!(finds_author_user {
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo").env("USER", "foo")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["foo"]"#));
});

test!(finds_author_user_escaped {
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo").env("USER", "foo \"bar\"")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["foo \"bar\""]"#));
});

test!(finds_author_username {
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo")
                                    .env_remove("USER")
                                    .env("USERNAME", "foo")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["foo"]"#));
});

test!(finds_author_priority {
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo")
                                    .env("USER", "bar2")
                                    .env("EMAIL", "baz2")
                                    .env("CARGO_NAME", "bar")
                                    .env("CARGO_EMAIL", "baz")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["bar <baz>"]"#));
});

test!(finds_author_email {
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo")
                                    .env("USER", "bar")
                                    .env("EMAIL", "baz")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["bar <baz>"]"#));
});

test!(finds_author_git {
    ::process("git").args(&["config", "--global", "user.name", "bar"])
                    .exec().unwrap();
    ::process("git").args(&["config", "--global", "user.email", "baz"])
                    .exec().unwrap();
    assert_that(cargo_process("new").arg("foo").env("USER", "foo"),
                execs().with_status(0));

    let toml = paths::root().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["bar <baz>"]"#));
});

test!(finds_git_email{
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo")
                                    .env("GIT_AUTHOR_NAME", "foo")
                                    .env("GIT_AUTHOR_EMAIL", "gitfoo")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["foo <gitfoo>"]"#), contents);
});

test!(finds_git_author{
    // Use a temp dir to make sure we don't pick up .cargo/config somewhere in
    // the hierarchy
    let td = TempDir::new("cargo").unwrap();
    assert_that(cargo_process("new").arg("foo")
                                    .env_remove("USER")
                                    .env("GIT_COMMITTER_NAME", "gitfoo")
                                    .cwd(td.path().clone()),
                execs().with_status(0));

    let toml = td.path().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["gitfoo"]"#));
});

test!(author_prefers_cargo {
    ::process("git").args(&["config", "--global", "user.name", "foo"])
                    .exec().unwrap();
    ::process("git").args(&["config", "--global", "user.email", "bar"])
                    .exec().unwrap();
    let root = paths::root();
    fs::create_dir(&root.join(".cargo")).unwrap();
    File::create(&root.join(".cargo/config")).unwrap().write_all(br#"
        [cargo-new]
        name = "new-foo"
        email = "new-bar"
        vcs = "none"
    "#).unwrap();

    assert_that(cargo_process("new").arg("foo").env("USER", "foo"),
                execs().with_status(0));

    let toml = paths::root().join("foo/Cargo.toml");
    let mut contents = String::new();
    File::open(&toml).unwrap().read_to_string(&mut contents).unwrap();
    assert!(contents.contains(r#"authors = ["new-foo <new-bar>"]"#));
    assert!(!root.join("foo/.gitignore").c_exists());
});

test!(git_prefers_command_line {
    let root = paths::root();
    let td = TempDir::new("cargo").unwrap();
    fs::create_dir(&root.join(".cargo")).unwrap();
    File::create(&root.join(".cargo/config")).unwrap().write_all(br#"
        [cargo-new]
        vcs = "none"
        name = "foo"
        email = "bar"
    "#).unwrap();

    assert_that(cargo_process("new").arg("foo").arg("--vcs").arg("git")
                                    .cwd(td.path())
                                    .env("USER", "foo"),
                execs().with_status(0));
    assert!(td.path().join("foo/.gitignore").c_exists());
});

test!(subpackage_no_git {
    assert_that(cargo_process("new").arg("foo").env("USER", "foo"),
                execs().with_status(0));

    let subpackage = paths::root().join("foo").join("components");
    fs::create_dir(&subpackage).unwrap();
    assert_that(cargo_process("new").arg("foo/components/subcomponent")
                                    .env("USER", "foo"),
                execs().with_status(0));

    assert_that(&paths::root().join("foo/components/subcomponent/.git"),
                 is_not(existing_file()));
    assert_that(&paths::root().join("foo/components/subcomponent/.gitignore"),
                 is_not(existing_file()));
});

test!(subpackage_git_with_vcs_arg {
    assert_that(cargo_process("new").arg("foo").env("USER", "foo"),
                execs().with_status(0));

    let subpackage = paths::root().join("foo").join("components");
    fs::create_dir(&subpackage).unwrap();
    assert_that(cargo_process("new").arg("foo/components/subcomponent")
                                    .arg("--vcs").arg("git")
                                    .env("USER", "foo"),
                execs().with_status(0));

    assert_that(&paths::root().join("foo/components/subcomponent/.git"),
                 existing_dir());
    assert_that(&paths::root().join("foo/components/subcomponent/.gitignore"),
                 existing_file());
});

test!(unknown_flags {
    assert_that(cargo_process("new").arg("foo").arg("--flag"),
                execs().with_status(1)
                       .with_stderr("\
[ERROR] Unknown flag: '--flag'

Usage:
    cargo new [..]
    cargo new [..]
"));
});
<|endoftext|>


<commit_before>//! This module defines the `Opcode` type, which contains all opcodes handled by the VM.

use std::fmt;
use std::ops::{Deref, DerefMut};

pub use self::Opcode::*;

/// An opcode that can be executed by the VM. This enum is always 32 bits large.
///
/// Documentation syntax:
///
/// * `R[n]` is the value stored in register (stack slot) `n`
/// * `U[n]` is the current value of the upvalue with the index `n`
/// * `C[n]` is the constant at index `n` (looked up in the current function's constant table)
/// * `PROTO[n]` is function prototype number `n`
/// * `A`, `B`, `C` are the values of the first, second and third `u8` parameters (resp.)
/// * `Xu` is the value of the operand of type `u16` (eXtended unsigned)
/// * `Xs` is the value of the operand of type `i16` (eXtended signed)
/// * `PC` is the program counter, which holds the number of the next opcode by default
#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash, RustcEncodable, RustcDecodable)]
#[repr(u8)]
pub enum Opcode {
    /// > R[A] := R[B]
    MOV(u8, u8),
    /// > R[A] := C[Xu]
    ///
    /// Loads the constant with index `Xu` into `R[A]`.
    LOADK(u8, u16),
    /// > R[A] := nil, R[A+1] := nil, ..., R[A+B] := nil
    ///
    /// Assigns `nil` to the B+1 registers starting at A. If B is 0, assigns `nil` only to `R[A]`.
    LOADNIL(u8, u8),
    /// > R[A], R[A+1], ..., R[A+B] := C
    ///
    /// Assigns a boolean to B+1 registers from `R[A]` to `R[A+B]`. If C is `false`, assign `false`
    /// to the registers, otherwise assign `true`.
    LOADBOOL(u8, u8, bool),
    /// > R[A] := {}
    ///
    /// Creates a new table and assigns it to `R[A]`.
    TABLE(u8),  // TODO Include initial capacity / Create from template
    /// > R[A] := []
    ///
    /// Creates a new array and assigns it to `R[A]`.
    ARRAY(u8),  // TODO Include initial capacity / Create from template
    /// > R[A] := closure of PROTO[Xu]
    ///
    /// Instantiates function prototype number `Xu` and stores a reference to the created closure
    /// in `R[A]`
    FUNC(u8, u16),
    /// > R[A], R[A+1], ..., R[A+C-2] := R\[A](R[A+1], ..., R[A+B-1])
    ///
    /// Calls `R[A]` with B-1 arguments stored in `R[A+1]` through `R[A+B-1]`. If B is 1, no
    /// arguments are passed. If B is 0, passes all arguments from A+1 to the top of the stack
    /// (this is used if the last argument is a function call, which may return an unknown number
    /// of values).
    ///
    /// Stores C-1 return values in `R[A]` through `R[A+C-2]`. If C is 1, no return values are
    /// stored. If C is 0, stores all return values in `R[A]`, `R[A+1]`, ... (if necessary,
    /// expanding the stack to fit all return values in).
    ///
    /// If the callee returns less than C-1 results, the rest is set to `nil`, except when C is 0,
    /// in which case 0 results may be returned (the top of the stack is set to A in that case).
    ///
    /// This might invoke a metamethod if `R[A]` has a metatable with a `__call` field.
    CALL(u8, u8, u8),
    /// > return R[A], R[A+1], ..., R[A+B-2]
    ///
    /// Returns B-1 values stored in `R[A]` through `R[A+B-2]` to the caller. If B is 1, returns no
    /// values and just leaves the current function, returning control to the caller. If B is 0,
    /// returns all values from A to the top of the stack.
    ///
    /// If the program's main function executes this opcode, the VM will pass the return values to
    /// the code that started the VM.
    RETURN(u8, u8),
    /// > close_upval(R[A], R[A+1], ...)
    ///
    /// Closes all open upvalues in registers `A` and higher.
    CLOSE(u8),
    /// > R[A], R[A+1], ..., R[A+B-1] := varargs
    ///
    /// Stores B arguments passed as variable arguments (in `...`) in `R[A]` through `R[A+B-1]`.
    /// If B is 0, stores all varargs in `R[A]`, `R[A+1]`, ... (expanding the stack if necessary).
    VARARGS(u8, u8),
    /// > PC += Xs
    ///
    /// Unconditional jump. Offsets `PC` by `Xs`, which may be negative.
    ///
    /// All jumps share the same property: If `Xs` is 0, PC will not be changed (the jump acts
    /// like a no-op). If `Xs` is -1, an endless loop is created.
    JMP(i16),
    /// > if R[A]: PC += Xs
    ///
    /// Conditional jump. If the value in `R[A]` is "truthy" (not `false` or `nil`), offsets `PC`
    /// by `Xs`.
    IF(u8, i16),
    /// > if not R[A]: PC += Xs
    ///
    /// If the value in `R[A]` is `false` or `nil`, offsets `PC` by `Xs`.
    IFNOT(u8, i16),
    /// > assert: R[A], R[A+1], R[A+2] are numeric
    /// > if (R[A+1] >= 0 && R[A] > R[A+2]) || (R[A+1] < 0 && R[A] < R[A+2]): PC += Xu
    ///
    /// Jumps out of a for-loop if the loop variable in `R[A]` is outside the range of the loop:
    /// If the loop step in `R[A+1]` is non-negative, this occurs exactly when the value is larger
    /// than the loop limit in `R[A+2]`. If the step is negative, this occurs when the value is
    /// smaller than the limit, since the loop runs backwards.
    FORCHECK(u8, u16),
    /// > R[A] := U[B]
    GETUPVAL(u8, u8),
    /// > U[A] := R[B]
    SETUPVAL(u8, u8),
    /// > R[A] := R[B][R[C]]
    ///
    /// Indexes `R[B]` with the value in `R[C]` and stores the result in `R[A]`.
    GETIDX(u8, u8, u8),
    /// > R[A][R[B]] := R[C]
    ///
    /// Stores `R[C]` in `R[A]` at index `R[B]`.
    SETIDX(u8, u8, u8),

    // Operator opcodes. Any of them can call a metamethod if the value in R[B] or R[C] has an
    // associated metatable that overrides the operator.

    /// > R[A] := R[B] + R[C]
    ADD(u8, u8, u8),
    /// > R[A] := R[B] - R[C]
    SUB(u8, u8, u8),
    /// > R[A] := R[B] * R[C]
    MUL(u8, u8, u8),
    /// > R[A] := R[B] / R[C]
    DIV(u8, u8, u8),
    /// > R[A] := R[B] % R[C]
    MOD(u8, u8, u8),
    /// > R[A] := R[B] ^ R[C]
    POW(u8, u8, u8),

    /// > R[A] := R[B] == R[C]
    EQ(u8, u8, u8),
    /// > R[A] := R[B] != R[C]
    NEQ(u8, u8, u8),
    /// > R[A] := R[B] <= R[C]
    LEQ(u8, u8, u8),
    /// > R[A] := R[B] >= R[C]
    GEQ(u8, u8, u8),
    /// > R[A] := R[B] < R[C]
    LESS(u8, u8, u8),
    /// > R[A] := R[B] > R[C]
    GREATER(u8, u8, u8),

    // LAND and LOR are implemented with jumps because they can shortcut. This also implies that
    // they cannot be overridden via a metatable.

    /// > R[A] := R[B] & R[C]
    BAND(u8, u8, u8),
    /// > R[A] := R[B] | R[C]
    BOR(u8, u8, u8),
    /// > R[A] := R[B] ~ R[C]
    BXOR(u8, u8, u8),
    /// > R[A] := R[B] << R[C]
    SHIFTL(u8, u8, u8),
    /// > R[A] := R[B] >> R[C]
    SHIFTR(u8, u8, u8),

    /// > R[A] := R[B] .. R[C]
    ///
    /// Concatenates the value in `R[B]` with the value in `R[C]`. Numbers are coerced to strings.
    CONCAT(u8, u8, u8),

    /// > R[A] := -R[B]
    NEG(u8, u8),
    /// > R[A] := ~R[B]
    INV(u8, u8),
    /// > R[A] := #R[B]
    LEN(u8, u8),
    /// > R[A] := !R[B]
    ///
    /// The values `false` and `nil` will be converted to `true`, any other value will be
    /// converted to `false`. This does not call a metamethod.
    NOT(u8, u8),
    /// Causes a VM panic. Used by the emitter for opcodes that will be replaced later in the
    /// compilation (such as forward jumps).
    INVALID,

    /// Testing and debugging opcode that will print the `Value` stored in the given stack slot.
    DEBUG(u8),
}

/// Newtype for `Vec<Opcode>` that overrides `Debug` to use at most one line per printed opcode
/// (when the `{:#?}` format specifier is used).
#[derive(Clone, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable)]
pub struct Opcodes(pub Vec<Opcode>);

impl fmt::Debug for Opcodes {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        let mut l = f.debug_list();
        for op in &self.0 {
            l.entry(&SingleLineDebug(op));
        }
        l.finish()
    }
}

impl Deref for Opcodes {
    type Target = Vec<Opcode>;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

impl DerefMut for Opcodes {
    fn deref_mut(&mut self) -> &mut <Self as Deref>::Target {
        &mut self.0
    }
}

struct SingleLineDebug<'a, T: fmt::Debug + 'a>(&'a T);

impl<'a, T: fmt::Debug + 'a> fmt::Debug for SingleLineDebug<'a, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{:?}", &self.0)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::mem;

    #[test]
    fn size() {
        // we guarantee that opcodes fit in 32 bits, regardless of the host platform
        assert_eq!(mem::size_of::<Opcode>(), 4);
    }

    #[test]
    fn format() {
        assert_eq!(format!("{:#?}", Opcodes(vec![
            MOV(0, 1),
            IFNOT(5, -1),
            CONCAT(2, 3, 4),
            LOADBOOL(7, 0, true),
        ])), "\
[
    MOV(0, 1),
    IFNOT(5, -1),
    CONCAT(2, 3, 4),
    LOADBOOL(7, 0, true)
]");
    }
}
<commit_msg>Make the opcode docs look nicer<commit_after>//! This module defines the `Opcode` type, which contains all opcodes handled by the VM.

use std::fmt;
use std::ops::{Deref, DerefMut};

pub use self::Opcode::*;

/// An opcode that can be executed by the VM. This enum is always 32 bits large.
///
/// Documentation syntax:
///
/// * `R[n]` is the value stored in register (stack slot) `n`
/// * `U[n]` is the current value of the upvalue with the index `n`
/// * `C[n]` is the constant at index `n` (looked up in the current function's constant table)
/// * `PROTO[n]` is function prototype number `n`
/// * `A`, `B`, `C` are the values of the first, second and third `u8` parameters (resp.)
/// * `Xu` is the value of the operand of type `u16` (eXtended unsigned)
/// * `Xs` is the value of the operand of type `i16` (eXtended signed)
/// * `PC` is the program counter, which holds the number of the next opcode by default
#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash, RustcEncodable, RustcDecodable)]
#[repr(u8)]
pub enum Opcode {
    /// **`R[A] := R[B]`**
    MOV(u8, u8),
    /// **`R[A] := C[Xu]`**
    ///
    /// Loads the constant with index `Xu` into `R[A]`.
    LOADK(u8, u16),
    /// **`R[A] := nil, R[A+1] := nil, ..., R[A+B] := nil`**
    ///
    /// Assigns `nil` to the B+1 registers starting at A. If B is 0, assigns `nil` only to `R[A]`.
    LOADNIL(u8, u8),
    /// **`R[A], R[A+1], ..., R[A+B] := C`**
    ///
    /// Assigns a boolean to B+1 registers from `R[A]` to `R[A+B]`. If C is `false`, assign `false`
    /// to the registers, otherwise assign `true`.
    LOADBOOL(u8, u8, bool),
    /// **`R[A] := {}`**
    ///
    /// Creates a new table and assigns it to `R[A]`.
    TABLE(u8),  // TODO Include initial capacity / Create from template
    /// **`R[A] := []`**
    ///
    /// Creates a new array and assigns it to `R[A]`.
    ARRAY(u8),  // TODO Include initial capacity / Create from template
    /// **`R[A] := closure of PROTO[Xu]`**
    ///
    /// Instantiates function prototype number `Xu` and stores a reference to the created closure
    /// in `R[A]`
    FUNC(u8, u16),
    /// **`R[A], R[A+1], ..., R[A+C-2] := R\[A](R[A+1], ..., R[A+B-1])`**
    ///
    /// Calls `R[A]` with B-1 arguments stored in `R[A+1]` through `R[A+B-1]`. If B is 1, no
    /// arguments are passed. If B is 0, passes all arguments from A+1 to the top of the stack
    /// (this is used if the last argument is a function call, which may return an unknown number
    /// of values).
    ///
    /// Stores C-1 return values in `R[A]` through `R[A+C-2]`. If C is 1, no return values are
    /// stored. If C is 0, stores all return values in `R[A]`, `R[A+1]`, ... (if necessary,
    /// expanding the stack to fit all return values in).
    ///
    /// If the callee returns less than C-1 results, the rest is set to `nil`, except when C is 0,
    /// in which case 0 results may be returned (the top of the stack is set to A in that case).
    ///
    /// This might invoke a metamethod if `R[A]` has a metatable with a `__call` field.
    CALL(u8, u8, u8),
    /// **`return R[A], R[A+1], ..., R[A+B-2]`**
    ///
    /// Returns B-1 values stored in `R[A]` through `R[A+B-2]` to the caller. If B is 1, returns no
    /// values and just leaves the current function, returning control to the caller. If B is 0,
    /// returns all values from A to the top of the stack.
    ///
    /// If the program's main function executes this opcode, the VM will pass the return values to
    /// the code that started the VM.
    RETURN(u8, u8),
    /// **`close_upval(R[A], R[A+1], ...)`**
    ///
    /// Closes all open upvalues in registers `A` and higher.
    CLOSE(u8),
    /// **`R[A], R[A+1], ..., R[A+B-1] := varargs`**
    ///
    /// Stores B arguments passed as variable arguments (in `...`) in `R[A]` through `R[A+B-1]`.
    /// If B is 0, stores all varargs in `R[A]`, `R[A+1]`, ... (expanding the stack if necessary).
    VARARGS(u8, u8),
    /// **`PC += Xs`**
    ///
    /// Unconditional jump. Offsets `PC` by `Xs`, which may be negative.
    ///
    /// All jumps share the same property: If `Xs` is 0, PC will not be changed (the jump acts
    /// like a no-op). If `Xs` is -1, an endless loop is created.
    JMP(i16),
    /// **`if R[A]: PC += Xs`**
    ///
    /// Conditional jump. If the value in `R[A]` is "truthy" (not `false` or `nil`), offsets `PC`
    /// by `Xs`.
    IF(u8, i16),
    /// **`if not R[A]: PC += Xs`**
    ///
    /// If the value in `R[A]` is `false` or `nil`, offsets `PC` by `Xs`.
    IFNOT(u8, i16),
    /// **`assert: R[A], R[A+1], R[A+2] are numeric`**
    ///
    /// **`if (R[A+1] >= 0 && R[A] > R[A+2]) || (R[A+1] < 0 && R[A] < R[A+2]): PC += Xu`**
    ///
    /// Jumps out of a for-loop if the loop variable in `R[A]` is outside the range of the loop:
    /// If the loop step in `R[A+1]` is non-negative, this occurs exactly when the value is larger
    /// than the loop limit in `R[A+2]`. If the step is negative, this occurs when the value is
    /// smaller than the limit, since the loop runs backwards.
    FORCHECK(u8, u16),
    /// **`R[A] := U[B]`**
    GETUPVAL(u8, u8),
    /// **`U[A] := R[B]`**
    SETUPVAL(u8, u8),
    /// **`R[A] := R[B][R[C]]`**
    ///
    /// Indexes `R[B]` with the value in `R[C]` and stores the result in `R[A]`.
    GETIDX(u8, u8, u8),
    /// **`R[A][R[B]] := R[C]`**
    ///
    /// Stores `R[C]` in `R[A]` at index `R[B]`.
    SETIDX(u8, u8, u8),

    /// **`R[A] := R[B] + R[C]`**
    ///
    /// All operator opcodes can call a metamethod, depending on the values in `R[B]` and `R[C]` and
    /// the specific operation.
    ADD(u8, u8, u8),
    /// **`R[A] := R[B] - R[C]`**
    SUB(u8, u8, u8),
    /// **`R[A] := R[B] * R[C]`**
    MUL(u8, u8, u8),
    /// **`R[A] := R[B] / R[C]`**
    DIV(u8, u8, u8),
    /// **`R[A] := R[B] % R[C]`**
    MOD(u8, u8, u8),
    /// **`R[A] := R[B] ^ R[C]`**
    POW(u8, u8, u8),

    /// **`R[A] := R[B] == R[C]`**
    EQ(u8, u8, u8),
    /// **`R[A] := R[B] != R[C]`**
    NEQ(u8, u8, u8),
    /// **`R[A] := R[B] <= R[C]`**
    LEQ(u8, u8, u8),
    /// **`R[A] := R[B] >= R[C]`**
    GEQ(u8, u8, u8),
    /// **`R[A] := R[B] < R[C]`**
    LESS(u8, u8, u8),
    /// **`R[A] := R[B] > R[C]`**
    GREATER(u8, u8, u8),

    /// **`R[A] := R[B] & R[C]`**
    ///
    /// Bitwise and. Logical `and` / `or` (`&&` / `||`) can shortcut and is implemented via
    /// test-and-jump. `&&` / `||` can also not be overridden via a metamethod.
    BAND(u8, u8, u8),
    /// **`R[A] := R[B] | R[C]`**
    BOR(u8, u8, u8),
    /// **`R[A] := R[B] ~ R[C]`**
    BXOR(u8, u8, u8),
    /// **`R[A] := R[B] << R[C]`**
    SHIFTL(u8, u8, u8),
    /// **`R[A] := R[B] >> R[C]`**
    SHIFTR(u8, u8, u8),

    /// **`R[A] := R[B] .. R[C]`**
    ///
    /// Concatenates the value in `R[B]` with the value in `R[C]`. Numbers are coerced to strings.
    /// If one of the values is neither a string nor a number, the `__concat` metamethod is called
    /// to convert the value.
    CONCAT(u8, u8, u8),

    /// **`R[A] := -R[B]`**
    NEG(u8, u8),
    /// **`R[A] := ~R[B]`**
    INV(u8, u8),
    /// **`R[A] := #R[B]`**
    LEN(u8, u8),
    /// **`R[A] := !R[B]`**
    ///
    /// The values `false` and `nil` will be converted to `true`, any other value will be
    /// converted to `false`. This does not call a metamethod.
    NOT(u8, u8),

    /// Causes a VM panic. Used by the emitter for opcodes that will be replaced later in the
    /// compilation (such as forward jumps).
    INVALID,

    /// Testing and debugging opcode that will print the `Value` stored in the given stack slot.
    DEBUG(u8),
}

/// Newtype for `Vec<Opcode>` that overrides `Debug` to use at most one line per printed opcode
/// (when the `{:#?}` format specifier is used).
#[derive(Clone, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable)]
pub struct Opcodes(pub Vec<Opcode>);

impl fmt::Debug for Opcodes {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        let mut l = f.debug_list();
        for op in &self.0 {
            l.entry(&SingleLineDebug(op));
        }
        l.finish()
    }
}

impl Deref for Opcodes {
    type Target = Vec<Opcode>;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

impl DerefMut for Opcodes {
    fn deref_mut(&mut self) -> &mut <Self as Deref>::Target {
        &mut self.0
    }
}

struct SingleLineDebug<'a, T: fmt::Debug + 'a>(&'a T);

impl<'a, T: fmt::Debug + 'a> fmt::Debug for SingleLineDebug<'a, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(f, "{:?}", &self.0)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    use std::mem;

    #[test]
    fn size() {
        // we guarantee that opcodes fit in 32 bits, regardless of the host platform
        assert_eq!(mem::size_of::<Opcode>(), 4);
    }

    #[test]
    fn format() {
        assert_eq!(format!("{:#?}", Opcodes(vec![
            MOV(0, 1),
            IFNOT(5, -1),
            CONCAT(2, 3, 4),
            LOADBOOL(7, 0, true),
        ])), "\
[
    MOV(0, 1),
    IFNOT(5, -1),
    CONCAT(2, 3, 4),
    LOADBOOL(7, 0, true)
]");
    }
}
<|endoftext|>


<commit_before>//! Script to check the validity of `href` links in our HTML documentation.
//!
//! In the past we've been quite error prone to writing in broken links as most
//! of them are manually rather than automatically added. As files move over
//! time or apis change old links become stale or broken. The purpose of this
//! script is to check all relative links in our documentation to make sure they
//! actually point to a valid place.
//!
//! Currently this doesn't actually do any HTML parsing or anything fancy like
//! that, it just has a simple "regex" to search for `href` and `id` tags.
//! These values are then translated to file URLs if possible and then the
//! destination is asserted to exist.
//!
//! A few exceptions are allowed as there's known bugs in rustdoc, but this
//! should catch the majority of "broken link" cases.

use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
use std::env;
use std::fs;
use std::path::{Component, Path, PathBuf};
use std::rc::Rc;

use crate::Redirect::*;

macro_rules! t {
    ($e:expr) => {
        match $e {
            Ok(e) => e,
            Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
        }
    };
}

fn main() {
    let docs = env::args_os().nth(1).unwrap();
    let docs = env::current_dir().unwrap().join(docs);
    let mut errors = false;
    walk(&mut HashMap::new(), &docs, &docs, &mut errors);
    if errors {
        panic!("found some broken links");
    }
}

#[derive(Debug)]
pub enum LoadError {
    IOError(std::io::Error),
    BrokenRedirect(PathBuf, std::io::Error),
    IsRedirect,
}

enum Redirect {
    SkipRedirect,
    FromRedirect(bool),
}

struct FileEntry {
    source: Rc<String>,
    ids: HashSet<String>,
}

type Cache = HashMap<PathBuf, FileEntry>;

fn small_url_encode(s: &str) -> String {
    s.replace("<", "%3C")
        .replace(">", "%3E")
        .replace(" ", "%20")
        .replace("?", "%3F")
        .replace("'", "%27")
        .replace("&", "%26")
        .replace(",", "%2C")
        .replace(":", "%3A")
        .replace(";", "%3B")
        .replace("[", "%5B")
        .replace("]", "%5D")
        .replace("\"", "%22")
}

impl FileEntry {
    fn parse_ids(&mut self, file: &Path, contents: &str, errors: &mut bool) {
        if self.ids.is_empty() {
            with_attrs_in_source(contents, " id", |fragment, i, _| {
                let frag = fragment.trim_start_matches("#").to_owned();
                let encoded = small_url_encode(&frag);
                if !self.ids.insert(frag) {
                    *errors = true;
                    println!("{}:{}: id is not unique: `{}`", file.display(), i, fragment);
                }
                // Just in case, we also add the encoded id.
                self.ids.insert(encoded);
            });
        }
    }
}

fn walk(cache: &mut Cache, root: &Path, dir: &Path, errors: &mut bool) {
    for entry in t!(dir.read_dir()).map(|e| t!(e)) {
        let path = entry.path();
        let kind = t!(entry.file_type());
        if kind.is_dir() {
            walk(cache, root, &path, errors);
        } else {
            let pretty_path = check(cache, root, &path, errors);
            if let Some(pretty_path) = pretty_path {
                let entry = cache.get_mut(&pretty_path).unwrap();
                // we don't need the source anymore,
                // so drop to reduce memory-usage
                entry.source = Rc::new(String::new());
            }
        }
    }
}

fn check(cache: &mut Cache, root: &Path, file: &Path, errors: &mut bool) -> Option<PathBuf> {
    // Ignore non-HTML files.
    if file.extension().and_then(|s| s.to_str()) != Some("html") {
        return None;
    }

    // Unfortunately we're not 100% full of valid links today to we need a few
    // exceptions to get this past `make check` today.
    // FIXME(#32129)
    if file.ends_with("std/io/struct.IoSlice.html")
    {
        return None;
    }

    // FIXME(#32130)
    if file.ends_with("alloc/collections/btree_map/struct.BTreeMap.html")
        || file.ends_with("alloc/collections/btree_set/struct.BTreeSet.html")
        || file.ends_with("std/collections/btree_map/struct.BTreeMap.html")
        || file.ends_with("std/collections/btree_set/struct.BTreeSet.html")
        || file.ends_with("std/collections/hash_set/struct.HashSet.html")
    {
        return None;
    }

    let res = load_file(cache, root, file, SkipRedirect);
    let (pretty_file, contents) = match res {
        Ok(res) => res,
        Err(_) => return None,
    };
    {
        cache.get_mut(&pretty_file).unwrap().parse_ids(&pretty_file, &contents, errors);
    }

    // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
    with_attrs_in_source(&contents, " href", |url, i, base| {
        // Ignore external URLs
        if url.starts_with("http:")
            || url.starts_with("https:")
            || url.starts_with("javascript:")
            || url.starts_with("ftp:")
            || url.starts_with("irc:")
            || url.starts_with("data:")
        {
            return;
        }
        let mut parts = url.splitn(2, "#");
        let url = parts.next().unwrap();
        let fragment = parts.next();
        let mut parts = url.splitn(2, "?");
        let url = parts.next().unwrap();

        // Once we've plucked out the URL, parse it using our base url and
        // then try to extract a file path.
        let mut path = file.to_path_buf();
        if !base.is_empty() || !url.is_empty() {
            path.pop();
            for part in Path::new(base).join(url).components() {
                match part {
                    Component::Prefix(_) | Component::RootDir => {
                        // Avoid absolute paths as they make the docs not
                        // relocatable by making assumptions on where the docs
                        // are hosted relative to the site root.
                        *errors = true;
                        println!(
                            "{}:{}: absolute path - {}",
                            pretty_file.display(),
                            i + 1,
                            Path::new(base).join(url).display()
                        );
                        return;
                    }
                    Component::CurDir => {}
                    Component::ParentDir => {
                        path.pop();
                    }
                    Component::Normal(s) => {
                        path.push(s);
                    }
                }
            }
        }

        // Alright, if we've found a file name then this file had better
        // exist! If it doesn't then we register and print an error.
        if path.exists() {
            if path.is_dir() {
                // Links to directories show as directory listings when viewing
                // the docs offline so it's best to avoid them.
                *errors = true;
                let pretty_path = path.strip_prefix(root).unwrap_or(&path);
                println!(
                    "{}:{}: directory link - {}",
                    pretty_file.display(),
                    i + 1,
                    pretty_path.display()
                );
                return;
            }
            if let Some(extension) = path.extension() {
                // Ignore none HTML files.
                if extension != "html" {
                    return;
                }
            }
            let res = load_file(cache, root, &path, FromRedirect(false));
            let (pretty_path, contents) = match res {
                Ok(res) => res,
                Err(LoadError::IOError(err)) => {
                    panic!("error loading {}: {}", path.display(), err);
                }
                Err(LoadError::BrokenRedirect(target, _)) => {
                    *errors = true;
                    println!(
                        "{}:{}: broken redirect to {}",
                        pretty_file.display(),
                        i + 1,
                        target.display()
                    );
                    return;
                }
                Err(LoadError::IsRedirect) => unreachable!(),
            };

            if let Some(ref fragment) = fragment {
                // Fragments like `#1-6` are most likely line numbers to be
                // interpreted by javascript, so we're ignoring these
                if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
                    return;
                }

                // These appear to be broken in mdbook right now?
                if fragment.starts_with("-") {
                    return;
                }

                let entry = &mut cache.get_mut(&pretty_path).unwrap();
                entry.parse_ids(&pretty_path, &contents, errors);

                if !entry.ids.contains(*fragment) {
                    *errors = true;
                    print!("{}:{}: broken link fragment ", pretty_file.display(), i + 1);
                    println!("`#{}` pointing to `{}`", fragment, pretty_path.display());
                };
            }
        } else {
            *errors = true;
            print!("{}:{}: broken link - ", pretty_file.display(), i + 1);
            let pretty_path = path.strip_prefix(root).unwrap_or(&path);
            println!("{}", pretty_path.display());
        }
    });
    Some(pretty_file)
}

fn load_file(
    cache: &mut Cache,
    root: &Path,
    file: &Path,
    redirect: Redirect,
) -> Result<(PathBuf, Rc<String>), LoadError> {
    let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file));

    let (maybe_redirect, contents) = match cache.entry(pretty_file.clone()) {
        Entry::Occupied(entry) => (None, entry.get().source.clone()),
        Entry::Vacant(entry) => {
            let contents = match fs::read_to_string(file) {
                Ok(s) => Rc::new(s),
                Err(err) => {
                    return Err(if let FromRedirect(true) = redirect {
                        LoadError::BrokenRedirect(file.to_path_buf(), err)
                    } else {
                        LoadError::IOError(err)
                    });
                }
            };

            let maybe = maybe_redirect(&contents);
            if maybe.is_some() {
                if let SkipRedirect = redirect {
                    return Err(LoadError::IsRedirect);
                }
            } else {
                entry.insert(FileEntry { source: contents.clone(), ids: HashSet::new() });
            }
            (maybe, contents)
        }
    };
    match maybe_redirect.map(|url| file.parent().unwrap().join(url)) {
        Some(redirect_file) => load_file(cache, root, &redirect_file, FromRedirect(true)),
        None => Ok((pretty_file, contents)),
    }
}

fn maybe_redirect(source: &str) -> Option<String> {
    const REDIRECT: &'static str = "
Redirecting to <a href=";

    let mut lines = source.lines();
    let redirect_line = lines.nth(6)?;

    redirect_line.find(REDIRECT).map(|i| {
        let rest = &redirect_line[(i + REDIRECT.len() + 1)..];
        let pos_quote = rest.find('"').unwrap();
        rest[..pos_quote].to_owned()
    })
}

fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(contents: &str, attr: &str, mut f: F) {
    let mut base = "";
    for (i, mut line) in contents.lines().enumerate() {
        while let Some(j) = line.find(attr) {
            let rest = &line[j + attr.len()..];
            // The base tag should always be the first link in the document so
            // we can get away with using one pass.
            let is_base = line[..j].ends_with("<base");
            line = rest;
            let pos_equals = match rest.find("=") {
                Some(i) => i,
                None => continue,
            };
            if rest[..pos_equals].trim_start_matches(" ") != "" {
                continue;
            }

            let rest = &rest[pos_equals + 1..];

            let pos_quote = match rest.find(&['"', '\''][..]) {
                Some(i) => i,
                None => continue,
            };
            let quote_delim = rest.as_bytes()[pos_quote] as char;

            if rest[..pos_quote].trim_start_matches(" ") != "" {
                continue;
            }
            let rest = &rest[pos_quote + 1..];
            let url = match rest.find(quote_delim) {
                Some(i) => &rest[..i],
                None => continue,
            };
            if is_base {
                base = url;
                continue;
            }
            f(url, i, base)
        }
    }
}
<commit_msg>Add explicit exception list to linkchecker<commit_after>//! Script to check the validity of `href` links in our HTML documentation.
//!
//! In the past we've been quite error prone to writing in broken links as most
//! of them are manually rather than automatically added. As files move over
//! time or apis change old links become stale or broken. The purpose of this
//! script is to check all relative links in our documentation to make sure they
//! actually point to a valid place.
//!
//! Currently this doesn't actually do any HTML parsing or anything fancy like
//! that, it just has a simple "regex" to search for `href` and `id` tags.
//! These values are then translated to file URLs if possible and then the
//! destination is asserted to exist.
//!
//! A few exceptions are allowed as there's known bugs in rustdoc, but this
//! should catch the majority of "broken link" cases.

use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
use std::env;
use std::fs;
use std::path::{Component, Path, PathBuf};
use std::rc::Rc;

use crate::Redirect::*;

// Add linkcheck exceptions here
// If at all possible you should use intra-doc links to avoid linkcheck issues. These
// are cases where that does not work
const LINKCHECK_EXCEPTIONS: &[(&str, &[&str])] = &[
    // These are methods on slice, and `Self` does not work on primitive impls
    // in intra-doc links (intra-doc links are weird)
    // https://github.com/rust-lang/rust/issues/62834 is necessary to be
    // able to link to slices
    (
        "std/io/struct.IoSlice.html",
        &[
            "#method.as_mut_ptr",
            "#method.sort_by_key",
            "#method.make_ascii_uppercase",
            "#method.make_ascii_lowercase",
        ],
    ),
    // These try to link to std::collections, but are defined in alloc
    // https://github.com/rust-lang/rust/issues/74481
    ("std/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
    ("std/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
    ("alloc/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
    ("alloc/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
];

macro_rules! t {
    ($e:expr) => {
        match $e {
            Ok(e) => e,
            Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
        }
    };
}

fn main() {
    let docs = env::args_os().nth(1).unwrap();
    let docs = env::current_dir().unwrap().join(docs);
    let mut errors = false;
    walk(&mut HashMap::new(), &docs, &docs, &mut errors);
    if errors {
        panic!("found some broken links");
    }
}

#[derive(Debug)]
pub enum LoadError {
    IOError(std::io::Error),
    BrokenRedirect(PathBuf, std::io::Error),
    IsRedirect,
}

enum Redirect {
    SkipRedirect,
    FromRedirect(bool),
}

struct FileEntry {
    source: Rc<String>,
    ids: HashSet<String>,
}

type Cache = HashMap<PathBuf, FileEntry>;

fn small_url_encode(s: &str) -> String {
    s.replace("<", "%3C")
        .replace(">", "%3E")
        .replace(" ", "%20")
        .replace("?", "%3F")
        .replace("'", "%27")
        .replace("&", "%26")
        .replace(",", "%2C")
        .replace(":", "%3A")
        .replace(";", "%3B")
        .replace("[", "%5B")
        .replace("]", "%5D")
        .replace("\"", "%22")
}

impl FileEntry {
    fn parse_ids(&mut self, file: &Path, contents: &str, errors: &mut bool) {
        if self.ids.is_empty() {
            with_attrs_in_source(contents, " id", |fragment, i, _| {
                let frag = fragment.trim_start_matches("#").to_owned();
                let encoded = small_url_encode(&frag);
                if !self.ids.insert(frag) {
                    *errors = true;
                    println!("{}:{}: id is not unique: `{}`", file.display(), i, fragment);
                }
                // Just in case, we also add the encoded id.
                self.ids.insert(encoded);
            });
        }
    }
}

fn walk(cache: &mut Cache, root: &Path, dir: &Path, errors: &mut bool) {
    for entry in t!(dir.read_dir()).map(|e| t!(e)) {
        let path = entry.path();
        let kind = t!(entry.file_type());
        if kind.is_dir() {
            walk(cache, root, &path, errors);
        } else {
            let pretty_path = check(cache, root, &path, errors);
            if let Some(pretty_path) = pretty_path {
                let entry = cache.get_mut(&pretty_path).unwrap();
                // we don't need the source anymore,
                // so drop to reduce memory-usage
                entry.source = Rc::new(String::new());
            }
        }
    }
}

fn is_exception(file: &Path, link: &str) -> bool {
    if let Some(entry) = LINKCHECK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
        entry.1.contains(&link)
    } else {
        false
    }
}

fn check(cache: &mut Cache, root: &Path, file: &Path, errors: &mut bool) -> Option<PathBuf> {
    // Ignore non-HTML files.
    if file.extension().and_then(|s| s.to_str()) != Some("html") {
        return None;
    }

    let res = load_file(cache, root, file, SkipRedirect);
    let (pretty_file, contents) = match res {
        Ok(res) => res,
        Err(_) => return None,
    };
    {
        cache.get_mut(&pretty_file).unwrap().parse_ids(&pretty_file, &contents, errors);
    }

    // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
    with_attrs_in_source(&contents, " href", |url, i, base| {
        // Ignore external URLs
        if url.starts_with("http:")
            || url.starts_with("https:")
            || url.starts_with("javascript:")
            || url.starts_with("ftp:")
            || url.starts_with("irc:")
            || url.starts_with("data:")
        {
            return;
        }
        let mut parts = url.splitn(2, "#");
        let url = parts.next().unwrap();
        let fragment = parts.next();
        let mut parts = url.splitn(2, "?");
        let url = parts.next().unwrap();

        // Once we've plucked out the URL, parse it using our base url and
        // then try to extract a file path.
        let mut path = file.to_path_buf();
        if !base.is_empty() || !url.is_empty() {
            path.pop();
            for part in Path::new(base).join(url).components() {
                match part {
                    Component::Prefix(_) | Component::RootDir => {
                        // Avoid absolute paths as they make the docs not
                        // relocatable by making assumptions on where the docs
                        // are hosted relative to the site root.
                        *errors = true;
                        println!(
                            "{}:{}: absolute path - {}",
                            pretty_file.display(),
                            i + 1,
                            Path::new(base).join(url).display()
                        );
                        return;
                    }
                    Component::CurDir => {}
                    Component::ParentDir => {
                        path.pop();
                    }
                    Component::Normal(s) => {
                        path.push(s);
                    }
                }
            }
        }

        // Alright, if we've found a file name then this file had better
        // exist! If it doesn't then we register and print an error.
        if path.exists() {
            if path.is_dir() {
                // Links to directories show as directory listings when viewing
                // the docs offline so it's best to avoid them.
                *errors = true;
                let pretty_path = path.strip_prefix(root).unwrap_or(&path);
                println!(
                    "{}:{}: directory link - {}",
                    pretty_file.display(),
                    i + 1,
                    pretty_path.display()
                );
                return;
            }
            if let Some(extension) = path.extension() {
                // Ignore none HTML files.
                if extension != "html" {
                    return;
                }
            }
            let res = load_file(cache, root, &path, FromRedirect(false));
            let (pretty_path, contents) = match res {
                Ok(res) => res,
                Err(LoadError::IOError(err)) => {
                    panic!("error loading {}: {}", path.display(), err);
                }
                Err(LoadError::BrokenRedirect(target, _)) => {
                    *errors = true;
                    println!(
                        "{}:{}: broken redirect to {}",
                        pretty_file.display(),
                        i + 1,
                        target.display()
                    );
                    return;
                }
                Err(LoadError::IsRedirect) => unreachable!(),
            };

            if let Some(ref fragment) = fragment {
                // Fragments like `#1-6` are most likely line numbers to be
                // interpreted by javascript, so we're ignoring these
                if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
                    return;
                }

                // These appear to be broken in mdbook right now?
                if fragment.starts_with("-") {
                    return;
                }

                let entry = &mut cache.get_mut(&pretty_path).unwrap();
                entry.parse_ids(&pretty_path, &contents, errors);

                if !entry.ids.contains(*fragment) && !is_exception(file, &format!("#{}", fragment))
                {
                    *errors = true;
                    print!("{}:{}: broken link fragment ", pretty_file.display(), i + 1);
                    println!("`#{}` pointing to `{}`", fragment, pretty_path.display());
                };
            }
        } else {
            let pretty_path = path.strip_prefix(root).unwrap_or(&path);
            if !is_exception(file, pretty_path.to_str().unwrap()) {
                *errors = true;
                print!("{}:{}: broken link - ", pretty_file.display(), i + 1);
                println!("{}", pretty_path.display());
            }
        }
    });
    Some(pretty_file)
}

fn load_file(
    cache: &mut Cache,
    root: &Path,
    file: &Path,
    redirect: Redirect,
) -> Result<(PathBuf, Rc<String>), LoadError> {
    let pretty_file = PathBuf::from(file.strip_prefix(root).unwrap_or(&file));

    let (maybe_redirect, contents) = match cache.entry(pretty_file.clone()) {
        Entry::Occupied(entry) => (None, entry.get().source.clone()),
        Entry::Vacant(entry) => {
            let contents = match fs::read_to_string(file) {
                Ok(s) => Rc::new(s),
                Err(err) => {
                    return Err(if let FromRedirect(true) = redirect {
                        LoadError::BrokenRedirect(file.to_path_buf(), err)
                    } else {
                        LoadError::IOError(err)
                    });
                }
            };

            let maybe = maybe_redirect(&contents);
            if maybe.is_some() {
                if let SkipRedirect = redirect {
                    return Err(LoadError::IsRedirect);
                }
            } else {
                entry.insert(FileEntry { source: contents.clone(), ids: HashSet::new() });
            }
            (maybe, contents)
        }
    };
    match maybe_redirect.map(|url| file.parent().unwrap().join(url)) {
        Some(redirect_file) => load_file(cache, root, &redirect_file, FromRedirect(true)),
        None => Ok((pretty_file, contents)),
    }
}

fn maybe_redirect(source: &str) -> Option<String> {
    const REDIRECT: &'static str = "
Redirecting to <a href=";

    let mut lines = source.lines();
    let redirect_line = lines.nth(6)?;

    redirect_line.find(REDIRECT).map(|i| {
        let rest = &redirect_line[(i + REDIRECT.len() + 1)..];
        let pos_quote = rest.find('"').unwrap();
        rest[..pos_quote].to_owned()
    })
}

fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(contents: &str, attr: &str, mut f: F) {
    let mut base = "";
    for (i, mut line) in contents.lines().enumerate() {
        while let Some(j) = line.find(attr) {
            let rest = &line[j + attr.len()..];
            // The base tag should always be the first link in the document so
            // we can get away with using one pass.
            let is_base = line[..j].ends_with("<base");
            line = rest;
            let pos_equals = match rest.find("=") {
                Some(i) => i,
                None => continue,
            };
            if rest[..pos_equals].trim_start_matches(" ") != "" {
                continue;
            }

            let rest = &rest[pos_equals + 1..];

            let pos_quote = match rest.find(&['"', '\''][..]) {
                Some(i) => i,
                None => continue,
            };
            let quote_delim = rest.as_bytes()[pos_quote] as char;

            if rest[..pos_quote].trim_start_matches(" ") != "" {
                continue;
            }
            let rest = &rest[pos_quote + 1..];
            let url = match rest.find(quote_delim) {
                Some(i) => &rest[..i],
                None => continue,
            };
            if is_base {
                base = url;
                continue;
            }
            f(url, i, base)
        }
    }
}
<|endoftext|>