<commit_before>// Copyright 2017 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! The module containing building blocks for creating blockchains powered by the
//! Exonum framework.
//!
//! Services are the main extension point for the Exonum framework. To create your own service on
//! top of Exonum blockchain you need to define following things:
//!
//! - Define your own information schema.
//! - Create one or more types of messages using a macro `message!` and implement
//! `Transaction` trait for them.
//! - Create data structure that implements `Service` trait.
//! - Optionally you can write api handlers.
//!
//! You may follow the [`minibank`][1] tutorial to get experience of programming your services.
//!
//! [1]: https://github.com/exonum/exonum-doc/blob/master/src/get-started/create-service.md

use vec_map::VecMap;
use byteorder::{ByteOrder, LittleEndian};
use mount::Mount;

use std::sync::Arc;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::mem;
use std::fmt;
use std::panic;
use std::net::SocketAddr;

use crypto::{self, Hash, PublicKey, SecretKey};
use messages::{CONSENSUS as CORE_SERVICE, Precommit, RawMessage, Connect};
use storage::{Database, Error, Fork, Patch, Snapshot};
use helpers::{Height, ValidatorId};
use node::ApiSender;

pub use self::block::{Block, BlockProof, SCHEMA_MAJOR_VERSION};
pub use self::schema::{gen_prefix, Schema, TxLocation};
pub use self::genesis::GenesisConfig;
pub use self::config::{ConsensusConfig, StoredConfiguration, TimeoutAdjusterConfig, ValidatorKeys};
pub use self::service::{ApiContext, Service, ServiceContext, SharedNodeState, Transaction};

mod block;
mod schema;
mod genesis;
mod service;
#[cfg(test)]
mod tests;

pub mod config;

/// Exonum blockchain instance with the concrete services set and data storage.
/// Only blockchains with the identical set of services and genesis block can be combined
/// into the single network.
pub struct Blockchain {
    db: Box<Database>,
    service_map: Arc<VecMap<Box<Service>>>,
    service_keypair: (PublicKey, SecretKey),
    api_sender: ApiSender,
}

impl Blockchain {
    /// Constructs a blockchain for the given `storage` and list of `services`.
    pub fn new(
        storage: Box<Database>,
        services: Vec<Box<Service>>,
        service_public_key: PublicKey,
        service_secret_key: SecretKey,
        api_sender: ApiSender,
    ) -> Blockchain {
        let mut service_map = VecMap::new();
        for service in services {
            let id = service.service_id() as usize;
            if service_map.contains_key(id) {
                panic!(
                    "Services have already contain service with id={}, please change it.",
                    id
                );
            }
            service_map.insert(id, service);
        }

        Blockchain {
            db: storage,
            service_map: Arc::new(service_map),
            service_keypair: (service_public_key, service_secret_key),
            api_sender,
        }
    }

    /// Returnts service `VecMap` for all our services.
    pub fn service_map(&self) -> &Arc<VecMap<Box<Service>>> {
        &self.service_map
    }

    /// Creates a readonly snapshot of the current storage state.
    pub fn snapshot(&self) -> Box<Snapshot> {
        self.db.snapshot()
    }

    /// Creates snapshot of the current storage state that can be later committed into storage
    /// via `merge` method.
    pub fn fork(&self) -> Fork {
        self.db.fork()
    }

    /// Tries to create a `Transaction` object from the given raw message.
    /// Raw message can be converted into `Transaction` object only
    /// if following conditions are met.
    ///
    /// - Blockchain has service with the `service_id` of given raw message.
    /// - Service can deserialize given raw message.
    pub fn tx_from_raw(&self, raw: RawMessage) -> Option<Box<Transaction>> {
        let id = raw.service_id() as usize;
        self.service_map.get(id).and_then(|service| {
            service.tx_from_raw(raw).ok()
        })
    }

    /// Commits changes from the patch to the blockchain storage.
    /// See [`Fork`](../storage/struct.Fork.html) for details.
    pub fn merge(&mut self, patch: Patch) -> Result<(), Error> {
        self.db.merge(patch)
    }

    /// Returns the hash of latest committed block.
    ///
    /// # Panics
    ///
    /// - If the genesis block was not committed.
    pub fn last_hash(&self) -> Hash {
        Schema::new(&self.snapshot())
            .block_hashes_by_height()
            .last()
            .unwrap_or_else(Hash::default)
    }

    /// Returns the latest committed block.
    ///
    /// # Panics
    ///
    /// - If the genesis block was not committed.
    pub fn last_block(&self) -> Block {
        Schema::new(&self.snapshot()).last_block().unwrap()
    }

    /// Creates and commits the genesis block for the given genesis configuration.
    pub fn create_genesis_block(&mut self, cfg: GenesisConfig) -> Result<(), Error> {
        let mut config_propose = StoredConfiguration {
            previous_cfg_hash: Hash::zero(),
            actual_from: Height::zero(),
            validator_keys: cfg.validator_keys,
            consensus: cfg.consensus,
            services: BTreeMap::new(),
        };

        let patch = {
            let mut fork = self.fork();
            // Update service tables
            for (_, service) in self.service_map.iter() {
                let cfg = service.initialize(&mut fork);
                let name = service.service_name();
                if config_propose.services.contains_key(name) {
                    panic!(
                        "Services already contain service with '{}' name, please change it",
                        name
                    );
                }
                config_propose.services.insert(name.into(), cfg);
            }
            // Commit actual configuration
            {
                let mut schema = Schema::new(&mut fork);
                if schema.block_hash_by_height(Height::zero()).is_some() {
                    // TODO create genesis block for MemoryDB and compare in hash with zero block
                    return Ok(());
                }
                schema.commit_configuration(config_propose);
            };
            self.merge(fork.into_patch())?;
            self.create_patch(ValidatorId::zero(), Height::zero(), &[], &BTreeMap::new())
                .1
        };
        self.merge(patch)?;
        Ok(())
    }

    /// Helper function to map tuple (`u16`, `u16`) of service table coordinates
    /// to 32 byte value for use as `MerklePatriciaTable` key (it currently
    /// supports only fixed size keys). `hash` function is used to distribute
    /// keys uniformly (compared to padding).
    /// # Arguments
    ///
    /// * `service_id` - `service_id` as returned by instance of type of
    /// `Service` trait
    /// * `table_idx` - index of service table in `Vec`, returned by
    /// `state_hash` method of instance of type of `Service` trait
    // also, it was the first idea around, to use `hash`
    pub fn service_table_unique_key(service_id: u16, table_idx: usize) -> Hash {
        debug_assert!(table_idx <= u16::max_value() as usize);
        let size = mem::size_of::<u16>();
        let mut vec = vec![0; 2 * size];
        LittleEndian::write_u16(&mut vec[0..size], service_id);
        LittleEndian::write_u16(&mut vec[size..2 * size], table_idx as u16);
        crypto::hash(&vec)
    }

    /// Executes the given transactions from pool.
    /// Then it collects the resulting changes from the current storage state and returns them
    /// with the hash of resulting block.
    pub fn create_patch(
        &self,
        proposer_id: ValidatorId,
        height: Height,
        tx_hashes: &[Hash],
        pool: &BTreeMap<Hash, Box<Transaction>>,
    ) -> (Hash, Patch) {
        // Create fork
        let mut fork = self.fork();

        let block_hash = {
            // Get last hash
            let last_hash = self.last_hash();
            // Save & execute transactions
            for (index, hash) in tx_hashes.iter().enumerate() {
                let tx = pool.get(hash).expect(
                    "BUG: Cannot find transaction in pool.",
                );

                fork.checkpoint();

                let r = panic::catch_unwind(panic::AssertUnwindSafe(|| { tx.execute(&mut fork); }));

                match r {
                    Ok(..) => fork.commit(),
                    Err(err) => {
                        if err.is::<Error>() {
                            // Continue panic unwind if the reason is StorageError
                            panic::resume_unwind(err);
                        }
                        fork.rollback();
                        error!("{:?} transaction execution failed: {:?}", tx, err);
                    }
                }

                let mut schema = Schema::new(&mut fork);
                schema.transactions_mut().put(hash, tx.raw().clone());
                schema.block_txs_mut(height).push(*hash);
                let location = TxLocation::new(height, index as u64);
                schema.tx_location_by_tx_hash_mut().put(hash, location);
            }

            // Get tx & state hash
            let (tx_hash, state_hash) = {
                let state_hashes = {
                    let schema = Schema::new(&fork);

                    let vec_core_state = schema.core_state_hash();
                    let mut state_hashes = Vec::new();

                    for (idx, core_table_hash) in vec_core_state.into_iter().enumerate() {
                        let key = Blockchain::service_table_unique_key(CORE_SERVICE, idx);
                        state_hashes.push((key, core_table_hash));
                    }

                    for service in self.service_map.values() {
                        let service_id = service.service_id();
                        let vec_service_state = service.state_hash(&fork);
                        for (idx, service_table_hash) in vec_service_state.into_iter().enumerate() {
                            let key = Blockchain::service_table_unique_key(service_id, idx);
                            state_hashes.push((key, service_table_hash));
                        }
                    }

                    state_hashes
                };

                let mut schema = Schema::new(&mut fork);

                let state_hash = {
                    let mut sum_table = schema.state_hash_aggregator_mut();
                    for (key, hash) in state_hashes {
                        sum_table.put(&key, hash)
                    }
                    sum_table.root_hash()
                };

                let tx_hash = schema.block_txs(height).root_hash();

                (tx_hash, state_hash)
            };

            // Create block
            let block = Block::new(
                SCHEMA_MAJOR_VERSION,
                proposer_id,
                height,
                tx_hashes.len() as u32,
                &last_hash,
                &tx_hash,
                &state_hash,
            );
            trace!("execute block = {:?}", block);
            // Eval block hash
            let block_hash = block.hash();
            // Update height
            let mut schema = Schema::new(&mut fork);
            schema.block_hashes_by_height_mut().push(block_hash);
            // Save block
            schema.blocks_mut().put(&block_hash, block);

            block_hash
        };

        (block_hash, fork.into_patch())
    }

    /// Commits to the storage block that proposes by node `State`.
    /// After that invokes `handle_commit` for each service in order of their identifiers
    /// and returns the list of transactions which which were created by the `handle_commit` event.
    #[cfg_attr(feature = "flame_profile", flame)]
    pub fn commit<'a, I>(
        &mut self,
        patch: &Patch,
        block_hash: Hash,
        precommits: I,
    ) -> Result<(), Error>
    where
        I: Iterator<Item = &'a Precommit>,
    {
        let patch = {
            let mut fork = {
                let mut fork = self.db.fork();
                fork.merge(patch.clone()); // FIXME: avoid cloning here
                fork
            };

            {
                let mut schema = Schema::new(&mut fork);
                for precommit in precommits {
                    schema.precommits_mut(&block_hash).push(precommit.clone());
                }
            }
            fork.into_patch()
        };
        self.merge(patch)?;
        // Initializes the context after merge.
        let context = ServiceContext::new(
            self.service_keypair.0,
            self.service_keypair.1.clone(),
            self.api_sender.clone(),
            self.fork(),
        );
        // Invokes `handle_commit` for each service in order of their identifiers
        for service in self.service_map.values() {
            service.handle_commit(&context);
        }
        Ok(())
    }

    /// Returns `Mount` object that aggregates public api handlers.
    pub fn mount_public_api(&self) -> Mount {
        let context = self.api_context();
        let mut mount = Mount::new();
        for service in self.service_map.values() {
            if let Some(handler) = service.public_api_handler(&context) {
                mount.mount(service.service_name(), handler);
            }
        }
        mount
    }

    /// Returns `Mount` object that aggregates private api handlers.
    pub fn mount_private_api(&self) -> Mount {
        let context = self.api_context();
        let mut mount = Mount::new();
        for service in self.service_map.values() {
            if let Some(handler) = service.private_api_handler(&context) {
                mount.mount(service.service_name(), handler);
            }
        }
        mount
    }

    fn api_context(&self) -> ApiContext {
        ApiContext::from_parts(
            self,
            self.api_sender.clone(),
            &self.service_keypair.0,
            &self.service_keypair.1,
        )
    }

    /// Saves peer to the peers cache
    pub fn save_peer(&mut self, pubkey: &PublicKey, peer: Connect) {
        let mut fork = self.fork();

        {
            let mut schema = Schema::new(&mut fork);
            schema.peers_cache_mut().put(pubkey, peer);
        }

        self.merge(fork.into_patch()).expect(
            "Unable to save peer to the peers cache",
        );
    }

    /// Removes peer from the peers cache
    pub fn remove_peer_with_addr(&mut self, addr: &SocketAddr) {
        let mut fork = self.fork();

        {
            let mut schema = Schema::new(&mut fork);
            let mut peers = schema.peers_cache_mut();
            let pubkey = peers.iter().find(|&(_, ref v)| v.addr() == *addr).map(|(k, _)| k);
            if let Some(pubkey) = pubkey {
                peers.remove(&pubkey);
            }
        }

        self.merge(fork.into_patch()).expect(
            "Unable to remove peer from the peers cache",
        );
    }

    /// Recover cached peers if any.
    pub fn get_saved_peers(&self) -> HashMap<PublicKey, Connect> {
        let schema = Schema::new(self.snapshot());
        let peers_cache = schema.peers_cache();
        let peers = peers_cache.iter()
            .map(|(k, v)| (k, v.clone()))
            .collect();
        peers
    }
}

impl fmt::Debug for Blockchain {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Blockchain(..)")
    }
}

impl Clone for Blockchain {
    fn clone(&self) -> Blockchain {
        Blockchain {
            db: self.db.clone(),
            service_map: Arc::clone(&self.service_map),
            api_sender: self.api_sender.clone(),
            service_keypair: self.service_keypair.clone(),
        }
    }
}
<commit_msg>Fix clippy issues<commit_after>// Copyright 2017 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! The module containing building blocks for creating blockchains powered by the
//! Exonum framework.
//!
//! Services are the main extension point for the Exonum framework. To create your own service on
//! top of Exonum blockchain you need to define following things:
//!
//! - Define your own information schema.
//! - Create one or more types of messages using a macro `message!` and implement
//! `Transaction` trait for them.
//! - Create data structure that implements `Service` trait.
//! - Optionally you can write api handlers.
//!
//! You may follow the [`minibank`][1] tutorial to get experience of programming your services.
//!
//! [1]: https://github.com/exonum/exonum-doc/blob/master/src/get-started/create-service.md

use vec_map::VecMap;
use byteorder::{ByteOrder, LittleEndian};
use mount::Mount;

use std::sync::Arc;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::mem;
use std::fmt;
use std::panic;
use std::net::SocketAddr;

use crypto::{self, Hash, PublicKey, SecretKey};
use messages::{CONSENSUS as CORE_SERVICE, Precommit, RawMessage, Connect};
use storage::{Database, Error, Fork, Patch, Snapshot};
use helpers::{Height, ValidatorId};
use node::ApiSender;

pub use self::block::{Block, BlockProof, SCHEMA_MAJOR_VERSION};
pub use self::schema::{gen_prefix, Schema, TxLocation};
pub use self::genesis::GenesisConfig;
pub use self::config::{ConsensusConfig, StoredConfiguration, TimeoutAdjusterConfig, ValidatorKeys};
pub use self::service::{ApiContext, Service, ServiceContext, SharedNodeState, Transaction};

mod block;
mod schema;
mod genesis;
mod service;
#[cfg(test)]
mod tests;

pub mod config;

/// Exonum blockchain instance with the concrete services set and data storage.
/// Only blockchains with the identical set of services and genesis block can be combined
/// into the single network.
pub struct Blockchain {
    db: Box<Database>,
    service_map: Arc<VecMap<Box<Service>>>,
    service_keypair: (PublicKey, SecretKey),
    api_sender: ApiSender,
}

impl Blockchain {
    /// Constructs a blockchain for the given `storage` and list of `services`.
    pub fn new(
        storage: Box<Database>,
        services: Vec<Box<Service>>,
        service_public_key: PublicKey,
        service_secret_key: SecretKey,
        api_sender: ApiSender,
    ) -> Blockchain {
        let mut service_map = VecMap::new();
        for service in services {
            let id = service.service_id() as usize;
            if service_map.contains_key(id) {
                panic!(
                    "Services have already contain service with id={}, please change it.",
                    id
                );
            }
            service_map.insert(id, service);
        }

        Blockchain {
            db: storage,
            service_map: Arc::new(service_map),
            service_keypair: (service_public_key, service_secret_key),
            api_sender,
        }
    }

    /// Returnts service `VecMap` for all our services.
    pub fn service_map(&self) -> &Arc<VecMap<Box<Service>>> {
        &self.service_map
    }

    /// Creates a readonly snapshot of the current storage state.
    pub fn snapshot(&self) -> Box<Snapshot> {
        self.db.snapshot()
    }

    /// Creates snapshot of the current storage state that can be later committed into storage
    /// via `merge` method.
    pub fn fork(&self) -> Fork {
        self.db.fork()
    }

    /// Tries to create a `Transaction` object from the given raw message.
    /// Raw message can be converted into `Transaction` object only
    /// if following conditions are met.
    ///
    /// - Blockchain has service with the `service_id` of given raw message.
    /// - Service can deserialize given raw message.
    pub fn tx_from_raw(&self, raw: RawMessage) -> Option<Box<Transaction>> {
        let id = raw.service_id() as usize;
        self.service_map.get(id).and_then(|service| {
            service.tx_from_raw(raw).ok()
        })
    }

    /// Commits changes from the patch to the blockchain storage.
    /// See [`Fork`](../storage/struct.Fork.html) for details.
    pub fn merge(&mut self, patch: Patch) -> Result<(), Error> {
        self.db.merge(patch)
    }

    /// Returns the hash of latest committed block.
    ///
    /// # Panics
    ///
    /// - If the genesis block was not committed.
    pub fn last_hash(&self) -> Hash {
        Schema::new(&self.snapshot())
            .block_hashes_by_height()
            .last()
            .unwrap_or_else(Hash::default)
    }

    /// Returns the latest committed block.
    ///
    /// # Panics
    ///
    /// - If the genesis block was not committed.
    pub fn last_block(&self) -> Block {
        Schema::new(&self.snapshot()).last_block().unwrap()
    }

    /// Creates and commits the genesis block for the given genesis configuration.
    pub fn create_genesis_block(&mut self, cfg: GenesisConfig) -> Result<(), Error> {
        let mut config_propose = StoredConfiguration {
            previous_cfg_hash: Hash::zero(),
            actual_from: Height::zero(),
            validator_keys: cfg.validator_keys,
            consensus: cfg.consensus,
            services: BTreeMap::new(),
        };

        let patch = {
            let mut fork = self.fork();
            // Update service tables
            for (_, service) in self.service_map.iter() {
                let cfg = service.initialize(&mut fork);
                let name = service.service_name();
                if config_propose.services.contains_key(name) {
                    panic!(
                        "Services already contain service with '{}' name, please change it",
                        name
                    );
                }
                config_propose.services.insert(name.into(), cfg);
            }
            // Commit actual configuration
            {
                let mut schema = Schema::new(&mut fork);
                if schema.block_hash_by_height(Height::zero()).is_some() {
                    // TODO create genesis block for MemoryDB and compare in hash with zero block
                    return Ok(());
                }
                schema.commit_configuration(config_propose);
            };
            self.merge(fork.into_patch())?;
            self.create_patch(ValidatorId::zero(), Height::zero(), &[], &BTreeMap::new())
                .1
        };
        self.merge(patch)?;
        Ok(())
    }

    /// Helper function to map tuple (`u16`, `u16`) of service table coordinates
    /// to 32 byte value for use as `MerklePatriciaTable` key (it currently
    /// supports only fixed size keys). `hash` function is used to distribute
    /// keys uniformly (compared to padding).
    /// # Arguments
    ///
    /// * `service_id` - `service_id` as returned by instance of type of
    /// `Service` trait
    /// * `table_idx` - index of service table in `Vec`, returned by
    /// `state_hash` method of instance of type of `Service` trait
    // also, it was the first idea around, to use `hash`
    pub fn service_table_unique_key(service_id: u16, table_idx: usize) -> Hash {
        debug_assert!(table_idx <= u16::max_value() as usize);
        let size = mem::size_of::<u16>();
        let mut vec = vec![0; 2 * size];
        LittleEndian::write_u16(&mut vec[0..size], service_id);
        LittleEndian::write_u16(&mut vec[size..2 * size], table_idx as u16);
        crypto::hash(&vec)
    }

    /// Executes the given transactions from pool.
    /// Then it collects the resulting changes from the current storage state and returns them
    /// with the hash of resulting block.
    pub fn create_patch(
        &self,
        proposer_id: ValidatorId,
        height: Height,
        tx_hashes: &[Hash],
        pool: &BTreeMap<Hash, Box<Transaction>>,
    ) -> (Hash, Patch) {
        // Create fork
        let mut fork = self.fork();

        let block_hash = {
            // Get last hash
            let last_hash = self.last_hash();
            // Save & execute transactions
            for (index, hash) in tx_hashes.iter().enumerate() {
                let tx = pool.get(hash).expect(
                    "BUG: Cannot find transaction in pool.",
                );

                fork.checkpoint();

                let r = panic::catch_unwind(panic::AssertUnwindSafe(|| { tx.execute(&mut fork); }));

                match r {
                    Ok(..) => fork.commit(),
                    Err(err) => {
                        if err.is::<Error>() {
                            // Continue panic unwind if the reason is StorageError
                            panic::resume_unwind(err);
                        }
                        fork.rollback();
                        error!("{:?} transaction execution failed: {:?}", tx, err);
                    }
                }

                let mut schema = Schema::new(&mut fork);
                schema.transactions_mut().put(hash, tx.raw().clone());
                schema.block_txs_mut(height).push(*hash);
                let location = TxLocation::new(height, index as u64);
                schema.tx_location_by_tx_hash_mut().put(hash, location);
            }

            // Get tx & state hash
            let (tx_hash, state_hash) = {
                let state_hashes = {
                    let schema = Schema::new(&fork);

                    let vec_core_state = schema.core_state_hash();
                    let mut state_hashes = Vec::new();

                    for (idx, core_table_hash) in vec_core_state.into_iter().enumerate() {
                        let key = Blockchain::service_table_unique_key(CORE_SERVICE, idx);
                        state_hashes.push((key, core_table_hash));
                    }

                    for service in self.service_map.values() {
                        let service_id = service.service_id();
                        let vec_service_state = service.state_hash(&fork);
                        for (idx, service_table_hash) in vec_service_state.into_iter().enumerate() {
                            let key = Blockchain::service_table_unique_key(service_id, idx);
                            state_hashes.push((key, service_table_hash));
                        }
                    }

                    state_hashes
                };

                let mut schema = Schema::new(&mut fork);

                let state_hash = {
                    let mut sum_table = schema.state_hash_aggregator_mut();
                    for (key, hash) in state_hashes {
                        sum_table.put(&key, hash)
                    }
                    sum_table.root_hash()
                };

                let tx_hash = schema.block_txs(height).root_hash();

                (tx_hash, state_hash)
            };

            // Create block
            let block = Block::new(
                SCHEMA_MAJOR_VERSION,
                proposer_id,
                height,
                tx_hashes.len() as u32,
                &last_hash,
                &tx_hash,
                &state_hash,
            );
            trace!("execute block = {:?}", block);
            // Eval block hash
            let block_hash = block.hash();
            // Update height
            let mut schema = Schema::new(&mut fork);
            schema.block_hashes_by_height_mut().push(block_hash);
            // Save block
            schema.blocks_mut().put(&block_hash, block);

            block_hash
        };

        (block_hash, fork.into_patch())
    }

    /// Commits to the storage block that proposes by node `State`.
    /// After that invokes `handle_commit` for each service in order of their identifiers
    /// and returns the list of transactions which which were created by the `handle_commit` event.
    #[cfg_attr(feature = "flame_profile", flame)]
    pub fn commit<'a, I>(
        &mut self,
        patch: &Patch,
        block_hash: Hash,
        precommits: I,
    ) -> Result<(), Error>
    where
        I: Iterator<Item = &'a Precommit>,
    {
        let patch = {
            let mut fork = {
                let mut fork = self.db.fork();
                fork.merge(patch.clone()); // FIXME: avoid cloning here
                fork
            };

            {
                let mut schema = Schema::new(&mut fork);
                for precommit in precommits {
                    schema.precommits_mut(&block_hash).push(precommit.clone());
                }
            }
            fork.into_patch()
        };
        self.merge(patch)?;
        // Initializes the context after merge.
        let context = ServiceContext::new(
            self.service_keypair.0,
            self.service_keypair.1.clone(),
            self.api_sender.clone(),
            self.fork(),
        );
        // Invokes `handle_commit` for each service in order of their identifiers
        for service in self.service_map.values() {
            service.handle_commit(&context);
        }
        Ok(())
    }

    /// Returns `Mount` object that aggregates public api handlers.
    pub fn mount_public_api(&self) -> Mount {
        let context = self.api_context();
        let mut mount = Mount::new();
        for service in self.service_map.values() {
            if let Some(handler) = service.public_api_handler(&context) {
                mount.mount(service.service_name(), handler);
            }
        }
        mount
    }

    /// Returns `Mount` object that aggregates private api handlers.
    pub fn mount_private_api(&self) -> Mount {
        let context = self.api_context();
        let mut mount = Mount::new();
        for service in self.service_map.values() {
            if let Some(handler) = service.private_api_handler(&context) {
                mount.mount(service.service_name(), handler);
            }
        }
        mount
    }

    fn api_context(&self) -> ApiContext {
        ApiContext::from_parts(
            self,
            self.api_sender.clone(),
            &self.service_keypair.0,
            &self.service_keypair.1,
        )
    }

    /// Saves peer to the peers cache
    pub fn save_peer(&mut self, pubkey: &PublicKey, peer: Connect) {
        let mut fork = self.fork();

        {
            let mut schema = Schema::new(&mut fork);
            schema.peers_cache_mut().put(pubkey, peer);
        }

        self.merge(fork.into_patch()).expect(
            "Unable to save peer to the peers cache",
        );
    }

    /// Removes peer from the peers cache
    pub fn remove_peer_with_addr(&mut self, addr: &SocketAddr) {
        let mut fork = self.fork();

        {
            let mut schema = Schema::new(&mut fork);
            let mut peers = schema.peers_cache_mut();
            let pubkey = peers.iter().find(|&(_, ref v)| v.addr() == *addr).map(|(k, _)| k);
            if let Some(pubkey) = pubkey {
                peers.remove(&pubkey);
            }
        }

        self.merge(fork.into_patch()).expect(
            "Unable to remove peer from the peers cache",
        );
    }

    /// Recover cached peers if any.
    pub fn get_saved_peers(&self) -> HashMap<PublicKey, Connect> {
        let schema = Schema::new(self.snapshot());
        let peers_cache = schema.peers_cache();
        let it = peers_cache.iter().map(|(k, v)| (k, v.clone()));
        it.collect()
    }
}

impl fmt::Debug for Blockchain {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "Blockchain(..)")
    }
}

impl Clone for Blockchain {
    fn clone(&self) -> Blockchain {
        Blockchain {
            db: self.db.clone(),
            service_map: Arc::clone(&self.service_map),
            api_sender: self.api_sender.clone(),
            service_keypair: self.service_keypair.clone(),
        }
    }
}
<|endoftext|>


<commit_before>use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};

use clap::{App, Arg, ArgMatches};
use walkdir::WalkDir;
use which::which;

use super::super::CliResult;
use super::Config;
use super::super::macho::is_macho_file;

fn invoke_dsymutil(path: &Path, output_path: &Path) -> CliResult<()> {
    let status = try!(Command::new("dsymutil")
        .arg("-o")
        .arg(output_path)
        .arg("--flat")
        .arg(&path)
        .stderr(Stdio::null())
        .status());
    if status.success() {
        Ok(())
    } else {
        fail!("dsymutil failed to extract symbols");
    }
}

fn extract_symbols(src: &Path, dst: &Path) -> CliResult<()> {
    for dent_rv in WalkDir::new(src) {
        let dent = try!(dent_rv);
        let md = try!(dent.metadata());
        if !md.is_file() || !is_macho_file(dent.path()) {
            continue;
        }

        let local_name = dent.path().strip_prefix(&src).unwrap();
        let full_path = dst.join(local_name);
        try!(fs::create_dir_all(&full_path.parent().unwrap()));
        try!(invoke_dsymutil(dent.path(), &full_path));

        println!("  {}", local_name.display());
    }
    Ok(())
}

pub fn make_app<'a, 'b: 'a>(app: App<'a, 'b>) -> App<'a, 'b>
{
    app
        .about("extract iOS device support symbols")
        .arg(Arg::with_name("version")
             .value_name("VERSION")
             .help("The iOS version to convert symbols for.")
             .required(true)
             .index(1))
        .arg(Arg::with_name("path")
             .value_name("PATH")
             .long("path")
             .short("p")
             .help("The path to the iOS Device Support folder if different."))
        .arg(Arg::with_name("output")
             .long("--output")
             .short("-o")
             .help("The output path folder.  If not provided a new folder in the \
                    current folder is used."))
}

pub fn execute<'a>(matches: &ArgMatches<'a>, _config: &Config) -> CliResult<()> {
    let version = matches.value_of("version").unwrap();
    let base_path = matches.value_of("path")
        .map(|p| PathBuf::from(p))
        .unwrap_or_else(|| {
            let mut path = env::home_dir().unwrap();
            path.push("Library/Developer/Xcode/iOS DeviceSupport");
            path
        });
    let source_path = try!(try!(base_path.read_dir()).filter_map(|ent_rv| {
        if let Ok(ent) = ent_rv {
            let name = ent.file_name();
            if name.to_string_lossy().split_whitespace().nth(0) == Some(version) {
                return Some(ent.path());
            }
        }
        None
    }).next().ok_or_else(|| format!("Could not find symbols for iOS version {} in {}",
                                    version, base_path.display())));
    let output = matches.value_of("output")
        .map(|p| PathBuf::from(p))
        .unwrap_or_else(|| {
            let mut path = env::current_dir().unwrap();
            path.push(format!("{}.symbols", version.replace(".", "_")));
            path
        });

    if output.exists() {
        fail!("Cannot proceed because the output path ({}) already exists.",
              output.display());
    }

    if which("dsymutil").is_err() {
        fail!("dsymutil is not installed on this machine but required.");
    }

    println!("Extracting iOS Device Support Symbols");
    println!("  iOS Version: {}", version);
    println!("  Source path: {}", source_path.display());
    println!("  Output path: {}", output.display());

    try!(extract_symbols(&source_path, &output));

    println!("All done!");

    Ok(())
}
<commit_msg>Added a help text<commit_after>use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};

use clap::{App, Arg, ArgMatches};
use walkdir::WalkDir;
use which::which;

use super::super::CliResult;
use super::Config;
use super::super::macho::is_macho_file;

fn invoke_dsymutil(path: &Path, output_path: &Path) -> CliResult<()> {
    let status = try!(Command::new("dsymutil")
        .arg("-o")
        .arg(output_path)
        .arg("--flat")
        .arg(&path)
        .stderr(Stdio::null())
        .status());
    if status.success() {
        Ok(())
    } else {
        fail!("dsymutil failed to extract symbols");
    }
}

fn extract_symbols(src: &Path, dst: &Path) -> CliResult<()> {
    for dent_rv in WalkDir::new(src) {
        let dent = try!(dent_rv);
        let md = try!(dent.metadata());
        if !md.is_file() || !is_macho_file(dent.path()) {
            continue;
        }

        let local_name = dent.path().strip_prefix(&src).unwrap();
        let full_path = dst.join(local_name);
        try!(fs::create_dir_all(&full_path.parent().unwrap()));
        try!(invoke_dsymutil(dent.path(), &full_path));

        println!("  {}", local_name.display());
    }
    Ok(())
}

pub fn make_app<'a, 'b: 'a>(app: App<'a, 'b>) -> App<'a, 'b>
{
    app
        .about("extract iOS device support symbols")
        .after_help("This command extracts debug symbols from the iOS Device Support \
                     folder of the system for a given version of IOS and stores them \
                     in a new folder.  This can then later be uploaded into the \
                     global symbol store of a Sentry installation.")
        .arg(Arg::with_name("version")
             .value_name("VERSION")
             .help("The iOS version to convert symbols for.")
             .required(true)
             .index(1))
        .arg(Arg::with_name("path")
             .value_name("PATH")
             .long("path")
             .short("p")
             .help("The path to the iOS Device Support folder if different."))
        .arg(Arg::with_name("output")
             .long("--output")
             .short("-o")
             .help("The output path folder.  If not provided a new folder in the \
                    current folder is used."))
}

pub fn execute<'a>(matches: &ArgMatches<'a>, _config: &Config) -> CliResult<()> {
    let version = matches.value_of("version").unwrap();
    let base_path = matches.value_of("path")
        .map(|p| PathBuf::from(p))
        .unwrap_or_else(|| {
            let mut path = env::home_dir().unwrap();
            path.push("Library/Developer/Xcode/iOS DeviceSupport");
            path
        });
    let source_path = try!(try!(base_path.read_dir()).filter_map(|ent_rv| {
        if let Ok(ent) = ent_rv {
            let name = ent.file_name();
            if name.to_string_lossy().split_whitespace().nth(0) == Some(version) {
                return Some(ent.path());
            }
        }
        None
    }).next().ok_or_else(|| format!("Could not find symbols for iOS version {} in {}",
                                    version, base_path.display())));
    let output = matches.value_of("output")
        .map(|p| PathBuf::from(p))
        .unwrap_or_else(|| {
            let mut path = env::current_dir().unwrap();
            path.push(format!("{}.symbols", version.replace(".", "_")));
            path
        });

    if output.exists() {
        fail!("Cannot proceed because the output path ({}) already exists.",
              output.display());
    }

    if which("dsymutil").is_err() {
        fail!("dsymutil is not installed on this machine but required.");
    }

    println!("Extracting iOS Device Support Symbols");
    println!("  iOS Version: {}", version);
    println!("  Source path: {}", source_path.display());
    println!("  Output path: {}", output.display());

    try!(extract_symbols(&source_path, &output));

    println!("All done!");

    Ok(())
}
<|endoftext|>


<commit_before>// Copyright 2013-2014 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! [*Unicode IDNA Compatibility Processing*
//! (Unicode Technical Standard #46)](http://www.unicode.org/reports/tr46/)

use self::Mapping::*;
use punycode;
#[allow(unused_imports, deprecated)]
use std::ascii::AsciiExt;
use std::cmp::Ordering::{Equal, Less, Greater};
use unicode_bidi::{BidiClass, bidi_class};
use unicode_normalization::UnicodeNormalization;
use unicode_normalization::char::is_combining_mark;

include!("uts46_mapping_table.rs");

pub static PUNYCODE_PREFIX: &'static str = "xn--";

#[derive(Debug)]
struct StringTableSlice {
    // Store these as separate fields so the structure will have an
    // alignment of 1 and thus pack better into the Mapping enum, below.
    byte_start_lo: u8,
    byte_start_hi: u8,
    byte_len: u8,
}

fn decode_slice(slice: &StringTableSlice) -> &'static str {
    let lo = slice.byte_start_lo as usize;
    let hi = slice.byte_start_hi as usize;
    let start = (hi << 8) | lo;
    let len = slice.byte_len as usize;
    &STRING_TABLE[start..(start + len)]
}

#[repr(u8)]
#[derive(Debug)]
enum Mapping {
    Valid,
    Ignored,
    Mapped(StringTableSlice),
    Deviation(StringTableSlice),
    Disallowed,
    DisallowedStd3Valid,
    DisallowedStd3Mapped(StringTableSlice),
}

struct Range {
    from: char,
    to: char,
}

fn find_char(codepoint: char) -> &'static Mapping {
    let r = TABLE.binary_search_by(|ref range| {
        if codepoint > range.to {
            Less
        } else if codepoint < range.from {
            Greater
        } else {
            Equal
        }
    });
    r.ok().map(|i| {
        const SINGLE_MARKER: u16 = 1 << 15;

        let x = INDEX_TABLE[i];
        let single = (x & SINGLE_MARKER) != 0;
        let offset = !SINGLE_MARKER & x;

        if single {
            &MAPPING_TABLE[offset as usize]
        } else {
            &MAPPING_TABLE[(offset + (codepoint as u16 - TABLE[i].from as u16)) as usize]
        }
    }).unwrap()
}

fn map_char(codepoint: char, flags: Flags, output: &mut String, errors: &mut Vec<Error>) {
    match *find_char(codepoint) {
        Mapping::Valid => output.push(codepoint),
        Mapping::Ignored => {},
        Mapping::Mapped(ref slice) => output.push_str(decode_slice(slice)),
        Mapping::Deviation(ref slice) => {
            if flags.transitional_processing {
                output.push_str(decode_slice(slice))
            } else {
                output.push(codepoint)
            }
        }
        Mapping::Disallowed => {
            errors.push(Error::DissallowedCharacter);
            output.push(codepoint);
        }
        Mapping::DisallowedStd3Valid => {
            if flags.use_std3_ascii_rules {
                errors.push(Error::DissallowedByStd3AsciiRules);
            }
            output.push(codepoint)
        }
        Mapping::DisallowedStd3Mapped(ref slice) => {
            if flags.use_std3_ascii_rules {
                errors.push(Error::DissallowedMappedInStd3);
            }
            output.push_str(decode_slice(slice))
        }
    }
}

// http://tools.ietf.org/html/rfc5893#section-2
fn passes_bidi(label: &str, is_bidi_domain: bool) -> bool {
    // Rule 0: Bidi Rules apply to Bidi Domain Names: a name with at least one RTL label.  A label
    // is RTL if it contains at least one character of bidi class R, AL or AN.
    if !is_bidi_domain {
        return true;
    }

    let mut chars = label.chars();
    let first_char_class = match chars.next() {
        Some(c) => bidi_class(c),
        None => return true, // empty string
    };

    match first_char_class {
        // LTR label
        BidiClass::L => {
            // Rule 5
            loop {
                match chars.next() {
                    Some(c) => {
                        if !matches!(bidi_class(c),
                                     BidiClass::L | BidiClass::EN |
                                     BidiClass::ES | BidiClass::CS |
                                     BidiClass::ET | BidiClass::ON |
                                     BidiClass::BN | BidiClass::NSM
                                    ) {
                            return false;
                        }
                    },
                    None => { break; },
                }
            }

            // Rule 6
            // must end in L or EN followed by 0 or more NSM
            let mut rev_chars = label.chars().rev();
            let mut last_non_nsm = rev_chars.next();
            loop {
                match last_non_nsm {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last_non_nsm = rev_chars.next();
                        continue;
                    }
                    _ => { break; },
                }
            }
            match last_non_nsm {
                Some(c) if bidi_class(c) == BidiClass::L
                    || bidi_class(c) == BidiClass::EN => {},
                Some(_) => { return false; },
                _ => {}
            }

        }

        // RTL label
        BidiClass::R | BidiClass::AL => {
            let mut found_en = false;
            let mut found_an = false;

            // Rule 2
            loop {
                match chars.next() {
                    Some(c) => {
                        let char_class = bidi_class(c);

                        if char_class == BidiClass::EN {
                            found_en = true;
                        }
                        if char_class == BidiClass::AN {
                            found_an = true;
                        }

                        if !matches!(char_class, BidiClass::R | BidiClass::AL |
                                     BidiClass::AN | BidiClass::EN |
                                     BidiClass::ES | BidiClass::CS |
                                     BidiClass::ET | BidiClass::ON |
                                     BidiClass::BN | BidiClass::NSM) {
                            return false;
                        }
                    },
                    None => { break; },
                }
            }
            // Rule 3
            let mut rev_chars = label.chars().rev();
            let mut last = rev_chars.next();
            loop { // must end in L or EN followed by 0 or more NSM
                match last {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last = rev_chars.next();
                        continue;
                    }
                    _ => { break; },
                }
            }
            match last {
                Some(c) if matches!(bidi_class(c), BidiClass::R | BidiClass::AL |
                                    BidiClass::EN | BidiClass::AN) => {},
                _ => { return false; }
            }

            // Rule 4
            if found_an && found_en {
                return false;
            }
        }

        // Rule 1: Should start with L or R/AL
        _ => {
            return false;
        }
    }

    return true;
}

/// http://www.unicode.org/reports/tr46/#Validity_Criteria
fn validate_full(label: &str, is_bidi_domain: bool, flags: Flags, errors: &mut Vec<Error>) {
    // V1: Must be in NFC form.
    if label.nfc().ne(label.chars()) {
        errors.push(Error::ValidityCriteria);
    } else {
        validate(label, is_bidi_domain, flags, errors);
    }
}

fn validate(label: &str, is_bidi_domain: bool, flags: Flags, errors: &mut Vec<Error>) {
    let first_char = label.chars().next();
    if first_char == None {
        // Empty string, pass
    }

    // V2: No U+002D HYPHEN-MINUS in both third and fourth positions.
    //
    // NOTE: Spec says that the label must not contain a HYPHEN-MINUS character in both the
    // third and fourth positions. But nobody follows this criteria. See the spec issue below:
    // https://github.com/whatwg/url/issues/53
    //
    // TODO: Add *CheckHyphens* flag.

    // V3: neither begin nor end with a U+002D HYPHEN-MINUS
    else if label.starts_with("-") || label.ends_with("-") {
        errors.push(Error::ValidityCriteria);
    }

    // V4: not contain a U+002E FULL STOP
    //
    // Here, label can't contain '.' since the input is from .split('.')

    // V5: not begin with a GC=Mark
    else if is_combining_mark(first_char.unwrap()) {
        errors.push(Error::ValidityCriteria);
    }

    // V6: Check against Mapping Table
    else if label.chars().any(|c| match *find_char(c) {
        Mapping::Valid => false,
        Mapping::Deviation(_) => flags.transitional_processing,
        Mapping::DisallowedStd3Valid => flags.use_std3_ascii_rules,
        _ => true,
    }) {
        errors.push(Error::ValidityCriteria);
    }

    // V7: ContextJ rules
    //
    // TODO: Implement rules and add *CheckJoiners* flag.

    // V8: Bidi rules
    //
    // TODO: Add *CheckBidi* flag
    else if !passes_bidi(label, is_bidi_domain)
    {
        errors.push(Error::ValidityCriteria);
    }
}

/// http://www.unicode.org/reports/tr46/#Processing
fn processing(domain: &str, flags: Flags, errors: &mut Vec<Error>) -> String {
    let mut mapped = String::with_capacity(domain.len());
    for c in domain.chars() {
        map_char(c, flags, &mut mapped, errors)
    }
    let mut normalized = String::with_capacity(mapped.len());
    normalized.extend(mapped.nfc());

    // Find out if it's a Bidi Domain Name
    //
    // First, check for literal bidi chars
    let mut is_bidi_domain = domain.chars().any(|c|
        matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
    );
    if !is_bidi_domain {
        // Then check for punycode-encoded bidi chars
        for label in normalized.split('.') {
            if label.starts_with(PUNYCODE_PREFIX) {
                match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                    Some(decoded_label) => {
                        if decoded_label.chars().any(|c|
                            matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
                        ) {
                            is_bidi_domain = true;
                        }
                    }
                    None => {
                        is_bidi_domain = true;
                    }
                }
            }
        }
    }

    let mut validated = String::new();
    let mut first = true;
    for label in normalized.split('.') {
        if !first {
            validated.push('.');
        }
        first = false;
        if label.starts_with(PUNYCODE_PREFIX) {
            match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                Some(decoded_label) => {
                    let flags = Flags { transitional_processing: false, ..flags };
                    validate_full(&decoded_label, is_bidi_domain, flags, errors);
                    validated.push_str(&decoded_label)
                }
                None => errors.push(Error::PunycodeError)
            }
        } else {
            // `normalized` is already `NFC` so we can skip that check
            validate(label, is_bidi_domain, flags, errors);
            validated.push_str(label)
        }
    }
    validated
}

#[derive(Clone, Copy)]
pub struct Config {
    flags: Flags,
}

impl From<Flags> for Config {
    #[inline]
    fn from(flags: Flags) -> Self {
        Self { flags }
    }
}

impl Config {
    #[inline]
    pub fn use_std3_ascii_rules(mut self, value: bool) -> Self {
        self.flags.use_std3_ascii_rules = value;
        self
    }

    #[inline]
    pub fn transitional_processing(mut self, value: bool) -> Self {
        self.flags.transitional_processing = value;
        self
    }

    #[inline]
    pub fn verify_dns_length(mut self, value: bool) -> Self {
        self.flags.verify_dns_length = value;
        self
    }

    /// http://www.unicode.org/reports/tr46/#ToASCII
    pub fn to_ascii(self, domain: &str) -> Result<String, Errors> {
        let mut errors = Vec::new();
        let mut result = String::new();
        let mut first = true;
        for label in processing(domain, self.flags, &mut errors).split('.') {
            if !first {
                result.push('.');
            }
            first = false;
            if label.is_ascii() {
                result.push_str(label);
            } else {
                match punycode::encode_str(label) {
                    Some(x) => {
                        result.push_str(PUNYCODE_PREFIX);
                        result.push_str(&x);
                    },
                    None => errors.push(Error::PunycodeError)
                }
            }
        }

        if self.flags.verify_dns_length {
            let domain = if result.ends_with(".") { &result[..result.len()-1] } else { &*result };
            if domain.len() < 1 || domain.split('.').any(|label| label.len() < 1) {
                errors.push(Error::TooShortForDns)
            }
            if domain.len() > 253 || domain.split('.').any(|label| label.len() > 63) {
                errors.push(Error::TooLongForDns)
            }
        }
        if errors.is_empty() {
            Ok(result)
        } else {
            Err(Errors(errors))
        }
    }

    /// http://www.unicode.org/reports/tr46/#ToUnicode
    pub fn to_unicode(self, domain: &str) -> (String, Result<(), Errors>) {
        let mut errors = Vec::new();
        let domain = processing(domain, self.flags, &mut errors);
        let errors = if errors.is_empty() {
            Ok(())
        } else {
            Err(Errors(errors))
        };
        (domain, errors)
    }

}

#[derive(Copy, Clone)]
pub struct Flags {
   pub use_std3_ascii_rules: bool,
   pub transitional_processing: bool,
   pub verify_dns_length: bool,
}

#[derive(PartialEq, Eq, Clone, Copy, Debug)]
enum Error {
    PunycodeError,
    ValidityCriteria,
    DissallowedByStd3AsciiRules,
    DissallowedMappedInStd3,
    DissallowedCharacter,
    TooLongForDns,
    TooShortForDns,
}

/// Errors recorded during UTS #46 processing.
///
/// This is opaque for now, only indicating the presence of at least one error.
/// More details may be exposed in the future.
#[derive(Debug)]
pub struct Errors(Vec<Error>);

/// http://www.unicode.org/reports/tr46/#ToASCII
pub fn to_ascii(domain: &str, flags: Flags) -> Result<String, Errors> {
    Config::from(flags).to_ascii(domain)
}

/// http://www.unicode.org/reports/tr46/#ToUnicode
///
/// Only `use_std3_ascii_rules` is used in `flags`.
pub fn to_unicode(domain: &str, mut flags: Flags) -> (String, Result<(), Errors>) {
    flags.transitional_processing = false;
    Config::from(flags).to_unicode(domain)
}
<commit_msg>Pass a Config value to validate in uts46<commit_after>// Copyright 2013-2014 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! [*Unicode IDNA Compatibility Processing*
//! (Unicode Technical Standard #46)](http://www.unicode.org/reports/tr46/)

use self::Mapping::*;
use punycode;
#[allow(unused_imports, deprecated)]
use std::ascii::AsciiExt;
use std::cmp::Ordering::{Equal, Less, Greater};
use unicode_bidi::{BidiClass, bidi_class};
use unicode_normalization::UnicodeNormalization;
use unicode_normalization::char::is_combining_mark;

include!("uts46_mapping_table.rs");

pub static PUNYCODE_PREFIX: &'static str = "xn--";

#[derive(Debug)]
struct StringTableSlice {
    // Store these as separate fields so the structure will have an
    // alignment of 1 and thus pack better into the Mapping enum, below.
    byte_start_lo: u8,
    byte_start_hi: u8,
    byte_len: u8,
}

fn decode_slice(slice: &StringTableSlice) -> &'static str {
    let lo = slice.byte_start_lo as usize;
    let hi = slice.byte_start_hi as usize;
    let start = (hi << 8) | lo;
    let len = slice.byte_len as usize;
    &STRING_TABLE[start..(start + len)]
}

#[repr(u8)]
#[derive(Debug)]
enum Mapping {
    Valid,
    Ignored,
    Mapped(StringTableSlice),
    Deviation(StringTableSlice),
    Disallowed,
    DisallowedStd3Valid,
    DisallowedStd3Mapped(StringTableSlice),
}

struct Range {
    from: char,
    to: char,
}

fn find_char(codepoint: char) -> &'static Mapping {
    let r = TABLE.binary_search_by(|ref range| {
        if codepoint > range.to {
            Less
        } else if codepoint < range.from {
            Greater
        } else {
            Equal
        }
    });
    r.ok().map(|i| {
        const SINGLE_MARKER: u16 = 1 << 15;

        let x = INDEX_TABLE[i];
        let single = (x & SINGLE_MARKER) != 0;
        let offset = !SINGLE_MARKER & x;

        if single {
            &MAPPING_TABLE[offset as usize]
        } else {
            &MAPPING_TABLE[(offset + (codepoint as u16 - TABLE[i].from as u16)) as usize]
        }
    }).unwrap()
}

fn map_char(codepoint: char, flags: Flags, output: &mut String, errors: &mut Vec<Error>) {
    match *find_char(codepoint) {
        Mapping::Valid => output.push(codepoint),
        Mapping::Ignored => {},
        Mapping::Mapped(ref slice) => output.push_str(decode_slice(slice)),
        Mapping::Deviation(ref slice) => {
            if flags.transitional_processing {
                output.push_str(decode_slice(slice))
            } else {
                output.push(codepoint)
            }
        }
        Mapping::Disallowed => {
            errors.push(Error::DissallowedCharacter);
            output.push(codepoint);
        }
        Mapping::DisallowedStd3Valid => {
            if flags.use_std3_ascii_rules {
                errors.push(Error::DissallowedByStd3AsciiRules);
            }
            output.push(codepoint)
        }
        Mapping::DisallowedStd3Mapped(ref slice) => {
            if flags.use_std3_ascii_rules {
                errors.push(Error::DissallowedMappedInStd3);
            }
            output.push_str(decode_slice(slice))
        }
    }
}

// http://tools.ietf.org/html/rfc5893#section-2
fn passes_bidi(label: &str, is_bidi_domain: bool) -> bool {
    // Rule 0: Bidi Rules apply to Bidi Domain Names: a name with at least one RTL label.  A label
    // is RTL if it contains at least one character of bidi class R, AL or AN.
    if !is_bidi_domain {
        return true;
    }

    let mut chars = label.chars();
    let first_char_class = match chars.next() {
        Some(c) => bidi_class(c),
        None => return true, // empty string
    };

    match first_char_class {
        // LTR label
        BidiClass::L => {
            // Rule 5
            loop {
                match chars.next() {
                    Some(c) => {
                        if !matches!(bidi_class(c),
                                     BidiClass::L | BidiClass::EN |
                                     BidiClass::ES | BidiClass::CS |
                                     BidiClass::ET | BidiClass::ON |
                                     BidiClass::BN | BidiClass::NSM
                                    ) {
                            return false;
                        }
                    },
                    None => { break; },
                }
            }

            // Rule 6
            // must end in L or EN followed by 0 or more NSM
            let mut rev_chars = label.chars().rev();
            let mut last_non_nsm = rev_chars.next();
            loop {
                match last_non_nsm {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last_non_nsm = rev_chars.next();
                        continue;
                    }
                    _ => { break; },
                }
            }
            match last_non_nsm {
                Some(c) if bidi_class(c) == BidiClass::L
                    || bidi_class(c) == BidiClass::EN => {},
                Some(_) => { return false; },
                _ => {}
            }

        }

        // RTL label
        BidiClass::R | BidiClass::AL => {
            let mut found_en = false;
            let mut found_an = false;

            // Rule 2
            loop {
                match chars.next() {
                    Some(c) => {
                        let char_class = bidi_class(c);

                        if char_class == BidiClass::EN {
                            found_en = true;
                        }
                        if char_class == BidiClass::AN {
                            found_an = true;
                        }

                        if !matches!(char_class, BidiClass::R | BidiClass::AL |
                                     BidiClass::AN | BidiClass::EN |
                                     BidiClass::ES | BidiClass::CS |
                                     BidiClass::ET | BidiClass::ON |
                                     BidiClass::BN | BidiClass::NSM) {
                            return false;
                        }
                    },
                    None => { break; },
                }
            }
            // Rule 3
            let mut rev_chars = label.chars().rev();
            let mut last = rev_chars.next();
            loop { // must end in L or EN followed by 0 or more NSM
                match last {
                    Some(c) if bidi_class(c) == BidiClass::NSM => {
                        last = rev_chars.next();
                        continue;
                    }
                    _ => { break; },
                }
            }
            match last {
                Some(c) if matches!(bidi_class(c), BidiClass::R | BidiClass::AL |
                                    BidiClass::EN | BidiClass::AN) => {},
                _ => { return false; }
            }

            // Rule 4
            if found_an && found_en {
                return false;
            }
        }

        // Rule 1: Should start with L or R/AL
        _ => {
            return false;
        }
    }

    return true;
}

/// http://www.unicode.org/reports/tr46/#Validity_Criteria
fn validate_full(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    // V1: Must be in NFC form.
    if label.nfc().ne(label.chars()) {
        errors.push(Error::ValidityCriteria);
    } else {
        validate(label, is_bidi_domain, config, errors);
    }
}

fn validate(label: &str, is_bidi_domain: bool, config: Config, errors: &mut Vec<Error>) {
    let first_char = label.chars().next();
    if first_char == None {
        // Empty string, pass
    }

    // V2: No U+002D HYPHEN-MINUS in both third and fourth positions.
    //
    // NOTE: Spec says that the label must not contain a HYPHEN-MINUS character in both the
    // third and fourth positions. But nobody follows this criteria. See the spec issue below:
    // https://github.com/whatwg/url/issues/53
    //
    // TODO: Add *CheckHyphens* flag.

    // V3: neither begin nor end with a U+002D HYPHEN-MINUS
    else if label.starts_with("-") || label.ends_with("-") {
        errors.push(Error::ValidityCriteria);
    }

    // V4: not contain a U+002E FULL STOP
    //
    // Here, label can't contain '.' since the input is from .split('.')

    // V5: not begin with a GC=Mark
    else if is_combining_mark(first_char.unwrap()) {
        errors.push(Error::ValidityCriteria);
    }

    // V6: Check against Mapping Table
    else if label.chars().any(|c| match *find_char(c) {
        Mapping::Valid => false,
        Mapping::Deviation(_) => config.flags.transitional_processing,
        Mapping::DisallowedStd3Valid => config.flags.use_std3_ascii_rules,
        _ => true,
    }) {
        errors.push(Error::ValidityCriteria);
    }

    // V7: ContextJ rules
    //
    // TODO: Implement rules and add *CheckJoiners* flag.

    // V8: Bidi rules
    //
    // TODO: Add *CheckBidi* flag
    else if !passes_bidi(label, is_bidi_domain)
    {
        errors.push(Error::ValidityCriteria);
    }
}

/// http://www.unicode.org/reports/tr46/#Processing
fn processing(domain: &str, config: Config, errors: &mut Vec<Error>) -> String {
    let mut mapped = String::with_capacity(domain.len());
    for c in domain.chars() {
        map_char(c, config.flags, &mut mapped, errors)
    }
    let mut normalized = String::with_capacity(mapped.len());
    normalized.extend(mapped.nfc());

    // Find out if it's a Bidi Domain Name
    //
    // First, check for literal bidi chars
    let mut is_bidi_domain = domain.chars().any(|c|
        matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
    );
    if !is_bidi_domain {
        // Then check for punycode-encoded bidi chars
        for label in normalized.split('.') {
            if label.starts_with(PUNYCODE_PREFIX) {
                match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                    Some(decoded_label) => {
                        if decoded_label.chars().any(|c|
                            matches!(bidi_class(c), BidiClass::R | BidiClass::AL | BidiClass::AN)
                        ) {
                            is_bidi_domain = true;
                        }
                    }
                    None => {
                        is_bidi_domain = true;
                    }
                }
            }
        }
    }

    let mut validated = String::new();
    let mut first = true;
    for label in normalized.split('.') {
        if !first {
            validated.push('.');
        }
        first = false;
        if label.starts_with(PUNYCODE_PREFIX) {
            match punycode::decode_to_string(&label[PUNYCODE_PREFIX.len()..]) {
                Some(decoded_label) => {
                    let config = config.transitional_processing(false);
                    validate_full(&decoded_label, is_bidi_domain, config, errors);
                    validated.push_str(&decoded_label)
                }
                None => errors.push(Error::PunycodeError)
            }
        } else {
            // `normalized` is already `NFC` so we can skip that check
            validate(label, is_bidi_domain, config, errors);
            validated.push_str(label)
        }
    }
    validated
}

#[derive(Clone, Copy)]
pub struct Config {
    flags: Flags,
}

impl From<Flags> for Config {
    #[inline]
    fn from(flags: Flags) -> Self {
        Self { flags }
    }
}

impl Config {
    #[inline]
    pub fn use_std3_ascii_rules(mut self, value: bool) -> Self {
        self.flags.use_std3_ascii_rules = value;
        self
    }

    #[inline]
    pub fn transitional_processing(mut self, value: bool) -> Self {
        self.flags.transitional_processing = value;
        self
    }

    #[inline]
    pub fn verify_dns_length(mut self, value: bool) -> Self {
        self.flags.verify_dns_length = value;
        self
    }

    /// http://www.unicode.org/reports/tr46/#ToASCII
    pub fn to_ascii(self, domain: &str) -> Result<String, Errors> {
        let mut errors = Vec::new();
        let mut result = String::new();
        let mut first = true;
        for label in processing(domain, self, &mut errors).split('.') {
            if !first {
                result.push('.');
            }
            first = false;
            if label.is_ascii() {
                result.push_str(label);
            } else {
                match punycode::encode_str(label) {
                    Some(x) => {
                        result.push_str(PUNYCODE_PREFIX);
                        result.push_str(&x);
                    },
                    None => errors.push(Error::PunycodeError)
                }
            }
        }

        if self.flags.verify_dns_length {
            let domain = if result.ends_with(".") { &result[..result.len()-1] } else { &*result };
            if domain.len() < 1 || domain.split('.').any(|label| label.len() < 1) {
                errors.push(Error::TooShortForDns)
            }
            if domain.len() > 253 || domain.split('.').any(|label| label.len() > 63) {
                errors.push(Error::TooLongForDns)
            }
        }
        if errors.is_empty() {
            Ok(result)
        } else {
            Err(Errors(errors))
        }
    }

    /// http://www.unicode.org/reports/tr46/#ToUnicode
    pub fn to_unicode(self, domain: &str) -> (String, Result<(), Errors>) {
        let mut errors = Vec::new();
        let domain = processing(domain, self, &mut errors);
        let errors = if errors.is_empty() {
            Ok(())
        } else {
            Err(Errors(errors))
        };
        (domain, errors)
    }

}

#[derive(Copy, Clone)]
pub struct Flags {
   pub use_std3_ascii_rules: bool,
   pub transitional_processing: bool,
   pub verify_dns_length: bool,
}

#[derive(PartialEq, Eq, Clone, Copy, Debug)]
enum Error {
    PunycodeError,
    ValidityCriteria,
    DissallowedByStd3AsciiRules,
    DissallowedMappedInStd3,
    DissallowedCharacter,
    TooLongForDns,
    TooShortForDns,
}

/// Errors recorded during UTS #46 processing.
///
/// This is opaque for now, only indicating the presence of at least one error.
/// More details may be exposed in the future.
#[derive(Debug)]
pub struct Errors(Vec<Error>);

/// http://www.unicode.org/reports/tr46/#ToASCII
pub fn to_ascii(domain: &str, flags: Flags) -> Result<String, Errors> {
    Config::from(flags).to_ascii(domain)
}

/// http://www.unicode.org/reports/tr46/#ToUnicode
///
/// Only `use_std3_ascii_rules` is used in `flags`.
pub fn to_unicode(domain: &str, mut flags: Flags) -> (String, Result<(), Errors>) {
    flags.transitional_processing = false;
    Config::from(flags).to_unicode(domain)
}
<|endoftext|>


<commit_before>use std::{
    collections::HashSet,
    path::{Path, PathBuf},
};

use xshell::Shell;

#[cfg(not(feature = "in-rust-tree"))]
use xshell::cmd;

#[cfg(not(feature = "in-rust-tree"))]
#[test]
fn check_code_formatting() {
    let sh = &Shell::new().unwrap();
    sh.change_dir(sourcegen::project_root());

    let out = cmd!(sh, "rustup run stable rustfmt --version").read().unwrap();
    if !out.contains("stable") {
        panic!(
            "Failed to run rustfmt from toolchain 'stable'. \
                 Please run `rustup component add rustfmt --toolchain stable` to install it.",
        )
    }

    let res = cmd!(sh, "rustup run stable cargo fmt -- --check").run();
    if res.is_err() {
        let _ = cmd!(sh, "rustup run stable cargo fmt").run();
    }
    res.unwrap()
}

#[test]
fn check_lsp_extensions_docs() {
    let sh = &Shell::new().unwrap();

    let expected_hash = {
        let lsp_ext_rs = sh
            .read_file(sourcegen::project_root().join("crates/rust-analyzer/src/lsp_ext.rs"))
            .unwrap();
        stable_hash(lsp_ext_rs.as_str())
    };

    let actual_hash = {
        let lsp_extensions_md =
            sh.read_file(sourcegen::project_root().join("docs/dev/lsp-extensions.md")).unwrap();
        let text = lsp_extensions_md
            .lines()
            .find_map(|line| line.strip_prefix("lsp_ext.rs hash:"))
            .unwrap()
            .trim();
        u64::from_str_radix(text, 16).unwrap()
    };

    if actual_hash != expected_hash {
        panic!(
            "
lsp_ext.rs was changed without touching lsp-extensions.md.

Expected hash: {:x}
Actual hash:   {:x}

Please adjust docs/dev/lsp-extensions.md.
",
            expected_hash, actual_hash
        )
    }
}

#[test]
fn files_are_tidy() {
    let sh = &Shell::new().unwrap();

    let files = sourcegen::list_files(&sourcegen::project_root().join("crates"));

    let mut tidy_docs = TidyDocs::default();
    let mut tidy_marks = TidyMarks::default();
    for path in files {
        let extension = path.extension().unwrap_or_default().to_str().unwrap_or_default();
        match extension {
            "rs" => {
                let text = sh.read_file(&path).unwrap();
                check_todo(&path, &text);
                check_dbg(&path, &text);
                check_test_attrs(&path, &text);
                check_trailing_ws(&path, &text);
                deny_clippy(&path, &text);
                tidy_docs.visit(&path, &text);
                tidy_marks.visit(&path, &text);
            }
            "toml" => {
                let text = sh.read_file(&path).unwrap();
                check_cargo_toml(&path, text);
            }
            _ => (),
        }
    }

    tidy_docs.finish();
    tidy_marks.finish();
}

fn check_cargo_toml(path: &Path, text: String) {
    let mut section = None;
    for (line_no, text) in text.lines().enumerate() {
        let text = text.trim();
        if text.starts_with('[') {
            if !text.ends_with(']') {
                panic!(
                    "\nplease don't add comments or trailing whitespace in section lines.\n\
                        {}:{}\n",
                    path.display(),
                    line_no + 1
                )
            }
            section = Some(text);
            continue;
        }
        let text: String = text.split_whitespace().collect();
        if !text.contains("path=") {
            continue;
        }
        match section {
            Some(s) if s.contains("dev-dependencies") => {
                if text.contains("version") {
                    panic!(
                        "\ncargo internal dev-dependencies should not have a version.\n\
                        {}:{}\n",
                        path.display(),
                        line_no + 1
                    );
                }
            }
            Some(s) if s.contains("dependencies") => {
                if !text.contains("version") {
                    panic!(
                        "\ncargo internal dependencies should have a version.\n\
                        {}:{}\n",
                        path.display(),
                        line_no + 1
                    );
                }
            }
            _ => {}
        }
    }
}

fn deny_clippy(path: &Path, text: &str) {
    let ignore = &[
        // The documentation in string literals may contain anything for its own purposes
        "ide-db/src/generated/lints.rs",
        // The tests test clippy lint hovers
        "ide/src/hover/tests.rs",
        // The tests test clippy lint completions
        "ide-completion/src/tests/attribute.rs",
    ];
    if ignore.iter().any(|p| path.ends_with(p)) {
        return;
    }

    if text.contains("\u{61}llow(clippy") {
        panic!(
            "\n\nallowing lints is forbidden: {}.
rust-analyzer intentionally doesn't check clippy on CI.
You can allow lint globally via `xtask clippy`.
See https://github.com/rust-lang/rust-clippy/issues/5537 for discussion.

",
            path.display()
        )
    }
}

#[cfg(not(feature = "in-rust-tree"))]
#[test]
fn check_licenses() {
    let sh = &Shell::new().unwrap();

    let expected = "
0BSD OR MIT OR Apache-2.0
Apache-2.0
Apache-2.0 OR BSL-1.0
Apache-2.0 OR MIT
Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT
Apache-2.0/MIT
BSD-3-Clause
BlueOak-1.0.0 OR MIT OR Apache-2.0
CC0-1.0 OR Artistic-2.0
ISC
MIT
MIT / Apache-2.0
MIT OR Apache-2.0
MIT OR Apache-2.0 OR Zlib
MIT OR Zlib OR Apache-2.0
MIT/Apache-2.0
Unlicense/MIT
Zlib OR Apache-2.0 OR MIT
"
    .lines()
    .filter(|it| !it.is_empty())
    .collect::<Vec<_>>();

    let meta = cmd!(sh, "cargo metadata --format-version 1").read().unwrap();
    let mut licenses = meta
        .split(|c| c == ',' || c == '{' || c == '}')
        .filter(|it| it.contains(r#""license""#))
        .map(|it| it.trim())
        .map(|it| it[r#""license":"#.len()..].trim_matches('"'))
        .collect::<Vec<_>>();
    licenses.sort_unstable();
    licenses.dedup();
    if licenses != expected {
        let mut diff = String::new();

        diff.push_str("New Licenses:\n");
        for &l in licenses.iter() {
            if !expected.contains(&l) {
                diff += &format!("  {}\n", l)
            }
        }

        diff.push_str("\nMissing Licenses:\n");
        for &l in expected.iter() {
            if !licenses.contains(&l) {
                diff += &format!("  {}\n", l)
            }
        }

        panic!("different set of licenses!\n{}", diff);
    }
    assert_eq!(licenses, expected);
}

fn check_todo(path: &Path, text: &str) {
    let need_todo = &[
        // This file itself obviously needs to use todo (<- like this!).
        "tests/tidy.rs",
        // Some of our assists generate `todo!()`.
        "handlers/add_turbo_fish.rs",
        "handlers/generate_function.rs",
        "handlers/add_missing_match_arms.rs",
        "handlers/replace_derive_with_manual_impl.rs",
        // To support generating `todo!()` in assists, we have `expr_todo()` in
        // `ast::make`.
        "ast/make.rs",
        // The documentation in string literals may contain anything for its own purposes
        "ide-db/src/generated/lints.rs",
        "ide-assists/src/utils/gen_trait_fn_body.rs",
        "ide-assists/src/tests/generated.rs",
        // The tests for missing fields
        "ide-diagnostics/src/handlers/missing_fields.rs",
    ];
    if need_todo.iter().any(|p| path.ends_with(p)) {
        return;
    }
    if text.contains("TODO") || text.contains("TOOD") || text.contains("todo!") {
        // Generated by an assist
        if text.contains("${0:todo!()}") {
            return;
        }

        panic!(
            "\nTODO markers or todo! macros should not be committed to the master branch,\n\
             use FIXME instead\n\
             {}\n",
            path.display(),
        )
    }
}

fn check_dbg(path: &Path, text: &str) {
    let need_dbg = &[
        // This file itself obviously needs to use dbg.
        "slow-tests/tidy.rs",
        // Assists to remove `dbg!()`
        "handlers/remove_dbg.rs",
        // We have .dbg postfix
        "ide-completion/src/completions/postfix.rs",
        "ide-completion/src/completions/keyword.rs",
        "ide-completion/src/tests/proc_macros.rs",
        // The documentation in string literals may contain anything for its own purposes
        "ide-completion/src/lib.rs",
        "ide-db/src/generated/lints.rs",
        // test for doc test for remove_dbg
        "src/tests/generated.rs",
    ];
    if need_dbg.iter().any(|p| path.ends_with(p)) {
        return;
    }
    if text.contains("dbg!") {
        panic!(
            "\ndbg! macros should not be committed to the master branch,\n\
             {}\n",
            path.display(),
        )
    }
}

fn check_test_attrs(path: &Path, text: &str) {
    let ignore_rule =
        "https://github.com/rust-lang/rust-analyzer/blob/master/docs/dev/style.md#ignore";
    let need_ignore: &[&str] = &[
        // This file.
        "slow-tests/tidy.rs",
        // Special case to run `#[ignore]` tests.
        "ide/src/runnables.rs",
        // A legit test which needs to be ignored, as it takes too long to run
        // :(
        "hir-def/src/nameres/collector.rs",
        // Long sourcegen test to generate lint completions.
        "ide-db/src/tests/sourcegen_lints.rs",
        // Obviously needs ignore.
        "ide-assists/src/handlers/toggle_ignore.rs",
        // See above.
        "ide-assists/src/tests/generated.rs",
    ];
    if text.contains("#[ignore") && !need_ignore.iter().any(|p| path.ends_with(p)) {
        panic!("\ndon't `#[ignore]` tests, see:\n\n    {}\n\n   {}\n", ignore_rule, path.display(),)
    }

    let panic_rule =
        "https://github.com/rust-lang/rust-analyzer/blob/master/docs/dev/style.md#should_panic";
    let need_panic: &[&str] = &[
        // This file.
        "slow-tests/tidy.rs",
        "test-utils/src/fixture.rs",
    ];
    if text.contains("#[should_panic") && !need_panic.iter().any(|p| path.ends_with(p)) {
        panic!(
            "\ndon't add `#[should_panic]` tests, see:\n\n    {}\n\n   {}\n",
            panic_rule,
            path.display(),
        )
    }
}

fn check_trailing_ws(path: &Path, text: &str) {
    if is_exclude_dir(path, &["test_data"]) {
        return;
    }
    for (line_number, line) in text.lines().enumerate() {
        if line.chars().last().map(char::is_whitespace) == Some(true) {
            panic!("Trailing whitespace in {} at line {}", path.display(), line_number + 1)
        }
    }
}

#[derive(Default)]
struct TidyDocs {
    missing_docs: Vec<String>,
    contains_fixme: Vec<PathBuf>,
}

impl TidyDocs {
    fn visit(&mut self, path: &Path, text: &str) {
        // Tests and diagnostic fixes don't need module level comments.
        if is_exclude_dir(path, &["tests", "test_data", "fixes", "grammar"]) {
            return;
        }

        if is_exclude_file(path) {
            return;
        }

        let first_line = match text.lines().next() {
            Some(it) => it,
            None => return,
        };

        if first_line.starts_with("//!") {
            if first_line.contains("FIXME") {
                self.contains_fixme.push(path.to_path_buf());
            }
        } else {
            if text.contains("// Feature:")
                || text.contains("// Assist:")
                || text.contains("// Diagnostic:")
            {
                return;
            }
            self.missing_docs.push(path.display().to_string());
        }

        fn is_exclude_file(d: &Path) -> bool {
            let file_names = ["tests.rs", "famous_defs_fixture.rs"];

            d.file_name()
                .unwrap_or_default()
                .to_str()
                .map(|f_n| file_names.iter().any(|name| *name == f_n))
                .unwrap_or(false)
        }
    }

    fn finish(self) {
        if !self.missing_docs.is_empty() {
            panic!(
                "\nMissing docs strings\n\n\
                 modules:\n{}\n\n",
                self.missing_docs.join("\n")
            )
        }

        for path in self.contains_fixme {
            panic!("FIXME doc in a fully-documented crate: {}", path.display())
        }
    }
}

fn is_exclude_dir(p: &Path, dirs_to_exclude: &[&str]) -> bool {
    p.strip_prefix(sourcegen::project_root())
        .unwrap()
        .components()
        .rev()
        .skip(1)
        .filter_map(|it| it.as_os_str().to_str())
        .any(|it| dirs_to_exclude.contains(&it))
}

#[derive(Default)]
struct TidyMarks {
    hits: HashSet<String>,
    checks: HashSet<String>,
}

impl TidyMarks {
    fn visit(&mut self, _path: &Path, text: &str) {
        find_marks(&mut self.hits, text, "hit");
        find_marks(&mut self.checks, text, "check");
        find_marks(&mut self.checks, text, "check_count");
    }

    fn finish(self) {
        assert!(!self.hits.is_empty());

        let diff: Vec<_> =
            self.hits.symmetric_difference(&self.checks).map(|it| it.as_str()).collect();

        if !diff.is_empty() {
            panic!("unpaired marks: {:?}", diff)
        }
    }
}

#[allow(deprecated)]
fn stable_hash(text: &str) -> u64 {
    use std::hash::{Hash, Hasher, SipHasher};

    let text = text.replace('\r', "");
    let mut hasher = SipHasher::default();
    text.hash(&mut hasher);
    hasher.finish()
}

fn find_marks(set: &mut HashSet<String>, text: &str, mark: &str) {
    let mut text = text;
    let mut prev_text = "";
    while text != prev_text {
        prev_text = text;
        if let Some(idx) = text.find(mark) {
            text = &text[idx + mark.len()..];
            if let Some(stripped_text) = text.strip_prefix("!(") {
                text = stripped_text.trim_start();
                if let Some(idx2) = text.find(|c: char| !(c.is_alphanumeric() || c == '_')) {
                    let mark_text = &text[..idx2];
                    set.insert(mark_text.to_string());
                    text = &text[idx2..];
                }
            }
        }
    }
}
<commit_msg>Add new license<commit_after>use std::{
    collections::HashSet,
    path::{Path, PathBuf},
};

use xshell::Shell;

#[cfg(not(feature = "in-rust-tree"))]
use xshell::cmd;

#[cfg(not(feature = "in-rust-tree"))]
#[test]
fn check_code_formatting() {
    let sh = &Shell::new().unwrap();
    sh.change_dir(sourcegen::project_root());

    let out = cmd!(sh, "rustup run stable rustfmt --version").read().unwrap();
    if !out.contains("stable") {
        panic!(
            "Failed to run rustfmt from toolchain 'stable'. \
                 Please run `rustup component add rustfmt --toolchain stable` to install it.",
        )
    }

    let res = cmd!(sh, "rustup run stable cargo fmt -- --check").run();
    if res.is_err() {
        let _ = cmd!(sh, "rustup run stable cargo fmt").run();
    }
    res.unwrap()
}

#[test]
fn check_lsp_extensions_docs() {
    let sh = &Shell::new().unwrap();

    let expected_hash = {
        let lsp_ext_rs = sh
            .read_file(sourcegen::project_root().join("crates/rust-analyzer/src/lsp_ext.rs"))
            .unwrap();
        stable_hash(lsp_ext_rs.as_str())
    };

    let actual_hash = {
        let lsp_extensions_md =
            sh.read_file(sourcegen::project_root().join("docs/dev/lsp-extensions.md")).unwrap();
        let text = lsp_extensions_md
            .lines()
            .find_map(|line| line.strip_prefix("lsp_ext.rs hash:"))
            .unwrap()
            .trim();
        u64::from_str_radix(text, 16).unwrap()
    };

    if actual_hash != expected_hash {
        panic!(
            "
lsp_ext.rs was changed without touching lsp-extensions.md.

Expected hash: {:x}
Actual hash:   {:x}

Please adjust docs/dev/lsp-extensions.md.
",
            expected_hash, actual_hash
        )
    }
}

#[test]
fn files_are_tidy() {
    let sh = &Shell::new().unwrap();

    let files = sourcegen::list_files(&sourcegen::project_root().join("crates"));

    let mut tidy_docs = TidyDocs::default();
    let mut tidy_marks = TidyMarks::default();
    for path in files {
        let extension = path.extension().unwrap_or_default().to_str().unwrap_or_default();
        match extension {
            "rs" => {
                let text = sh.read_file(&path).unwrap();
                check_todo(&path, &text);
                check_dbg(&path, &text);
                check_test_attrs(&path, &text);
                check_trailing_ws(&path, &text);
                deny_clippy(&path, &text);
                tidy_docs.visit(&path, &text);
                tidy_marks.visit(&path, &text);
            }
            "toml" => {
                let text = sh.read_file(&path).unwrap();
                check_cargo_toml(&path, text);
            }
            _ => (),
        }
    }

    tidy_docs.finish();
    tidy_marks.finish();
}

fn check_cargo_toml(path: &Path, text: String) {
    let mut section = None;
    for (line_no, text) in text.lines().enumerate() {
        let text = text.trim();
        if text.starts_with('[') {
            if !text.ends_with(']') {
                panic!(
                    "\nplease don't add comments or trailing whitespace in section lines.\n\
                        {}:{}\n",
                    path.display(),
                    line_no + 1
                )
            }
            section = Some(text);
            continue;
        }
        let text: String = text.split_whitespace().collect();
        if !text.contains("path=") {
            continue;
        }
        match section {
            Some(s) if s.contains("dev-dependencies") => {
                if text.contains("version") {
                    panic!(
                        "\ncargo internal dev-dependencies should not have a version.\n\
                        {}:{}\n",
                        path.display(),
                        line_no + 1
                    );
                }
            }
            Some(s) if s.contains("dependencies") => {
                if !text.contains("version") {
                    panic!(
                        "\ncargo internal dependencies should have a version.\n\
                        {}:{}\n",
                        path.display(),
                        line_no + 1
                    );
                }
            }
            _ => {}
        }
    }
}

fn deny_clippy(path: &Path, text: &str) {
    let ignore = &[
        // The documentation in string literals may contain anything for its own purposes
        "ide-db/src/generated/lints.rs",
        // The tests test clippy lint hovers
        "ide/src/hover/tests.rs",
        // The tests test clippy lint completions
        "ide-completion/src/tests/attribute.rs",
    ];
    if ignore.iter().any(|p| path.ends_with(p)) {
        return;
    }

    if text.contains("\u{61}llow(clippy") {
        panic!(
            "\n\nallowing lints is forbidden: {}.
rust-analyzer intentionally doesn't check clippy on CI.
You can allow lint globally via `xtask clippy`.
See https://github.com/rust-lang/rust-clippy/issues/5537 for discussion.

",
            path.display()
        )
    }
}

#[cfg(not(feature = "in-rust-tree"))]
#[test]
fn check_licenses() {
    let sh = &Shell::new().unwrap();

    let expected = "
(MIT OR Apache-2.0) AND Unicode-DFS-2016
0BSD OR MIT OR Apache-2.0
Apache-2.0
Apache-2.0 OR BSL-1.0
Apache-2.0 OR MIT
Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT
Apache-2.0/MIT
BSD-3-Clause
BlueOak-1.0.0 OR MIT OR Apache-2.0
CC0-1.0 OR Artistic-2.0
ISC
MIT
MIT / Apache-2.0
MIT OR Apache-2.0
MIT OR Apache-2.0 OR Zlib
MIT OR Zlib OR Apache-2.0
MIT/Apache-2.0
Unlicense/MIT
Zlib OR Apache-2.0 OR MIT
"
    .lines()
    .filter(|it| !it.is_empty())
    .collect::<Vec<_>>();

    let meta = cmd!(sh, "cargo metadata --format-version 1").read().unwrap();
    let mut licenses = meta
        .split(|c| c == ',' || c == '{' || c == '}')
        .filter(|it| it.contains(r#""license""#))
        .map(|it| it.trim())
        .map(|it| it[r#""license":"#.len()..].trim_matches('"'))
        .collect::<Vec<_>>();
    licenses.sort_unstable();
    licenses.dedup();
    if licenses != expected {
        let mut diff = String::new();

        diff.push_str("New Licenses:\n");
        for &l in licenses.iter() {
            if !expected.contains(&l) {
                diff += &format!("  {}\n", l)
            }
        }

        diff.push_str("\nMissing Licenses:\n");
        for &l in expected.iter() {
            if !licenses.contains(&l) {
                diff += &format!("  {}\n", l)
            }
        }

        panic!("different set of licenses!\n{}", diff);
    }
    assert_eq!(licenses, expected);
}

fn check_todo(path: &Path, text: &str) {
    let need_todo = &[
        // This file itself obviously needs to use todo (<- like this!).
        "tests/tidy.rs",
        // Some of our assists generate `todo!()`.
        "handlers/add_turbo_fish.rs",
        "handlers/generate_function.rs",
        "handlers/add_missing_match_arms.rs",
        "handlers/replace_derive_with_manual_impl.rs",
        // To support generating `todo!()` in assists, we have `expr_todo()` in
        // `ast::make`.
        "ast/make.rs",
        // The documentation in string literals may contain anything for its own purposes
        "ide-db/src/generated/lints.rs",
        "ide-assists/src/utils/gen_trait_fn_body.rs",
        "ide-assists/src/tests/generated.rs",
        // The tests for missing fields
        "ide-diagnostics/src/handlers/missing_fields.rs",
    ];
    if need_todo.iter().any(|p| path.ends_with(p)) {
        return;
    }
    if text.contains("TODO") || text.contains("TOOD") || text.contains("todo!") {
        // Generated by an assist
        if text.contains("${0:todo!()}") {
            return;
        }

        panic!(
            "\nTODO markers or todo! macros should not be committed to the master branch,\n\
             use FIXME instead\n\
             {}\n",
            path.display(),
        )
    }
}

fn check_dbg(path: &Path, text: &str) {
    let need_dbg = &[
        // This file itself obviously needs to use dbg.
        "slow-tests/tidy.rs",
        // Assists to remove `dbg!()`
        "handlers/remove_dbg.rs",
        // We have .dbg postfix
        "ide-completion/src/completions/postfix.rs",
        "ide-completion/src/completions/keyword.rs",
        "ide-completion/src/tests/proc_macros.rs",
        // The documentation in string literals may contain anything for its own purposes
        "ide-completion/src/lib.rs",
        "ide-db/src/generated/lints.rs",
        // test for doc test for remove_dbg
        "src/tests/generated.rs",
    ];
    if need_dbg.iter().any(|p| path.ends_with(p)) {
        return;
    }
    if text.contains("dbg!") {
        panic!(
            "\ndbg! macros should not be committed to the master branch,\n\
             {}\n",
            path.display(),
        )
    }
}

fn check_test_attrs(path: &Path, text: &str) {
    let ignore_rule =
        "https://github.com/rust-lang/rust-analyzer/blob/master/docs/dev/style.md#ignore";
    let need_ignore: &[&str] = &[
        // This file.
        "slow-tests/tidy.rs",
        // Special case to run `#[ignore]` tests.
        "ide/src/runnables.rs",
        // A legit test which needs to be ignored, as it takes too long to run
        // :(
        "hir-def/src/nameres/collector.rs",
        // Long sourcegen test to generate lint completions.
        "ide-db/src/tests/sourcegen_lints.rs",
        // Obviously needs ignore.
        "ide-assists/src/handlers/toggle_ignore.rs",
        // See above.
        "ide-assists/src/tests/generated.rs",
    ];
    if text.contains("#[ignore") && !need_ignore.iter().any(|p| path.ends_with(p)) {
        panic!("\ndon't `#[ignore]` tests, see:\n\n    {}\n\n   {}\n", ignore_rule, path.display(),)
    }

    let panic_rule =
        "https://github.com/rust-lang/rust-analyzer/blob/master/docs/dev/style.md#should_panic";
    let need_panic: &[&str] = &[
        // This file.
        "slow-tests/tidy.rs",
        "test-utils/src/fixture.rs",
    ];
    if text.contains("#[should_panic") && !need_panic.iter().any(|p| path.ends_with(p)) {
        panic!(
            "\ndon't add `#[should_panic]` tests, see:\n\n    {}\n\n   {}\n",
            panic_rule,
            path.display(),
        )
    }
}

fn check_trailing_ws(path: &Path, text: &str) {
    if is_exclude_dir(path, &["test_data"]) {
        return;
    }
    for (line_number, line) in text.lines().enumerate() {
        if line.chars().last().map(char::is_whitespace) == Some(true) {
            panic!("Trailing whitespace in {} at line {}", path.display(), line_number + 1)
        }
    }
}

#[derive(Default)]
struct TidyDocs {
    missing_docs: Vec<String>,
    contains_fixme: Vec<PathBuf>,
}

impl TidyDocs {
    fn visit(&mut self, path: &Path, text: &str) {
        // Tests and diagnostic fixes don't need module level comments.
        if is_exclude_dir(path, &["tests", "test_data", "fixes", "grammar"]) {
            return;
        }

        if is_exclude_file(path) {
            return;
        }

        let first_line = match text.lines().next() {
            Some(it) => it,
            None => return,
        };

        if first_line.starts_with("//!") {
            if first_line.contains("FIXME") {
                self.contains_fixme.push(path.to_path_buf());
            }
        } else {
            if text.contains("// Feature:")
                || text.contains("// Assist:")
                || text.contains("// Diagnostic:")
            {
                return;
            }
            self.missing_docs.push(path.display().to_string());
        }

        fn is_exclude_file(d: &Path) -> bool {
            let file_names = ["tests.rs", "famous_defs_fixture.rs"];

            d.file_name()
                .unwrap_or_default()
                .to_str()
                .map(|f_n| file_names.iter().any(|name| *name == f_n))
                .unwrap_or(false)
        }
    }

    fn finish(self) {
        if !self.missing_docs.is_empty() {
            panic!(
                "\nMissing docs strings\n\n\
                 modules:\n{}\n\n",
                self.missing_docs.join("\n")
            )
        }

        for path in self.contains_fixme {
            panic!("FIXME doc in a fully-documented crate: {}", path.display())
        }
    }
}

fn is_exclude_dir(p: &Path, dirs_to_exclude: &[&str]) -> bool {
    p.strip_prefix(sourcegen::project_root())
        .unwrap()
        .components()
        .rev()
        .skip(1)
        .filter_map(|it| it.as_os_str().to_str())
        .any(|it| dirs_to_exclude.contains(&it))
}

#[derive(Default)]
struct TidyMarks {
    hits: HashSet<String>,
    checks: HashSet<String>,
}

impl TidyMarks {
    fn visit(&mut self, _path: &Path, text: &str) {
        find_marks(&mut self.hits, text, "hit");
        find_marks(&mut self.checks, text, "check");
        find_marks(&mut self.checks, text, "check_count");
    }

    fn finish(self) {
        assert!(!self.hits.is_empty());

        let diff: Vec<_> =
            self.hits.symmetric_difference(&self.checks).map(|it| it.as_str()).collect();

        if !diff.is_empty() {
            panic!("unpaired marks: {:?}", diff)
        }
    }
}

#[allow(deprecated)]
fn stable_hash(text: &str) -> u64 {
    use std::hash::{Hash, Hasher, SipHasher};

    let text = text.replace('\r', "");
    let mut hasher = SipHasher::default();
    text.hash(&mut hasher);
    hasher.finish()
}

fn find_marks(set: &mut HashSet<String>, text: &str, mark: &str) {
    let mut text = text;
    let mut prev_text = "";
    while text != prev_text {
        prev_text = text;
        if let Some(idx) = text.find(mark) {
            text = &text[idx + mark.len()..];
            if let Some(stripped_text) = text.strip_prefix("!(") {
                text = stripped_text.trim_start();
                if let Some(idx2) = text.find(|c: char| !(c.is_alphanumeric() || c == '_')) {
                    let mark_text = &text[..idx2];
                    set.insert(mark_text.to_string());
                    text = &text[idx2..];
                }
            }
        }
    }
}
<|endoftext|>