<commit_before>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

use failure::format_err;
use futures::stream::StreamExt;
use slog_scope::{info, warn};

use mimir2::domain::ports::primary::list_documents::ListDocuments;
use mimir2::domain::{
    model::index::IndexVisibility, ports::primary::generate_index::GenerateIndex,
};
use mimir2::{
    adapters::secondary::elasticsearch::{self, ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ},
    domain::ports::secondary::remote::Remote,
};
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use mimirsbrunn::osm_reader::make_osm_reader;
use mimirsbrunn::osm_reader::poi::{add_address, compute_weight, pois, PoiConfig};
use mimirsbrunn::osm_reader::street::{compute_street_weight, streets};
use mimirsbrunn::settings::osm2mimir::{Args, Settings};
use places::poi::Poi;

const POI_REVERSE_GEOCODING_CONCURRENCY: usize = 8;

async fn run(args: Args) -> Result<(), mimirsbrunn::Error> {
    let input = args.input.clone(); // we save the input, because args will be consumed by settings.
    validate_args(&args)?;
    let settings = &Settings::new(args.clone())?;

    let mut osm_reader = make_osm_reader(&input)?;

    let import_streets_enabled = settings
        .street
        .as_ref()
        .map(|street| street.import)
        .unwrap_or_else(|| false);

    let import_poi_enabled = settings
        .poi
        .as_ref()
        .map(|poi| poi.import)
        .unwrap_or_else(|| false);

    if !import_streets_enabled && !import_poi_enabled {
        return Err(format_err!(
            "Neither streets nor POIs import is enabled. Nothing to do.\n\
             Use --import-way=true or --import-poi=true"
        ));
    }

    let pool =
        elasticsearch::remote::connection_pool_url(&settings.elasticsearch.connection_string)
            .await
            .map_err(|err| {
                format_err!(
                    "could not create elasticsearch connection pool: {}",
                    err.to_string()
                )
            })?;

    let client = pool
        .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
        .await
        .map_err(|err| format_err!("could not connect elasticsearch pool: {}", err.to_string()))?;

    let admins_geofinder: AdminGeoFinder = match client.list_documents().await {
        Ok(stream) => {
            stream
                .map(|admin| admin.expect("could not parse admin"))
                .collect()
                .await
        }
        Err(err) => {
            warn!("administratives regions not found in es db. {:?}", err);
            std::iter::empty().collect()
        }
    };

    if import_streets_enabled {
        info!("Extracting streets from osm");
        let mut streets = streets(&mut osm_reader, &admins_geofinder, settings)?;

        info!("computing street weight");
        compute_street_weight(&mut streets);

        let streets = futures::stream::iter(streets);
        let street_config = args.get_street_config()?;

        client
            .generate_index(street_config, streets, IndexVisibility::Public)
            .await
            .map_err(|err| format_err!("could not generate index: {}", err.to_string()))?;
    }

    if import_poi_enabled {
        let config = settings
            .poi
            .as_ref()
            .and_then(|poi| poi.config.clone())
            .unwrap_or_else(PoiConfig::default);

        // Ideally, this pois function would create a stream, which would then map and do other
        // stuff, and then be indexed
        info!("Extracting pois from osm");
        let pois = pois(&mut osm_reader, &config, &admins_geofinder);

        let pois: Vec<Poi> = futures::stream::iter(pois)
            .map(compute_weight)
            .map(|poi| add_address(&client, poi))
            .buffer_unordered(POI_REVERSE_GEOCODING_CONCURRENCY)
            .collect()
            .await;

        let poi_config = args.get_poi_config()?;

        client
            .generate_index(
                poi_config,
                futures::stream::iter(pois),
                IndexVisibility::Public,
            )
            .await
            .map_err(|err| format_err!("could not generate index: {}", err.to_string()))?;
    }
    Ok(())
}

// We need to allow for unused variables, because currently all the checks on
// args require the db-storage feature. If this feature is not used, then there
// is a warning
#[allow(unused_variables)]
fn validate_args(args: &Args) -> Result<(), mimirsbrunn::Error> {
    #[cfg(feature = "db-storage")]
    if args.db_file.is_some() {
        // If the user specified db_file, he must also specify db_buffer_size, or else!
        if args.db_buffer_size.is_none() {
            return Err(failure::format_err!("You need to specify database buffer size if you want to use database storage. Use --db-buffer-size"));
        }
    }
    #[cfg(feature = "db-storage")]
    if args.db_buffer_size.is_some() {
        // If the user specified db_buffer_size, he must also specify db_file, or else!
        if args.db_file.is_none() {
            return Err(failure::format_err!("You need to specify database file if you want to use database storage. Use --db-file"));
        }
    }
    Ok(())
}

#[tokio::main]
async fn main() {
    mimirsbrunn::utils::launch_async(Box::new(run)).await;
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::elasticsearch::ElasticsearchStorage;
    use common::config::load_es_config_for;
    use common::document::ContainerDocument;
    use futures::TryStreamExt;
    use mimir2::domain::model::query::Query;
    use mimir2::domain::ports::primary::list_documents::ListDocuments;
    use mimir2::domain::ports::primary::search_documents::SearchDocuments;
    use mimir2::{adapters::secondary::elasticsearch::remote, utils::docker};
    use mimirsbrunn::admin::index_cosmogony;
    use places::{admin::Admin, street::Street, Place};
    use structopt::StructOpt;

    fn elasticsearch_test_url() -> String {
        std::env::var(elasticsearch::remote::ES_TEST_KEY).expect("env var")
    }

    async fn index_cosmogony_admins(client: &ElasticsearchStorage) {
        index_cosmogony(
            "./tests/fixtures/cosmogony.json".into(),
            vec![],
            load_es_config_for::<Admin>(
                None,
                None,
                vec!["container.dataset=osm2mimir-test".into()],
            )
            .unwrap(),
            client,
        )
        .await
        .unwrap()
    }

    #[tokio::test]
    async fn should_correctly_index_osm_streets_and_pois() {
        let _guard = docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        // Now we query the index we just created. Since it's a small cosmogony file with few entries,
        // we'll just list all the documents in the index, and check them.
        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");

        let client = pool
            .conn()
            .await
            .expect("Elasticsearch Connection Established");

        index_cosmogony_admins(&client).await;

        let storage_args = if cfg!(feature = "db-storage") {
            vec!["--db-file=test-db.sqlite3", "--db-buffer-size=10"]
        } else {
            vec![]
        };

        let args = Args::from_iter(
            [
                "osm2mimir",
                "--input=./tests/fixtures/osm_fixture.osm.pbf",
                "--dataset=osm2mimir-test",
                "--import-way=true",
                "--import-poi=true",
                &format!("-c={}", elasticsearch_test_url()),
            ]
            .iter()
            .copied()
            .chain(storage_args),
        );

        let _res = mimirsbrunn::utils::launch_async_args(run, args).await;

        let search = |query: &str| {
            let client = client.clone();
            let query: String = query.into();
            async move {
                client
                    .search_documents(
                        vec![
                            Street::static_doc_type().into(),
                            Poi::static_doc_type().into(),
                        ],
                        Query::QueryString(format!("full_label.prefix:({})", query)),
                    )
                    .await
                    .unwrap()
                    .into_iter()
                    .map(|json| serde_json::from_value::<Place>(json).unwrap())
                    .collect::<Vec<Place>>()
            }
        };

        let streets: Vec<Street> = client
            .list_documents()
            .await
            .unwrap()
            .try_collect()
            .await
            .unwrap();
        assert_eq!(streets.len(), 13);

        // Basic street search
        let results = search("Rue des Près").await;
        assert_eq!(results[0].label(), "Rue des Près (Livry-sur-Seine)");
        assert_eq!(
            results
                .iter()
                .filter(
                    |place| place.is_street() && place.label() == "Rue des Près (Livry-sur-Seine)"
                )
                .count(),
            1,
            "Only 1 'Rue des Près' is expected"
        );

        // All ways with same name in the same city are merged into a single street
        let results = search("Rue du Four à Chaux").await;
        assert_eq!(
            results.iter()
                .filter(|place| place.label() == "Rue du Four à Chaux (Livry-sur-Seine)")
                .count(),
            1,
            "Only 1 'Rue du Four à Chaux' is expected as all ways the same name should be merged into 1 street."
        );
        assert_eq!(
            results[0].id(),
            "street:osm:way:40812939",
            "The way with minimum way_id should be used as street id."
        );

        // Street admin is based on a middle node.
        // (Here the first node is located outside Melun)
        let results = search("Rue Marcel Houdet").await;
        assert_eq!(results[0].label(), "Rue Marcel Houdet (Melun)");
        assert!(results[0]
            .admins()
            .iter()
            .filter(|a| a.is_city())
            .any(|a| a.name == "Melun"));

        // Basic search for Poi by label
        let res = search("Le-Mée-sur-Seine Courtilleraies").await;
        assert_eq!(
            res[0].poi().expect("Place should be a poi").poi_type.id,
            "poi_type:amenity:post_office"
        );

        // highway=bus_stop should not be indexed
        let res = search("Grand Châtelet").await;
        assert!(
            res.is_empty(),
            "'Grand Châtelet' (highway=bus_stop) should not be found."
        );

        // "Rue de Villiers" is at the exact neighborhood between two cities, a
        // document must be added for both.
        let results = search("Rue de Villiers").await;
        assert!(["Neuilly-sur-Seine", "Levallois-Perret"]
            .iter()
            .all(|city| {
                results.iter().any(|poi| {
                    poi.admins()
                        .iter()
                        .filter(|a| a.is_city())
                        .any(|admin| &admin.name == city)
                })
            }));
    }
}
<commit_msg>fix missing conn arguments<commit_after>// Copyright © 2016, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

use failure::format_err;
use futures::stream::StreamExt;
use slog_scope::{info, warn};

use mimir2::domain::ports::primary::list_documents::ListDocuments;
use mimir2::domain::{
    model::index::IndexVisibility, ports::primary::generate_index::GenerateIndex,
};
use mimir2::{
    adapters::secondary::elasticsearch::{self, ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ},
    domain::ports::secondary::remote::Remote,
};
use mimirsbrunn::admin_geofinder::AdminGeoFinder;
use mimirsbrunn::osm_reader::make_osm_reader;
use mimirsbrunn::osm_reader::poi::{add_address, compute_weight, pois, PoiConfig};
use mimirsbrunn::osm_reader::street::{compute_street_weight, streets};
use mimirsbrunn::settings::osm2mimir::{Args, Settings};
use places::poi::Poi;

const POI_REVERSE_GEOCODING_CONCURRENCY: usize = 8;

async fn run(args: Args) -> Result<(), mimirsbrunn::Error> {
    let input = args.input.clone(); // we save the input, because args will be consumed by settings.
    validate_args(&args)?;
    let settings = &Settings::new(args.clone())?;

    let mut osm_reader = make_osm_reader(&input)?;

    let import_streets_enabled = settings
        .street
        .as_ref()
        .map(|street| street.import)
        .unwrap_or_else(|| false);

    let import_poi_enabled = settings
        .poi
        .as_ref()
        .map(|poi| poi.import)
        .unwrap_or_else(|| false);

    if !import_streets_enabled && !import_poi_enabled {
        return Err(format_err!(
            "Neither streets nor POIs import is enabled. Nothing to do.\n\
             Use --import-way=true or --import-poi=true"
        ));
    }

    let pool =
        elasticsearch::remote::connection_pool_url(&settings.elasticsearch.connection_string)
            .await
            .map_err(|err| {
                format_err!(
                    "could not create elasticsearch connection pool: {}",
                    err.to_string()
                )
            })?;

    let client = pool
        .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
        .await
        .map_err(|err| format_err!("could not connect elasticsearch pool: {}", err.to_string()))?;

    let admins_geofinder: AdminGeoFinder = match client.list_documents().await {
        Ok(stream) => {
            stream
                .map(|admin| admin.expect("could not parse admin"))
                .collect()
                .await
        }
        Err(err) => {
            warn!("administratives regions not found in es db. {:?}", err);
            std::iter::empty().collect()
        }
    };

    if import_streets_enabled {
        info!("Extracting streets from osm");
        let mut streets = streets(&mut osm_reader, &admins_geofinder, settings)?;

        info!("computing street weight");
        compute_street_weight(&mut streets);

        let streets = futures::stream::iter(streets);
        let street_config = args.get_street_config()?;

        client
            .generate_index(street_config, streets, IndexVisibility::Public)
            .await
            .map_err(|err| format_err!("could not generate index: {}", err.to_string()))?;
    }

    if import_poi_enabled {
        let config = settings
            .poi
            .as_ref()
            .and_then(|poi| poi.config.clone())
            .unwrap_or_else(PoiConfig::default);

        // Ideally, this pois function would create a stream, which would then map and do other
        // stuff, and then be indexed
        info!("Extracting pois from osm");
        let pois = pois(&mut osm_reader, &config, &admins_geofinder);

        let pois: Vec<Poi> = futures::stream::iter(pois)
            .map(compute_weight)
            .map(|poi| add_address(&client, poi))
            .buffer_unordered(POI_REVERSE_GEOCODING_CONCURRENCY)
            .collect()
            .await;

        let poi_config = args.get_poi_config()?;

        client
            .generate_index(
                poi_config,
                futures::stream::iter(pois),
                IndexVisibility::Public,
            )
            .await
            .map_err(|err| format_err!("could not generate index: {}", err.to_string()))?;
    }
    Ok(())
}

// We need to allow for unused variables, because currently all the checks on
// args require the db-storage feature. If this feature is not used, then there
// is a warning
#[allow(unused_variables)]
fn validate_args(args: &Args) -> Result<(), mimirsbrunn::Error> {
    #[cfg(feature = "db-storage")]
    if args.db_file.is_some() {
        // If the user specified db_file, he must also specify db_buffer_size, or else!
        if args.db_buffer_size.is_none() {
            return Err(failure::format_err!("You need to specify database buffer size if you want to use database storage. Use --db-buffer-size"));
        }
    }
    #[cfg(feature = "db-storage")]
    if args.db_buffer_size.is_some() {
        // If the user specified db_buffer_size, he must also specify db_file, or else!
        if args.db_file.is_none() {
            return Err(failure::format_err!("You need to specify database file if you want to use database storage. Use --db-file"));
        }
    }
    Ok(())
}

#[tokio::main]
async fn main() {
    mimirsbrunn::utils::launch_async(Box::new(run)).await;
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::elasticsearch::ElasticsearchStorage;
    use common::config::load_es_config_for;
    use common::document::ContainerDocument;
    use futures::TryStreamExt;
    use mimir2::domain::model::query::Query;
    use mimir2::domain::ports::primary::list_documents::ListDocuments;
    use mimir2::domain::ports::primary::search_documents::SearchDocuments;
    use mimir2::{adapters::secondary::elasticsearch::remote, utils::docker};
    use mimirsbrunn::admin::index_cosmogony;
    use places::{admin::Admin, street::Street, Place};
    use structopt::StructOpt;

    fn elasticsearch_test_url() -> String {
        std::env::var(elasticsearch::remote::ES_TEST_KEY).expect("env var")
    }

    async fn index_cosmogony_admins(client: &ElasticsearchStorage) {
        index_cosmogony(
            "./tests/fixtures/cosmogony.json".into(),
            vec![],
            load_es_config_for::<Admin>(
                None,
                None,
                vec!["container.dataset=osm2mimir-test".into()],
            )
            .unwrap(),
            client,
        )
        .await
        .unwrap()
    }

    #[tokio::test]
    async fn should_correctly_index_osm_streets_and_pois() {
        let _guard = docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        // Now we query the index we just created. Since it's a small cosmogony file with few entries,
        // we'll just list all the documents in the index, and check them.
        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");

        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");

        index_cosmogony_admins(&client).await;

        let storage_args = if cfg!(feature = "db-storage") {
            vec!["--db-file=test-db.sqlite3", "--db-buffer-size=10"]
        } else {
            vec![]
        };

        let args = Args::from_iter(
            [
                "osm2mimir",
                "--input=./tests/fixtures/osm_fixture.osm.pbf",
                "--dataset=osm2mimir-test",
                "--import-way=true",
                "--import-poi=true",
                &format!("-c={}", elasticsearch_test_url()),
            ]
            .iter()
            .copied()
            .chain(storage_args),
        );

        let _res = mimirsbrunn::utils::launch_async_args(run, args).await;

        let search = |query: &str| {
            let client = client.clone();
            let query: String = query.into();
            async move {
                client
                    .search_documents(
                        vec![
                            Street::static_doc_type().into(),
                            Poi::static_doc_type().into(),
                        ],
                        Query::QueryString(format!("full_label.prefix:({})", query)),
                    )
                    .await
                    .unwrap()
                    .into_iter()
                    .map(|json| serde_json::from_value::<Place>(json).unwrap())
                    .collect::<Vec<Place>>()
            }
        };

        let streets: Vec<Street> = client
            .list_documents()
            .await
            .unwrap()
            .try_collect()
            .await
            .unwrap();
        assert_eq!(streets.len(), 13);

        // Basic street search
        let results = search("Rue des Près").await;
        assert_eq!(results[0].label(), "Rue des Près (Livry-sur-Seine)");
        assert_eq!(
            results
                .iter()
                .filter(
                    |place| place.is_street() && place.label() == "Rue des Près (Livry-sur-Seine)"
                )
                .count(),
            1,
            "Only 1 'Rue des Près' is expected"
        );

        // All ways with same name in the same city are merged into a single street
        let results = search("Rue du Four à Chaux").await;
        assert_eq!(
            results.iter()
                .filter(|place| place.label() == "Rue du Four à Chaux (Livry-sur-Seine)")
                .count(),
            1,
            "Only 1 'Rue du Four à Chaux' is expected as all ways the same name should be merged into 1 street."
        );
        assert_eq!(
            results[0].id(),
            "street:osm:way:40812939",
            "The way with minimum way_id should be used as street id."
        );

        // Street admin is based on a middle node.
        // (Here the first node is located outside Melun)
        let results = search("Rue Marcel Houdet").await;
        assert_eq!(results[0].label(), "Rue Marcel Houdet (Melun)");
        assert!(results[0]
            .admins()
            .iter()
            .filter(|a| a.is_city())
            .any(|a| a.name == "Melun"));

        // Basic search for Poi by label
        let res = search("Le-Mée-sur-Seine Courtilleraies").await;
        assert_eq!(
            res[0].poi().expect("Place should be a poi").poi_type.id,
            "poi_type:amenity:post_office"
        );

        // highway=bus_stop should not be indexed
        let res = search("Grand Châtelet").await;
        assert!(
            res.is_empty(),
            "'Grand Châtelet' (highway=bus_stop) should not be found."
        );

        // "Rue de Villiers" is at the exact neighborhood between two cities, a
        // document must be added for both.
        let results = search("Rue de Villiers").await;
        assert!(["Neuilly-sur-Seine", "Levallois-Perret"]
            .iter()
            .all(|city| {
                results.iter().any(|poi| {
                    poi.admins()
                        .iter()
                        .filter(|a| a.is_city())
                        .any(|admin| &admin.name == city)
                })
            }));
    }
}
<|endoftext|>


<commit_before>// Copyright © 2019, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

use slog_scope::{error, info};

use failure::format_err;
use lazy_static::lazy_static;
use mimir::objects::{Coord, I18nProperties, Poi, PoiType, Property};
use mimir::rubber::{IndexSettings, IndexVisibility, Rubber, TypedIndex};
use mimirsbrunn::{admin_geofinder::AdminGeoFinder, labels, utils};
use navitia_poi_model::{Model as NavitiaModel, Poi as NavitiaPoi, PoiType as NavitiaPoiType};
use std::collections::HashMap;
use std::ops::Deref;
use std::path::PathBuf;
use structopt::StructOpt;

lazy_static! {
    static ref DEFAULT_NB_THREADS: String = num_cpus::get().to_string();
}

// This function takes a Poi from the navitia model, ie from the CSV deserialization, and returns
// a Poi from the mimir model, with all the contextual information added.
fn into_mimir_poi(
    poi: NavitiaPoi,
    poi_types: &HashMap<String, NavitiaPoiType>,
    rubber: &mut Rubber,
    admins_geofinder: &AdminGeoFinder,
) -> Result<Poi, mimirsbrunn::Error> {
    let poi_type = poi_types
        .get(&poi.poi_type_id)
        .ok_or_else(|| format_err!("could not find Poi Type '{}'", poi.poi_type_id))
        .map(PoiType::from)?;

    let coord = Coord::from(&poi.coord);

    let place = rubber
        .get_address(&coord) // No timeout
        .ok()
        .and_then(|addrs| addrs.into_iter().next()); // Take the first place

    let addr = place.as_ref().and_then(|place| place.address());

    // We the the admins from the address, or, if we don't have any, from the geofinder.
    let admins = place.map_or_else(|| admins_geofinder.get(&poi.coord), |addr| addr.admins());

    if admins.is_empty() {
        return Err(format_err!("Could not find admins for POI {}", &poi.id));
    }

    // The weight is that of the city, or 0.0 if there is no such admin.
    let weight: f64 = admins
        .iter()
        .filter(|adm| adm.is_city())
        .map(|adm| adm.weight)
        .next()
        .unwrap_or(0.0);

    let country_codes = utils::find_country_codes(admins.iter().map(|a| a.deref()));

    let label =
        labels::format_poi_label(&poi.name, admins.iter().map(|a| a.deref()), &country_codes);

    let poi = Poi {
        id: mimir::objects::normalize_id("poi", &poi.id),
        label,
        name: poi.name,
        coord,
        approx_coord: Some(coord.into()),
        administrative_regions: admins,
        weight,
        zip_codes: vec![],
        poi_type,
        properties: poi.properties.into_iter().map(Property::from).collect(),
        address: addr,
        country_codes,
        names: I18nProperties::default(),
        labels: I18nProperties::default(),
        distance: None,
        context: None,
    };

    Ok(poi)
}

fn import_pois(
    rubber: &mut Rubber,
    index: &TypedIndex<Poi>,
    admins_geofinder: AdminGeoFinder,
    file: &PathBuf,
) -> Result<(), mimirsbrunn::Error>
where
{
    info!("Add data in elasticsearch db.");

    let model = NavitiaModel::try_from_path(file)?;
    let poi_types = model.poi_types;

    // Note: We're ignoring those POIs that fail to be enriched.
    let pois: Vec<_> = model
        .pois
        .into_iter()
        .filter_map(|(id, poi)| {
            into_mimir_poi(poi, &poi_types, rubber, &admins_geofinder)
                .map_err(|err| info!("Could not extract information for POI '{}': {}", id, err))
                .ok()
        })
        .collect(); // TODO Can we get rid of collect, and chain with the following rubber...?

    let count = rubber
        .bulk_index(&index, pois.into_iter())
        .map_err(|err| format_err!("Failed bulk insertion {}", err))?;

    info!("importing POIs: {} POIs added.", count);

    Ok(())
}

/// This function initializes the ES context: It creates an index for this dataset,
/// and then import the POIs in it.
fn index_poi(
    cnx_string: &str,
    dataset: &str,
    file: &PathBuf,
    visibility: IndexVisibility,
    nb_shards: usize,
    nb_replicas: usize,
) -> Result<(), mimirsbrunn::Error>
where
{
    let mut rubber = Rubber::new(cnx_string);
    rubber.initialize_templates()?;

    let settings = IndexSettings {
        nb_shards,
        nb_replicas,
    };

    let index = rubber.make_index(dataset, &settings)?;

    let admins = rubber.get_all_admins().map_err(|err| {
        error!("Administratives regions not found in es db");
        err
    })?;
    let admins_geofinder = admins.into_iter().collect();

    import_pois(&mut rubber, &index, admins_geofinder, file)?;

    rubber
        .publish_index(dataset, index, visibility)
        .map_err(|err| format_err!("Failed to publish index {}.", err))
}

#[derive(StructOpt, Debug)]
struct Args {
    /// POI file
    #[structopt(short = "i", long = "input", parse(from_os_str))]
    input: PathBuf,

    /// Elasticsearch parameters.
    #[structopt(
        short = "c",
        long = "connection-string",
        default_value = "http://localhost:9200/munin"
    )]
    connection_string: String,

    /// Name of the dataset.
    /// A dataset is a label, that can be used for filtering the data.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,

    /// Indicate if the POI dataset is private
    #[structopt(short = "p", long = "private")]
    private: bool,

    /// Number of threads to use
    #[structopt(
        short = "t",
        long = "nb-threads",
        default_value = &DEFAULT_NB_THREADS
    )]
    nb_threads: usize,

    /// Number of shards for the es index
    #[structopt(short = "s", long = "nb-shards", default_value = "1")]
    nb_shards: usize,

    /// Number of replicas for the es index
    #[structopt(short = "r", long = "nb-replicas", default_value = "1")]
    nb_replicas: usize,
}

fn run(args: Args) -> Result<(), mimirsbrunn::Error> {
    let visibility = if args.private {
        IndexVisibility::Private
    } else {
        IndexVisibility::Public
    };

    index_poi(
        &args.connection_string,
        &args.dataset,
        &args.input,
        visibility,
        args.nb_shards,
        args.nb_replicas,
    )
}
fn main() {
    mimirsbrunn::utils::launch_run(run);
}
<commit_msg>Fixed linting<commit_after>// Copyright © 2019, Canal TP and/or its affiliates. All rights reserved.
//
// This file is part of Navitia,
//     the software to build cool stuff with public transport.
//
// Hope you'll enjoy and contribute to this project,
//     powered by Canal TP (www.canaltp.fr).
// Help us simplify mobility and open public transport:
//     a non ending quest to the responsive locomotion way of traveling!
//
// LICENCE: This program is free software; you can redistribute it
// and/or modify it under the terms of the GNU Affero General Public
// License as published by the Free Software Foundation, either
// version 3 of the License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see
// <http://www.gnu.org/licenses/>.
//
// Stay tuned using
// twitter @navitia
// IRC #navitia on freenode
// https://groups.google.com/d/forum/navitia
// www.navitia.io

use slog_scope::{error, info};

use failure::format_err;
use lazy_static::lazy_static;
use mimir::objects::{Coord, I18nProperties, Poi, PoiType, Property};
use mimir::rubber::{IndexSettings, IndexVisibility, Rubber, TypedIndex};
use mimirsbrunn::{admin_geofinder::AdminGeoFinder, labels, utils};
use navitia_poi_model::{Model as NavitiaModel, Poi as NavitiaPoi, PoiType as NavitiaPoiType};
use std::collections::HashMap;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use structopt::StructOpt;

lazy_static! {
    static ref DEFAULT_NB_THREADS: String = num_cpus::get().to_string();
}

// This function takes a Poi from the navitia model, ie from the CSV deserialization, and returns
// a Poi from the mimir model, with all the contextual information added.
fn into_mimir_poi(
    poi: NavitiaPoi,
    poi_types: &HashMap<String, NavitiaPoiType>,
    rubber: &mut Rubber,
    admins_geofinder: &AdminGeoFinder,
) -> Result<Poi, mimirsbrunn::Error> {
    let poi_type = poi_types
        .get(&poi.poi_type_id)
        .ok_or_else(|| format_err!("could not find Poi Type '{}'", poi.poi_type_id))
        .map(PoiType::from)?;

    let coord = Coord::from(&poi.coord);

    let place = rubber
        .get_address(&coord) // No timeout
        .ok()
        .and_then(|addrs| addrs.into_iter().next()); // Take the first place

    let addr = place.as_ref().and_then(|place| place.address());

    // We the the admins from the address, or, if we don't have any, from the geofinder.
    let admins = place.map_or_else(|| admins_geofinder.get(&poi.coord), |addr| addr.admins());

    if admins.is_empty() {
        return Err(format_err!("Could not find admins for POI {}", &poi.id));
    }

    // The weight is that of the city, or 0.0 if there is no such admin.
    let weight: f64 = admins
        .iter()
        .filter(|adm| adm.is_city())
        .map(|adm| adm.weight)
        .next()
        .unwrap_or(0.0);

    let country_codes = utils::find_country_codes(admins.iter().map(|a| a.deref()));

    let label =
        labels::format_poi_label(&poi.name, admins.iter().map(|a| a.deref()), &country_codes);

    let poi = Poi {
        id: mimir::objects::normalize_id("poi", &poi.id),
        label,
        name: poi.name,
        coord,
        approx_coord: Some(coord.into()),
        administrative_regions: admins,
        weight,
        zip_codes: vec![],
        poi_type,
        properties: poi.properties.into_iter().map(Property::from).collect(),
        address: addr,
        country_codes,
        names: I18nProperties::default(),
        labels: I18nProperties::default(),
        distance: None,
        context: None,
    };

    Ok(poi)
}

fn import_pois(
    rubber: &mut Rubber,
    index: &TypedIndex<Poi>,
    admins_geofinder: AdminGeoFinder,
    file: &Path,
) -> Result<(), mimirsbrunn::Error>
where
{
    info!("Add data in elasticsearch db.");

    let model = NavitiaModel::try_from_path(file)?;
    let poi_types = model.poi_types;

    // Note: We're ignoring those POIs that fail to be enriched.
    let pois: Vec<_> = model
        .pois
        .into_iter()
        .filter_map(|(id, poi)| {
            into_mimir_poi(poi, &poi_types, rubber, &admins_geofinder)
                .map_err(|err| info!("Could not extract information for POI '{}': {}", id, err))
                .ok()
        })
        .collect(); // TODO Can we get rid of collect, and chain with the following rubber...?

    let count = rubber
        .bulk_index(&index, pois.into_iter())
        .map_err(|err| format_err!("Failed bulk insertion {}", err))?;

    info!("importing POIs: {} POIs added.", count);

    Ok(())
}

/// This function initializes the ES context: It creates an index for this dataset,
/// and then import the POIs in it.
fn index_poi(
    cnx_string: &str,
    dataset: &str,
    file: &Path,
    visibility: IndexVisibility,
    nb_shards: usize,
    nb_replicas: usize,
) -> Result<(), mimirsbrunn::Error>
where
{
    let mut rubber = Rubber::new(cnx_string);
    rubber.initialize_templates()?;

    let settings = IndexSettings {
        nb_shards,
        nb_replicas,
    };

    let index = rubber.make_index(dataset, &settings)?;

    let admins = rubber.get_all_admins().map_err(|err| {
        error!("Administratives regions not found in es db");
        err
    })?;
    let admins_geofinder = admins.into_iter().collect();

    import_pois(&mut rubber, &index, admins_geofinder, file)?;

    rubber
        .publish_index(dataset, index, visibility)
        .map_err(|err| format_err!("Failed to publish index {}.", err))
}

#[derive(StructOpt, Debug)]
struct Args {
    /// POI file
    #[structopt(short = "i", long = "input", parse(from_os_str))]
    input: PathBuf,

    /// Elasticsearch parameters.
    #[structopt(
        short = "c",
        long = "connection-string",
        default_value = "http://localhost:9200/munin"
    )]
    connection_string: String,

    /// Name of the dataset.
    /// A dataset is a label, that can be used for filtering the data.
    #[structopt(short = "d", long = "dataset", default_value = "fr")]
    dataset: String,

    /// Indicate if the POI dataset is private
    #[structopt(short = "p", long = "private")]
    private: bool,

    /// Number of threads to use
    #[structopt(
        short = "t",
        long = "nb-threads",
        default_value = &DEFAULT_NB_THREADS
    )]
    nb_threads: usize,

    /// Number of shards for the es index
    #[structopt(short = "s", long = "nb-shards", default_value = "1")]
    nb_shards: usize,

    /// Number of replicas for the es index
    #[structopt(short = "r", long = "nb-replicas", default_value = "1")]
    nb_replicas: usize,
}

fn run(args: Args) -> Result<(), mimirsbrunn::Error> {
    let visibility = if args.private {
        IndexVisibility::Private
    } else {
        IndexVisibility::Public
    };

    index_poi(
        &args.connection_string,
        &args.dataset,
        &args.input,
        visibility,
        args.nb_shards,
        args.nb_replicas,
    )
}
fn main() {
    mimirsbrunn::utils::launch_run(run);
}
<|endoftext|>


<commit_before>extern crate libsqlite3_sys as ffi;

use std::collections::HashMap;
use std::os::raw as libc;
use std::ptr::NonNull;
use std::{slice, str};

use crate::row::*;
use crate::sqlite::{Sqlite, SqliteType};

/// Raw sqlite value as received from the database
///
/// Use existing `FromSql` implementations to convert this into
/// rust values:
#[allow(missing_debug_implementations, missing_copy_implementations)]
pub struct SqliteValue {
    value: ffi::sqlite3_value,
}

pub struct SqliteRow {
    stmt: NonNull<ffi::sqlite3_stmt>,
    next_col_index: libc::c_int,
}

impl SqliteValue {
    #[allow(clippy::new_ret_no_self)]
    pub(crate) unsafe fn new<'a>(inner: *mut ffi::sqlite3_value) -> Option<&'a Self> {
        (inner as *const _ as *const Self).as_ref().and_then(|v| {
            if v.is_null() {
                None
            } else {
                Some(v)
            }
        })
    }

    pub(crate) fn read_text(&self) -> &str {
        unsafe {
            let ptr = ffi::sqlite3_value_text(self.value());
            let len = ffi::sqlite3_value_bytes(self.value());
            let bytes = slice::from_raw_parts(ptr as *const u8, len as usize);
            // The string is guaranteed to be utf8 according to
            // https://www.sqlite.org/c3ref/value_blob.html
            str::from_utf8_unchecked(bytes)
        }
    }

    pub(crate) fn read_blob(&self) -> &[u8] {
        unsafe {
            let ptr = ffi::sqlite3_value_blob(self.value());
            let len = ffi::sqlite3_value_bytes(self.value());
            slice::from_raw_parts(ptr as *const u8, len as usize)
        }
    }

    pub(crate) fn read_integer(&self) -> i32 {
        unsafe { ffi::sqlite3_value_int(self.value()) as i32 }
    }

    pub(crate) fn read_long(&self) -> i64 {
        unsafe { ffi::sqlite3_value_int64(self.value()) as i64 }
    }

    pub(crate) fn read_double(&self) -> f64 {
        unsafe { ffi::sqlite3_value_double(self.value()) as f64 }
    }

    pub(crate) fn is_null(&self) -> bool {
        self.value_type().is_none()
    }

    /// Get the type of the value as returned by sqlite
    pub fn value_type(&self) -> Option<SqliteType> {
        let tpe = unsafe { ffi::sqlite3_value_type(self.value()) };
        match tpe {
            ffi::SQLITE_TEXT => Some(SqliteType::Text),
            ffi::SQLITE_INTEGER => Some(SqliteType::Long),
            ffi::SQLITE_FLOAT => Some(SqliteType::Double),
            ffi::SQLITE_BLOB => Some(SqliteType::Binary),
            ffi::SQLITE_NULL => None,
            _ => unreachable!("Sqlite does saying this is not reachable"),
        }
    }

    fn value(&self) -> *mut ffi::sqlite3_value {
        &self.value as *const _ as _
    }
}

impl SqliteRow {
    pub(crate) fn new(inner_statement: NonNull<ffi::sqlite3_stmt>) -> Self {
        SqliteRow {
            stmt: inner_statement,
            next_col_index: 0,
        }
    }

    pub fn into_named<'a>(self, indices: &'a HashMap<&'a str, usize>) -> SqliteNamedRow<'a> {
        SqliteNamedRow {
            stmt: self.stmt,
            column_indices: indices,
        }
    }
}

impl Row<Sqlite> for SqliteRow {
    fn take(&mut self) -> Option<&SqliteValue> {
        let col_index = self.next_col_index;
        self.next_col_index += 1;

        unsafe {
            let ptr = ffi::sqlite3_column_value(self.stmt.as_ptr(), col_index);
            SqliteValue::new(ptr)
        }
    }

    fn next_is_null(&self, count: usize) -> bool {
        (0..count).all(|i| {
            let idx = self.next_col_index + i as libc::c_int;
            let tpe = unsafe { ffi::sqlite3_column_type(self.stmt.as_ptr(), idx) };
            tpe == ffi::SQLITE_NULL
        })
    }

    fn column_name(&self) -> Option<&str> {
        unsafe {
            let ptr = ffi::sqlite3_column_name(self.stmt.as_ptr(), self.next_col_index);
            Some(std::ffi::CStr::from_ptr(ptr).to_str().expect(
                "The Sqlite documentation states that this is UTF8. \
                 If you see this error message something has gone \
                 horribliy wrong. Please open an issue at the \
                 diesel repository.",
            ))
        }
    }

    fn column_count(&self) -> usize {
        unsafe { ffi::sqlite3_column_count(self.stmt.as_ptr()) as usize }
    }
}

pub struct SqliteNamedRow<'a> {
    stmt: NonNull<ffi::sqlite3_stmt>,
    column_indices: &'a HashMap<&'a str, usize>,
}

impl<'a> NamedRow<Sqlite> for SqliteNamedRow<'a> {
    fn index_of(&self, column_name: &str) -> Option<usize> {
        self.column_indices.get(column_name).cloned()
    }

    fn get_raw_value(&self, idx: usize) -> Option<&SqliteValue> {
        unsafe {
            let ptr = ffi::sqlite3_column_value(self.stmt.as_ptr(), idx as libc::c_int);
            SqliteValue::new(ptr)
        }
    }
}
<commit_msg>Implement `NamedRow` change for sqlite<commit_after>extern crate libsqlite3_sys as ffi;

use std::collections::HashMap;
use std::os::raw as libc;
use std::ptr::NonNull;
use std::{slice, str};

use crate::row::*;
use crate::sqlite::{Sqlite, SqliteType};

/// Raw sqlite value as received from the database
///
/// Use existing `FromSql` implementations to convert this into
/// rust values:
#[allow(missing_debug_implementations, missing_copy_implementations)]
pub struct SqliteValue {
    value: ffi::sqlite3_value,
}

pub struct SqliteRow {
    stmt: NonNull<ffi::sqlite3_stmt>,
    next_col_index: libc::c_int,
}

impl SqliteValue {
    #[allow(clippy::new_ret_no_self)]
    pub(crate) unsafe fn new<'a>(inner: *mut ffi::sqlite3_value) -> Option<&'a Self> {
        (inner as *const _ as *const Self).as_ref().and_then(|v| {
            if v.is_null() {
                None
            } else {
                Some(v)
            }
        })
    }

    pub(crate) fn read_text(&self) -> &str {
        unsafe {
            let ptr = ffi::sqlite3_value_text(self.value());
            let len = ffi::sqlite3_value_bytes(self.value());
            let bytes = slice::from_raw_parts(ptr as *const u8, len as usize);
            // The string is guaranteed to be utf8 according to
            // https://www.sqlite.org/c3ref/value_blob.html
            str::from_utf8_unchecked(bytes)
        }
    }

    pub(crate) fn read_blob(&self) -> &[u8] {
        unsafe {
            let ptr = ffi::sqlite3_value_blob(self.value());
            let len = ffi::sqlite3_value_bytes(self.value());
            slice::from_raw_parts(ptr as *const u8, len as usize)
        }
    }

    pub(crate) fn read_integer(&self) -> i32 {
        unsafe { ffi::sqlite3_value_int(self.value()) as i32 }
    }

    pub(crate) fn read_long(&self) -> i64 {
        unsafe { ffi::sqlite3_value_int64(self.value()) as i64 }
    }

    pub(crate) fn read_double(&self) -> f64 {
        unsafe { ffi::sqlite3_value_double(self.value()) as f64 }
    }

    pub(crate) fn is_null(&self) -> bool {
        self.value_type().is_none()
    }

    /// Get the type of the value as returned by sqlite
    pub fn value_type(&self) -> Option<SqliteType> {
        let tpe = unsafe { ffi::sqlite3_value_type(self.value()) };
        match tpe {
            ffi::SQLITE_TEXT => Some(SqliteType::Text),
            ffi::SQLITE_INTEGER => Some(SqliteType::Long),
            ffi::SQLITE_FLOAT => Some(SqliteType::Double),
            ffi::SQLITE_BLOB => Some(SqliteType::Binary),
            ffi::SQLITE_NULL => None,
            _ => unreachable!("Sqlite does saying this is not reachable"),
        }
    }

    fn value(&self) -> *mut ffi::sqlite3_value {
        &self.value as *const _ as _
    }
}

impl SqliteRow {
    pub(crate) fn new(inner_statement: NonNull<ffi::sqlite3_stmt>) -> Self {
        SqliteRow {
            stmt: inner_statement,
            next_col_index: 0,
        }
    }

    pub fn into_named<'a>(self, indices: &'a HashMap<&'a str, usize>) -> SqliteNamedRow<'a> {
        SqliteNamedRow {
            stmt: self.stmt,
            column_indices: indices,
        }
    }
}

impl Row<Sqlite> for SqliteRow {
    fn take(&mut self) -> Option<&SqliteValue> {
        let col_index = self.next_col_index;
        self.next_col_index += 1;

        unsafe {
            let ptr = ffi::sqlite3_column_value(self.stmt.as_ptr(), col_index);
            SqliteValue::new(ptr)
        }
    }

    fn next_is_null(&self, count: usize) -> bool {
        (0..count).all(|i| {
            let idx = self.next_col_index + i as libc::c_int;
            let tpe = unsafe { ffi::sqlite3_column_type(self.stmt.as_ptr(), idx) };
            tpe == ffi::SQLITE_NULL
        })
    }

    fn column_name(&self) -> Option<&str> {
        column_name(self.stmt, self.next_col_index)
    }

    fn column_count(&self) -> usize {
        column_count(self.stmt) as usize
    }
}

pub struct SqliteNamedRow<'a> {
    stmt: NonNull<ffi::sqlite3_stmt>,
    column_indices: &'a HashMap<&'a str, usize>,
}

impl<'a> NamedRow<Sqlite> for SqliteNamedRow<'a> {
    fn index_of(&self, column_name: &str) -> Option<usize> {
        self.column_indices.get(column_name).cloned()
    }

    fn get_raw_value(&self, idx: usize) -> Option<&SqliteValue> {
        unsafe {
            let ptr = ffi::sqlite3_column_value(self.stmt.as_ptr(), idx as libc::c_int);
            SqliteValue::new(ptr)
        }
    }

    fn field_names(&self) -> Vec<&str> {
        (0..column_count(self.stmt))
            .filter_map(|c| column_name(self.stmt, c))
            .collect()
    }
}

fn column_name<'a>(stmt: NonNull<ffi::sqlite3_stmt>, field_number: i32) -> Option<&'a str> {
    unsafe {
        let ptr = ffi::sqlite3_column_name(stmt.as_ptr(), field_number);
        if ptr.is_null() {
            None
        } else {
            Some(std::ffi::CStr::from_ptr(ptr).to_str().expect(
                "The Sqlite documentation states that this is UTF8. \
                 If you see this error message something has gone \
                 horribliy wrong. Please open an issue at the \
                 diesel repository.",
            ))
        }
    }
}

fn column_count(stmt: NonNull<ffi::sqlite3_stmt>) -> i32 {
    unsafe { ffi::sqlite3_column_count(stmt.as_ptr()) }
}
<|endoftext|>


<commit_before>//! [PUT /_matrix/federation/v1/send/{txnId}](https://matrix.org/docs/spec/server_server/r0.1.3#put-matrix-federation-v1-send-txnid)

use std::{collections::BTreeMap, time::SystemTime};

use ruma_api::ruma_api;
use ruma_events::pdu::Pdu;
use ruma_identifiers::{EventId, ServerName};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;

ruma_api! {
    metadata: {
        description: "Send transaction messages to another server",
        name: "send_transaction_message",
        method: PUT,
        path: "/_matrix/federation/v1/send/:transaction_id",
        rate_limited: false,
        authentication: ServerSignatures,
    }

    request: {
        /// A transaction ID unique between sending and receiving homeservers.
        #[ruma_api(path)]
        pub transaction_id: &'a str,

        /// The server_name of the homeserver sending this transaction.
        pub origin: &'a ServerName,

        /// POSIX timestamp in milliseconds on the originating homeserver when this transaction
        /// started.
        #[serde(with = "ruma_serde::time::ms_since_unix_epoch")]
        pub origin_server_ts: SystemTime,

        /// List of persistent updates to rooms.
        ///
        /// Must not be more than 50 items.
        pub pdus: &'a [Pdu],

        /// List of ephemeral messages.
        ///
        /// Must not be more than 100 items.
        #[serde(default, skip_serializing_if = "<[_]>::is_empty")]
        pub edus: &'a [Edu],
    }

    #[derive(Default)]
    response: {
        /// Map of event IDs and response for each PDU given in the request.
        #[serde(with = "crate::serde::pdu_process_response")]
        pub pdus: BTreeMap<EventId, Result<(), String>>,
    }
}

impl<'a> Request<'a> {
    /// Creates a new `Request` with the given transaction ID, origin, timestamp.
    ///
    /// The PDU and EDU lists will start off empty.
    pub fn new(
        transaction_id: &'a str,
        origin: &'a ServerName,
        origin_server_ts: SystemTime,
    ) -> Self {
        Self { transaction_id, origin, origin_server_ts, pdus: &[], edus: &[] }
    }
}

impl Response {
    /// Creates a new `Response` with the given PDUs.
    pub fn new(pdus: BTreeMap<EventId, Result<(), String>>) -> Self {
        Self { pdus }
    }
}

/// Type for passing ephemeral data to homeservers.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Edu {
    /// Type of the ephemeral message.
    pub edu_type: String,

    /// Content of ephemeral message
    pub content: JsonValue,
}
<commit_msg>federation-api: Fix missing Raw<> for Pdu, Edu<commit_after>//! [PUT /_matrix/federation/v1/send/{txnId}](https://matrix.org/docs/spec/server_server/r0.1.3#put-matrix-federation-v1-send-txnid)

use std::{collections::BTreeMap, time::SystemTime};

use ruma_api::ruma_api;
use ruma_common::Raw;
use ruma_events::pdu::Pdu;
use ruma_identifiers::{EventId, ServerName};
use serde::{Deserialize, Serialize};
use serde_json::Value as JsonValue;

ruma_api! {
    metadata: {
        description: "Send transaction messages to another server",
        name: "send_transaction_message",
        method: PUT,
        path: "/_matrix/federation/v1/send/:transaction_id",
        rate_limited: false,
        authentication: ServerSignatures,
    }

    request: {
        /// A transaction ID unique between sending and receiving homeservers.
        #[ruma_api(path)]
        pub transaction_id: &'a str,

        /// The server_name of the homeserver sending this transaction.
        pub origin: &'a ServerName,

        /// POSIX timestamp in milliseconds on the originating homeserver when this transaction
        /// started.
        #[serde(with = "ruma_serde::time::ms_since_unix_epoch")]
        pub origin_server_ts: SystemTime,

        /// List of persistent updates to rooms.
        ///
        /// Must not be more than 50 items.
        pub pdus: &'a [Raw<Pdu>],

        /// List of ephemeral messages.
        ///
        /// Must not be more than 100 items.
        #[serde(default, skip_serializing_if = "<[_]>::is_empty")]
        pub edus: &'a [Raw<Edu>],
    }

    #[derive(Default)]
    response: {
        /// Map of event IDs and response for each PDU given in the request.
        #[serde(with = "crate::serde::pdu_process_response")]
        pub pdus: BTreeMap<EventId, Result<(), String>>,
    }
}

impl<'a> Request<'a> {
    /// Creates a new `Request` with the given transaction ID, origin, timestamp.
    ///
    /// The PDU and EDU lists will start off empty.
    pub fn new(
        transaction_id: &'a str,
        origin: &'a ServerName,
        origin_server_ts: SystemTime,
    ) -> Self {
        Self { transaction_id, origin, origin_server_ts, pdus: &[], edus: &[] }
    }
}

impl Response {
    /// Creates a new `Response` with the given PDUs.
    pub fn new(pdus: BTreeMap<EventId, Result<(), String>>) -> Self {
        Self { pdus }
    }
}

/// Type for passing ephemeral data to homeservers.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Edu {
    /// Type of the ephemeral message.
    pub edu_type: String,

    /// Content of ephemeral message
    pub content: JsonValue,
}
<|endoftext|>


<commit_before>use std::ffi::OsStr;
use std::path::PathBuf;

use regex::RegexSet;

#[derive(Debug)]
struct Message {
    path: PathBuf,
    bad_lines: Vec<String>,
}

impl Message {
    fn new(path: PathBuf) -> Self {
        let content: String = std::fs::read_to_string(&path).unwrap();
        // we don't want the first letter after "error: ", "help: " ... to be capitalized
        // also no puncutation (except for "?" ?) at the end of a line
        let regex_set: RegexSet = RegexSet::new(&[
            r"error: [A-Z]",
            r"help: [A-Z]",
            r"warning: [A-Z]",
            r"note: [A-Z]",
            r"try this: [A-Z]",
            r"error: .*[.!]$",
            r"help: .*[.!]$",
            r"warning: .*[.!]$",
            r"note: .*[.!]$",
            r"try this: .*[.!]$",
        ])
        .unwrap();

        // sometimes the first character is capitalized and it is legal (like in "Iterator...") or
        // we want to ask a question ending in "?"
        let exceptions_set: RegexSet = RegexSet::new(&[
            r".*error: I see you're using a LinkedList! Perhaps you meant some other data structure?",
            r".*C-like enum variant discriminant is not portable to 32-bit targets",
            r".*Iterator::step_by(0) will panic at runtime",
            r".*did you mean `unix`?",
            r".*the arguments may be inverted...",
            r".*Intel x86 assembly syntax used",
            r".*AT&T x86 assembly syntax used",
            r".*remove .*the return type...",
            r"note: Clippy version: .*",
            r"the compiler unexpectedly panicked. this is a bug.",
            r".*help: I think you meant: .*",
            r"Iterator.* will panic at runtime",
        ])
        .unwrap();

        let bad_lines = content
            .lines()
            .filter(|line| regex_set.matches(line).matched_any())
            // ignore exceptions
            .filter(|line| !exceptions_set.matches(line).matched_any())
            .map(ToOwned::to_owned)
            .collect::<Vec<String>>();

        Message { path, bad_lines }
    }
}

#[test]
fn lint_message_convention() {
    // make sure that lint messages:
    // * are not capitalized
    // * don't have puncuation at the end of the last sentence

    // these directories have interesting tests
    let test_dirs = ["ui", "ui-cargo", "ui-internal", "ui-toml"]
        .iter()
        .map(PathBuf::from)
        .map(|p| {
            let base = PathBuf::from("tests");
            base.join(p)
        });

    // gather all .stderr files
    let tests = test_dirs
        .flat_map(|dir| {
            std::fs::read_dir(dir)
                .expect("failed to read dir")
                .map(|direntry| direntry.unwrap().path())
        })
        .filter(|file| matches!(file.extension().map(OsStr::to_str), Some(Some("stderr"))));

    // get all files that have any "bad lines" in them
    let bad_tests: Vec<Message> = tests
        .map(Message::new)
        .filter(|message| !message.bad_lines.is_empty())
        .collect();

    bad_tests.iter().for_each(|message| {
        eprintln!(
            "error: the test '{}' contained the following nonconforming lines :",
            message.path.display()
        );
        message.bad_lines.iter().for_each(|line| eprintln!("{}", line));
        eprintln!("\n\n");
    });

    eprintln!("\n\n\nLint message should not start with a capital letter and should not have punctuation at the end of the message unless multiple sentences are needed.");
    eprintln!("Check out the rustc-dev-guide for more information:");
    eprintln!("https://rustc-dev-guide.rust-lang.org/diagnostics.html#diagnostic-structure\n\n\n");

    assert!(bad_tests.is_empty());
}
<commit_msg>disable lint_message_convention test inside the rustc test suite<commit_after>use std::ffi::OsStr;
use std::path::PathBuf;

use regex::RegexSet;

#[derive(Debug)]
struct Message {
    path: PathBuf,
    bad_lines: Vec<String>,
}

impl Message {
    fn new(path: PathBuf) -> Self {
        let content: String = std::fs::read_to_string(&path).unwrap();
        // we don't want the first letter after "error: ", "help: " ... to be capitalized
        // also no puncutation (except for "?" ?) at the end of a line
        let regex_set: RegexSet = RegexSet::new(&[
            r"error: [A-Z]",
            r"help: [A-Z]",
            r"warning: [A-Z]",
            r"note: [A-Z]",
            r"try this: [A-Z]",
            r"error: .*[.!]$",
            r"help: .*[.!]$",
            r"warning: .*[.!]$",
            r"note: .*[.!]$",
            r"try this: .*[.!]$",
        ])
        .unwrap();

        // sometimes the first character is capitalized and it is legal (like in "Iterator...") or
        // we want to ask a question ending in "?"
        let exceptions_set: RegexSet = RegexSet::new(&[
            r".*error: I see you're using a LinkedList! Perhaps you meant some other data structure?",
            r".*C-like enum variant discriminant is not portable to 32-bit targets",
            r".*Iterator::step_by(0) will panic at runtime",
            r".*did you mean `unix`?",
            r".*the arguments may be inverted...",
            r".*Intel x86 assembly syntax used",
            r".*AT&T x86 assembly syntax used",
            r".*remove .*the return type...",
            r"note: Clippy version: .*",
            r"the compiler unexpectedly panicked. this is a bug.",
            r".*help: I think you meant: .*",
            r"Iterator.* will panic at runtime",
        ])
        .unwrap();

        let bad_lines = content
            .lines()
            .filter(|line| regex_set.matches(line).matched_any())
            // ignore exceptions
            .filter(|line| !exceptions_set.matches(line).matched_any())
            .map(ToOwned::to_owned)
            .collect::<Vec<String>>();

        Message { path, bad_lines }
    }
}

#[test]
fn lint_message_convention() {
    // disable the test inside the rustc test suite
    if option_env!("RUSTC_TEST_SUITE").is_some() {
        return;
    }

    // make sure that lint messages:
    // * are not capitalized
    // * don't have puncuation at the end of the last sentence

    // these directories have interesting tests
    let test_dirs = ["ui", "ui-cargo", "ui-internal", "ui-toml"]
        .iter()
        .map(PathBuf::from)
        .map(|p| {
            let base = PathBuf::from("tests");
            base.join(p)
        });

    // gather all .stderr files
    let tests = test_dirs
        .flat_map(|dir| {
            std::fs::read_dir(dir)
                .expect("failed to read dir")
                .map(|direntry| direntry.unwrap().path())
        })
        .filter(|file| matches!(file.extension().map(OsStr::to_str), Some(Some("stderr"))));

    // get all files that have any "bad lines" in them
    let bad_tests: Vec<Message> = tests
        .map(Message::new)
        .filter(|message| !message.bad_lines.is_empty())
        .collect();

    bad_tests.iter().for_each(|message| {
        eprintln!(
            "error: the test '{}' contained the following nonconforming lines :",
            message.path.display()
        );
        message.bad_lines.iter().for_each(|line| eprintln!("{}", line));
        eprintln!("\n\n");
    });

    eprintln!("\n\n\nLint message should not start with a capital letter and should not have punctuation at the end of the message unless multiple sentences are needed.");
    eprintln!("Check out the rustc-dev-guide for more information:");
    eprintln!("https://rustc-dev-guide.rust-lang.org/diagnostics.html#diagnostic-structure\n\n\n");

    assert!(bad_tests.is_empty());
}
<|endoftext|>


<commit_before>//! Tests for custom cargo commands and other global command features.

use std::env;
use std::fs::{self, File};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::str;

use cargo_test_support::cargo_process;
use cargo_test_support::paths::{self, CargoPathExt};
use cargo_test_support::registry::Package;
use cargo_test_support::{basic_bin_manifest, basic_manifest, cargo_exe, project, Project};

#[cfg_attr(windows, allow(dead_code))]
enum FakeKind<'a> {
    Executable,
    Symlink { target: &'a Path },
}

/// Adds an empty file with executable flags (and platform-dependent suffix).
//
// TODO: move this to `Project` if other cases using this emerge.
fn fake_file(proj: Project, dir: &Path, name: &str, kind: &FakeKind<'_>) -> Project {
    let path = proj
        .root()
        .join(dir)
        .join(&format!("{}{}", name, env::consts::EXE_SUFFIX));
    path.parent().unwrap().mkdir_p();
    match *kind {
        FakeKind::Executable => {
            File::create(&path).unwrap();
            make_executable(&path);
        }
        FakeKind::Symlink { target } => {
            make_symlink(&path, target);
        }
    }
    return proj;

    #[cfg(unix)]
    fn make_executable(p: &Path) {
        use std::os::unix::prelude::*;

        let mut perms = fs::metadata(p).unwrap().permissions();
        let mode = perms.mode();
        perms.set_mode(mode | 0o111);
        fs::set_permissions(p, perms).unwrap();
    }
    #[cfg(windows)]
    fn make_executable(_: &Path) {}
    #[cfg(unix)]
    fn make_symlink(p: &Path, t: &Path) {
        ::std::os::unix::fs::symlink(t, p).expect("Failed to create symlink");
    }
    #[cfg(windows)]
    fn make_symlink(_: &Path, _: &Path) {
        panic!("Not supported")
    }
}

fn path() -> Vec<PathBuf> {
    env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect()
}

#[cargo_test]
fn list_commands_with_descriptions() {
    let p = project().build();
    p.cargo("--list")
        .with_stdout_contains(
            "    build                Compile a local package and all of its dependencies",
        )
        // Assert that `read-manifest` prints the right one-line description followed by another
        // command, indented.
        .with_stdout_contains(
            "    read-manifest        Print a JSON representation of a Cargo.toml manifest.",
        )
        .run();
}

#[cargo_test]
fn list_command_looks_at_path() {
    let proj = project().build();
    let proj = fake_file(
        proj,
        Path::new("path-test"),
        "cargo-1",
        &FakeKind::Executable,
    );

    let mut path = path();
    path.push(proj.root().join("path-test"));
    let path = env::join_paths(path.iter()).unwrap();
    let output = cargo_process("-v --list")
        .env("PATH", &path)
        .exec_with_output()
        .unwrap();
    let output = str::from_utf8(&output.stdout).unwrap();
    assert!(
        output.contains("\n    1                   "),
        "missing 1: {}",
        output
    );
}

// Windows and symlinks don't currently mix well.
#[cfg(unix)]
#[cargo_test]
fn list_command_resolves_symlinks() {
    let proj = project().build();
    let proj = fake_file(
        proj,
        Path::new("path-test"),
        "cargo-2",
        &FakeKind::Symlink {
            target: &cargo_exe(),
        },
    );

    let mut path = path();
    path.push(proj.root().join("path-test"));
    let path = env::join_paths(path.iter()).unwrap();
    let output = cargo_process("-v --list")
        .env("PATH", &path)
        .exec_with_output()
        .unwrap();
    let output = str::from_utf8(&output.stdout).unwrap();
    assert!(
        output.contains("\n    2                   "),
        "missing 2: {}",
        output
    );
}

#[cargo_test]
fn find_closest_biuld_to_build() {
    cargo_process("biuld")
        .with_status(101)
        .with_stderr_contains(
            "\
error: no such subcommand: `biuld`

<tab>Did you mean `build`?
",
        )
        .run();

    // But, if we actually have `biuld`, it must work!
    // https://github.com/rust-lang/cargo/issues/5201
    Package::new("cargo-biuld", "1.0.0")
        .file(
            "src/main.rs",
            r#"
            fn main() {
                println!("Similar, but not identical to, build");
            }
        "#,
        )
        .publish();

    cargo_process("install cargo-biuld").run();
    cargo_process("biuld")
        .with_stdout("Similar, but not identical to, build\n")
        .run();
    cargo_process("--list")
        .with_stdout_contains(
            "    build                Compile a local package and all of its dependencies\n",
        )
        .with_stdout_contains("    biuld\n")
        .run();
}

#[cargo_test]
fn find_closest_alias() {
    let root = paths::root();
    let my_home = root.join("my_home");
    fs::create_dir(&my_home).unwrap();
    fs::write(
        &my_home.join("config"),
        r#"
            [alias]
            myalias = "build"
        "#,
    )
    .unwrap();

    cargo_process("myalais")
        .env("CARGO_HOME", &my_home)
        .with_status(101)
        .with_stderr_contains(
            "\
error: no such subcommand: `myalais`

<tab>Did you mean `myalias`?
",
        )
        .run();

    // But, if no alias is defined, it must not suggest one!
    cargo_process("myalais")
        .with_status(101)
        .with_stderr_contains(
            "\
error: no such subcommand: `myalais`
",
        )
        .with_stderr_does_not_contain(
            "\
<tab>Did you mean `myalias`?
",
        )
        .run();
}

// If a subcommand is more than an edit distance of 3 away, we don't make a suggestion.
#[cargo_test]
fn find_closest_dont_correct_nonsense() {
    cargo_process("there-is-no-way-that-there-is-a-command-close-to-this")
        .cwd(&paths::root())
        .with_status(101)
        .with_stderr(
            "[ERROR] no such subcommand: \
                        `there-is-no-way-that-there-is-a-command-close-to-this`
",
        )
        .run();
}

#[cargo_test]
fn displays_subcommand_on_error() {
    cargo_process("invalid-command")
        .with_status(101)
        .with_stderr("[ERROR] no such subcommand: `invalid-command`\n")
        .run();
}

#[cargo_test]
fn override_cargo_home() {
    let root = paths::root();
    let my_home = root.join("my_home");
    fs::create_dir(&my_home).unwrap();
    fs::write(
        &my_home.join("config"),
        r#"
            [cargo-new]
            name = "foo"
            email = "bar"
            git = false
        "#,
    )
    .unwrap();

    cargo_process("new foo")
        .env("USER", "foo")
        .env("CARGO_HOME", &my_home)
        .run();

    let toml = paths::root().join("foo/Cargo.toml");
    let contents = fs::read_to_string(&toml).unwrap();
    assert!(contents.contains(r#"authors = ["foo <bar>"]"#));
}

#[cargo_test]
fn cargo_subcommand_env() {
    let src = format!(
        r#"
        use std::env;

        fn main() {{
            println!("{{}}", env::var("{}").unwrap());
        }}
        "#,
        cargo::CARGO_ENV
    );

    let p = project()
        .at("cargo-envtest")
        .file("Cargo.toml", &basic_bin_manifest("cargo-envtest"))
        .file("src/main.rs", &src)
        .build();

    let target_dir = p.target_debug_dir();

    p.cargo("build").run();
    assert!(p.bin("cargo-envtest").is_file());

    let cargo = cargo_exe().canonicalize().unwrap();
    let mut path = path();
    path.push(target_dir);
    let path = env::join_paths(path.iter()).unwrap();

    cargo_process("envtest")
        .env("PATH", &path)
        .with_stdout(cargo.to_str().unwrap())
        .run();
}

#[cargo_test]
fn cargo_subcommand_args() {
    let p = project()
        .at("cargo-foo")
        .file("Cargo.toml", &basic_manifest("cargo-foo", "0.0.1"))
        .file(
            "src/main.rs",
            r#"
            fn main() {
                let args: Vec<_> = ::std::env::args().collect();
                println!("{:?}", args);
            }
        "#,
        )
        .build();

    p.cargo("build").run();
    let cargo_foo_bin = p.bin("cargo-foo");
    assert!(cargo_foo_bin.is_file());

    let mut path = path();
    path.push(p.target_debug_dir());
    let path = env::join_paths(path.iter()).unwrap();

    cargo_process("foo bar -v --help")
        .env("PATH", &path)
        .with_stdout(
            r#"["[CWD]/cargo-foo/target/debug/cargo-foo[EXE]", "foo", "bar", "-v", "--help"]"#,
        )
        .run();
}

#[cargo_test]
fn cargo_help() {
    cargo_process("").run();
    cargo_process("help").run();
    cargo_process("-h").run();
    cargo_process("help build").run();
    cargo_process("build -h").run();
    cargo_process("help help").run();
}

#[cargo_test]
fn cargo_help_external_subcommand() {
    Package::new("cargo-fake-help", "1.0.0")
        .file(
            "src/main.rs",
            r#"
            fn main() {
                if ::std::env::args().nth(2) == Some(String::from("--help")) {
                    println!("fancy help output");
                }
            }"#,
        )
        .publish();
    cargo_process("install cargo-fake-help").run();
    cargo_process("help fake-help")
        .with_stdout("fancy help output\n")
        .run();
}

#[cargo_test]
fn explain() {
    cargo_process("--explain E0001")
        .with_stdout_contains(
            "This error suggests that the expression arm corresponding to the noted pattern",
        )
        .run();
}

// Test that the output of `cargo -Z help` shows a different help screen with
// all the `-Z` flags.
#[cargo_test]
fn z_flags_help() {
    cargo_process("-Z help")
        .with_stdout_contains("    -Z unstable-options -- Allow the usage of unstable options")
        .run();
}

#[cargo_test]
fn closed_output_ok() {
    // Checks that closed output doesn't cause an error.
    let mut p = cargo_process("--list").build_command();
    p.stdout(Stdio::piped()).stderr(Stdio::piped());
    let mut child = p.spawn().unwrap();
    // Close stdout
    drop(child.stdout.take());
    // Read stderr
    let mut s = String::new();
    child
        .stderr
        .as_mut()
        .unwrap()
        .read_to_string(&mut s)
        .unwrap();
    let status = child.wait().unwrap();
    assert!(status.success());
    assert!(s.is_empty(), s);
}
<commit_msg>Add test for listing builtin aliases<commit_after>//! Tests for custom cargo commands and other global command features.

use std::env;
use std::fs::{self, File};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use std::str;

use cargo_test_support::cargo_process;
use cargo_test_support::paths::{self, CargoPathExt};
use cargo_test_support::registry::Package;
use cargo_test_support::{basic_bin_manifest, basic_manifest, cargo_exe, project, Project};

#[cfg_attr(windows, allow(dead_code))]
enum FakeKind<'a> {
    Executable,
    Symlink { target: &'a Path },
}

/// Adds an empty file with executable flags (and platform-dependent suffix).
//
// TODO: move this to `Project` if other cases using this emerge.
fn fake_file(proj: Project, dir: &Path, name: &str, kind: &FakeKind<'_>) -> Project {
    let path = proj
        .root()
        .join(dir)
        .join(&format!("{}{}", name, env::consts::EXE_SUFFIX));
    path.parent().unwrap().mkdir_p();
    match *kind {
        FakeKind::Executable => {
            File::create(&path).unwrap();
            make_executable(&path);
        }
        FakeKind::Symlink { target } => {
            make_symlink(&path, target);
        }
    }
    return proj;

    #[cfg(unix)]
    fn make_executable(p: &Path) {
        use std::os::unix::prelude::*;

        let mut perms = fs::metadata(p).unwrap().permissions();
        let mode = perms.mode();
        perms.set_mode(mode | 0o111);
        fs::set_permissions(p, perms).unwrap();
    }
    #[cfg(windows)]
    fn make_executable(_: &Path) {}
    #[cfg(unix)]
    fn make_symlink(p: &Path, t: &Path) {
        ::std::os::unix::fs::symlink(t, p).expect("Failed to create symlink");
    }
    #[cfg(windows)]
    fn make_symlink(_: &Path, _: &Path) {
        panic!("Not supported")
    }
}

fn path() -> Vec<PathBuf> {
    env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect()
}

#[cargo_test]
fn list_commands_with_descriptions() {
    let p = project().build();
    p.cargo("--list")
        .with_stdout_contains(
            "    build                Compile a local package and all of its dependencies",
        )
        // Assert that `read-manifest` prints the right one-line description followed by another
        // command, indented.
        .with_stdout_contains(
            "    read-manifest        Print a JSON representation of a Cargo.toml manifest.",
        )
        .run();
}

#[cargo_test]
fn list_aliases_with_descriptions() {
    let p = project().build();
    p.cargo("--list")
        .with_stdout_contains("    b                    alias: build")
        .with_stdout_contains("    c                    alias: check")
        .with_stdout_contains("    r                    alias: run")
        .with_stdout_contains("    t                    alias: test")
        .run();
}

#[cargo_test]
fn list_command_looks_at_path() {
    let proj = project().build();
    let proj = fake_file(
        proj,
        Path::new("path-test"),
        "cargo-1",
        &FakeKind::Executable,
    );

    let mut path = path();
    path.push(proj.root().join("path-test"));
    let path = env::join_paths(path.iter()).unwrap();
    let output = cargo_process("-v --list")
        .env("PATH", &path)
        .exec_with_output()
        .unwrap();
    let output = str::from_utf8(&output.stdout).unwrap();
    assert!(
        output.contains("\n    1                   "),
        "missing 1: {}",
        output
    );
}

// Windows and symlinks don't currently mix well.
#[cfg(unix)]
#[cargo_test]
fn list_command_resolves_symlinks() {
    let proj = project().build();
    let proj = fake_file(
        proj,
        Path::new("path-test"),
        "cargo-2",
        &FakeKind::Symlink {
            target: &cargo_exe(),
        },
    );

    let mut path = path();
    path.push(proj.root().join("path-test"));
    let path = env::join_paths(path.iter()).unwrap();
    let output = cargo_process("-v --list")
        .env("PATH", &path)
        .exec_with_output()
        .unwrap();
    let output = str::from_utf8(&output.stdout).unwrap();
    assert!(
        output.contains("\n    2                   "),
        "missing 2: {}",
        output
    );
}

#[cargo_test]
fn find_closest_biuld_to_build() {
    cargo_process("biuld")
        .with_status(101)
        .with_stderr_contains(
            "\
error: no such subcommand: `biuld`

<tab>Did you mean `build`?
",
        )
        .run();

    // But, if we actually have `biuld`, it must work!
    // https://github.com/rust-lang/cargo/issues/5201
    Package::new("cargo-biuld", "1.0.0")
        .file(
            "src/main.rs",
            r#"
            fn main() {
                println!("Similar, but not identical to, build");
            }
        "#,
        )
        .publish();

    cargo_process("install cargo-biuld").run();
    cargo_process("biuld")
        .with_stdout("Similar, but not identical to, build\n")
        .run();
    cargo_process("--list")
        .with_stdout_contains(
            "    build                Compile a local package and all of its dependencies\n",
        )
        .with_stdout_contains("    biuld\n")
        .run();
}

#[cargo_test]
fn find_closest_alias() {
    let root = paths::root();
    let my_home = root.join("my_home");
    fs::create_dir(&my_home).unwrap();
    fs::write(
        &my_home.join("config"),
        r#"
            [alias]
            myalias = "build"
        "#,
    )
    .unwrap();

    cargo_process("myalais")
        .env("CARGO_HOME", &my_home)
        .with_status(101)
        .with_stderr_contains(
            "\
error: no such subcommand: `myalais`

<tab>Did you mean `myalias`?
",
        )
        .run();

    // But, if no alias is defined, it must not suggest one!
    cargo_process("myalais")
        .with_status(101)
        .with_stderr_contains(
            "\
error: no such subcommand: `myalais`
",
        )
        .with_stderr_does_not_contain(
            "\
<tab>Did you mean `myalias`?
",
        )
        .run();
}

// If a subcommand is more than an edit distance of 3 away, we don't make a suggestion.
#[cargo_test]
fn find_closest_dont_correct_nonsense() {
    cargo_process("there-is-no-way-that-there-is-a-command-close-to-this")
        .cwd(&paths::root())
        .with_status(101)
        .with_stderr(
            "[ERROR] no such subcommand: \
                        `there-is-no-way-that-there-is-a-command-close-to-this`
",
        )
        .run();
}

#[cargo_test]
fn displays_subcommand_on_error() {
    cargo_process("invalid-command")
        .with_status(101)
        .with_stderr("[ERROR] no such subcommand: `invalid-command`\n")
        .run();
}

#[cargo_test]
fn override_cargo_home() {
    let root = paths::root();
    let my_home = root.join("my_home");
    fs::create_dir(&my_home).unwrap();
    fs::write(
        &my_home.join("config"),
        r#"
            [cargo-new]
            name = "foo"
            email = "bar"
            git = false
        "#,
    )
    .unwrap();

    cargo_process("new foo")
        .env("USER", "foo")
        .env("CARGO_HOME", &my_home)
        .run();

    let toml = paths::root().join("foo/Cargo.toml");
    let contents = fs::read_to_string(&toml).unwrap();
    assert!(contents.contains(r#"authors = ["foo <bar>"]"#));
}

#[cargo_test]
fn cargo_subcommand_env() {
    let src = format!(
        r#"
        use std::env;

        fn main() {{
            println!("{{}}", env::var("{}").unwrap());
        }}
        "#,
        cargo::CARGO_ENV
    );

    let p = project()
        .at("cargo-envtest")
        .file("Cargo.toml", &basic_bin_manifest("cargo-envtest"))
        .file("src/main.rs", &src)
        .build();

    let target_dir = p.target_debug_dir();

    p.cargo("build").run();
    assert!(p.bin("cargo-envtest").is_file());

    let cargo = cargo_exe().canonicalize().unwrap();
    let mut path = path();
    path.push(target_dir);
    let path = env::join_paths(path.iter()).unwrap();

    cargo_process("envtest")
        .env("PATH", &path)
        .with_stdout(cargo.to_str().unwrap())
        .run();
}

#[cargo_test]
fn cargo_subcommand_args() {
    let p = project()
        .at("cargo-foo")
        .file("Cargo.toml", &basic_manifest("cargo-foo", "0.0.1"))
        .file(
            "src/main.rs",
            r#"
            fn main() {
                let args: Vec<_> = ::std::env::args().collect();
                println!("{:?}", args);
            }
        "#,
        )
        .build();

    p.cargo("build").run();
    let cargo_foo_bin = p.bin("cargo-foo");
    assert!(cargo_foo_bin.is_file());

    let mut path = path();
    path.push(p.target_debug_dir());
    let path = env::join_paths(path.iter()).unwrap();

    cargo_process("foo bar -v --help")
        .env("PATH", &path)
        .with_stdout(
            r#"["[CWD]/cargo-foo/target/debug/cargo-foo[EXE]", "foo", "bar", "-v", "--help"]"#,
        )
        .run();
}

#[cargo_test]
fn cargo_help() {
    cargo_process("").run();
    cargo_process("help").run();
    cargo_process("-h").run();
    cargo_process("help build").run();
    cargo_process("build -h").run();
    cargo_process("help help").run();
}

#[cargo_test]
fn cargo_help_external_subcommand() {
    Package::new("cargo-fake-help", "1.0.0")
        .file(
            "src/main.rs",
            r#"
            fn main() {
                if ::std::env::args().nth(2) == Some(String::from("--help")) {
                    println!("fancy help output");
                }
            }"#,
        )
        .publish();
    cargo_process("install cargo-fake-help").run();
    cargo_process("help fake-help")
        .with_stdout("fancy help output\n")
        .run();
}

#[cargo_test]
fn explain() {
    cargo_process("--explain E0001")
        .with_stdout_contains(
            "This error suggests that the expression arm corresponding to the noted pattern",
        )
        .run();
}

// Test that the output of `cargo -Z help` shows a different help screen with
// all the `-Z` flags.
#[cargo_test]
fn z_flags_help() {
    cargo_process("-Z help")
        .with_stdout_contains("    -Z unstable-options -- Allow the usage of unstable options")
        .run();
}

#[cargo_test]
fn closed_output_ok() {
    // Checks that closed output doesn't cause an error.
    let mut p = cargo_process("--list").build_command();
    p.stdout(Stdio::piped()).stderr(Stdio::piped());
    let mut child = p.spawn().unwrap();
    // Close stdout
    drop(child.stdout.take());
    // Read stderr
    let mut s = String::new();
    child
        .stderr
        .as_mut()
        .unwrap()
        .read_to_string(&mut s)
        .unwrap();
    let status = child.wait().unwrap();
    assert!(status.success());
    assert!(s.is_empty(), s);
}
<|endoftext|>