<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::{
    auth::with_auth,
    constants::MAX_CONTENT_LENGTH,
    context::Context,
    errors::{MetricsIngestError, ServiceError},
    metrics::METRICS_INGEST_BACKEND_REQUEST_DURATION,
    types::{auth::Claims, common::NodeType},
};
use crate::{debug, error};
use reqwest::{header::CONTENT_ENCODING, StatusCode};
use tokio::time::Instant;
use warp::{filters::BoxedFilter, hyper::body::Bytes, reject, reply, Filter, Rejection, Reply};

/// TODO: Cleanup after v1 API is ramped up
pub fn metrics_ingest_legacy(context: Context) -> BoxedFilter<(impl Reply,)> {
    warp::path!("push-metrics")
        .and(warp::post())
        .and(context.clone().filter())
        .and(with_auth(
            context,
            vec![
                NodeType::Validator,
                NodeType::ValidatorFullNode,
                NodeType::PublicFullNode,
            ],
        ))
        .and(warp::header::optional(CONTENT_ENCODING.as_str()))
        .and(warp::body::content_length_limit(MAX_CONTENT_LENGTH))
        .and(warp::body::bytes())
        .and_then(handle_metrics_ingest)
        .boxed()
}

pub fn metrics_ingest(context: Context) -> BoxedFilter<(impl Reply,)> {
    warp::path!("ingest" / "metrics")
        .and(warp::post())
        .and(context.clone().filter())
        .and(with_auth(
            context,
            vec![
                NodeType::Validator,
                NodeType::ValidatorFullNode,
                NodeType::PublicFullNode,
            ],
        ))
        .and(warp::header::optional(CONTENT_ENCODING.as_str()))
        .and(warp::body::content_length_limit(MAX_CONTENT_LENGTH))
        .and(warp::body::bytes())
        .and_then(handle_metrics_ingest)
        .boxed()
}

pub async fn handle_metrics_ingest(
    context: Context,
    claims: Claims,
    encoding: Option<String>,
    metrics_body: Bytes,
) -> anyhow::Result<impl Reply, Rejection> {
    debug!("handling prometheus metrics ingest");

    let extra_labels = claims_to_extra_labels(
        &claims,
        context
            .peer_identities()
            .get(&claims.chain_id)
            .and_then(|peers| peers.get(&claims.peer_id)),
    );

    let start_timer = Instant::now();

    let post_futures = context.metrics_client().values().map(|client| {
        client.post_prometheus_metrics(
            metrics_body.clone(),
            extra_labels.clone(),
            encoding.clone().unwrap_or_default(),
        )
    });

    let results = futures::future::join_all(post_futures)
        .await
        .into_iter()
        .zip(context.metrics_client().keys())
        .map(|(res, name)| {
            match res {
                Ok(res) => {
                    METRICS_INGEST_BACKEND_REQUEST_DURATION
                        .with_label_values(&[name, res.status().as_str()])
                        .observe(start_timer.elapsed().as_secs_f64());
                    if res.status().is_success() {
                        debug!("remote write to victoria metrics succeeded");
                    } else {
                        error!(
                            "remote write failed to victoria_metrics for client {}: {}",
                            name,
                            res.error_for_status().err().unwrap()
                        );
                        return Err(());
                    }
                }
                Err(err) => {
                    METRICS_INGEST_BACKEND_REQUEST_DURATION
                        .with_label_values(&[name, "Unknown"])
                        .observe(start_timer.elapsed().as_secs_f64());
                    error!(
                        "error sending remote write request for client {}: {}",
                        name, err
                    );
                    return Err(());
                }
            }
            Ok(())
        });

    #[allow(clippy::unnecessary_fold)]
    if results.fold(true, |acc, r| acc && r.is_err()) {
        return Err(reject::custom(ServiceError::internal(
            MetricsIngestError::IngestionError.into(),
        )));
    }

    Ok(reply::with_status(reply::reply(), StatusCode::CREATED))
}

fn claims_to_extra_labels(claims: &Claims, common_name: Option<&String>) -> Vec<String> {
    let chain_name = if claims.chain_id.id() == 3 {
        format!("chain_name={}", claims.chain_id.id())
    } else {
        format!("chain_name={}", claims.chain_id)
    };
    let pod_name = if let Some(common_name) = common_name {
        format!(
            "kubernetes_pod_name=peer_id:{}//{}",
            common_name,
            claims.peer_id.to_hex_literal()
        )
    } else {
        // for community nodes we cannot determine which pod name they run in (or whether they run in k8s at all),
        // so we use the peer id as an approximation/replacement for pod_name
        // This works well with our existing grafana dashboards
        format!(
            "kubernetes_pod_name=peer_id:{}",
            claims.peer_id.to_hex_literal()
        )
    };
    vec![
        format!("role={}", claims.node_type),
        chain_name,
        format!("namespace={}", "telemetry-service"),
        pod_name,
    ]
}

#[cfg(test)]
mod test {
    use std::str::FromStr;

    use crate::tests::test_context;
    use crate::MetricsClient;

    use super::*;
    use aptos_types::{chain_id::ChainId, PeerId};
    use httpmock::MockServer;
    use reqwest::Url;

    #[test]
    fn verify_labels() {
        let claims = claims_to_extra_labels(
            &super::Claims {
                chain_id: ChainId::new(25),
                peer_id: PeerId::from_str("0x1").unwrap(),
                node_type: NodeType::Validator,
                epoch: 3,
                exp: 123,
                iat: 123,
            },
            Some(&String::from("test_name")),
        );
        assert_eq!(
            claims,
            vec![
                "role=validator",
                "chain_name=25",
                "namespace=telemetry-service",
                "kubernetes_pod_name=peer_id:test_name//0x1",
            ]
        );

        let claims = claims_to_extra_labels(
            &super::Claims {
                chain_id: ChainId::new(25),
                peer_id: PeerId::from_str("0x1").unwrap(),
                node_type: NodeType::Validator,
                epoch: 3,
                exp: 123,
                iat: 123,
            },
            None,
        );
        assert_eq!(
            claims,
            vec![
                "role=validator",
                "chain_name=25",
                "namespace=telemetry-service",
                "kubernetes_pod_name=peer_id:0x1",
            ]
        );
    }

    #[tokio::test]
    async fn test_metrics_ingest_all_success() {
        let mut test_context = test_context::new_test_context().await;
        let claims = Claims::test();
        let body = Bytes::from_static(b"hello");

        let server1 = MockServer::start();
        let mock1 = server1.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(200);
        });

        let server2 = MockServer::start();
        let mock2 = server2.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(200);
        });

        let clients = test_context.inner.metrics_client_mut();
        clients.insert(
            "endpoint1".into(),
            MetricsClient::new(Url::parse(&server1.base_url()).unwrap(), "token1".into()),
        );
        clients.insert(
            "endpoint2".into(),
            MetricsClient::new(Url::parse(&server2.base_url()).unwrap(), "token2".into()),
        );

        let result =
            handle_metrics_ingest(test_context.inner, claims, Some("gzip".into()), body).await;

        mock1.assert();
        mock2.assert();
        assert!(result.is_ok());
    }

    #[tokio::test]
    async fn test_metrics_ingest_partial_success() {
        let mut test_context = test_context::new_test_context().await;
        let claims = Claims::test();
        let body = Bytes::from_static(b"hello");

        let server1 = MockServer::start();
        let mock1 = server1.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(200);
        });

        let server2 = MockServer::start();
        let mock2 = server2.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(500);
        });

        let clients = test_context.inner.metrics_client_mut();
        clients.insert(
            "endpoint1".into(),
            MetricsClient::new(Url::parse(&server1.base_url()).unwrap(), "token1".into()),
        );
        clients.insert(
            "endpoint2".into(),
            MetricsClient::new(Url::parse(&server2.base_url()).unwrap(), "token2".into()),
        );

        let result =
            handle_metrics_ingest(test_context.inner, claims, Some("gzip".into()), body).await;

        mock1.assert();
        mock2.assert_hits(4);
        assert!(result.is_ok());
    }

    #[tokio::test]
    async fn test_metrics_ingest_all_failure() {
        let mut test_context = test_context::new_test_context().await;
        let claims = Claims::test();
        let body = Bytes::from_static(b"hello");

        let server1 = MockServer::start();
        let mock1 = server1.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(500);
        });

        let server2 = MockServer::start();
        let mock2 = server2.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(401);
        });

        let clients = test_context.inner.metrics_client_mut();
        clients.insert(
            "endpoint1".into(),
            MetricsClient::new(Url::parse(&server1.base_url()).unwrap(), "token1".into()),
        );
        clients.insert(
            "endpoint2".into(),
            MetricsClient::new(Url::parse(&server2.base_url()).unwrap(), "token2".into()),
        );

        let result =
            handle_metrics_ingest(test_context.inner, claims, Some("gzip".into()), body).await;

        mock1.assert_hits(4);
        mock2.assert();
        assert!(result.is_err());
    }
}
<commit_msg>special handling of chain ID 3 and mainnet push metrics<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

use crate::{
    auth::with_auth,
    constants::MAX_CONTENT_LENGTH,
    context::Context,
    errors::{MetricsIngestError, ServiceError},
    metrics::METRICS_INGEST_BACKEND_REQUEST_DURATION,
    types::{auth::Claims, common::NodeType},
};
use crate::{debug, error};
use aptos_types::chain_id::ChainId;
use reqwest::{header::CONTENT_ENCODING, StatusCode};
use tokio::time::Instant;
use warp::{filters::BoxedFilter, hyper::body::Bytes, reject, reply, Filter, Rejection, Reply};

/// TODO: Cleanup after v1 API is ramped up
pub fn metrics_ingest_legacy(context: Context) -> BoxedFilter<(impl Reply,)> {
    warp::path!("push-metrics")
        .and(warp::post())
        .and(context.clone().filter())
        .and(with_auth(
            context,
            vec![
                NodeType::Validator,
                NodeType::ValidatorFullNode,
                NodeType::PublicFullNode,
            ],
        ))
        .and(warp::header::optional(CONTENT_ENCODING.as_str()))
        .and(warp::body::content_length_limit(MAX_CONTENT_LENGTH))
        .and(warp::body::bytes())
        .and_then(handle_metrics_ingest)
        .boxed()
}

pub fn metrics_ingest(context: Context) -> BoxedFilter<(impl Reply,)> {
    warp::path!("ingest" / "metrics")
        .and(warp::post())
        .and(context.clone().filter())
        .and(with_auth(
            context,
            vec![
                NodeType::Validator,
                NodeType::ValidatorFullNode,
                NodeType::PublicFullNode,
            ],
        ))
        .and(warp::header::optional(CONTENT_ENCODING.as_str()))
        .and(warp::body::content_length_limit(MAX_CONTENT_LENGTH))
        .and(warp::body::bytes())
        .and_then(handle_metrics_ingest)
        .boxed()
}

pub async fn handle_metrics_ingest(
    context: Context,
    claims: Claims,
    encoding: Option<String>,
    metrics_body: Bytes,
) -> anyhow::Result<impl Reply, Rejection> {
    debug!("handling prometheus metrics ingest");

    let extra_labels = claims_to_extra_labels(
        &claims,
        context
            .peer_identities()
            .get(&claims.chain_id)
            .and_then(|peers| peers.get(&claims.peer_id)),
    );

    let start_timer = Instant::now();

    let filtered_clients = context.metrics_client().iter().filter(|(name, _)| {
        if claims.chain_id.id() == 3 {
            return true;
        } else if claims.chain_id == ChainId::mainnet() {
            return !name.starts_with("default");
        }
        name.starts_with("default")
    });

    let post_futures = filtered_clients.clone().map(|(_, client)| {
        client.post_prometheus_metrics(
            metrics_body.clone(),
            extra_labels.clone(),
            encoding.clone().unwrap_or_default(),
        )
    });

    let results = futures::future::join_all(post_futures)
        .await
        .into_iter()
        .zip(filtered_clients.map(|(name, _)| name))
        .map(|(res, name)| {
            match res {
                Ok(res) => {
                    METRICS_INGEST_BACKEND_REQUEST_DURATION
                        .with_label_values(&[name, res.status().as_str()])
                        .observe(start_timer.elapsed().as_secs_f64());
                    if res.status().is_success() {
                        debug!("remote write to victoria metrics succeeded");
                    } else {
                        error!(
                            "remote write failed to victoria_metrics for client {}: {}",
                            name,
                            res.error_for_status().err().unwrap()
                        );
                        return Err(());
                    }
                }
                Err(err) => {
                    METRICS_INGEST_BACKEND_REQUEST_DURATION
                        .with_label_values(&[name, "Unknown"])
                        .observe(start_timer.elapsed().as_secs_f64());
                    error!(
                        "error sending remote write request for client {}: {}",
                        name, err
                    );
                    return Err(());
                }
            }
            Ok(())
        });

    #[allow(clippy::unnecessary_fold)]
    if results.fold(true, |acc, r| acc && r.is_err()) {
        return Err(reject::custom(ServiceError::internal(
            MetricsIngestError::IngestionError.into(),
        )));
    }

    Ok(reply::with_status(reply::reply(), StatusCode::CREATED))
}

fn claims_to_extra_labels(claims: &Claims, common_name: Option<&String>) -> Vec<String> {
    let chain_name = if claims.chain_id.id() == 3 {
        format!("chain_name={}", claims.chain_id.id())
    } else {
        format!("chain_name={}", claims.chain_id)
    };
    let pod_name = if let Some(common_name) = common_name {
        format!(
            "kubernetes_pod_name=peer_id:{}//{}",
            common_name,
            claims.peer_id.to_hex_literal()
        )
    } else {
        // for community nodes we cannot determine which pod name they run in (or whether they run in k8s at all),
        // so we use the peer id as an approximation/replacement for pod_name
        // This works well with our existing grafana dashboards
        format!(
            "kubernetes_pod_name=peer_id:{}",
            claims.peer_id.to_hex_literal()
        )
    };
    vec![
        format!("role={}", claims.node_type),
        chain_name,
        format!("namespace={}", "telemetry-service"),
        pod_name,
    ]
}

#[cfg(test)]
mod test {
    use std::str::FromStr;

    use crate::tests::test_context;
    use crate::MetricsClient;

    use super::*;
    use aptos_types::{chain_id::ChainId, PeerId};
    use httpmock::MockServer;
    use reqwest::Url;

    #[test]
    fn verify_labels() {
        let claims = claims_to_extra_labels(
            &super::Claims {
                chain_id: ChainId::new(25),
                peer_id: PeerId::from_str("0x1").unwrap(),
                node_type: NodeType::Validator,
                epoch: 3,
                exp: 123,
                iat: 123,
            },
            Some(&String::from("test_name")),
        );
        assert_eq!(
            claims,
            vec![
                "role=validator",
                "chain_name=25",
                "namespace=telemetry-service",
                "kubernetes_pod_name=peer_id:test_name//0x1",
            ]
        );

        let claims = claims_to_extra_labels(
            &super::Claims {
                chain_id: ChainId::new(25),
                peer_id: PeerId::from_str("0x1").unwrap(),
                node_type: NodeType::Validator,
                epoch: 3,
                exp: 123,
                iat: 123,
            },
            None,
        );
        assert_eq!(
            claims,
            vec![
                "role=validator",
                "chain_name=25",
                "namespace=telemetry-service",
                "kubernetes_pod_name=peer_id:0x1",
            ]
        );
    }

    #[tokio::test]
    async fn test_metrics_ingest_all_success() {
        let mut test_context = test_context::new_test_context().await;
        let claims = Claims::test();
        let body = Bytes::from_static(b"hello");

        let server1 = MockServer::start();
        let mock1 = server1.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(200);
        });

        let server2 = MockServer::start();
        let mock2 = server2.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(200);
        });

        let clients = test_context.inner.metrics_client_mut();
        clients.insert(
            "default1".into(),
            MetricsClient::new(Url::parse(&server1.base_url()).unwrap(), "token1".into()),
        );
        clients.insert(
            "default2".into(),
            MetricsClient::new(Url::parse(&server2.base_url()).unwrap(), "token2".into()),
        );

        let result =
            handle_metrics_ingest(test_context.inner, claims, Some("gzip".into()), body).await;

        mock1.assert();
        mock2.assert();
        assert!(result.is_ok());
    }

    #[tokio::test]
    async fn test_metrics_ingest_partial_success() {
        let mut test_context = test_context::new_test_context().await;
        let claims = Claims::test();
        let body = Bytes::from_static(b"hello");

        let server1 = MockServer::start();
        let mock1 = server1.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(200);
        });

        let server2 = MockServer::start();
        let mock2 = server2.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(500);
        });

        let clients = test_context.inner.metrics_client_mut();
        clients.insert(
            "default1".into(),
            MetricsClient::new(Url::parse(&server1.base_url()).unwrap(), "token1".into()),
        );
        clients.insert(
            "default2".into(),
            MetricsClient::new(Url::parse(&server2.base_url()).unwrap(), "token2".into()),
        );

        let result =
            handle_metrics_ingest(test_context.inner, claims, Some("gzip".into()), body).await;

        mock1.assert();
        mock2.assert_hits(4);
        assert!(result.is_ok());
    }

    #[tokio::test]
    async fn test_metrics_ingest_all_failure() {
        let mut test_context = test_context::new_test_context().await;
        let claims = Claims::test();
        let body = Bytes::from_static(b"hello");

        let server1 = MockServer::start();
        let mock1 = server1.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(500);
        });

        let server2 = MockServer::start();
        let mock2 = server2.mock(|when, then| {
            when.method("POST").path("/api/v1/import/prometheus");
            then.status(401);
        });

        let clients = test_context.inner.metrics_client_mut();
        clients.insert(
            "default1".into(),
            MetricsClient::new(Url::parse(&server1.base_url()).unwrap(), "token1".into()),
        );
        clients.insert(
            "default2".into(),
            MetricsClient::new(Url::parse(&server2.base_url()).unwrap(), "token2".into()),
        );

        let result =
            handle_metrics_ingest(test_context.inner, claims, Some("gzip".into()), body).await;

        mock1.assert_hits(4);
        mock2.assert();
        assert!(result.is_err());
    }
}
<|endoftext|>


<commit_before>use byteorder::{LittleEndian, WriteBytesExt};
use std::collections::HashSet;
use std::io::prelude::*;
use std::io::{Cursor, Seek};
use std::iter::FromIterator;
use zip::write::FileOptions;
use zip::CompressionMethod;

// This test asserts that after creating a zip file, then reading its contents back out,
// the extracted data will *always* be exactly the same as the original data.
#[test]
fn end_to_end() {
    let file = &mut Cursor::new(Vec::new());

    write_to_zip(file).expect("file written");

    check_zip_contents(file, ENTRY_NAME);
}

// This test asserts that after copying a `ZipFile` to a new `ZipWriter`, then reading its
// contents back out, the extracted data will *always* be exactly the same as the original data.
#[test]
fn copy() {
    let src_file = &mut Cursor::new(Vec::new());
    write_to_zip(src_file).expect("file written");

    let mut tgt_file = &mut Cursor::new(Vec::new());

    {
        let mut src_archive = zip::ZipArchive::new(src_file).unwrap();
        let mut zip = zip::ZipWriter::new(&mut tgt_file);

        {
            let file = src_archive.by_name(ENTRY_NAME).expect("file found");
            zip.raw_copy_file(file).unwrap();
        }

        {
            let file = src_archive.by_name(ENTRY_NAME).expect("file found");
            zip.raw_copy_file_rename(file, COPY_ENTRY_NAME).unwrap();
        }
    }

    let mut tgt_archive = zip::ZipArchive::new(tgt_file).unwrap();

    check_zip_file_contents(&mut tgt_archive, ENTRY_NAME);
    check_zip_file_contents(&mut tgt_archive, COPY_ENTRY_NAME);
}

// This test asserts that after appending to a `ZipWriter`, then reading its contents back out,
// both the prior data and the appended data will be exactly the same as their originals.
#[test]
fn append() {
    let mut file = &mut Cursor::new(Vec::new());
    write_to_zip(file).expect("file written");

    {
        let mut zip = zip::ZipWriter::new_append(&mut file).unwrap();
        zip.start_file(COPY_ENTRY_NAME, Default::default()).unwrap();
        zip.write_all(LOREM_IPSUM).unwrap();
        zip.finish().unwrap();
    }

    let mut zip = zip::ZipArchive::new(&mut file).unwrap();
    check_zip_file_contents(&mut zip, ENTRY_NAME);
    check_zip_file_contents(&mut zip, COPY_ENTRY_NAME);
}

fn write_to_zip(file: &mut Cursor<Vec<u8>>) -> zip::result::ZipResult<()> {
    let mut zip = zip::ZipWriter::new(file);

    zip.add_directory("test/", Default::default())?;

    let options = FileOptions::default()
        .compression_method(CompressionMethod::Stored)
        .unix_permissions(0o755);
    zip.start_file("test/☃.txt", options)?;
    zip.write_all(b"Hello, World!\n")?;

    zip.start_file_with_extra_data("test_with_extra_data/🐢.txt", Default::default())?;
    zip.write_u16::<LittleEndian>(0xbeef)?;
    zip.write_u16::<LittleEndian>(EXTRA_DATA.len() as u16)?;
    zip.write_all(EXTRA_DATA)?;
    zip.end_extra_data()?;
    zip.write_all(b"Hello, World! Again.\n")?;

    zip.start_file(ENTRY_NAME, Default::default())?;
    zip.write_all(LOREM_IPSUM)?;

    zip.finish()?;
    Ok(())
}

fn read_zip<R: Read + Seek>(zip_file: R) -> zip::result::ZipResult<zip::ZipArchive<R>> {
    let mut archive = zip::ZipArchive::new(zip_file).unwrap();

    let expected_file_names = [
        "test/",
        "test/☃.txt",
        "test_with_extra_data/🐢.txt",
        ENTRY_NAME,
    ];
    let expected_file_names = HashSet::from_iter(expected_file_names.iter().copied());
    let file_names = archive.file_names().collect::<HashSet<_>>();
    assert_eq!(file_names, expected_file_names);

    {
        let file_with_extra_data = archive.by_name("test_with_extra_data/🐢.txt")?;
        let mut extra_data = Vec::new();
        extra_data.write_u16::<LittleEndian>(0xbeef)?;
        extra_data.write_u16::<LittleEndian>(EXTRA_DATA.len() as u16)?;
        extra_data.write_all(EXTRA_DATA)?;
        assert_eq!(file_with_extra_data.extra_data(), extra_data.as_slice());
    }

    Ok(archive)
}

fn read_zip_file<R: Read + Seek>(
    archive: &mut zip::ZipArchive<R>,
    name: &str,
) -> zip::result::ZipResult<String> {
    let mut file = archive.by_name(name)?;

    let mut contents = String::new();
    file.read_to_string(&mut contents).unwrap();
    Ok(contents)
}

fn check_zip_contents(zip_file: &mut Cursor<Vec<u8>>, name: &str) {
    let mut archive = read_zip(zip_file).unwrap();
    check_zip_file_contents(&mut archive, name);
}

fn check_zip_file_contents<R: Read + Seek>(archive: &mut zip::ZipArchive<R>, name: &str) {
    let file_contents: String = read_zip_file(archive, name).unwrap();
    assert_eq!(file_contents.as_bytes(), LOREM_IPSUM);
}

const LOREM_IPSUM : &[u8] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. In tellus elit, tristique vitae mattis egestas, ultricies vitae risus. Quisque sit amet quam ut urna aliquet
molestie. Proin blandit ornare dui, a tempor nisl accumsan in. Praesent a consequat felis. Morbi metus diam, auctor in auctor vel, feugiat id odio. Curabitur ex ex,
dictum quis auctor quis, suscipit id lorem. Aliquam vestibulum dolor nec enim vehicula, porta tristique augue tincidunt. Vivamus ut gravida est. Sed pellentesque, dolor
vitae tristique consectetur, neque lectus pulvinar dui, sed feugiat purus diam id lectus. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per
inceptos himenaeos. Maecenas feugiat velit in ex ultrices scelerisque id id neque.
";

const EXTRA_DATA: &[u8] = b"Extra Data";

const ENTRY_NAME: &str = "test/lorem_ipsum.txt";

const COPY_ENTRY_NAME: &str = "test/lorem_ipsum_renamed.txt";
<commit_msg>Add test changes from other branch<commit_after>use byteorder::{LittleEndian, WriteBytesExt};
use std::collections::HashSet;
use std::io::prelude::*;
use std::io::{Cursor, Seek};
use std::iter::FromIterator;
use zip::write::FileOptions;
use zip::{CompressionMethod, SUPPORTED_METHODS};

// This test asserts that after creating a zip file, then reading its contents back out,
// the extracted data will *always* be exactly the same as the original data.
#[test]
fn end_to_end() {
    for &method in SUPPORTED_METHODS {
        let file = &mut Cursor::new(Vec::new());

        println!("Writing file with {} compression", method);
        write_to_zip(file, method).expect("Couldn't write to test file");

        println!("Checking file contents");
        check_zip_contents(file, ENTRY_NAME, Some(method));
    }
}

// This test asserts that after copying a `ZipFile` to a new `ZipWriter`, then reading its
// contents back out, the extracted data will *always* be exactly the same as the original data.
#[test]
fn copy() {
    for &method in SUPPORTED_METHODS {
        let src_file = &mut Cursor::new(Vec::new());
        write_to_zip(src_file, method).expect("Couldn't write to test file");

        let mut tgt_file = &mut Cursor::new(Vec::new());

        {
            let mut src_archive = zip::ZipArchive::new(src_file).unwrap();
            let mut zip = zip::ZipWriter::new(&mut tgt_file);

            {
                let file = src_archive
                    .by_name(ENTRY_NAME)
                    .expect("Missing expected file");

                zip.raw_copy_file(file).expect("Couldn't copy file");
            }

            {
                let file = src_archive
                    .by_name(ENTRY_NAME)
                    .expect("Missing expected file");

                zip.raw_copy_file_rename(file, COPY_ENTRY_NAME)
                    .expect("Couldn't copy and rename file");
            }
        }

        let mut tgt_archive = zip::ZipArchive::new(tgt_file).unwrap();

        check_zip_file_contents(&mut tgt_archive, ENTRY_NAME);
        check_zip_file_contents(&mut tgt_archive, COPY_ENTRY_NAME);
    }
}

// This test asserts that after appending to a `ZipWriter`, then reading its contents back out,
// both the prior data and the appended data will be exactly the same as their originals.
#[test]
fn append() {
    for &method in SUPPORTED_METHODS {
        let mut file = &mut Cursor::new(Vec::new());
        write_to_zip(file, method).expect("Couldn't write to test file");

        {
            let mut zip = zip::ZipWriter::new_append(&mut file).unwrap();
            zip.start_file(
                COPY_ENTRY_NAME,
                FileOptions::default().compression_method(method),
            )
            .unwrap();
            zip.write_all(LOREM_IPSUM).unwrap();
            zip.finish().unwrap();
        }

        let mut zip = zip::ZipArchive::new(&mut file).unwrap();
        check_zip_file_contents(&mut zip, ENTRY_NAME);
        check_zip_file_contents(&mut zip, COPY_ENTRY_NAME);
    }
}

fn write_to_zip(
    file: &mut Cursor<Vec<u8>>,
    method: CompressionMethod,
) -> zip::result::ZipResult<()> {
    let mut zip = zip::ZipWriter::new(file);

    zip.add_directory("test/", Default::default())?;

    let options = FileOptions::default()
        .compression_method(method)
        .unix_permissions(0o755);

    zip.start_file("test/☃.txt", options)?;
    zip.write_all(b"Hello, World!\n")?;

    zip.start_file_with_extra_data("test_with_extra_data/🐢.txt", options)?;
    zip.write_u16::<LittleEndian>(0xbeef)?;
    zip.write_u16::<LittleEndian>(EXTRA_DATA.len() as u16)?;
    zip.write_all(EXTRA_DATA)?;
    zip.end_extra_data()?;
    zip.write_all(b"Hello, World! Again.\n")?;

    zip.start_file(ENTRY_NAME, options)?;
    zip.write_all(LOREM_IPSUM)?;

    zip.finish()?;
    Ok(())
}

fn read_zip<R: Read + Seek>(zip_file: R) -> zip::result::ZipResult<zip::ZipArchive<R>> {
    let mut archive = zip::ZipArchive::new(zip_file).unwrap();

    let expected_file_names = [
        "test/",
        "test/☃.txt",
        "test_with_extra_data/🐢.txt",
        ENTRY_NAME,
    ];
    let expected_file_names = HashSet::from_iter(expected_file_names.iter().copied());
    let file_names = archive.file_names().collect::<HashSet<_>>();
    assert_eq!(file_names, expected_file_names);

    {
        let file_with_extra_data = archive.by_name("test_with_extra_data/🐢.txt")?;
        let mut extra_data = Vec::new();
        extra_data.write_u16::<LittleEndian>(0xbeef)?;
        extra_data.write_u16::<LittleEndian>(EXTRA_DATA.len() as u16)?;
        extra_data.write_all(EXTRA_DATA)?;
        assert_eq!(file_with_extra_data.extra_data(), extra_data.as_slice());
    }

    Ok(archive)
}

fn read_zip_file<R: Read + Seek>(
    archive: &mut zip::ZipArchive<R>,
    name: &str,
) -> zip::result::ZipResult<String> {
    let mut file = archive.by_name(name)?;

    let mut contents = String::new();
    file.read_to_string(&mut contents).unwrap();
    Ok(contents)
}

fn check_zip_contents(
    zip_file: &mut Cursor<Vec<u8>>,
    name: &str,
    expected_method: Option<CompressionMethod>,
) {
    let mut archive = read_zip(zip_file).unwrap();

    match expected_method {
        Some(method) => {
            let file = archive.by_name(name).unwrap();

            assert_eq!(
                method,
                file.compression(),
                "File does not have expected compression method"
            )
        }
        None => {}
    }

    check_zip_file_contents(&mut archive, name);
}

fn check_zip_file_contents<R: Read + Seek>(archive: &mut zip::ZipArchive<R>, name: &str) {
    let file_contents: String = read_zip_file(archive, name).unwrap();
    assert_eq!(file_contents.as_bytes(), LOREM_IPSUM);
}

const LOREM_IPSUM : &[u8] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. In tellus elit, tristique vitae mattis egestas, ultricies vitae risus. Quisque sit amet quam ut urna aliquet
molestie. Proin blandit ornare dui, a tempor nisl accumsan in. Praesent a consequat felis. Morbi metus diam, auctor in auctor vel, feugiat id odio. Curabitur ex ex,
dictum quis auctor quis, suscipit id lorem. Aliquam vestibulum dolor nec enim vehicula, porta tristique augue tincidunt. Vivamus ut gravida est. Sed pellentesque, dolor
vitae tristique consectetur, neque lectus pulvinar dui, sed feugiat purus diam id lectus. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per
inceptos himenaeos. Maecenas feugiat velit in ex ultrices scelerisque id id neque.
";

const EXTRA_DATA: &[u8] = b"Extra Data";

const ENTRY_NAME: &str = "test/lorem_ipsum.txt";

const COPY_ENTRY_NAME: &str = "test/lorem_ipsum_renamed.txt";
<|endoftext|>


<commit_before>#![feature(prelude_import)]
//! Test
#[prelude_import]
use std::prelude::v1::*;
#[macro_use]
extern crate std;
/// Test
pub fn test() {}
<commit_msg>Update test case for nightly's new prelude import<commit_after>#![feature(prelude_import)]
//! Test
#[prelude_import]
use std::prelude::rust_2018::*;
#[macro_use]
extern crate std;
/// Test
pub fn test() {}
<|endoftext|>


<commit_before>extern crate serde_json;
extern crate toml_edit;

use serde_json::Value as Json;
use serde_json::Map as JsonMap;

use toml_edit::{Document, Item, Value, Iter};

fn pair_to_json((key, value): (&str, Item)) -> (String, Json) {
    fn typed_json(s: &str, json: Json) -> Json {
        let mut map = JsonMap::new();
        map.insert("type".to_owned(), Json::String(s.into()));
        map.insert("value".to_owned(), json);
        Json::Object(map)
    }
    fn value_to_json(value: &Value) -> Json {
        match *value {
            Value::String(ref s) => typed_json("string", Json::String(s.value().clone())),
            Value::Integer(ref i) => typed_json("integer", Json::String(format!("{}", i.value()))),
            Value::Float(ref f) => typed_json("float", Json::String(format!("{}", f.value()))),
            Value::Boolean(ref b) => typed_json("bool", Json::String(b.raw().into())),
            Value::DateTime(ref d) => typed_json("datetime", Json::String(d.raw().into())),
            Value::Array(ref a) => {
                let json = Json::Array(a.iter().map(value_to_json).collect::<Vec<_>>());
                typed_json("array", json)
            }
            Value::InlineTable(ref t) => {
                to_json(Box::new(t.iter().map(|(k, v)| (k, Item::Value(v.clone())))))
            }
        }
    }
    let json = match value {
        Item::Value(ref v) => value_to_json(v),
        Item::ArrayOfTables(ref arr) => {
            Json::Array(arr.iter().map(|t| to_json(iter_to_owned(t.iter()))).collect::<Vec<_>>())
        }
        Item::Table(ref table) => to_json(iter_to_owned(table.iter())),
        Item::None => Json::Null,
    };
    (key.to_owned(), json)
}

fn iter_to_owned(iter: Iter) -> OwnedIter {
    Box::new(
        iter.map(|(k, v)| (k, v.clone()))
    )
}

type OwnedIter<'s> = Box<Iterator<Item = (&'s str, Item)> + 's>;

fn to_json(iter: OwnedIter) -> Json {
    Json::Object(iter.map(pair_to_json).collect())
}

fn run(json: &str, toml: &str) {
    let doc = toml.parse::<Document>();
    assert!(doc.is_ok());
    let doc = doc.unwrap();

    let json: Json = serde_json::from_str(json).unwrap();
    let toml_json = to_json(iter_to_owned(doc.iter()));
    // compare structure with jsons
    assert_eq!(json, toml_json);

    // check round-trip equality
    let toml = doc.to_string();
    let doc = toml.parse::<Document>();
    assert!(doc.is_ok());
    let doc = doc.unwrap();
    assert_eq!(doc.to_string(), toml);
}

macro_rules! t(
    ($name:ident, $json:expr, $toml:expr) => (
        #[test]
        fn $name() {
            run(include_str!($json),
                include_str!($toml));
        }
    )
);

#[test]
fn table_reordering() {
    let toml = r#"
[[bin]] # bin 1
[a.b.c.e]
[a]
[other.table]
[[bin]] # bin 2
[a.b.c.d]
[a.b.c]
[[bin]] # bin 3
"#;
    let expected = r#"
[[bin]] # bin 1
[[bin]] # bin 2
[[bin]] # bin 3
[a]
[a.b.c]
[a.b.c.e]
[a.b.c.d]
[other.table]
"#;
    let doc = toml.parse::<Document>();
    assert!(doc.is_ok());
    let doc = doc.unwrap();
    assert_eq!(doc.to_string(), expected);
}

t!(
    test_array_empty,
    "fixtures/valid/array-empty.json",
    "fixtures/valid/array-empty.toml"
);
t!(
    test_array_nopspaces,
    "fixtures/valid/array-nospaces.json",
    "fixtures/valid/array-nospaces.toml"
);
t!(
    test_arrays_hetergeneous,
    "fixtures/valid/arrays-hetergeneous.json",
    "fixtures/valid/arrays-hetergeneous.toml"
);
t!(
    test_arrays,
    "fixtures/valid/arrays.json",
    "fixtures/valid/arrays.toml"
);
t!(
    test_arrays_nested,
    "fixtures/valid/arrays-nested.json",
    "fixtures/valid/arrays-nested.toml"
);
t!(
    test_bool,
    "fixtures/valid/bool.json",
    "fixtures/valid/bool.toml"
);
t!(
    test_comment_at_eof2,
    "fixtures/valid/comments-at-eof2.json",
    "fixtures/valid/comments-at-eof2.toml"
);
t!(
    test_comment_at_eof,
    "fixtures/valid/comments-at-eof.json",
    "fixtures/valid/comments-at-eof.toml"
);
t!(
    test_comments_everywhere,
    "fixtures/valid/comments-everywhere.json",
    "fixtures/valid/comments-everywhere.toml"
);
t!(
    test_datetime,
    "fixtures/valid/datetime.json",
    "fixtures/valid/datetime.toml"
);
t!(
    test_empty,
    "fixtures/valid/empty.json",
    "fixtures/valid/empty.toml"
);
t!(
    test_example,
    "fixtures/valid/example.json",
    "fixtures/valid/example.toml"
);
t!(
    test_float_exponent,
    "fixtures/valid/float-exponent.json",
    "fixtures/valid/float-exponent.toml"
);
t!(
    test_float,
    "fixtures/valid/float.json",
    "fixtures/valid/float.toml"
);
t!(
    test_float_underscore,
    "fixtures/valid/float-underscore.json",
    "fixtures/valid/float-underscore.toml"
);
t!(
    test_implicit_and_explicit_after,
    "fixtures/valid/implicit-and-explicit-after.json",
    "fixtures/valid/implicit-and-explicit-after.toml"
);
t!(
    test_implicit_and_explicit_before,
    "fixtures/valid/implicit-and-explicit-before.json",
    "fixtures/valid/implicit-and-explicit-before.toml"
);
t!(
    test_implicit_groups,
    "fixtures/valid/implicit-groups.json",
    "fixtures/valid/implicit-groups.toml"
);
t!(
    test_inline_table,
    "fixtures/valid/inline-table.json",
    "fixtures/valid/inline-table.toml"
);
t!(
    test_integer,
    "fixtures/valid/integer.json",
    "fixtures/valid/integer.toml"
);
t!(
    test_integer_underscore,
    "fixtures/valid/integer-underscore.json",
    "fixtures/valid/integer-underscore.toml"
);
t!(
    test_keys_equal_nospace,
    "fixtures/valid/key-equals-nospace.json",
    "fixtures/valid/key-equals-nospace.toml"
);
t!(
    test_key_numeric,
    "fixtures/valid/key-numeric.json",
    "fixtures/valid/key-numeric.toml"
);
t!(
    test_key_space,
    "fixtures/valid/key-space.json",
    "fixtures/valid/key-space.toml"
);
t!(
    test_key_special_chars,
    "fixtures/valid/key-special-chars.json",
    "fixtures/valid/key-special-chars.toml"
);
t!(
    test_long_float,
    "fixtures/valid/long-float.json",
    "fixtures/valid/long-float.toml"
);
t!(
    test_long_integer,
    "fixtures/valid/long-integer.json",
    "fixtures/valid/long-integer.toml"
);
t!(
    test_multiline_string,
    "fixtures/valid/multiline-string.json",
    "fixtures/valid/multiline-string.toml"
);
t!(
    test_raw_multiline_string,
    "fixtures/valid/raw-multiline-string.json",
    "fixtures/valid/raw-multiline-string.toml"
);
t!(
    test_raw_string,
    "fixtures/valid/raw-string.json",
    "fixtures/valid/raw-string.toml"
);
t!(
    test_string_empty,
    "fixtures/valid/string-empty.json",
    "fixtures/valid/string-empty.toml"
);
t!(
    test_string_escapes,
    "fixtures/valid/string-escapes.json",
    "fixtures/valid/string-escapes.toml"
);
t!(
    test_string_nl,
    "fixtures/valid/string-nl.json",
    "fixtures/valid/string-nl.toml"
);
t!(
    test_string_simple,
    "fixtures/valid/string-simple.json",
    "fixtures/valid/string-simple.toml"
);
t!(
    test_string_with_pound,
    "fixtures/valid/string-with-pound.json",
    "fixtures/valid/string-with-pound.toml"
);
t!(
    test_table_array_implicit,
    "fixtures/valid/table-array-implicit.json",
    "fixtures/valid/table-array-implicit.toml"
);
t!(
    test_table_array_many,
    "fixtures/valid/table-array-many.json",
    "fixtures/valid/table-array-many.toml"
);
t!(
    test_table_array_nest,
    "fixtures/valid/table-array-nest.json",
    "fixtures/valid/table-array-nest.toml"
);
t!(
    test_array_one,
    "fixtures/valid/table-array-one.json",
    "fixtures/valid/table-array-one.toml"
);
t!(
    test_table_empty,
    "fixtures/valid/table-empty.json",
    "fixtures/valid/table-empty.toml"
);
t!(
    test_table_no_eol,
    "fixtures/valid/table-no-eol.json",
    "fixtures/valid/table-no-eol.toml"
);
t!(
    test_table_sub_empty,
    "fixtures/valid/table-sub-empty.json",
    "fixtures/valid/table-sub-empty.toml"
);
t!(
    test_whitespace,
    "fixtures/valid/table-whitespace.json",
    "fixtures/valid/table-whitespace.toml"
);
t!(
    test_table_with_literal_string,
    "fixtures/valid/table-with-literal-string.json",
    "fixtures/valid/table-with-literal-string.toml"
);
t!(
    test_table_with_poung,
    "fixtures/valid/table-with-pound.json",
    "fixtures/valid/table-with-pound.toml"
);
t!(
    test_table_with_single_quotes,
    "fixtures/valid/table-with-single-quotes.json",
    "fixtures/valid/table-with-single-quotes.toml"
);
t!(
    test_unicode_escape,
    "fixtures/valid/unicode-escape.json",
    "fixtures/valid/unicode-escape.toml"
);
t!(
    test_unicode_literal,
    "fixtures/valid/unicode-literal.json",
    "fixtures/valid/unicode-literal.toml"
);
t!(
    test_windows_path,
    "fixtures/valid/windows-path.json",
    "fixtures/valid/windows-path.toml"
);
<commit_msg>test(parser): Test the first encountered key's quote type is used<commit_after>extern crate serde_json;
extern crate toml_edit;

use serde_json::Value as Json;
use serde_json::Map as JsonMap;

use toml_edit::{Document, Item, Value, Iter};

fn pair_to_json((key, value): (&str, Item)) -> (String, Json) {
    fn typed_json(s: &str, json: Json) -> Json {
        let mut map = JsonMap::new();
        map.insert("type".to_owned(), Json::String(s.into()));
        map.insert("value".to_owned(), json);
        Json::Object(map)
    }
    fn value_to_json(value: &Value) -> Json {
        match *value {
            Value::String(ref s) => typed_json("string", Json::String(s.value().clone())),
            Value::Integer(ref i) => typed_json("integer", Json::String(format!("{}", i.value()))),
            Value::Float(ref f) => typed_json("float", Json::String(format!("{}", f.value()))),
            Value::Boolean(ref b) => typed_json("bool", Json::String(b.raw().into())),
            Value::DateTime(ref d) => typed_json("datetime", Json::String(d.raw().into())),
            Value::Array(ref a) => {
                let json = Json::Array(a.iter().map(value_to_json).collect::<Vec<_>>());
                typed_json("array", json)
            }
            Value::InlineTable(ref t) => {
                to_json(Box::new(t.iter().map(|(k, v)| (k, Item::Value(v.clone())))))
            }
        }
    }
    let json = match value {
        Item::Value(ref v) => value_to_json(v),
        Item::ArrayOfTables(ref arr) => {
            Json::Array(arr.iter().map(|t| to_json(iter_to_owned(t.iter()))).collect::<Vec<_>>())
        }
        Item::Table(ref table) => to_json(iter_to_owned(table.iter())),
        Item::None => Json::Null,
    };
    (key.to_owned(), json)
}

fn iter_to_owned(iter: Iter) -> OwnedIter {
    Box::new(
        iter.map(|(k, v)| (k, v.clone()))
    )
}

type OwnedIter<'s> = Box<Iterator<Item = (&'s str, Item)> + 's>;

fn to_json(iter: OwnedIter) -> Json {
    Json::Object(iter.map(pair_to_json).collect())
}

fn run(json: &str, toml: &str) {
    let doc = toml.parse::<Document>();
    assert!(doc.is_ok());
    let doc = doc.unwrap();

    let json: Json = serde_json::from_str(json).unwrap();
    let toml_json = to_json(iter_to_owned(doc.iter()));
    // compare structure with jsons
    assert_eq!(json, toml_json);

    // check round-trip equality
    let toml = doc.to_string();
    let doc = toml.parse::<Document>();
    assert!(doc.is_ok());
    let doc = doc.unwrap();
    assert_eq!(doc.to_string(), toml);
}

macro_rules! t(
    ($name:ident, $json:expr, $toml:expr) => (
        #[test]
        fn $name() {
            run(include_str!($json),
                include_str!($toml));
        }
    )
);

#[test]
fn table_reordering() {
    let toml = r#"
[[bin]] # bin 1
[a.'b'.c.e]
[a]
[other.table]
[[bin]] # bin 2
[a.b.c.d]
[a."b".c]
[[bin]] # bin 3
"#;
    let expected = r#"
[[bin]] # bin 1
[[bin]] # bin 2
[[bin]] # bin 3
[a]
[a.'b'.c]
[a.'b'.c.e]
[a.'b'.c.d]
[other.table]
"#;
    let doc = toml.parse::<Document>();
    assert!(doc.is_ok());
    let doc = doc.unwrap();
    assert_eq!(doc.to_string(), expected);
}

t!(
    test_array_empty,
    "fixtures/valid/array-empty.json",
    "fixtures/valid/array-empty.toml"
);
t!(
    test_array_nopspaces,
    "fixtures/valid/array-nospaces.json",
    "fixtures/valid/array-nospaces.toml"
);
t!(
    test_arrays_hetergeneous,
    "fixtures/valid/arrays-hetergeneous.json",
    "fixtures/valid/arrays-hetergeneous.toml"
);
t!(
    test_arrays,
    "fixtures/valid/arrays.json",
    "fixtures/valid/arrays.toml"
);
t!(
    test_arrays_nested,
    "fixtures/valid/arrays-nested.json",
    "fixtures/valid/arrays-nested.toml"
);
t!(
    test_bool,
    "fixtures/valid/bool.json",
    "fixtures/valid/bool.toml"
);
t!(
    test_comment_at_eof2,
    "fixtures/valid/comments-at-eof2.json",
    "fixtures/valid/comments-at-eof2.toml"
);
t!(
    test_comment_at_eof,
    "fixtures/valid/comments-at-eof.json",
    "fixtures/valid/comments-at-eof.toml"
);
t!(
    test_comments_everywhere,
    "fixtures/valid/comments-everywhere.json",
    "fixtures/valid/comments-everywhere.toml"
);
t!(
    test_datetime,
    "fixtures/valid/datetime.json",
    "fixtures/valid/datetime.toml"
);
t!(
    test_empty,
    "fixtures/valid/empty.json",
    "fixtures/valid/empty.toml"
);
t!(
    test_example,
    "fixtures/valid/example.json",
    "fixtures/valid/example.toml"
);
t!(
    test_float_exponent,
    "fixtures/valid/float-exponent.json",
    "fixtures/valid/float-exponent.toml"
);
t!(
    test_float,
    "fixtures/valid/float.json",
    "fixtures/valid/float.toml"
);
t!(
    test_float_underscore,
    "fixtures/valid/float-underscore.json",
    "fixtures/valid/float-underscore.toml"
);
t!(
    test_implicit_and_explicit_after,
    "fixtures/valid/implicit-and-explicit-after.json",
    "fixtures/valid/implicit-and-explicit-after.toml"
);
t!(
    test_implicit_and_explicit_before,
    "fixtures/valid/implicit-and-explicit-before.json",
    "fixtures/valid/implicit-and-explicit-before.toml"
);
t!(
    test_implicit_groups,
    "fixtures/valid/implicit-groups.json",
    "fixtures/valid/implicit-groups.toml"
);
t!(
    test_inline_table,
    "fixtures/valid/inline-table.json",
    "fixtures/valid/inline-table.toml"
);
t!(
    test_integer,
    "fixtures/valid/integer.json",
    "fixtures/valid/integer.toml"
);
t!(
    test_integer_underscore,
    "fixtures/valid/integer-underscore.json",
    "fixtures/valid/integer-underscore.toml"
);
t!(
    test_keys_equal_nospace,
    "fixtures/valid/key-equals-nospace.json",
    "fixtures/valid/key-equals-nospace.toml"
);
t!(
    test_key_numeric,
    "fixtures/valid/key-numeric.json",
    "fixtures/valid/key-numeric.toml"
);
t!(
    test_key_space,
    "fixtures/valid/key-space.json",
    "fixtures/valid/key-space.toml"
);
t!(
    test_key_special_chars,
    "fixtures/valid/key-special-chars.json",
    "fixtures/valid/key-special-chars.toml"
);
t!(
    test_long_float,
    "fixtures/valid/long-float.json",
    "fixtures/valid/long-float.toml"
);
t!(
    test_long_integer,
    "fixtures/valid/long-integer.json",
    "fixtures/valid/long-integer.toml"
);
t!(
    test_multiline_string,
    "fixtures/valid/multiline-string.json",
    "fixtures/valid/multiline-string.toml"
);
t!(
    test_raw_multiline_string,
    "fixtures/valid/raw-multiline-string.json",
    "fixtures/valid/raw-multiline-string.toml"
);
t!(
    test_raw_string,
    "fixtures/valid/raw-string.json",
    "fixtures/valid/raw-string.toml"
);
t!(
    test_string_empty,
    "fixtures/valid/string-empty.json",
    "fixtures/valid/string-empty.toml"
);
t!(
    test_string_escapes,
    "fixtures/valid/string-escapes.json",
    "fixtures/valid/string-escapes.toml"
);
t!(
    test_string_nl,
    "fixtures/valid/string-nl.json",
    "fixtures/valid/string-nl.toml"
);
t!(
    test_string_simple,
    "fixtures/valid/string-simple.json",
    "fixtures/valid/string-simple.toml"
);
t!(
    test_string_with_pound,
    "fixtures/valid/string-with-pound.json",
    "fixtures/valid/string-with-pound.toml"
);
t!(
    test_table_array_implicit,
    "fixtures/valid/table-array-implicit.json",
    "fixtures/valid/table-array-implicit.toml"
);
t!(
    test_table_array_many,
    "fixtures/valid/table-array-many.json",
    "fixtures/valid/table-array-many.toml"
);
t!(
    test_table_array_nest,
    "fixtures/valid/table-array-nest.json",
    "fixtures/valid/table-array-nest.toml"
);
t!(
    test_array_one,
    "fixtures/valid/table-array-one.json",
    "fixtures/valid/table-array-one.toml"
);
t!(
    test_table_empty,
    "fixtures/valid/table-empty.json",
    "fixtures/valid/table-empty.toml"
);
t!(
    test_table_no_eol,
    "fixtures/valid/table-no-eol.json",
    "fixtures/valid/table-no-eol.toml"
);
t!(
    test_table_sub_empty,
    "fixtures/valid/table-sub-empty.json",
    "fixtures/valid/table-sub-empty.toml"
);
t!(
    test_whitespace,
    "fixtures/valid/table-whitespace.json",
    "fixtures/valid/table-whitespace.toml"
);
t!(
    test_table_with_literal_string,
    "fixtures/valid/table-with-literal-string.json",
    "fixtures/valid/table-with-literal-string.toml"
);
t!(
    test_table_with_poung,
    "fixtures/valid/table-with-pound.json",
    "fixtures/valid/table-with-pound.toml"
);
t!(
    test_table_with_single_quotes,
    "fixtures/valid/table-with-single-quotes.json",
    "fixtures/valid/table-with-single-quotes.toml"
);
t!(
    test_unicode_escape,
    "fixtures/valid/unicode-escape.json",
    "fixtures/valid/unicode-escape.toml"
);
t!(
    test_unicode_literal,
    "fixtures/valid/unicode-literal.json",
    "fixtures/valid/unicode-literal.toml"
);
t!(
    test_windows_path,
    "fixtures/valid/windows-path.json",
    "fixtures/valid/windows-path.toml"
);
<|endoftext|>


<commit_before>#![deny(warnings)]
extern crate extra;

use std::io;
use std::io::{Write, BufRead, BufReader, Stderr};
use std::env;
use std::fs::File;
use std::path::Path;
use std::error::Error;
use std::process::exit;

use extra::option::OptionalExt;
use extra::io::WriteExt;

static MAN_PAGE: &'static str = /* @MANSTART{grep} */ r#"
NAME
    grep - print lines matching a pattern

SYNOPSIS
    grep [-h | --help] [-n --line-number] PATTERN [FILE...]

DESCRIPTION
    grep searches the named input FILEs for lines containing a match to the given PATTERN. If no
    files are specified, grep searches the standard input. grep prints the matching lines.

OPTIONS
    -h
    --help
        Print this manual page.

    -n
    --line-number
        Prefix each line of output with the line number of the match.
"#; /* @MANEND */

#[derive(Copy, Clone)]
struct Flags {
    line_numbers: bool,
}

impl Flags {
    fn new() -> Flags {
        Flags { line_numbers: false }
    }
}

fn main() {
    let args = env::args().skip(1);
    let stdout = io::stdout();
    let mut stdout = stdout.lock();
    let mut stderr = io::stderr();
    let stdin = io::stdin();
    let stdin = stdin.lock();

    let mut flags = Flags::new();
    let mut pattern = String::new();
    let mut files = Vec::with_capacity(args.len());
    for arg in args {
        if arg.starts_with("-") {
            match arg.as_str() {
                "-h" | "--help" => {
                    stdout.writeln(MAN_PAGE.as_bytes()).try(&mut stderr);
                },
                "-n" | "--line-number" => flags.line_numbers = true,
                _ => {
                    stderr.write(b"Unknown option: ").try(&mut stderr);
                    stderr.write(arg.as_bytes()).try(&mut stderr);
                    stderr.write(b"\n").try(&mut stderr);
                    let _ = stderr.flush();
                    exit(1);
                }
            }
        } else if pattern.is_empty() {
            pattern = arg.clone();
        } else {
            match File::open(&Path::new(&arg)) {
                Ok(f) => files.push(f),
                Err(e) => {
                    stderr.write(b"Error opening ").try(&mut stderr);
                    stderr.write(arg.as_bytes()).try(&mut stderr);
                    stderr.write(b": ").try(&mut stderr);
                    stderr.write(e.description().as_bytes()).try(&mut stderr);
                    stderr.write(b"\n").try(&mut stderr);
                    let _ = stderr.flush();
                    exit(1);
                }
            }
        }
    }

    if pattern.is_empty() {
        stderr.write_all(b"You must provide a pattern\n").try(&mut stderr);
        exit(1);
    }

    if files.is_empty() {
        do_simple_search(BufReader::new(stdin), &pattern, &mut stdout, &mut stderr, flags);
    } else {
        for f in files {
            do_simple_search(BufReader::new(f), &pattern, &mut stdout, &mut stderr, flags);
        }
    }
}

fn do_simple_search<T: BufRead, O: Write + WriteExt>(reader: T, pattern: &str, out: &mut O, stderr: &mut Stderr, flags: Flags) {
    let mut line_num = 0;
    for result in reader.lines() {
        line_num += 1;
        if let Ok(line) = result {
            if line.contains(pattern) {
                if flags.line_numbers {
                    out.write_all((line_num.to_string() + ": ").as_bytes()).try(stderr);
                }
                out.writeln(line.as_bytes()).try(stderr);
            }
        }
    }
}
<commit_msg>Add -v argument to grep<commit_after>#![deny(warnings)]
extern crate extra;

use std::io;
use std::io::{Write, BufRead, BufReader, Stderr};
use std::env;
use std::fs::File;
use std::path::Path;
use std::error::Error;
use std::process::exit;

use extra::option::OptionalExt;
use extra::io::WriteExt;

static MAN_PAGE: &'static str = /* @MANSTART{grep} */ r#"
NAME
    grep - print lines matching a pattern

SYNOPSIS
    grep [-h | --help] [-n --line-number] PATTERN [FILE...]

DESCRIPTION
    grep searches the named input FILEs for lines containing a match to the given PATTERN. If no
    files are specified, grep searches the standard input. grep prints the matching lines.

OPTIONS
    -h
    --help
        Print this manual page.

    -v
    --invert-match
        Invert matching.

    -n
    --line-number
        Prefix each line of output with the line number of the match.
"#; /* @MANEND */

#[derive(Copy, Clone)]
struct Flags {
    line_numbers: bool,
    invert_match: bool,
}

impl Flags {
    fn new() -> Flags {
        Flags { line_numbers: false, invert_match: false }
    }
}

fn main() {
    let args = env::args().skip(1);
    let stdout = io::stdout();
    let mut stdout = stdout.lock();
    let mut stderr = io::stderr();
    let stdin = io::stdin();
    let stdin = stdin.lock();

    let mut flags = Flags::new();
    let mut pattern = String::new();
    let mut files = Vec::with_capacity(args.len());
    for arg in args {
        if arg.starts_with("-") {
            match arg.as_str() {
                "-h" | "--help" => {
                    stdout.writeln(MAN_PAGE.as_bytes()).try(&mut stderr);
                },
                "-n" | "--line-number" => flags.line_numbers = true,
                "-v" | "--invert-match" => flags.invert_match = true,
                _ => {
                    stderr.write(b"Unknown option: ").try(&mut stderr);
                    stderr.write(arg.as_bytes()).try(&mut stderr);
                    stderr.write(b"\n").try(&mut stderr);
                    let _ = stderr.flush();
                    exit(1);
                }
            }
        } else if pattern.is_empty() {
            pattern = arg.clone();
        } else {
            match File::open(&Path::new(&arg)) {
                Ok(f) => files.push(f),
                Err(e) => {
                    stderr.write(b"Error opening ").try(&mut stderr);
                    stderr.write(arg.as_bytes()).try(&mut stderr);
                    stderr.write(b": ").try(&mut stderr);
                    stderr.write(e.description().as_bytes()).try(&mut stderr);
                    stderr.write(b"\n").try(&mut stderr);
                    let _ = stderr.flush();
                    exit(1);
                }
            }
        }
    }

    if pattern.is_empty() {
        stderr.write_all(b"You must provide a pattern\n").try(&mut stderr);
        exit(1);
    }

    if files.is_empty() {
        do_simple_search(BufReader::new(stdin), &pattern, &mut stdout, &mut stderr, flags);
    } else {
        for f in files {
            do_simple_search(BufReader::new(f), &pattern, &mut stdout, &mut stderr, flags);
        }
    }
}

fn do_simple_search<T: BufRead, O: Write + WriteExt>(reader: T, pattern: &str, out: &mut O, stderr: &mut Stderr, flags: Flags) {
    let mut line_num = 0;
    for result in reader.lines() {
        line_num += 1;
        if let Ok(line) = result {
            let is_match = if flags.invert_match {
                !line.contains(pattern)
            } else {
                line.contains(pattern)
            };
            if is_match {
                if flags.line_numbers {
                    out.write_all((line_num.to_string() + ": ").as_bytes()).try(stderr);
                }
                out.writeln(line.as_bytes()).try(stderr);
            }
        }
    }
}
<|endoftext|>


<commit_before>#![deny(warnings)]

// TODO support reading from standard input

extern crate coreutils;
extern crate termion;

use std::env::args;
use std::fs::File;
use std::io::{self, Write, Read, StdoutLock, Stderr};
use std::path::Path;

use coreutils::extra::OptionalExt;

use termion::{terminal_size, TermRead, TermWrite, IntoRawMode, Key, Style, RawTerminal};

static LONG_HELP: &'static str = r#"
    NAME
        less - view a text file.
    SYNOPSIS
        less [-h | --help] [input]
    DESCRIPTION
        This utility views text files. If no input file is specified as an argument, standard input is used.
    OPTIONS
        --help, -h
            Print this manual page.
    AUTHOR
        This program was written by MovingtoMars for Redox OS. Bugs, issues, or feature requests should be reported in the Github repository, 'redox-os/extrautils'.
    COPYRIGHT
        Copyright (c) 2016 MovingtoMars

        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"#;

fn main() {
    let mut args = args().skip(1);
    let mut stdout = io::stdout().lock();
    let mut stdin = io::stdin().lock();
    let mut stderr = io::stderr();

    if let Some(x) = args.next() {
        match x.as_str() {
            "--help" | "-h" => {
                // Print help.
                stdout.write(LONG_HELP.as_bytes()).try(&mut stderr);
                return;
            },
            filename => {
                let mut file = File::open(Path::new(filename)).try(&mut stderr);
                run(&mut file, &mut stdin, &mut stdout, &mut stderr);
            }
        }

        if let Some(x) = args.next() {
            let mut file = File::open(Path::new(x.as_str())).try(&mut stderr);
            run(&mut file, &mut stdin, &mut stdout, &mut stderr);
        }
    } else {
        writeln!(stderr, "Readin from stdin is not yet supported").try(&mut stderr);
        //run(&mut stdin, t, &mut stdout, &mut stderr);
    };
}

struct Buffer {
    lines: Vec<String>,
    y_off: u16,
}

impl Buffer {
    fn new() -> Buffer {
        Buffer{
            lines: Vec::new(),
            y_off: 0,
        }
    }

    fn read(&mut self, from: &mut Read) -> std::io::Result<usize> {
        let mut tmp = String::new();
        let res = from.read_to_string(&mut tmp);
        try!(res);

        self.lines.append(&mut tmp
            .as_str()
            .split("\n")
            .map(|x| {String::from(x)})
            .collect());

        return res;
    }

    fn draw(&self, to: &mut RawTerminal<&mut StdoutLock>, w: u16, h: u16) -> std::io::Result<usize> {
        try!(to.goto(0, 0));

        let mut y = 0;
        let mut bytes_written = 0;

        let lines = if self.lines.len() <= h as usize {
            &self.lines
        } else if self.y_off as usize >= self.lines.len() {
            &self.lines[0..0]
        } else {
            &self.lines[self.y_off as usize..]
        };

        for line in lines {
            if line.len() > w as usize {
                bytes_written += try!(to.write(line[..w as usize].as_bytes()));
            } else {
                bytes_written += try!(to.write(line.as_bytes()));
            }

            y += 1;
            try!(to.goto(0, y));
            if y >= h {
                break;
            }
        }

        Ok(bytes_written)
    }

    fn scroll_up(&mut self) {
        if self.y_off != 0 {
            self.y_off -= 1;
        }
    }

    fn scroll_down(&mut self) {
        if (self.y_off as usize) < self.lines.len() - 1 {
            self.y_off += 1;
        }
    }
}

fn run(file: &mut Read, controls: &mut Read, stdout: &mut StdoutLock, stderr: &mut Stderr) {
    let mut stdout = stdout.into_raw_mode().try(stderr);

    let (w, h) = {
        let (w, h) = terminal_size().try(stderr);
        (w as u16, h as u16)
    };

    stdout.clear().try(stderr);
    stdout.goto(0, h - 1).try(stderr);
    stdout.style(Style::Bold).try(stderr);
    stdout.write(b"Press q to exit.").try(stderr);
    stdout.goto(0, h - 1).try(stderr);
    stdout.reset().try(stderr);

    let mut buffer = Buffer::new();
    buffer.read(file).try(stderr);

    buffer.draw(&mut stdout, w, h - 1).try(stderr);
    stdout.goto(0, h - 1).try(stderr);
    stdout.flush().try(stderr);

    for c in controls.keys() {
        match c {
            Key::Char('q') => {
                stdout.clear().try(stderr);
                stdout.goto(0, 0).try(stderr);
                return
            },
            Key::Up => buffer.scroll_up(),
            Key::Down => buffer.scroll_down(),
            _ => {},
        }

        stdout.clear().try(stderr);
        buffer.draw(&mut stdout, w, h - 1).try(stderr);
        stdout.goto(0, h - 1).try(stderr);
        stdout.style(Style::Bold).try(stderr);
        stdout.write(b"Press q to exit.").try(stderr);
        stdout.goto(0, h - 1).try(stderr);
        stdout.reset().try(stderr);
        stdout.flush().try(stderr);
    }
}
<commit_msg>Limit scroll to end of file, correct redraw on redox<commit_after>#![deny(warnings)]

// TODO support reading from standard input

extern crate coreutils;
extern crate termion;

use std::env::args;
use std::fs::File;
use std::io::{self, Write, Read, StdoutLock, Stderr};
use std::path::Path;

use coreutils::extra::OptionalExt;

use termion::{terminal_size, TermRead, TermWrite, IntoRawMode, Key, Style, RawTerminal};

static LONG_HELP: &'static str = r#"
    NAME
        less - view a text file.
    SYNOPSIS
        less [-h | --help] [input]
    DESCRIPTION
        This utility views text files. If no input file is specified as an argument, standard input is used.
    OPTIONS
        --help, -h
            Print this manual page.
    AUTHOR
        This program was written by MovingtoMars for Redox OS. Bugs, issues, or feature requests should be reported in the Github repository, 'redox-os/extrautils'.
    COPYRIGHT
        Copyright (c) 2016 MovingtoMars

        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"#;

fn main() {
    let mut args = args().skip(1);
    let mut stdout = io::stdout().lock();
    let mut stdin = io::stdin().lock();
    let mut stderr = io::stderr();

    if let Some(x) = args.next() {
        match x.as_str() {
            "--help" | "-h" => {
                // Print help.
                stdout.write(LONG_HELP.as_bytes()).try(&mut stderr);
                return;
            },
            filename => {
                let mut file = File::open(Path::new(filename)).try(&mut stderr);
                run(&mut file, &mut stdin, &mut stdout, &mut stderr);
            }
        }

        if let Some(x) = args.next() {
            let mut file = File::open(Path::new(x.as_str())).try(&mut stderr);
            run(&mut file, &mut stdin, &mut stdout, &mut stderr);
        }
    } else {
        writeln!(stderr, "Readin from stdin is not yet supported").try(&mut stderr);
        //run(&mut stdin, &mut stdin, &mut stdout, &mut stderr);
    };
}

struct Buffer {
    lines: Vec<String>,
    y_off: u16,
}

impl Buffer {
    fn new() -> Buffer {
        Buffer{
            lines: Vec::new(),
            y_off: 0,
        }
    }

    fn read(&mut self, from: &mut Read) -> std::io::Result<usize> {
        let mut tmp = String::new();
        let res = from.read_to_string(&mut tmp);
        try!(res);

        self.lines.append(&mut tmp
            .as_str()
            .split("\n")
            .map(|x| {String::from(x)})
            .collect());

        return res;
    }

    fn draw(&self, to: &mut RawTerminal<&mut StdoutLock>, w: u16, h: u16) -> std::io::Result<usize> {
        try!(to.goto(0, 0));

        let mut y = 0;
        let mut bytes_written = 0;

        let lines = if self.lines.len() <= h as usize {
            &self.lines
        } else if self.y_off as usize >= self.lines.len() {
            &self.lines[0..0]
        } else {
            &self.lines[self.y_off as usize..]
        };

        for line in lines {
            if line.len() > w as usize {
                bytes_written += try!(to.write(line[..w as usize].as_bytes()));
            } else {
                bytes_written += try!(to.write(line.as_bytes()));
            }

            y += 1;
            try!(to.goto(0, y));
            if y >= h {
                break;
            }
        }

        Ok(bytes_written)
    }

    fn scroll_up(&mut self) {
        if self.y_off > 0 {
            self.y_off -= 1;
        }
    }

    fn scroll_down(&mut self, height: u16) {
        if ((self.y_off + height) as usize) <= self.lines.len() {
            self.y_off += 1;
        }
    }
}

fn run(file: &mut Read, controls: &mut Read, stdout: &mut StdoutLock, stderr: &mut Stderr) {
    let mut stdout = stdout.into_raw_mode().try(stderr);

    let (w, h) = {
        let (w, h) = terminal_size().try(stderr);
        (w as u16, h as u16)
    };

    stdout.clear().try(stderr);
    stdout.reset().try(stderr);

    let mut buffer = Buffer::new();
    buffer.read(file).try(stderr);

    buffer.draw(&mut stdout, w, h - 1).try(stderr);
    stdout.goto(0, h - 1).try(stderr);
    stdout.style(Style::Bold).try(stderr);
    stdout.write(b"Press q to exit.").try(stderr);
    stdout.reset().try(stderr);
    stdout.flush().try(stderr);

    for c in controls.keys() {
        match c {
            Key::Char('q') => {
                stdout.clear().try(stderr);
                stdout.reset().try(stderr);
                stdout.goto(0, 0).try(stderr);
                return
            },
            Key::Up => buffer.scroll_up(),
            Key::Down => buffer.scroll_down(h),
            _ => {},
        }

        stdout.clear().try(stderr);
        stdout.reset().try(stderr);
        buffer.draw(&mut stdout, w, h - 1).try(stderr);
        stdout.goto(0, h - 1).try(stderr);
        stdout.style(Style::Bold).try(stderr);
        stdout.write(b"Press q to exit.").try(stderr);
        stdout.reset().try(stderr);
        stdout.flush().try(stderr);
    }
}
<|endoftext|>


<commit_before>use rustc::hir::*;
use rustc::hir::intravisit::FnKind;
use rustc::lint::*;
use rustc::ty::{self, RegionKind, TypeFoldable};
use rustc::traits;
use rustc::middle::expr_use_visitor as euv;
use rustc::middle::mem_categorization as mc;
use syntax::ast::NodeId;
use syntax_pos::Span;
use syntax::errors::DiagnosticBuilder;
use utils::{get_trait_def_id, implements_trait, in_macro, is_copy, is_self, match_type, multispan_sugg, paths,
            snippet, snippet_opt, span_lint_and_then};
use utils::ptr::get_spans;
use std::collections::{HashMap, HashSet};
use std::borrow::Cow;

/// **What it does:** Checks for functions taking arguments by value, but not
/// consuming them in its
/// body.
///
/// **Why is this bad?** Taking arguments by reference is more flexible and can
/// sometimes avoid
/// unnecessary allocations.
///
/// **Known problems:** Hopefully none.
///
/// **Example:**
/// ```rust
/// fn foo(v: Vec<i32>) {
///     assert_eq!(v.len(), 42);
/// }
/// ```
declare_lint! {
    pub NEEDLESS_PASS_BY_VALUE,
    Warn,
    "functions taking arguments by value, but not consuming them in its body"
}

pub struct NeedlessPassByValue;

impl LintPass for NeedlessPassByValue {
    fn get_lints(&self) -> LintArray {
        lint_array![NEEDLESS_PASS_BY_VALUE]
    }
}

macro_rules! need {
    ($e: expr) => { if let Some(x) = $e { x } else { return; } };
}

impl<'a, 'tcx> LateLintPass<'a, 'tcx> for NeedlessPassByValue {
    fn check_fn(
        &mut self,
        cx: &LateContext<'a, 'tcx>,
        kind: FnKind<'tcx>,
        decl: &'tcx FnDecl,
        body: &'tcx Body,
        span: Span,
        node_id: NodeId,
    ) {
        if in_macro(span) {
            return;
        }

        match kind {
            FnKind::ItemFn(.., attrs) => for a in attrs {
                if_let_chain!{[
                    a.meta_item_list().is_some(),
                    let Some(name) = a.name(),
                    name == "proc_macro_derive",
                ], {
                    return;
                }}
            },
            _ => return,
        }

        // Allow `Borrow` or functions to be taken by value
        let borrow_trait = need!(get_trait_def_id(cx, &paths::BORROW_TRAIT));
        let fn_traits = [
            need!(cx.tcx.lang_items().fn_trait()),
            need!(cx.tcx.lang_items().fn_once_trait()),
            need!(cx.tcx.lang_items().fn_mut_trait()),
        ];

        let sized_trait = need!(cx.tcx.lang_items().sized_trait());

        let fn_def_id = cx.tcx.hir.local_def_id(node_id);

        let preds = traits::elaborate_predicates(cx.tcx, cx.param_env.caller_bounds.to_vec())
            .filter(|p| !p.is_global())
            .filter_map(|pred| if let ty::Predicate::Trait(poly_trait_ref) = pred {
                if poly_trait_ref.def_id() == sized_trait || poly_trait_ref.skip_binder().has_escaping_regions() {
                    return None;
                }
                Some(poly_trait_ref)
            } else {
                None
            })
            .collect::<Vec<_>>();

        // Collect moved variables and spans which will need dereferencings from the
        // function body.
        let MovedVariablesCtxt {
            moved_vars,
            spans_need_deref,
            ..
        } = {
            let mut ctx = MovedVariablesCtxt::new(cx);
            let region_scope_tree = &cx.tcx.region_scope_tree(fn_def_id);
            euv::ExprUseVisitor::new(&mut ctx, cx.tcx, cx.param_env, region_scope_tree, cx.tables).consume_body(body);
            ctx
        };

        let fn_sig = cx.tcx.fn_sig(fn_def_id);
        let fn_sig = cx.tcx.erase_late_bound_regions(&fn_sig);

        for (idx, ((input, &ty), arg)) in decl.inputs
            .iter()
            .zip(fn_sig.inputs())
            .zip(&body.arguments)
            .enumerate()
        {
            // * Exclude a type that is specifically bounded by `Borrow`.
            // * Exclude a type whose reference also fulfills its bound.
            //   (e.g. `std::convert::AsRef`, `serde::Serialize`)
            let (implements_borrow_trait, all_borrowable_trait) = {
                let preds = preds
                    .iter()
                    .filter(|t| t.skip_binder().self_ty() == ty)
                    .collect::<Vec<_>>();

                (
                    preds.iter().any(|t| t.def_id() == borrow_trait),
                    !preds.is_empty() && preds.iter().all(|t| {
                        implements_trait(
                            cx,
                            cx.tcx.mk_imm_ref(&RegionKind::ReErased, ty),
                            t.def_id(),
                            &t.skip_binder().input_types().skip(1).collect::<Vec<_>>(),
                        )
                    }),
                )
            };

            if_let_chain! {[
                !is_self(arg),
                !ty.is_mutable_pointer(),
                !is_copy(cx, ty),
                !fn_traits.iter().any(|&t| implements_trait(cx, ty, t, &[])),
                !implements_borrow_trait,
                !all_borrowable_trait,

                let PatKind::Binding(mode, canonical_id, ..) = arg.pat.node,
                !moved_vars.contains(&canonical_id),
            ], {
                if mode == BindingAnnotation::Mutable || mode == BindingAnnotation::RefMut {
                    continue;
                }

                // Dereference suggestion
                let sugg = |db: &mut DiagnosticBuilder| {
                    let deref_span = spans_need_deref.get(&canonical_id);
                    if_let_chain! {[
                        match_type(cx, ty, &paths::VEC),
                        let Some(clone_spans) =
                            get_spans(cx, Some(body.id()), idx, &[("clone", ".to_owned()")]),
                        let TyPath(QPath::Resolved(_, ref path)) = input.node,
                        let Some(elem_ty) = path.segments.iter()
                            .find(|seg| seg.name == "Vec")
                            .and_then(|ps| ps.parameters.as_ref())
                            .map(|params| &params.types[0]),
                    ], {
                        let slice_ty = format!("&[{}]", snippet(cx, elem_ty.span, "_"));
                        db.span_suggestion(input.span,
                                        "consider changing the type to",
                                        slice_ty);

                        for (span, suggestion) in clone_spans {
                            db.span_suggestion(
                                span,
                                &snippet_opt(cx, span)
                                    .map_or(
                                        "change the call to".into(),
                                        |x| Cow::from(format!("change `{}` to", x)),
                                    ),
                                suggestion.into()
                            );
                        }

                        // cannot be destructured, no need for `*` suggestion
                        assert!(deref_span.is_none());
                        return;
                    }}

                    if match_type(cx, ty, &paths::STRING) {
                        if let Some(clone_spans) =
                            get_spans(cx, Some(body.id()), idx, &[("clone", ".to_string()"), ("as_str", "")]) {
                            db.span_suggestion(input.span, "consider changing the type to", "&str".to_string());

                            for (span, suggestion) in clone_spans {
                                db.span_suggestion(
                                    span,
                                    &snippet_opt(cx, span)
                                        .map_or(
                                            "change the call to".into(),
                                            |x| Cow::from(format!("change `{}` to", x))
                                        ),
                                    suggestion.into(),
                                );
                            }

                            assert!(deref_span.is_none());
                            return;
                        }
                    }

                    let mut spans = vec![(input.span, format!("&{}", snippet(cx, input.span, "_")))];

                    // Suggests adding `*` to dereference the added reference.
                    if let Some(deref_span) = deref_span {
                        spans.extend(
                            deref_span
                                .iter()
                                .cloned()
                                .map(|span| (span, format!("*{}", snippet(cx, span, "<expr>")))),
                        );
                        spans.sort_by_key(|&(span, _)| span);
                    }
                    multispan_sugg(db, "consider taking a reference instead".to_string(), spans);
                };

                span_lint_and_then(
                    cx,
                    NEEDLESS_PASS_BY_VALUE,
                    input.span,
                    "this argument is passed by value, but not consumed in the function body",
                    sugg,
                );
            }}
        }
    }
}

struct MovedVariablesCtxt<'a, 'tcx: 'a> {
    cx: &'a LateContext<'a, 'tcx>,
    moved_vars: HashSet<NodeId>,
    /// Spans which need to be prefixed with `*` for dereferencing the
    /// suggested additional reference.
    spans_need_deref: HashMap<NodeId, HashSet<Span>>,
}

impl<'a, 'tcx> MovedVariablesCtxt<'a, 'tcx> {
    fn new(cx: &'a LateContext<'a, 'tcx>) -> Self {
        Self {
            cx: cx,
            moved_vars: HashSet::new(),
            spans_need_deref: HashMap::new(),
        }
    }

    fn move_common(&mut self, _consume_id: NodeId, _span: Span, cmt: mc::cmt<'tcx>) {
        let cmt = unwrap_downcast_or_interior(cmt);

        if let mc::Categorization::Local(vid) = cmt.cat {
            self.moved_vars.insert(vid);
        }
    }

    fn non_moving_pat(&mut self, matched_pat: &Pat, cmt: mc::cmt<'tcx>) {
        let cmt = unwrap_downcast_or_interior(cmt);

        if let mc::Categorization::Local(vid) = cmt.cat {
            let mut id = matched_pat.id;
            loop {
                let parent = self.cx.tcx.hir.get_parent_node(id);
                if id == parent {
                    // no parent
                    return;
                }
                id = parent;

                if let Some(node) = self.cx.tcx.hir.find(id) {
                    match node {
                        map::Node::NodeExpr(e) => {
                            // `match` and `if let`
                            if let ExprMatch(ref c, ..) = e.node {
                                self.spans_need_deref
                                    .entry(vid)
                                    .or_insert_with(HashSet::new)
                                    .insert(c.span);
                            }
                        },

                        map::Node::NodeStmt(s) => {
                            // `let <pat> = x;`
                            if_let_chain! {[
                                let StmtDecl(ref decl, _) = s.node,
                                let DeclLocal(ref local) = decl.node,
                            ], {
                                self.spans_need_deref
                                    .entry(vid)
                                    .or_insert_with(HashSet::new)
                                    .insert(local.init
                                        .as_ref()
                                        .map(|e| e.span)
                                        .expect("`let` stmt without init aren't caught by match_pat"));
                            }}
                        },

                        _ => {},
                    }
                }
            }
        }
    }
}

impl<'a, 'tcx> euv::Delegate<'tcx> for MovedVariablesCtxt<'a, 'tcx> {
    fn consume(&mut self, consume_id: NodeId, consume_span: Span, cmt: mc::cmt<'tcx>, mode: euv::ConsumeMode) {
        if let euv::ConsumeMode::Move(_) = mode {
            self.move_common(consume_id, consume_span, cmt);
        }
    }

    fn matched_pat(&mut self, matched_pat: &Pat, cmt: mc::cmt<'tcx>, mode: euv::MatchMode) {
        if let euv::MatchMode::MovingMatch = mode {
            self.move_common(matched_pat.id, matched_pat.span, cmt);
        } else {
            self.non_moving_pat(matched_pat, cmt);
        }
    }

    fn consume_pat(&mut self, consume_pat: &Pat, cmt: mc::cmt<'tcx>, mode: euv::ConsumeMode) {
        if let euv::ConsumeMode::Move(_) = mode {
            self.move_common(consume_pat.id, consume_pat.span, cmt);
        }
    }

    fn borrow(&mut self, _: NodeId, _: Span, _: mc::cmt<'tcx>, _: ty::Region, _: ty::BorrowKind, _: euv::LoanCause) {}

    fn mutate(&mut self, _: NodeId, _: Span, _: mc::cmt<'tcx>, _: euv::MutateMode) {}

    fn decl_without_init(&mut self, _: NodeId, _: Span) {}
}

fn unwrap_downcast_or_interior(mut cmt: mc::cmt) -> mc::cmt {
    loop {
        match cmt.cat.clone() {
            mc::Categorization::Downcast(c, _) | mc::Categorization::Interior(c, _) => {
                cmt = c;
            },
            _ => return cmt,
        }
    }
}
<commit_msg>Detect proc-macro in needless_pass_by_value<commit_after>use rustc::hir::*;
use rustc::hir::intravisit::FnKind;
use rustc::lint::*;
use rustc::ty::{self, RegionKind, TypeFoldable};
use rustc::traits;
use rustc::middle::expr_use_visitor as euv;
use rustc::middle::mem_categorization as mc;
use syntax::ast::NodeId;
use syntax_pos::Span;
use syntax::errors::DiagnosticBuilder;
use utils::{get_trait_def_id, implements_trait, in_macro, is_copy, is_self, match_type, multispan_sugg, paths,
            snippet, snippet_opt, span_lint_and_then};
use utils::ptr::get_spans;
use std::collections::{HashMap, HashSet};
use std::borrow::Cow;

/// **What it does:** Checks for functions taking arguments by value, but not
/// consuming them in its
/// body.
///
/// **Why is this bad?** Taking arguments by reference is more flexible and can
/// sometimes avoid
/// unnecessary allocations.
///
/// **Known problems:** Hopefully none.
///
/// **Example:**
/// ```rust
/// fn foo(v: Vec<i32>) {
///     assert_eq!(v.len(), 42);
/// }
/// ```
declare_lint! {
    pub NEEDLESS_PASS_BY_VALUE,
    Warn,
    "functions taking arguments by value, but not consuming them in its body"
}

pub struct NeedlessPassByValue;

impl LintPass for NeedlessPassByValue {
    fn get_lints(&self) -> LintArray {
        lint_array![NEEDLESS_PASS_BY_VALUE]
    }
}

macro_rules! need {
    ($e: expr) => { if let Some(x) = $e { x } else { return; } };
}

impl<'a, 'tcx> LateLintPass<'a, 'tcx> for NeedlessPassByValue {
    fn check_fn(
        &mut self,
        cx: &LateContext<'a, 'tcx>,
        kind: FnKind<'tcx>,
        decl: &'tcx FnDecl,
        body: &'tcx Body,
        span: Span,
        node_id: NodeId,
    ) {
        if in_macro(span) {
            return;
        }

        match kind {
            FnKind::ItemFn(.., attrs) => for a in attrs {
                if_let_chain!{[
                    a.meta_item_list().is_some(),
                    let Some(name) = a.name(),
                    name == "proc_macro_derive",
                ], {
                    return;
                }}
            },
            _ => return,
        }

        // Allow `Borrow` or functions to be taken by value
        let borrow_trait = need!(get_trait_def_id(cx, &paths::BORROW_TRAIT));
        let fn_traits = [
            need!(cx.tcx.lang_items().fn_trait()),
            need!(cx.tcx.lang_items().fn_once_trait()),
            need!(cx.tcx.lang_items().fn_mut_trait()),
        ];

        let sized_trait = need!(cx.tcx.lang_items().sized_trait());

        let fn_def_id = cx.tcx.hir.local_def_id(node_id);

        let preds = traits::elaborate_predicates(cx.tcx, cx.param_env.caller_bounds.to_vec())
            .filter(|p| !p.is_global())
            .filter_map(|pred| if let ty::Predicate::Trait(poly_trait_ref) = pred {
                if poly_trait_ref.def_id() == sized_trait || poly_trait_ref.skip_binder().has_escaping_regions() {
                    return None;
                }
                Some(poly_trait_ref)
            } else {
                None
            })
            .collect::<Vec<_>>();

        // Collect moved variables and spans which will need dereferencings from the
        // function body.
        let MovedVariablesCtxt {
            moved_vars,
            spans_need_deref,
            ..
        } = {
            let mut ctx = MovedVariablesCtxt::new(cx);
            let region_scope_tree = &cx.tcx.region_scope_tree(fn_def_id);
            euv::ExprUseVisitor::new(&mut ctx, cx.tcx, cx.param_env, region_scope_tree, cx.tables).consume_body(body);
            ctx
        };

        let fn_sig = cx.tcx.fn_sig(fn_def_id);
        let fn_sig = cx.tcx.erase_late_bound_regions(&fn_sig);

        for (idx, ((input, &ty), arg)) in decl.inputs
            .iter()
            .zip(fn_sig.inputs())
            .zip(&body.arguments)
            .enumerate()
        {
            // All spans generated from a proc-macro invocation are the same...
            if span == input.span {
                return;
            }

            // * Exclude a type that is specifically bounded by `Borrow`.
            // * Exclude a type whose reference also fulfills its bound.
            //   (e.g. `std::convert::AsRef`, `serde::Serialize`)
            let (implements_borrow_trait, all_borrowable_trait) = {
                let preds = preds
                    .iter()
                    .filter(|t| t.skip_binder().self_ty() == ty)
                    .collect::<Vec<_>>();

                (
                    preds.iter().any(|t| t.def_id() == borrow_trait),
                    !preds.is_empty() && preds.iter().all(|t| {
                        implements_trait(
                            cx,
                            cx.tcx.mk_imm_ref(&RegionKind::ReErased, ty),
                            t.def_id(),
                            &t.skip_binder().input_types().skip(1).collect::<Vec<_>>(),
                        )
                    }),
                )
            };

            if_let_chain! {[
                !is_self(arg),
                !ty.is_mutable_pointer(),
                !is_copy(cx, ty),
                !fn_traits.iter().any(|&t| implements_trait(cx, ty, t, &[])),
                !implements_borrow_trait,
                !all_borrowable_trait,

                let PatKind::Binding(mode, canonical_id, ..) = arg.pat.node,
                !moved_vars.contains(&canonical_id),
            ], {
                if mode == BindingAnnotation::Mutable || mode == BindingAnnotation::RefMut {
                    continue;
                }

                // Dereference suggestion
                let sugg = |db: &mut DiagnosticBuilder| {
                    let deref_span = spans_need_deref.get(&canonical_id);
                    if_let_chain! {[
                        match_type(cx, ty, &paths::VEC),
                        let Some(clone_spans) =
                            get_spans(cx, Some(body.id()), idx, &[("clone", ".to_owned()")]),
                        let TyPath(QPath::Resolved(_, ref path)) = input.node,
                        let Some(elem_ty) = path.segments.iter()
                            .find(|seg| seg.name == "Vec")
                            .and_then(|ps| ps.parameters.as_ref())
                            .map(|params| &params.types[0]),
                    ], {
                        let slice_ty = format!("&[{}]", snippet(cx, elem_ty.span, "_"));
                        db.span_suggestion(input.span,
                                        "consider changing the type to",
                                        slice_ty);

                        for (span, suggestion) in clone_spans {
                            db.span_suggestion(
                                span,
                                &snippet_opt(cx, span)
                                    .map_or(
                                        "change the call to".into(),
                                        |x| Cow::from(format!("change `{}` to", x)),
                                    ),
                                suggestion.into()
                            );
                        }

                        // cannot be destructured, no need for `*` suggestion
                        assert!(deref_span.is_none());
                        return;
                    }}

                    if match_type(cx, ty, &paths::STRING) {
                        if let Some(clone_spans) =
                            get_spans(cx, Some(body.id()), idx, &[("clone", ".to_string()"), ("as_str", "")]) {
                            db.span_suggestion(input.span, "consider changing the type to", "&str".to_string());

                            for (span, suggestion) in clone_spans {
                                db.span_suggestion(
                                    span,
                                    &snippet_opt(cx, span)
                                        .map_or(
                                            "change the call to".into(),
                                            |x| Cow::from(format!("change `{}` to", x))
                                        ),
                                    suggestion.into(),
                                );
                            }

                            assert!(deref_span.is_none());
                            return;
                        }
                    }

                    let mut spans = vec![(input.span, format!("&{}", snippet(cx, input.span, "_")))];

                    // Suggests adding `*` to dereference the added reference.
                    if let Some(deref_span) = deref_span {
                        spans.extend(
                            deref_span
                                .iter()
                                .cloned()
                                .map(|span| (span, format!("*{}", snippet(cx, span, "<expr>")))),
                        );
                        spans.sort_by_key(|&(span, _)| span);
                    }
                    multispan_sugg(db, "consider taking a reference instead".to_string(), spans);
                };

                span_lint_and_then(
                    cx,
                    NEEDLESS_PASS_BY_VALUE,
                    input.span,
                    "this argument is passed by value, but not consumed in the function body",
                    sugg,
                );
            }}
        }
    }
}

struct MovedVariablesCtxt<'a, 'tcx: 'a> {
    cx: &'a LateContext<'a, 'tcx>,
    moved_vars: HashSet<NodeId>,
    /// Spans which need to be prefixed with `*` for dereferencing the
    /// suggested additional reference.
    spans_need_deref: HashMap<NodeId, HashSet<Span>>,
}

impl<'a, 'tcx> MovedVariablesCtxt<'a, 'tcx> {
    fn new(cx: &'a LateContext<'a, 'tcx>) -> Self {
        Self {
            cx: cx,
            moved_vars: HashSet::new(),
            spans_need_deref: HashMap::new(),
        }
    }

    fn move_common(&mut self, _consume_id: NodeId, _span: Span, cmt: mc::cmt<'tcx>) {
        let cmt = unwrap_downcast_or_interior(cmt);

        if let mc::Categorization::Local(vid) = cmt.cat {
            self.moved_vars.insert(vid);
        }
    }

    fn non_moving_pat(&mut self, matched_pat: &Pat, cmt: mc::cmt<'tcx>) {
        let cmt = unwrap_downcast_or_interior(cmt);

        if let mc::Categorization::Local(vid) = cmt.cat {
            let mut id = matched_pat.id;
            loop {
                let parent = self.cx.tcx.hir.get_parent_node(id);
                if id == parent {
                    // no parent
                    return;
                }
                id = parent;

                if let Some(node) = self.cx.tcx.hir.find(id) {
                    match node {
                        map::Node::NodeExpr(e) => {
                            // `match` and `if let`
                            if let ExprMatch(ref c, ..) = e.node {
                                self.spans_need_deref
                                    .entry(vid)
                                    .or_insert_with(HashSet::new)
                                    .insert(c.span);
                            }
                        },

                        map::Node::NodeStmt(s) => {
                            // `let <pat> = x;`
                            if_let_chain! {[
                                let StmtDecl(ref decl, _) = s.node,
                                let DeclLocal(ref local) = decl.node,
                            ], {
                                self.spans_need_deref
                                    .entry(vid)
                                    .or_insert_with(HashSet::new)
                                    .insert(local.init
                                        .as_ref()
                                        .map(|e| e.span)
                                        .expect("`let` stmt without init aren't caught by match_pat"));
                            }}
                        },

                        _ => {},
                    }
                }
            }
        }
    }
}

impl<'a, 'tcx> euv::Delegate<'tcx> for MovedVariablesCtxt<'a, 'tcx> {
    fn consume(&mut self, consume_id: NodeId, consume_span: Span, cmt: mc::cmt<'tcx>, mode: euv::ConsumeMode) {
        if let euv::ConsumeMode::Move(_) = mode {
            self.move_common(consume_id, consume_span, cmt);
        }
    }

    fn matched_pat(&mut self, matched_pat: &Pat, cmt: mc::cmt<'tcx>, mode: euv::MatchMode) {
        if let euv::MatchMode::MovingMatch = mode {
            self.move_common(matched_pat.id, matched_pat.span, cmt);
        } else {
            self.non_moving_pat(matched_pat, cmt);
        }
    }

    fn consume_pat(&mut self, consume_pat: &Pat, cmt: mc::cmt<'tcx>, mode: euv::ConsumeMode) {
        if let euv::ConsumeMode::Move(_) = mode {
            self.move_common(consume_pat.id, consume_pat.span, cmt);
        }
    }

    fn borrow(&mut self, _: NodeId, _: Span, _: mc::cmt<'tcx>, _: ty::Region, _: ty::BorrowKind, _: euv::LoanCause) {}

    fn mutate(&mut self, _: NodeId, _: Span, _: mc::cmt<'tcx>, _: euv::MutateMode) {}

    fn decl_without_init(&mut self, _: NodeId, _: Span) {}
}

fn unwrap_downcast_or_interior(mut cmt: mc::cmt) -> mc::cmt {
    loop {
        match cmt.cat.clone() {
            mc::Categorization::Downcast(c, _) | mc::Categorization::Interior(c, _) => {
                cmt = c;
            },
            _ => return cmt,
        }
    }
}
<|endoftext|>