<commit_before>#![deny(warnings)]
#![feature(std_misc, thread_sleep)]

extern crate cargo_registry;
extern crate postgres;
extern crate semver;
extern crate time;

use std::env;
use std::collections::HashMap;
use std::time::Duration;

use cargo_registry::{VersionDownload, Version, Model};

static LIMIT: i64 = 10000;

#[allow(dead_code)] // dead in tests
fn main() {
    let daemon = env::args().nth(1).as_ref().map(|s| &s[..])
                    == Some("daemon");
    let sleep = env::args().nth(2).map(|s| s.parse::<i64>().unwrap());
    loop {
        let conn = postgres::Connection::connect(&env("DATABASE_URL")[..],
                                                 &postgres::SslMode::None).unwrap();
        {
            let tx = conn.transaction().unwrap();
            update(&tx).unwrap();
            tx.set_commit();
            tx.finish().unwrap();
        }
        drop(conn);
        if daemon {
            #[allow(deprecated)]
            fn do_sleep(sleep: Option<i64>) {
                std::thread::sleep(Duration::seconds(sleep.unwrap()));
            }
            do_sleep(sleep);
        } else {
            break
        }
    }
}

fn env(s: &str) -> String {
    match env::var(s).ok() {
        Some(s) => s,
        None => panic!("must have `{}` defined", s),
    }
}

fn update(tx: &postgres::Transaction) -> postgres::Result<()> {
    let mut max = 0;
    loop {
        let tx = try!(tx.transaction());
        {
            let stmt = try!(tx.prepare("SELECT * FROM version_downloads \
                                        WHERE processed = FALSE AND id > $1
                                        ORDER BY id ASC
                                        LIMIT $2"));
            let mut rows = try!(stmt.query(&[&max, &LIMIT]));
            match try!(collect(&tx, &mut rows)) {
                None => break,
                Some(m) => max = m,
            }
        }
        tx.set_commit();
        try!(tx.finish());
    }
    Ok(())
}

fn collect(tx: &postgres::Transaction,
           rows: &mut postgres::Rows) -> postgres::Result<Option<i32>> {

    // Anything older than 24 hours ago will be frozen and will not be queried
    // against again.
    let cutoff = time::now_utc().to_timespec();
    let cutoff = cutoff + Duration::days(-1);

    let mut map = HashMap::new();
    for row in rows.iter() {
        let download: VersionDownload = Model::from_row(&row);
        assert!(map.insert(download.id, download).is_none());
    }
    println!("updating {} versions", map.len());
    if map.len() == 0 { return Ok(None) }

    let mut max = 0;
    let mut total = 0;
    for (id, download) in map.iter() {
        if *id > max { max = *id; }
        if download.counted == download.downloads { continue }
        let amt = download.downloads - download.counted;

        let crate_id = Version::find(tx, download.version_id).unwrap().crate_id;

        // Update the total number of version downloads
        try!(tx.execute("UPDATE versions
                         SET downloads = downloads + $1
                         WHERE id = $2",
                        &[&amt, &download.version_id]));
        // Update the total number of crate downloads
        try!(tx.execute("UPDATE crates SET downloads = downloads + $1
                         WHERE id = $2", &[&amt, &crate_id]));

        // Update the total number of crate downloads for today
        let cnt = try!(tx.execute("UPDATE crate_downloads
                                   SET downloads = downloads + $2
                                   WHERE crate_id = $1 AND date = date($3)",
                                  &[&crate_id, &amt, &download.date]));
        if cnt == 0 {
            try!(tx.execute("INSERT INTO crate_downloads
                             (crate_id, downloads, date)
                             VALUES ($1, $2, $3)",
                            &[&crate_id, &amt, &download.date]));
        }

        // Flag this row as having been processed if we're passed the cutoff,
        // and unconditionally increment the number of counted downloads.
        try!(tx.execute("UPDATE version_downloads
                         SET processed = $2, counted = downloads
                         WHERE id = $1",
                        &[id, &(download.date < cutoff)]));
        total += amt as i64;
    }

    // After everything else is done, update the global counter of total
    // downloads.
    try!(tx.execute("UPDATE metadata SET total_downloads = total_downloads + $1",
                    &[&total]));

    Ok(Some(max))
}

#[cfg(test)]
mod test {
    use std::collections::HashMap;

    use postgres;
    use semver;

    use cargo_registry::{Version, Crate, User};

    fn conn() -> postgres::Connection {
        postgres::Connection::connect(&::env("TEST_DATABASE_URL")[..],
                                      &postgres::SslMode::None).unwrap()
    }

    fn user(conn: &postgres::Transaction) -> User{
        User::find_or_insert(conn, "login", None, None, None,
                             "access_token", "api_token").unwrap()
    }

    fn crate_downloads(tx: &postgres::Transaction, id: i32, expected: usize) {
        let stmt = tx.prepare("SELECT * FROM crate_downloads
                               WHERE crate_id = $1").unwrap();
        let dl: i32 = stmt.query(&[&id]).unwrap().iter()
                          .next().unwrap().get("downloads");
        assert_eq!(dl, expected as i32);
    }

    #[test]
    fn increment() {
        let conn = conn();
        let tx = conn.transaction().unwrap();
        let user = user(&tx);
        let krate = Crate::find_or_insert(&tx, "foo", user.id, &None, &None,
                                          &None, &None, &[], &None, &None,
                                          &None).unwrap();
        let version = Version::insert(&tx, krate.id,
                                      &semver::Version::parse("1.0.0").unwrap(),
                                      &HashMap::new(), &[]).unwrap();
        tx.execute("INSERT INTO version_downloads \
                    (version_id, downloads, counted, date, processed)
                    VALUES ($1, 1, 0, current_date, false)",
                   &[&version.id]).unwrap();
        tx.execute("INSERT INTO version_downloads \
                    (version_id, downloads, counted, date, processed)
                    VALUES ($1, 1, 0, current_date, true)",
                   &[&version.id]).unwrap();
        ::update(&tx).unwrap();
        assert_eq!(Version::find(&tx, version.id).unwrap().downloads, 1);
        assert_eq!(Crate::find(&tx, krate.id).unwrap().downloads, 1);
        crate_downloads(&tx, krate.id, 1);
        ::update(&tx).unwrap();
        assert_eq!(Version::find(&tx, version.id).unwrap().downloads, 1);
    }

    #[test]
    fn increment_a_little() {
        let conn = conn();
        let tx = conn.transaction().unwrap();
        let user = user(&tx);
        let krate = Crate::find_or_insert(&tx, "foo", user.id, &None,
                                          &None, &None, &None, &[], &None,
                                          &None, &None).unwrap();
        let version = Version::insert(&tx, krate.id,
                                      &semver::Version::parse("1.0.0").unwrap(),
                                      &HashMap::new(), &[]).unwrap();
        tx.execute("INSERT INTO version_downloads \
                    (version_id, downloads, counted, date, processed)
                    VALUES ($1, 2, 1, current_date, false)",
                   &[&version.id]).unwrap();
        tx.execute("INSERT INTO version_downloads \
                    (version_id, downloads, counted, date, processed)
                    VALUES ($1, 1, 0, current_date, false)",
                   &[&version.id]).unwrap();
        ::update(&tx).unwrap();
        assert_eq!(Version::find(&tx, version.id).unwrap().downloads, 2);
        assert_eq!(Crate::find(&tx, krate.id).unwrap().downloads, 2);
        crate_downloads(&tx, krate.id, 2);
        ::update(&tx).unwrap();
        assert_eq!(Version::find(&tx, version.id).unwrap().downloads, 2);
    }
}
<commit_msg>Fix flagging downloads as processed<commit_after>#![deny(warnings)]
#![feature(std_misc, thread_sleep)]

extern crate cargo_registry;
extern crate postgres;
extern crate semver;
extern crate time;

use std::env;
use std::collections::HashMap;
use std::time::Duration;

use cargo_registry::{VersionDownload, Version, Model};

static LIMIT: i64 = 10000;

#[allow(dead_code)] // dead in tests
fn main() {
    let daemon = env::args().nth(1).as_ref().map(|s| &s[..])
                    == Some("daemon");
    let sleep = env::args().nth(2).map(|s| s.parse::<i64>().unwrap());
    loop {
        let conn = postgres::Connection::connect(&env("DATABASE_URL")[..],
                                                 &postgres::SslMode::None).unwrap();
        {
            let tx = conn.transaction().unwrap();
            update(&tx).unwrap();
            tx.set_commit();
            tx.finish().unwrap();
        }
        drop(conn);
        if daemon {
            #[allow(deprecated)]
            fn do_sleep(sleep: Option<i64>) {
                std::thread::sleep(Duration::seconds(sleep.unwrap()));
            }
            do_sleep(sleep);
        } else {
            break
        }
    }
}

fn env(s: &str) -> String {
    match env::var(s).ok() {
        Some(s) => s,
        None => panic!("must have `{}` defined", s),
    }
}

fn update(tx: &postgres::Transaction) -> postgres::Result<()> {
    let mut max = 0;
    loop {
        let tx = try!(tx.transaction());
        {
            let stmt = try!(tx.prepare("SELECT * FROM version_downloads \
                                        WHERE processed = FALSE AND id > $1
                                        ORDER BY id ASC
                                        LIMIT $2"));
            let mut rows = try!(stmt.query(&[&max, &LIMIT]));
            match try!(collect(&tx, &mut rows)) {
                None => break,
                Some(m) => max = m,
            }
        }
        tx.set_commit();
        try!(tx.finish());
    }
    Ok(())
}

fn collect(tx: &postgres::Transaction,
           rows: &mut postgres::Rows) -> postgres::Result<Option<i32>> {

    // Anything older than 24 hours ago will be frozen and will not be queried
    // against again.
    let cutoff = time::now_utc().to_timespec();
    let cutoff = cutoff + Duration::days(-1);

    let mut map = HashMap::new();
    for row in rows.iter() {
        let download: VersionDownload = Model::from_row(&row);
        assert!(map.insert(download.id, download).is_none());
    }
    println!("updating {} versions (cutoff {})", map.len(),
             time::at(cutoff).rfc822());
    if map.len() == 0 { return Ok(None) }

    let mut max = 0;
    let mut total = 0;
    for (id, download) in map.iter() {
        if *id > max { max = *id; }
        if download.date > cutoff && download.counted == download.downloads {
            continue
        }
        let amt = download.downloads - download.counted;

        let crate_id = Version::find(tx, download.version_id).unwrap().crate_id;

        // Update the total number of version downloads
        try!(tx.execute("UPDATE versions
                         SET downloads = downloads + $1
                         WHERE id = $2",
                        &[&amt, &download.version_id]));
        // Update the total number of crate downloads
        try!(tx.execute("UPDATE crates SET downloads = downloads + $1
                         WHERE id = $2", &[&amt, &crate_id]));

        // Update the total number of crate downloads for today
        let cnt = try!(tx.execute("UPDATE crate_downloads
                                   SET downloads = downloads + $2
                                   WHERE crate_id = $1 AND date = date($3)",
                                  &[&crate_id, &amt, &download.date]));
        if cnt == 0 {
            try!(tx.execute("INSERT INTO crate_downloads
                             (crate_id, downloads, date)
                             VALUES ($1, $2, $3)",
                            &[&crate_id, &amt, &download.date]));
        }

        // Flag this row as having been processed if we're passed the cutoff,
        // and unconditionally increment the number of counted downloads.
        try!(tx.execute("UPDATE version_downloads
                         SET processed = $2, counted = downloads
                         WHERE id = $1",
                        &[id, &(download.date < cutoff)]));
        total += amt as i64;
    }

    // After everything else is done, update the global counter of total
    // downloads.
    try!(tx.execute("UPDATE metadata SET total_downloads = total_downloads + $1",
                    &[&total]));

    Ok(Some(max))
}

#[cfg(test)]
mod test {
    use std::collections::HashMap;

    use postgres;
    use semver;

    use cargo_registry::{Version, Crate, User};

    fn conn() -> postgres::Connection {
        postgres::Connection::connect(&::env("TEST_DATABASE_URL")[..],
                                      &postgres::SslMode::None).unwrap()
    }

    fn user(conn: &postgres::Transaction) -> User{
        User::find_or_insert(conn, "login", None, None, None,
                             "access_token", "api_token").unwrap()
    }

    fn crate_downloads(tx: &postgres::Transaction, id: i32, expected: usize) {
        let stmt = tx.prepare("SELECT * FROM crate_downloads
                               WHERE crate_id = $1").unwrap();
        let dl: i32 = stmt.query(&[&id]).unwrap().iter()
                          .next().unwrap().get("downloads");
        assert_eq!(dl, expected as i32);
    }

    #[test]
    fn increment() {
        let conn = conn();
        let tx = conn.transaction().unwrap();
        let user = user(&tx);
        let krate = Crate::find_or_insert(&tx, "foo", user.id, &None, &None,
                                          &None, &None, &[], &None, &None,
                                          &None).unwrap();
        let version = Version::insert(&tx, krate.id,
                                      &semver::Version::parse("1.0.0").unwrap(),
                                      &HashMap::new(), &[]).unwrap();
        tx.execute("INSERT INTO version_downloads \
                    (version_id, downloads, counted, date, processed)
                    VALUES ($1, 1, 0, current_date, false)",
                   &[&version.id]).unwrap();
        tx.execute("INSERT INTO version_downloads \
                    (version_id, downloads, counted, date, processed)
                    VALUES ($1, 1, 0, current_date, true)",
                   &[&version.id]).unwrap();
        ::update(&tx).unwrap();
        assert_eq!(Version::find(&tx, version.id).unwrap().downloads, 1);
        assert_eq!(Crate::find(&tx, krate.id).unwrap().downloads, 1);
        crate_downloads(&tx, krate.id, 1);
        ::update(&tx).unwrap();
        assert_eq!(Version::find(&tx, version.id).unwrap().downloads, 1);
    }

    #[test]
    fn set_processed_true() {
        let conn = conn();
        let tx = conn.transaction().unwrap();
        let user = user(&tx);
        let krate = Crate::find_or_insert(&tx, "foo", user.id, &None,
                                          &None, &None, &None, &[], &None,
                                          &None, &None).unwrap();
        let version = Version::insert(&tx, krate.id,
                                      &semver::Version::parse("1.0.0").unwrap(),
                                      &HashMap::new(), &[]).unwrap();
        tx.execute("INSERT INTO version_downloads \
                    (version_id, downloads, counted, date, processed)
                    VALUES ($1, 2, 2, current_date - interval '2 days', false)",
                   &[&version.id]).unwrap();
        ::update(&tx).unwrap();
        let stmt = tx.prepare("SELECT processed FROM version_downloads
                               WHERE version_id = $1").unwrap();
        let processed: bool = stmt.query(&[&version.id]).unwrap().iter()
                                  .next().unwrap().get("processed");
        assert!(processed);
    }

    #[test]
    fn increment_a_little() {
        let conn = conn();
        let tx = conn.transaction().unwrap();
        let user = user(&tx);
        let krate = Crate::find_or_insert(&tx, "foo", user.id, &None,
                                          &None, &None, &None, &[], &None,
                                          &None, &None).unwrap();
        let version = Version::insert(&tx, krate.id,
                                      &semver::Version::parse("1.0.0").unwrap(),
                                      &HashMap::new(), &[]).unwrap();
        tx.execute("INSERT INTO version_downloads \
                    (version_id, downloads, counted, date, processed)
                    VALUES ($1, 2, 1, current_date, false)",
                   &[&version.id]).unwrap();
        tx.execute("INSERT INTO version_downloads \
                    (version_id, downloads, counted, date, processed)
                    VALUES ($1, 1, 0, current_date, false)",
                   &[&version.id]).unwrap();
        ::update(&tx).unwrap();
        assert_eq!(Version::find(&tx, version.id).unwrap().downloads, 2);
        assert_eq!(Crate::find(&tx, krate.id).unwrap().downloads, 2);
        crate_downloads(&tx, krate.id, 2);
        ::update(&tx).unwrap();
        assert_eq!(Version::find(&tx, version.id).unwrap().downloads, 2);
    }
}
<|endoftext|>


<commit_before>use std::collections::hash_map::HashMap;
use std::convert::TryFrom;

use futures::future::*;
use http::{HeaderMap, HeaderValue, Method};
use http::header::{HeaderName, InvalidHeaderName, InvalidHeaderValue};
use http::header::CONTENT_TYPE;
use http::method::InvalidMethod;
use itertools::Itertools;
use log::*;
use reqwest::{Client, Error, RequestBuilder};

use pact_matching::models::*;
use pact_matching::s;
use pact_models::content_types::ContentType;
use pact_models::bodies::OptionalBody;

use super::*;

#[derive(Debug)]
#[allow(dead_code)]
pub enum ProviderClientError {
    RequestMethodError(String, InvalidMethod),
    RequestHeaderNameError(String, InvalidHeaderName),
    RequestHeaderValueError(String, InvalidHeaderValue),
    RequestBodyError(String),
    ResponseError(String),
    ResponseStatusCodeError(u16),
}

impl From<reqwest::Error> for ProviderClientError {
  fn from(err: Error) -> Self {
    ProviderClientError::ResponseError(err.to_string())
  }
}

pub fn join_paths(base: &str, path: String) -> String {
    let mut full_path = s!(base.trim_end_matches('/'));
    full_path.push('/');
    full_path.push_str(path.trim_start_matches('/'));
    full_path
}

fn create_native_request(client: &Client, base_url: &str, request: &Request) -> Result<RequestBuilder, ProviderClientError> {
  let url = join_paths(base_url, request.path.clone());
  let mut builder = client.request(Method::from_bytes(
    &request.method.clone().into_bytes()).unwrap_or(Method::GET), &url);

  if let Some(query) = &request.query {
    builder = builder.query(&query.iter()
      .sorted_by(|a, b| Ord::cmp(&a.0, &b.0))
      .flat_map(|(k, v)| {
        v.iter().map(|v| (k, v)).collect_vec()
      }).collect_vec());
  }

  if let Some(headers) = &request.headers {
    let mut header_map = HeaderMap::new();
    for (k, vals) in headers {
      for header_value in vals {
        let header_name = HeaderName::try_from(k)
          .map_err(|err| ProviderClientError::RequestHeaderNameError(
            format!("Failed to parse header value: {}", header_value), err))?;
        header_map.append(header_name,  HeaderValue::from_str(header_value.as_str())
          .map_err(|err| ProviderClientError::RequestHeaderValueError(
            format!("Failed to parse header value: {}", header_value), err))?);
      }
    }
    builder = builder.headers(header_map);
  }

  match request.body {
    OptionalBody::Present(ref s, _) => builder = builder.body(s.clone()),
    OptionalBody::Null => {
      if request.content_type().unwrap_or_default().is_json() {
        builder = builder.body("null");
      }
    },
    _ => ()
  };

  Ok(builder)
}

fn extract_headers(headers: &HeaderMap) -> Option<HashMap<String, Vec<String>>> {
  if !headers.is_empty() {
    let result = headers.keys()
      .map(|name| {
        let values = headers.get_all(name);
        let parsed_vals: Vec<Result<String, ()>> = values.iter()
          .map(|val| val.to_str()
            .map(|v| v.to_string())
            .map_err(|err| log::warn!("Failed to parse HTTP header value: {}", err))
          ).collect();
       (name.as_str().into(), parsed_vals.iter().cloned()
            .filter(|val| val.is_ok())
            .map(|val| val.unwrap_or_default())
            .flat_map(|val| val.split(",").map(|v| v.to_string()).collect::<Vec<String>>())
            .map(|val| val.trim().to_string())
            .collect())
      })
      .collect();

    Some(result)
  } else {
    None
  }
}

async fn extract_body(response: reqwest::Response, pact_response: &Response) -> Result<OptionalBody, reqwest::Error> {
  let body = response.bytes().await?;
  if !body.is_empty() {
    Ok(OptionalBody::Present(body, pact_response.content_type()))
  } else {
    Ok(OptionalBody::Empty)
  }
}

async fn native_response_to_pact_response(
    native_response: reqwest::Response
) -> Result<Response, reqwest::Error> {
  debug!("Received native response: {:?}", native_response);

  let status = native_response.status().as_u16();
  let headers = extract_headers(native_response.headers());
  let response = Response {
    status,
    headers,
    .. Response::default()
  };

  let body = extract_body(native_response, &response).await?;

  let response = Response {
    body, .. response.clone()
  };
  info!("Received response: {}", response);
  Ok(response)
}

/// This function makes the actual request to the provider, executing any request filter before
/// executing the request
pub async fn make_provider_request<F: RequestFilterExecutor>(
  provider: &ProviderInfo,
  request: &Request,
  options: &VerificationOptions<F>,
  client: &reqwest::Client
) -> Result<Response, ProviderClientError> {
  let request_filter_option = options.request_filter.clone();
  let request = if request_filter_option.is_some() {
    let request_filter = request_filter_option.unwrap();
    info!("Invoking request filter for request");
    request_filter.call(request)
  } else {
    request.clone()
  };

  let base_url = match provider.port {
    Some(port) => format!("{}://{}:{}{}", provider.protocol, provider.host, port, provider.path),
    None => format!("{}://{}{}", provider.protocol, provider.host, provider.path),
  };

  info!("Sending request to provider at {}", base_url);
  debug!("Provider details = {:?}", provider);
  debug!("Sending request {}", request);
  trace!("body: {}", request.body.str_value());
  let request = create_native_request(client, &base_url, &request)?;

  let response = request.send()
    .and_then(native_response_to_pact_response)
    .await
    .map_err(|err| ProviderClientError::ResponseError(err.to_string()))?;

  Ok(response)
}

/// Make a state change request. If the response returns a JSON body, convert that into a HashMap
/// and return it
pub async fn make_state_change_request(
  client: &reqwest::Client,
  state_change_url: &str,
  request: &Request
) -> Result<HashMap<String, Value>, ProviderClientError> {
  log::debug!("Sending {} to state change handler", request);

  let request = create_native_request(client, state_change_url, request)?;
  let result = request.send().await;

  match result {
    Ok(response) => {
      debug!("State change request: {:?}", response);
      if response.status().is_success() {
        if let Some(content_type) = response.headers().get(CONTENT_TYPE) {
          if let Ok(content_type) = ContentType::parse(content_type.to_str().unwrap_or_default()) {
            if content_type.is_json() {
              let body = response.bytes().await?;
              match serde_json::from_slice::<Value>(&body) {
                Ok(body) => match body {
                  Value::Object(map) => Ok(map.iter().map(|(k, v)| (k.clone(), v.clone())).collect()),
                  _ => Ok(hashmap!{})
                },
                Err(_) => Ok(hashmap!{})
              }
            } else {
              Ok(hashmap!{})
            }
          } else {
            Ok(hashmap!{})
          }
        } else {
          Ok(hashmap!{})
        }
      } else {
        Err(ProviderClientError::ResponseStatusCodeError(response.status().as_u16()))
      }
    },
    Err(err) => {
      debug!("State change request failed with error {}", err);
      Err(ProviderClientError::ResponseError(err.to_string()))
    }
  }
}

pub fn provider_client_error_to_string(err: ProviderClientError) -> String {
  match err {
    ProviderClientError::RequestMethodError(ref method, _) =>
      format!("Invalid request method: '{}'", method),
    ProviderClientError::RequestHeaderNameError(ref name, _) =>
      format!("Invalid header name: '{}'", name),
    ProviderClientError::RequestHeaderValueError(ref value, _) =>
      format!("Invalid header value: '{}'", value),
    ProviderClientError::RequestBodyError(ref message) =>
      format!("Invalid request body: '{}'", message),
    ProviderClientError::ResponseError(ref message) =>
      format!("Invalid response: {}", message),
    ProviderClientError::ResponseStatusCodeError(ref code) =>
      format!("Invalid status code: {}", code)
  }
}

#[cfg(test)]
mod tests {
  use expectest::expect;
  use expectest::prelude::*;
  use itertools::Itertools;
  use maplit::*;

  use pact_matching::models::Request;
  use pact_matching::s;
  use pact_models::bodies::OptionalBody;

  use super::{create_native_request, join_paths};

  #[test]
  fn join_paths_test() {
      expect!(join_paths(&s!(""), s!(""))).to(be_equal_to(s!("/")));
      expect!(join_paths(&s!("/"), s!(""))).to(be_equal_to(s!("/")));
      expect!(join_paths(&s!(""), s!("/"))).to(be_equal_to(s!("/")));
      expect!(join_paths(&s!("/"), s!("/"))).to(be_equal_to(s!("/")));
      expect!(join_paths(&s!("/a/b"), s!("/c/d"))).to(be_equal_to(s!("/a/b/c/d")));
  }

  #[test]
  fn convert_request_to_native_request_test() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request::default();
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/"));
    expect!(request_builder.body()).to(be_none());
  }

  #[test]
  fn convert_request_to_native_request_with_query_parameters() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request {
      query: Some(hashmap!{
        "a".to_string() => vec!["b".to_string()],
        "c".to_string() => vec!["d".to_string(), "e".to_string()]
      }),
      .. Request::default()
    };
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/?a=b&c=d&c=e"));
  }

  #[test]
  fn convert_request_to_native_request_with_headers() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request {
      headers: Some(hashmap! {
        "A".to_string() => vec!["B".to_string()],
        "B".to_string() => vec!["C".to_string(), "D".to_string()]
      }),
      .. Request::default()
    };
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/"));

    let headers = request_builder.headers();
    expect!(headers.len()).to(be_equal_to(3));
    expect!(&headers["A"]).to(be_equal_to("B"));
    expect!(&headers["B"]).to(be_equal_to("C"));
    expect!(headers.get_all("B").iter().map(|v| v.to_str().unwrap()).collect_vec())
      .to(be_equal_to(vec!["C", "D"]));
  }

  #[test]
  fn convert_request_to_native_request_with_body() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request {
      body: OptionalBody::from("body"),
      .. Request::default()
    };
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/"));
    expect!(request_builder.body().unwrap().as_bytes()).to(be_some().value("body".as_bytes()));
  }

  #[test]
  fn convert_request_to_native_request_with_null_body() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request {
      body: OptionalBody::Null,
      .. Request::default()
    };
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/"));
    expect!(request_builder.body()).to(be_none());
  }

  #[test]
  fn convert_request_to_native_request_with_json_null_body() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request {
      headers: Some(hashmap! {
        "Content-Type".to_string() => vec!["application/json".to_string()]
      }),
      body: OptionalBody::Null,
      .. Request::default()
    };
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/"));
    expect!(request_builder.body().unwrap().as_bytes()).to(be_some().value("null".as_bytes()));
  }
}
<commit_msg>test: extract_headers function, specially with comma separated values<commit_after>use std::collections::hash_map::HashMap;
use std::convert::TryFrom;

use futures::future::*;
use http::{HeaderMap, HeaderValue, Method};
use http::header::{HeaderName, InvalidHeaderName, InvalidHeaderValue};
use http::header::CONTENT_TYPE;
use http::method::InvalidMethod;
use itertools::Itertools;
use log::*;
use reqwest::{Client, Error, RequestBuilder};

use pact_matching::models::*;
use pact_matching::s;
use pact_models::content_types::ContentType;
use pact_models::bodies::OptionalBody;

use super::*;

#[derive(Debug)]
#[allow(dead_code)]
pub enum ProviderClientError {
    RequestMethodError(String, InvalidMethod),
    RequestHeaderNameError(String, InvalidHeaderName),
    RequestHeaderValueError(String, InvalidHeaderValue),
    RequestBodyError(String),
    ResponseError(String),
    ResponseStatusCodeError(u16),
}

impl From<reqwest::Error> for ProviderClientError {
  fn from(err: Error) -> Self {
    ProviderClientError::ResponseError(err.to_string())
  }
}

pub fn join_paths(base: &str, path: String) -> String {
    let mut full_path = s!(base.trim_end_matches('/'));
    full_path.push('/');
    full_path.push_str(path.trim_start_matches('/'));
    full_path
}

fn create_native_request(client: &Client, base_url: &str, request: &Request) -> Result<RequestBuilder, ProviderClientError> {
  let url = join_paths(base_url, request.path.clone());
  let mut builder = client.request(Method::from_bytes(
    &request.method.clone().into_bytes()).unwrap_or(Method::GET), &url);

  if let Some(query) = &request.query {
    builder = builder.query(&query.iter()
      .sorted_by(|a, b| Ord::cmp(&a.0, &b.0))
      .flat_map(|(k, v)| {
        v.iter().map(|v| (k, v)).collect_vec()
      }).collect_vec());
  }

  if let Some(headers) = &request.headers {
    let mut header_map = HeaderMap::new();
    for (k, vals) in headers {
      for header_value in vals {
        let header_name = HeaderName::try_from(k)
          .map_err(|err| ProviderClientError::RequestHeaderNameError(
            format!("Failed to parse header value: {}", header_value), err))?;
        header_map.append(header_name,  HeaderValue::from_str(header_value.as_str())
          .map_err(|err| ProviderClientError::RequestHeaderValueError(
            format!("Failed to parse header value: {}", header_value), err))?);
      }
    }
    builder = builder.headers(header_map);
  }

  match request.body {
    OptionalBody::Present(ref s, _) => builder = builder.body(s.clone()),
    OptionalBody::Null => {
      if request.content_type().unwrap_or_default().is_json() {
        builder = builder.body("null");
      }
    },
    _ => ()
  };

  Ok(builder)
}

fn extract_headers(headers: &HeaderMap) -> Option<HashMap<String, Vec<String>>> {
  if !headers.is_empty() {
    let result = headers.keys()
      .map(|name| {
        let values = headers.get_all(name);
        let parsed_vals: Vec<Result<String, ()>> = values.iter()
          .map(|val| val.to_str()
            .map(|v| v.to_string())
            .map_err(|err| log::warn!("Failed to parse HTTP header value: {}", err))
          ).collect();
       (name.as_str().into(), parsed_vals.iter().cloned()
            .filter(|val| val.is_ok())
            .map(|val| val.unwrap_or_default())
            .flat_map(|val| val.split(",").map(|v| v.to_string()).collect::<Vec<String>>())
            .map(|val| val.trim().to_string())
            .collect())
      })
      .collect();

    Some(result)
  } else {
    None
  }
}

async fn extract_body(response: reqwest::Response, pact_response: &Response) -> Result<OptionalBody, reqwest::Error> {
  let body = response.bytes().await?;
  if !body.is_empty() {
    Ok(OptionalBody::Present(body, pact_response.content_type()))
  } else {
    Ok(OptionalBody::Empty)
  }
}

async fn native_response_to_pact_response(
    native_response: reqwest::Response
) -> Result<Response, reqwest::Error> {
  debug!("Received native response: {:?}", native_response);

  let status = native_response.status().as_u16();
  let headers = extract_headers(native_response.headers());
  let response = Response {
    status,
    headers,
    .. Response::default()
  };

  let body = extract_body(native_response, &response).await?;

  let response = Response {
    body, .. response.clone()
  };
  info!("Received response: {}", response);
  Ok(response)
}

/// This function makes the actual request to the provider, executing any request filter before
/// executing the request
pub async fn make_provider_request<F: RequestFilterExecutor>(
  provider: &ProviderInfo,
  request: &Request,
  options: &VerificationOptions<F>,
  client: &reqwest::Client
) -> Result<Response, ProviderClientError> {
  let request_filter_option = options.request_filter.clone();
  let request = if request_filter_option.is_some() {
    let request_filter = request_filter_option.unwrap();
    info!("Invoking request filter for request");
    request_filter.call(request)
  } else {
    request.clone()
  };

  let base_url = match provider.port {
    Some(port) => format!("{}://{}:{}{}", provider.protocol, provider.host, port, provider.path),
    None => format!("{}://{}{}", provider.protocol, provider.host, provider.path),
  };

  info!("Sending request to provider at {}", base_url);
  debug!("Provider details = {:?}", provider);
  debug!("Sending request {}", request);
  trace!("body: {}", request.body.str_value());
  let request = create_native_request(client, &base_url, &request)?;

  let response = request.send()
    .and_then(native_response_to_pact_response)
    .await
    .map_err(|err| ProviderClientError::ResponseError(err.to_string()))?;

  Ok(response)
}

/// Make a state change request. If the response returns a JSON body, convert that into a HashMap
/// and return it
pub async fn make_state_change_request(
  client: &reqwest::Client,
  state_change_url: &str,
  request: &Request
) -> Result<HashMap<String, Value>, ProviderClientError> {
  log::debug!("Sending {} to state change handler", request);

  let request = create_native_request(client, state_change_url, request)?;
  let result = request.send().await;

  match result {
    Ok(response) => {
      debug!("State change request: {:?}", response);
      if response.status().is_success() {
        if let Some(content_type) = response.headers().get(CONTENT_TYPE) {
          if let Ok(content_type) = ContentType::parse(content_type.to_str().unwrap_or_default()) {
            if content_type.is_json() {
              let body = response.bytes().await?;
              match serde_json::from_slice::<Value>(&body) {
                Ok(body) => match body {
                  Value::Object(map) => Ok(map.iter().map(|(k, v)| (k.clone(), v.clone())).collect()),
                  _ => Ok(hashmap!{})
                },
                Err(_) => Ok(hashmap!{})
              }
            } else {
              Ok(hashmap!{})
            }
          } else {
            Ok(hashmap!{})
          }
        } else {
          Ok(hashmap!{})
        }
      } else {
        Err(ProviderClientError::ResponseStatusCodeError(response.status().as_u16()))
      }
    },
    Err(err) => {
      debug!("State change request failed with error {}", err);
      Err(ProviderClientError::ResponseError(err.to_string()))
    }
  }
}

pub fn provider_client_error_to_string(err: ProviderClientError) -> String {
  match err {
    ProviderClientError::RequestMethodError(ref method, _) =>
      format!("Invalid request method: '{}'", method),
    ProviderClientError::RequestHeaderNameError(ref name, _) =>
      format!("Invalid header name: '{}'", name),
    ProviderClientError::RequestHeaderValueError(ref value, _) =>
      format!("Invalid header value: '{}'", value),
    ProviderClientError::RequestBodyError(ref message) =>
      format!("Invalid request body: '{}'", message),
    ProviderClientError::ResponseError(ref message) =>
      format!("Invalid response: {}", message),
    ProviderClientError::ResponseStatusCodeError(ref code) =>
      format!("Invalid status code: {}", code)
  }
}

#[cfg(test)]
mod tests {
  use expectest::expect;
  use expectest::prelude::*;
  use itertools::Itertools;
  use maplit::*;

  use pact_matching::models::Request;
  use pact_matching::s;
  use pact_models::bodies::OptionalBody;

  use super::{create_native_request, join_paths, extract_headers};
  use http::{HeaderMap, HeaderValue};

  #[test]
  fn extract_headers_tests() {
    let mut headers = HeaderMap::new();
    headers.insert("HOST", "example.com".parse().unwrap());
    headers.insert("CONTENT_LENGTH", "123".parse().unwrap());
    let response = extract_headers(&headers).unwrap();
    expect!(&response["host"][0]).to(be_equal_to(&"example.com"));
    expect!(&response["content_length"][0]).to(be_equal_to(&"123"));
  }

  #[test]
  fn extract_headers_return_none_if_headers_are_empty() {
    expect!(extract_headers(&HeaderMap::new())).to(be_none());
  }

  #[test]
  fn extract_headers_when_header_value_is_a_comma_separated_string() {
    let mut headers = HeaderMap::new();
    headers.insert(
      "Access-Control-Expose-Headers",
      "Content-Length, Content-Type, Expires".parse().unwrap()
    );
    let response = extract_headers(&headers).unwrap();
    expect!(&response["access-control-expose-headers"][0]).to(be_equal_to(&"Content-Length"));
    expect!(&response["access-control-expose-headers"][1]).to(be_equal_to(&"Content-Type"));
    expect!(&response["access-control-expose-headers"][2]).to(be_equal_to(&"Expires"));
  }

  #[test]
  fn join_paths_test() {
      expect!(join_paths(&s!(""), s!(""))).to(be_equal_to(s!("/")));
      expect!(join_paths(&s!("/"), s!(""))).to(be_equal_to(s!("/")));
      expect!(join_paths(&s!(""), s!("/"))).to(be_equal_to(s!("/")));
      expect!(join_paths(&s!("/"), s!("/"))).to(be_equal_to(s!("/")));
      expect!(join_paths(&s!("/a/b"), s!("/c/d"))).to(be_equal_to(s!("/a/b/c/d")));
  }

  #[test]
  fn convert_request_to_native_request_test() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request::default();
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/"));
    expect!(request_builder.body()).to(be_none());
  }

  #[test]
  fn convert_request_to_native_request_with_query_parameters() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request {
      query: Some(hashmap!{
        "a".to_string() => vec!["b".to_string()],
        "c".to_string() => vec!["d".to_string(), "e".to_string()]
      }),
      .. Request::default()
    };
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/?a=b&c=d&c=e"));
  }

  #[test]
  fn convert_request_to_native_request_with_headers() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request {
      headers: Some(hashmap! {
        "A".to_string() => vec!["B".to_string()],
        "B".to_string() => vec!["C".to_string(), "D".to_string()]
      }),
      .. Request::default()
    };
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/"));

    let headers = request_builder.headers();
    expect!(headers.len()).to(be_equal_to(3));
    expect!(&headers["A"]).to(be_equal_to("B"));
    expect!(&headers["B"]).to(be_equal_to("C"));
    expect!(headers.get_all("B").iter().map(|v| v.to_str().unwrap()).collect_vec())
      .to(be_equal_to(vec!["C", "D"]));
  }

  #[test]
  fn convert_request_to_native_request_with_body() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request {
      body: OptionalBody::from("body"),
      .. Request::default()
    };
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/"));
    expect!(request_builder.body().unwrap().as_bytes()).to(be_some().value("body".as_bytes()));
  }

  #[test]
  fn convert_request_to_native_request_with_null_body() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request {
      body: OptionalBody::Null,
      .. Request::default()
    };
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/"));
    expect!(request_builder.body()).to(be_none());
  }

  #[test]
  fn convert_request_to_native_request_with_json_null_body() {
    let client = reqwest::Client::new();
    let base_url = "http://example.test:8080".to_string();
    let request = Request {
      headers: Some(hashmap! {
        "Content-Type".to_string() => vec!["application/json".to_string()]
      }),
      body: OptionalBody::Null,
      .. Request::default()
    };
    let request_builder = create_native_request(&client, &base_url, &request).unwrap().build().unwrap();

    expect!(request_builder.method()).to(be_equal_to("GET"));
    expect!(request_builder.url().as_str()).to(be_equal_to("http://example.test:8080/"));
    expect!(request_builder.body().unwrap().as_bytes()).to(be_some().value("null".as_bytes()));
  }
}
<|endoftext|>


<commit_before>#![feature(plugin, io, path, collections)]

#[plugin]
#[macro_use]
#[no_link]
extern crate rustful_macros;

extern crate rustful;

use std::old_io::{File, IoResult};
use std::sync::{Arc, RwLock};
use std::error::Error;

use rustful::{Server, Context, Response, Handler, TreeRouter};
use rustful::cache::{CachedValue, CachedProcessedFile};
use rustful::Method::Get;
use rustful::StatusCode::InternalServerError;
use rustful::header::ContentType;

fn main() {
    println!("Visit http://localhost:8080 to try this example.");

    //Cache the page
    let page = Arc::new(CachedProcessedFile::new(Path::new("examples/handler_storage/page.html"), None, read_string));

    //The shared counter state
    let value = Arc::new(RwLock::new(0));

    let router = insert_routes!{
        TreeRouter::new(): {
            "/" => Get: Counter{
                page: page.clone(),
                value: value.clone(),
                operation: None
            },
            "/add" => Get: Counter{
                page: page.clone(),
                value: value.clone(),
                operation: Some(add as fn(i32) -> i32)
            },
            "/sub" => Get: Counter{
                page: page.clone(),
                value: value.clone(),
                operation: Some(sub as fn(i32) -> i32)
            }
        }
    };

    let server_result = Server::new().port(8080).handlers(router).run();

    match server_result {
        Ok(_server) => {},
        Err(e) => println!("could not start server: {}", e.description())
    }
}

fn add(value: i32) -> i32 {
    value + 1
}

fn sub(value: i32) -> i32 {
    value - 1
}

fn read_string(mut file: IoResult<File>) -> IoResult<Option<String>> {
    //Read file into a string
    file.read_to_string().map(|s| Some(s))
}

struct Counter {
    //We are using the handler to cache the page in this exmaple
    page: Arc<CachedProcessedFile<String>>,

    value: Arc<RwLock<i32>>,
    operation: Option<fn(i32) -> i32>
}

impl Handler for Counter {
    type Cache = ();

    fn handle_request(&self, _context: Context, mut response: Response) {
        self.operation.map(|o| {
            //Lock the value for writing and update it
            let mut value = self.value.write().unwrap();
            *value = (o)(*value);
        });

        response.set_header(ContentType(content_type!("text", "html", ("charset", "UTF-8"))));

        //Insert the value into the page and write it to the response
        match *self.page.borrow() {
            Some(ref page) => {
                let count = self.value.read().unwrap().to_string();

                try_send!(response.into_writer(), page.replace("{}", &count[]));
            },
            None => {
                //Oh no! The page was not loaded!
                response.set_status(InternalServerError);
            }
        }
    }
}<commit_msg>Remove unused feature from example<commit_after>#![feature(plugin, io, path)]

#[plugin]
#[macro_use]
#[no_link]
extern crate rustful_macros;

extern crate rustful;

use std::old_io::{File, IoResult};
use std::sync::{Arc, RwLock};
use std::error::Error;

use rustful::{Server, Context, Response, Handler, TreeRouter};
use rustful::cache::{CachedValue, CachedProcessedFile};
use rustful::Method::Get;
use rustful::StatusCode::InternalServerError;
use rustful::header::ContentType;

fn main() {
    println!("Visit http://localhost:8080 to try this example.");

    //Cache the page
    let page = Arc::new(CachedProcessedFile::new(Path::new("examples/handler_storage/page.html"), None, read_string));

    //The shared counter state
    let value = Arc::new(RwLock::new(0));

    let router = insert_routes!{
        TreeRouter::new(): {
            "/" => Get: Counter{
                page: page.clone(),
                value: value.clone(),
                operation: None
            },
            "/add" => Get: Counter{
                page: page.clone(),
                value: value.clone(),
                operation: Some(add as fn(i32) -> i32)
            },
            "/sub" => Get: Counter{
                page: page.clone(),
                value: value.clone(),
                operation: Some(sub as fn(i32) -> i32)
            }
        }
    };

    let server_result = Server::new().port(8080).handlers(router).run();

    match server_result {
        Ok(_server) => {},
        Err(e) => println!("could not start server: {}", e.description())
    }
}

fn add(value: i32) -> i32 {
    value + 1
}

fn sub(value: i32) -> i32 {
    value - 1
}

fn read_string(mut file: IoResult<File>) -> IoResult<Option<String>> {
    //Read file into a string
    file.read_to_string().map(|s| Some(s))
}

struct Counter {
    //We are using the handler to cache the page in this exmaple
    page: Arc<CachedProcessedFile<String>>,

    value: Arc<RwLock<i32>>,
    operation: Option<fn(i32) -> i32>
}

impl Handler for Counter {
    type Cache = ();

    fn handle_request(&self, _context: Context, mut response: Response) {
        self.operation.map(|o| {
            //Lock the value for writing and update it
            let mut value = self.value.write().unwrap();
            *value = (o)(*value);
        });

        response.set_header(ContentType(content_type!("text", "html", ("charset", "UTF-8"))));

        //Insert the value into the page and write it to the response
        match *self.page.borrow() {
            Some(ref page) => {
                let count = self.value.read().unwrap().to_string();

                try_send!(response.into_writer(), page.replace("{}", &count[]));
            },
            None => {
                //Oh no! The page was not loaded!
                response.set_status(InternalServerError);
            }
        }
    }
}<|endoftext|>


<commit_before>extern crate cargotest;
extern crate hamcrest;

use std::fs::{self, File};
use std::io::prelude::*;

use cargotest::support::{project, execs};
use hamcrest::{assert_that, existing_file, is_not};

#[test]
fn adding_and_removing_packages() {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]
            name = "foo"
            authors = []
            version = "0.0.1"
        "#)
        .file("src/main.rs", "fn main() {}")
        .file("bar/Cargo.toml", r#"
            [package]
            name = "bar"
            authors = []
            version = "0.0.1"
        "#)
        .file("bar/src/lib.rs", "")
        .build();

    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));

    let toml = p.root().join("Cargo.toml");
    let lock1 = p.read_lockfile();

    // add a dep
    File::create(&toml).unwrap().write_all(br#"
        [package]
        name = "foo"
        authors = []
        version = "0.0.1"

        [dependencies.bar]
        path = "bar"
    "#).unwrap();
    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));
    let lock2 = p.read_lockfile();
    assert!(lock1 != lock2);

    // change the dep
    File::create(&p.root().join("bar/Cargo.toml")).unwrap().write_all(br#"
        [package]
        name = "bar"
        authors = []
        version = "0.0.2"
    "#).unwrap();
    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));
    let lock3 = p.read_lockfile();
    assert!(lock1 != lock3);
    assert!(lock2 != lock3);

    // remove the dep
    println!("lock4");
    File::create(&toml).unwrap().write_all(br#"
        [package]
        name = "foo"
        authors = []
        version = "0.0.1"
    "#).unwrap();
    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));
    let lock4 = p.read_lockfile();
    assert_eq!(lock1, lock4);
}

#[test]
fn preserve_metadata() {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]
            name = "foo"
            authors = []
            version = "0.0.1"
        "#)
        .file("src/main.rs", "fn main() {}")
        .file("bar/Cargo.toml", r#"
            [package]
            name = "bar"
            authors = []
            version = "0.0.1"
        "#)
        .file("bar/src/lib.rs", "")
        .build();

    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));

    let metadata = r#"
[metadata]
bar = "baz"
foo = "bar"
"#;
    let lockfile = p.root().join("Cargo.lock");
    let lock = p.read_lockfile();
    let data = lock + metadata;
    File::create(&lockfile).unwrap().write_all(data.as_bytes()).unwrap();

    // Build and make sure the metadata is still there
    assert_that(p.cargo("build"),
                execs().with_status(0));
    let lock = p.read_lockfile();
    assert!(lock.contains(metadata.trim()), "{}", lock);

    // Update and make sure the metadata is still there
    assert_that(p.cargo("update"),
                execs().with_status(0));
    let lock = p.read_lockfile();
    assert!(lock.contains(metadata.trim()), "{}", lock);
}

#[test]
fn preserve_line_endings_issue_2076() {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]
            name = "foo"
            authors = []
            version = "0.0.1"
        "#)
        .file("src/main.rs", "fn main() {}")
        .file("bar/Cargo.toml", r#"
            [package]
            name = "bar"
            authors = []
            version = "0.0.1"
        "#)
        .file("bar/src/lib.rs", "")
        .build();

    let lockfile = p.root().join("Cargo.lock");
    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));
    assert_that(&lockfile,
                existing_file());
    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));

    let lock0 = p.read_lockfile();

    assert!(lock0.starts_with("[[package]]\n"));

    let lock1 = lock0.replace("\n", "\r\n");
    {
        File::create(&lockfile).unwrap().write_all(lock1.as_bytes()).unwrap();
    }

    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));

    let lock2 = p.read_lockfile();

    assert!(lock2.starts_with("[[package]]\r\n"));
    assert_eq!(lock1, lock2);
}

#[test]
fn cargo_update_generate_lockfile() {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]
            name = "foo"
            authors = []
            version = "0.0.1"
        "#)
        .file("src/main.rs", "fn main() {}")
        .build();

    let lockfile = p.root().join("Cargo.lock");
    assert_that(&lockfile, is_not(existing_file()));
    assert_that(p.cargo("update"), execs().with_status(0).with_stdout(""));
    assert_that(&lockfile, existing_file());

    fs::remove_file(p.root().join("Cargo.lock")).unwrap();

    assert_that(&lockfile, is_not(existing_file()));
    assert_that(p.cargo("update"), execs().with_status(0).with_stdout(""));
    assert_that(&lockfile, existing_file());

}
<commit_msg>add a test<commit_after>extern crate cargotest;
extern crate hamcrest;

use std::fs::{self, File};
use std::io::prelude::*;

use cargotest::support::{project, execs};
use hamcrest::{assert_that, existing_file, is_not};

#[test]
fn adding_and_removing_packages() {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]
            name = "foo"
            authors = []
            version = "0.0.1"
        "#)
        .file("src/main.rs", "fn main() {}")
        .file("bar/Cargo.toml", r#"
            [package]
            name = "bar"
            authors = []
            version = "0.0.1"
        "#)
        .file("bar/src/lib.rs", "")
        .build();

    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));

    let toml = p.root().join("Cargo.toml");
    let lock1 = p.read_lockfile();

    // add a dep
    File::create(&toml).unwrap().write_all(br#"
        [package]
        name = "foo"
        authors = []
        version = "0.0.1"

        [dependencies.bar]
        path = "bar"
    "#).unwrap();
    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));
    let lock2 = p.read_lockfile();
    assert!(lock1 != lock2);

    // change the dep
    File::create(&p.root().join("bar/Cargo.toml")).unwrap().write_all(br#"
        [package]
        name = "bar"
        authors = []
        version = "0.0.2"
    "#).unwrap();
    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));
    let lock3 = p.read_lockfile();
    assert!(lock1 != lock3);
    assert!(lock2 != lock3);

    // remove the dep
    println!("lock4");
    File::create(&toml).unwrap().write_all(br#"
        [package]
        name = "foo"
        authors = []
        version = "0.0.1"
    "#).unwrap();
    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));
    let lock4 = p.read_lockfile();
    assert_eq!(lock1, lock4);
}

#[test]
fn no_index_update() {
    use cargotest::ChannelChanger;
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]
            name = "foo"
            authors = []
            version = "0.0.1"

            [dependencies]
            serde = "1.0"
        "#)
        .file("src/main.rs", "fn main() {}")
        .build();

    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0).with_stdout("")
                    .with_stderr_contains("    Updating registry `https://github.com/rust-lang/crates.io-index`"));

    assert_that(p.cargo("generate-lockfile").masquerade_as_nightly_cargo().arg("-Zno-index-update"),
                execs().with_status(0).with_stdout("")
                    .with_stderr_does_not_contain("    Updating registry `https://github.com/rust-lang/crates.io-index`"));
}

#[test]
fn preserve_metadata() {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]
            name = "foo"
            authors = []
            version = "0.0.1"
        "#)
        .file("src/main.rs", "fn main() {}")
        .file("bar/Cargo.toml", r#"
            [package]
            name = "bar"
            authors = []
            version = "0.0.1"
        "#)
        .file("bar/src/lib.rs", "")
        .build();

    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));

    let metadata = r#"
[metadata]
bar = "baz"
foo = "bar"
"#;
    let lockfile = p.root().join("Cargo.lock");
    let lock = p.read_lockfile();
    let data = lock + metadata;
    File::create(&lockfile).unwrap().write_all(data.as_bytes()).unwrap();

    // Build and make sure the metadata is still there
    assert_that(p.cargo("build"),
                execs().with_status(0));
    let lock = p.read_lockfile();
    assert!(lock.contains(metadata.trim()), "{}", lock);

    // Update and make sure the metadata is still there
    assert_that(p.cargo("update"),
                execs().with_status(0));
    let lock = p.read_lockfile();
    assert!(lock.contains(metadata.trim()), "{}", lock);
}

#[test]
fn preserve_line_endings_issue_2076() {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]
            name = "foo"
            authors = []
            version = "0.0.1"
        "#)
        .file("src/main.rs", "fn main() {}")
        .file("bar/Cargo.toml", r#"
            [package]
            name = "bar"
            authors = []
            version = "0.0.1"
        "#)
        .file("bar/src/lib.rs", "")
        .build();

    let lockfile = p.root().join("Cargo.lock");
    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));
    assert_that(&lockfile,
                existing_file());
    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));

    let lock0 = p.read_lockfile();

    assert!(lock0.starts_with("[[package]]\n"));

    let lock1 = lock0.replace("\n", "\r\n");
    {
        File::create(&lockfile).unwrap().write_all(lock1.as_bytes()).unwrap();
    }

    assert_that(p.cargo("generate-lockfile"),
                execs().with_status(0));

    let lock2 = p.read_lockfile();

    assert!(lock2.starts_with("[[package]]\r\n"));
    assert_eq!(lock1, lock2);
}

#[test]
fn cargo_update_generate_lockfile() {
    let p = project("foo")
        .file("Cargo.toml", r#"
            [package]
            name = "foo"
            authors = []
            version = "0.0.1"
        "#)
        .file("src/main.rs", "fn main() {}")
        .build();

    let lockfile = p.root().join("Cargo.lock");
    assert_that(&lockfile, is_not(existing_file()));
    assert_that(p.cargo("update"), execs().with_status(0).with_stdout(""));
    assert_that(&lockfile, existing_file());

    fs::remove_file(p.root().join("Cargo.lock")).unwrap();

    assert_that(&lockfile, is_not(existing_file()));
    assert_that(p.cargo("update"), execs().with_status(0).with_stdout(""));
    assert_that(&lockfile, existing_file());

}
<|endoftext|>


<commit_before>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let graph = GraphMMap::<u32>::new(&filename);

        let nodes = graph.nodes();

        let mut src = vec![1.0; 1 + (nodes / peers as usize)];  // local rank accumulation
        let mut dst = vec![0.0; nodes];                         // local rank accumulation

        let mut start = time::precise_time_s();

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    // /---- should look familiar! ----\
                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node];
                    }

                    for node in 0..src.len() {
                        let edges = graph.edges(index + peers * node);
                        let value = src[node] / edges.len() as f32;
                        for &b in edges {
                            dst[b as usize] += value;
                        }
                    }
                    // \------ end familiar part ------/

                    output.give_at(&iter, dst.drain_temp()
                                             .enumerate()
                                             .filter(|&(_,f)| f != 0.0)
                                             .map(|(u,f)| (u as u32, f)));

                    for _ in 0..(graph.nodes() + 1) { dst.push(0.0); }

                    println!("iteration {:?}: {}s", iter, time::precise_time_s() - start);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        let local_index = index as usize % workers;

        let mut acc = vec![0.0; 1 + (nodes / workers)];

        ranks
        // .unary_notify(
        //     Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
        //     format!("Aggregation"),
        //     vec![],
        //     move |input, output, iterator| {
        //         while let Some((iter, data)) = input.pull() {
        //             iterator.notify_at(&iter);
        //             for (node, rank) in data.drain_temp() {
        //                 acc[node as usize / workers] += rank;
        //             }
        //         }
        //
        //         while let Some((item, _)) = iterator.next() {
        //
        //             output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
        //                                      .map(|(u,f)| (((u * workers + local_index) as u32), f)));
        //
        //             for _ in 0..(1 + (nodes/workers)) { acc.push(0.0); }
        //             assert!(acc.len() == (1 + (nodes/workers)));
        //         }
        //     }
        // )
        .connect_loop(helper);
    }

    while root.step() { }
}
<commit_msg>aggregation added<commit_after>// #![feature(scoped)]
// #![feature(collections)]

extern crate mmap;
extern crate time;
extern crate timely;
extern crate columnar;
extern crate dataflow_join;

extern crate docopt;
use docopt::Docopt;

use std::thread;

use dataflow_join::graph::{GraphTrait, GraphMMap};

use timely::progress::timestamp::RootTimestamp;
use timely::progress::scope::Scope;
use timely::progress::nested::Summary::Local;
use timely::example_static::*;
use timely::communication::*;
use timely::communication::pact::Exchange;

use timely::networking::initialize_networking;
use timely::networking::initialize_networking_from_file;

use timely::drain::DrainExt;

static USAGE: &'static str = "
Usage: pagerank <source> [options] [<arguments>...]

Options:
    -w <arg>, --workers <arg>    number of workers per process [default: 1]
    -p <arg>, --processid <arg>  identity of this process      [default: 0]
    -n <arg>, --processes <arg>  number of processes involved  [default: 1]
    -h <arg>, --hosts <arg>      list of host:port for workers
";

fn main () {
    let args = Docopt::new(USAGE).and_then(|dopt| dopt.parse()).unwrap_or_else(|e| e.exit());

    // let workers = if let Ok(threads) = args.get_str("<workers>").parse() { threads }
    //               else { panic!("invalid setting for workers: {}", args.get_str("<workers>")) };
    // println!("starting pagerank dataflow with {:?} worker{}", workers, if workers == 1 { "" } else { "s" });
    let source = args.get_str("<source>").to_owned();

    let workers: u64 = if let Ok(threads) = args.get_str("-w").parse() { threads }
                       else { panic!("invalid setting for --workers: {}", args.get_str("-t")) };
    let process_id: u64 = if let Ok(proc_id) = args.get_str("-p").parse() { proc_id }
                          else { panic!("invalid setting for --processid: {}", args.get_str("-p")) };
    let processes: u64 = if let Ok(processes) = args.get_str("-n").parse() { processes }
                         else { panic!("invalid setting for --processes: {}", args.get_str("-n")) };

    println!("Starting pagerank dataflow with");
    println!("\tworkers:\t{}", workers);
    println!("\tprocesses:\t{}", processes);
    println!("\tprocessid:\t{}", process_id);

    // vector holding communicators to use; one per local worker.
    if processes > 1 {
        println!("Initializing BinaryCommunicator");

        let hosts = args.get_str("-h");
        let communicators = if hosts != "" {
            initialize_networking_from_file(hosts, process_id, workers).ok().expect("error initializing networking")
        }
        else {
            let addresses = (0..processes).map(|index| format!("localhost:{}", 2101 + index).to_string()).collect();
            initialize_networking(addresses, process_id, workers).ok().expect("error initializing networking")
        };

        pagerank_multi(communicators, source);
    }
    else if workers > 1 {
        println!("Initializing ProcessCommunicator");
        pagerank_multi(ProcessCommunicator::new_vector(workers), source);
    }
    else {
        println!("Initializing ThreadCommunicator");
        pagerank_multi(vec![ThreadCommunicator], source);
    };
}

fn pagerank_multi<C>(communicators: Vec<C>, filename: String)
where C: Communicator+Send {
    let mut guards = Vec::new();
    let workers = communicators.len();
    for communicator in communicators.into_iter() {
        let filename = filename.clone();
        guards.push(thread::Builder::new().name(format!("timely worker {}", communicator.index()))
                                          .spawn(move || pagerank(communicator, filename, workers))
                                          .unwrap());
    }

    for guard in guards { guard.join().unwrap(); }
}

fn pagerank<C>(communicator: C, filename: String, workers: usize)
where C: Communicator {
    let index = communicator.index() as usize;
    let peers = communicator.peers() as usize;

    let mut root = GraphRoot::new(communicator);

    {   // new scope avoids long borrow on root
        let mut builder = root.new_subgraph();

        // establish the beginnings of a loop,
        // 20 iterations, each time around += 1.
        let (helper, stream) = builder.loop_variable::<(u32, f32)>(RootTimestamp::new(20), Local(1));

        let graph = GraphMMap::<u32>::new(&filename);

        let nodes = graph.nodes();

        let mut src = vec![1.0; 1 + (nodes / peers as usize)];  // local rank accumulation
        let mut dst = vec![0.0; nodes];                         // local rank accumulation

        let mut start = time::precise_time_s();

        // from feedback, place an operator that
        // aggregates and broadcasts ranks along edges.
        let ranks = stream.enable(builder).unary_notify(

            Exchange::new(|x: &(u32, f32)| x.0 as u64),     // 1. how data should be exchanged
            format!("PageRank"),                            // 2. a tasteful, descriptive name
            vec![RootTimestamp::new(0)],                    // 3. indicate an initial capability
            move |input, output, iterator| {                // 4. provide the operator logic

                while let Some((iter, _)) = iterator.next() {

                    // /---- should look familiar! ----\
                    for node in 0..src.len() {
                        src[node] = 0.15 + 0.85 * src[node];
                    }

                    for node in 0..src.len() {
                        let edges = graph.edges(index + peers * node);
                        let value = src[node] / edges.len() as f32;
                        for &b in edges {
                            dst[b as usize] += value;
                        }
                    }
                    // \------ end familiar part ------/

                    output.give_at(&iter, dst.drain_temp()
                                             .enumerate()
                                             .filter(|&(_,f)| f != 0.0)
                                             .map(|(u,f)| (u as u32, f)));

                    for _ in 0..(graph.nodes() + 1) { dst.push(0.0); }

                    println!("iteration {:?}: {}s", iter, time::precise_time_s() - start);
                    start = time::precise_time_s();
                }

                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        src[node as usize / peers] += rank;
                    }
                }
            }
        );

        let local_index = index as usize % workers;

        let mut acc = vec![0.0; 1 + (nodes / workers)];

        ranks
        .unary_notify(
            Exchange::new(move |x: &(u32, f32)| (workers * (index / workers)) as u64 + (x.0 as u64 % workers as u64)),
            format!("Aggregation"),
            vec![],
            move |input, output, iterator| {
                while let Some((iter, data)) = input.pull() {
                    iterator.notify_at(&iter);
                    for (node, rank) in data.drain_temp() {
                        acc[node as usize / workers] += rank;
                    }
                }

                while let Some((item, _)) = iterator.next() {

                    output.give_at(&item, acc.drain_temp().enumerate().filter(|x| x.1 != 0.0)
                                             .map(|(u,f)| (((u * workers + local_index) as u32), f)));

                    for _ in 0..(1 + (nodes/workers)) { acc.push(0.0); }
                    assert!(acc.len() == (1 + (nodes/workers)));
                }
            }
        )
        .connect_loop(helper);
    }

    while root.step() { }
}
<|endoftext|>


<commit_before>use std::future::Future;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::thread::{self, Thread};
use std::time::Duration;

use tokio::time::Instant;

pub(crate) fn timeout<F, I, E>(fut: F, timeout: Option<Duration>) -> Result<I, Waited<E>>
where
    F: Future<Output = Result<I, E>>,
{
    enter();

    let deadline = timeout.map(|d| {
        log::trace!("wait at most {:?}", d);
        Instant::now() + d
    });

    let thread = ThreadWaker(thread::current());
    // Arc shouldn't be necessary, since `Thread` is reference counted internally,
    // but let's just stay safe for now.
    let waker = futures_util::task::waker(Arc::new(thread));
    let mut cx = Context::from_waker(&waker);

    futures_util::pin_mut!(fut);

    loop {
        match fut.as_mut().poll(&mut cx) {
            Poll::Ready(Ok(val)) => return Ok(val),
            Poll::Ready(Err(err)) => return Err(Waited::Inner(err)),
            Poll::Pending => (), // fallthrough
        }

        if let Some(deadline) = deadline {
            let now = Instant::now();
            if now >= deadline {
                log::trace!("wait timeout exceeded");
                return Err(Waited::TimedOut(crate::error::TimedOut));
            }

            log::trace!(
                "({:?}) park timeout {:?}",
                thread::current().id(),
                deadline - now
            );
            thread::park_timeout(deadline - now);
        } else {
            log::trace!("({:?}) park without timeout", thread::current().id());
            thread::park();
        }
    }
}

#[derive(Debug)]
pub(crate) enum Waited<E> {
    TimedOut(crate::error::TimedOut),
    Inner(E),
}

struct ThreadWaker(Thread);

impl futures_util::task::ArcWake for ThreadWaker {
    fn wake_by_ref(arc_self: &Arc<Self>) {
        arc_self.0.unpark();
    }
}

fn enter() {
    // Check we aren't already in a runtime
    #[cfg(debug_assertions)]
    {
        tokio::runtime::Builder::new_current_thread()
            .build()
            .expect("build shell runtime")
            .enter();
    }
}
<commit_msg>Fix unused lint for tokio's enter guard in blocking::wait (#1223)<commit_after>use std::future::Future;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::thread::{self, Thread};
use std::time::Duration;

use tokio::time::Instant;

pub(crate) fn timeout<F, I, E>(fut: F, timeout: Option<Duration>) -> Result<I, Waited<E>>
where
    F: Future<Output = Result<I, E>>,
{
    enter();

    let deadline = timeout.map(|d| {
        log::trace!("wait at most {:?}", d);
        Instant::now() + d
    });

    let thread = ThreadWaker(thread::current());
    // Arc shouldn't be necessary, since `Thread` is reference counted internally,
    // but let's just stay safe for now.
    let waker = futures_util::task::waker(Arc::new(thread));
    let mut cx = Context::from_waker(&waker);

    futures_util::pin_mut!(fut);

    loop {
        match fut.as_mut().poll(&mut cx) {
            Poll::Ready(Ok(val)) => return Ok(val),
            Poll::Ready(Err(err)) => return Err(Waited::Inner(err)),
            Poll::Pending => (), // fallthrough
        }

        if let Some(deadline) = deadline {
            let now = Instant::now();
            if now >= deadline {
                log::trace!("wait timeout exceeded");
                return Err(Waited::TimedOut(crate::error::TimedOut));
            }

            log::trace!(
                "({:?}) park timeout {:?}",
                thread::current().id(),
                deadline - now
            );
            thread::park_timeout(deadline - now);
        } else {
            log::trace!("({:?}) park without timeout", thread::current().id());
            thread::park();
        }
    }
}

#[derive(Debug)]
pub(crate) enum Waited<E> {
    TimedOut(crate::error::TimedOut),
    Inner(E),
}

struct ThreadWaker(Thread);

impl futures_util::task::ArcWake for ThreadWaker {
    fn wake_by_ref(arc_self: &Arc<Self>) {
        arc_self.0.unpark();
    }
}

fn enter() {
    // Check we aren't already in a runtime
    #[cfg(debug_assertions)]
    {
        let _enter = tokio::runtime::Builder::new_current_thread()
            .build()
            .expect("build shell runtime")
            .enter();
    }
}
<|endoftext|>


<commit_before>use super::{function, trait_impls};
use crate::{
    analysis::flags::Info,
    analysis::special_functions::Type,
    codegen::general::{
        self, cfg_condition, cfg_condition_doc, cfg_condition_no_doc, cfg_condition_string,
        cfg_deprecated, derives, doc_alias, version_condition, version_condition_doc,
        version_condition_no_doc, version_condition_string,
    },
    config::gobjects::GObject,
    env::Env,
    file_saver,
    library::*,
    nameutil::{bitfield_member_name, use_glib_type},
    traits::*,
};
use std::{
    io::{prelude::*, Result},
    path::Path,
};

pub fn generate(env: &Env, root_path: &Path, mod_rs: &mut Vec<String>) {
    if !env
        .analysis
        .flags
        .iter()
        .any(|f| env.config.objects[&f.full_name].status.need_generate())
    {
        return;
    }

    let path = root_path.join("flags.rs");
    file_saver::save_to_file(path, env.config.make_backup, |w| {
        general::start_comments(w, &env.config)?;
        general::uses(w, env, &env.analysis.flags_imports, None)?;
        writeln!(w)?;

        mod_rs.push("\nmod flags;".into());
        for flags_analysis in &env.analysis.flags {
            let config = &env.config.objects[&flags_analysis.full_name];
            if !config.status.need_generate() {
                continue;
            }
            let flags = flags_analysis.type_(&env.library);

            if let Some(cfg) = version_condition_string(env, None, flags.version, false, 0) {
                mod_rs.push(cfg);
            }
            if let Some(cfg) = cfg_condition_string(config.cfg_condition.as_ref(), false, 0) {
                mod_rs.push(cfg);
            }
            mod_rs.push(format!("pub use self::flags::{};", flags.name));
            generate_flags(env, w, flags, config, flags_analysis)?;
        }

        Ok(())
    });
}

#[allow(clippy::write_literal)]
fn generate_flags(
    env: &Env,
    w: &mut dyn Write,
    flags: &Bitfield,
    config: &GObject,
    analysis: &Info,
) -> Result<()> {
    let sys_crate_name = env.main_sys_crate_name();
    cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
    version_condition_no_doc(w, env, None, flags.version, false, 0)?;
    writeln!(w, "bitflags! {{")?;
    cfg_condition_doc(w, config.cfg_condition.as_ref(), false, 1)?;
    version_condition_doc(w, env, flags.version, false, 1)?;
    cfg_deprecated(
        w,
        env,
        Some(analysis.type_id),
        flags.deprecated_version,
        false,
        1,
    )?;
    if config.must_use {
        writeln!(w, "    #[must_use]")?;
    }

    if let Some(ref d) = config.derives {
        derives(w, d, 1)?;
    }

    doc_alias(w, &flags.c_type, "", 1)?;
    writeln!(w, "    pub struct {}: u32 {{", flags.name)?;
    for member in &flags.members {
        let member_config = config.members.matched(&member.name);
        if member.status.ignored() {
            continue;
        }

        let name = bitfield_member_name(&member.name);
        let deprecated_version = member_config
            .iter()
            .find_map(|m| m.deprecated_version)
            .or(member.deprecated_version);
        let version = member_config
            .iter()
            .find_map(|m| m.version)
            .or(member.version);
        let cfg_cond = member_config.iter().find_map(|m| m.cfg_condition.as_ref());
        cfg_deprecated(w, env, Some(analysis.type_id), deprecated_version, false, 2)?;
        version_condition(w, env, None, version, false, 2)?;
        cfg_condition(w, cfg_cond, false, 2)?;
        if member.c_identifier != member.name {
            doc_alias(w, &member.c_identifier, "", 2)?;
        }
        writeln!(
            w,
            "\t\tconst {} = {}::{} as u32;",
            name, sys_crate_name, member.c_identifier,
        )?;
    }

    writeln!(
        w,
        "    }}
}}"
    )?;

    let functions = analysis
        .functions
        .iter()
        .filter(|f| f.status.need_generate())
        .collect::<Vec<_>>();

    if !functions.is_empty() {
        writeln!(w)?;
        version_condition(w, env, None, flags.version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        write!(w, "impl {} {{", analysis.name)?;
        for func_analysis in functions {
            function::generate(
                w,
                env,
                func_analysis,
                Some(&analysis.specials),
                flags.version,
                false,
                false,
                1,
            )?;
        }
        writeln!(w, "}}")?;
    }

    trait_impls::generate(
        w,
        env,
        &analysis.name,
        &analysis.functions,
        &analysis.specials,
        None,
        None,
        config.cfg_condition.as_ref(),
    )?;

    writeln!(w)?;

    if config.generate_display_trait && !analysis.specials.has_trait(Type::Display) {
        // Generate Display trait implementation.
        version_condition(w, env, None, flags.version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        writeln!(
            w,
            "impl fmt::Display for {0} {{\n\
            \tfn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {{\n\
            \t\t<Self as fmt::Debug>::fmt(self, f)\n\
            \t}}\n\
            }}\n",
            flags.name
        )?;
    }

    version_condition(w, env, None, flags.version, false, 0)?;
    cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
    writeln!(
        w,
        "#[doc(hidden)]
impl IntoGlib for {name} {{
    type GlibType = {sys_crate_name}::{ffi_name};

    fn into_glib(self) -> {sys_crate_name}::{ffi_name} {{
        self.bits()
    }}
}}
",
        sys_crate_name = sys_crate_name,
        name = flags.name,
        ffi_name = flags.c_type
    )?;

    let assert = if env.config.generate_safety_asserts {
        "skip_assert_initialized!();\n\t\t"
    } else {
        ""
    };

    version_condition(w, env, None, flags.version, false, 0)?;
    cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
    writeln!(
        w,
        "#[doc(hidden)]
impl FromGlib<{sys_crate_name}::{ffi_name}> for {name} {{
    unsafe fn from_glib(value: {sys_crate_name}::{ffi_name}) -> Self {{
        {assert}Self::from_bits_truncate(value)
    }}
}}
",
        sys_crate_name = sys_crate_name,
        name = flags.name,
        ffi_name = flags.c_type,
        assert = assert
    )?;

    if let Some(ref get_type) = flags.glib_get_type {
        let configured_functions = config.functions.matched("get_type");
        let version = std::iter::once(flags.version)
            .chain(configured_functions.iter().map(|f| f.version))
            .max()
            .flatten();

        version_condition(w, env, None, version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        writeln!(
            w,
            "impl StaticType for {name} {{
    fn static_type() -> Type {{
        unsafe {{ from_glib({sys_crate_name}::{get_type}()) }}
    }}
}}",
            sys_crate_name = sys_crate_name,
            name = flags.name,
            get_type = get_type
        )?;
        writeln!(w)?;

        version_condition(w, env, None, version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        writeln!(
            w,
            "impl {valuetype} for {name} {{
    type Type = Self;
}}",
            name = flags.name,
            valuetype = use_glib_type(env, "value::ValueType"),
        )?;
        writeln!(w)?;

        version_condition(w, env, None, version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        writeln!(
            w,
            "unsafe impl<'a> FromValue<'a> for {name} {{
    type Checker = {genericwrongvaluetypechecker}<Self>;

    unsafe fn from_value(value: &'a {gvalue}) -> Self {{
        {assert}from_glib({glib}(value.to_glib_none().0))
    }}
}}",
            name = flags.name,
            glib = use_glib_type(env, "gobject_ffi::g_value_get_flags"),
            gvalue = use_glib_type(env, "Value"),
            genericwrongvaluetypechecker = use_glib_type(env, "value::GenericValueTypeChecker"),
            assert = assert,
        )?;
        writeln!(w)?;

        version_condition(w, env, None, version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        writeln!(
            w,
            "impl ToValue for {name} {{
    fn to_value(&self) -> {gvalue} {{
        let mut value = {gvalue}::for_value_type::<Self>();
        unsafe {{
            {glib}(value.to_glib_none_mut().0, self.into_glib());
        }}
        value
    }}

    fn value_type(&self) -> {gtype} {{
        Self::static_type()
    }}
}}",
            name = flags.name,
            glib = use_glib_type(env, "gobject_ffi::g_value_set_flags"),
            gvalue = use_glib_type(env, "Value"),
            gtype = use_glib_type(env, "Type"),
        )?;
        writeln!(w)?;
    }

    Ok(())
}
<commit_msg>flags: allow setting visibility<commit_after>use super::{function, trait_impls};
use crate::{
    analysis::flags::Info,
    analysis::special_functions::Type,
    codegen::general::{
        self, cfg_condition, cfg_condition_doc, cfg_condition_no_doc, cfg_condition_string,
        cfg_deprecated, derives, doc_alias, version_condition, version_condition_doc,
        version_condition_no_doc, version_condition_string,
    },
    config::gobjects::GObject,
    env::Env,
    file_saver,
    library::*,
    nameutil::{bitfield_member_name, use_glib_type},
    traits::*,
};
use std::{
    io::{prelude::*, Result},
    path::Path,
};

pub fn generate(env: &Env, root_path: &Path, mod_rs: &mut Vec<String>) {
    if !env
        .analysis
        .flags
        .iter()
        .any(|f| env.config.objects[&f.full_name].status.need_generate())
    {
        return;
    }

    let path = root_path.join("flags.rs");
    file_saver::save_to_file(path, env.config.make_backup, |w| {
        general::start_comments(w, &env.config)?;
        general::uses(w, env, &env.analysis.flags_imports, None)?;
        writeln!(w)?;

        mod_rs.push("\nmod flags;".into());
        for flags_analysis in &env.analysis.flags {
            let config = &env.config.objects[&flags_analysis.full_name];
            if !config.status.need_generate() {
                continue;
            }
            let flags = flags_analysis.type_(&env.library);

            if let Some(cfg) = version_condition_string(env, None, flags.version, false, 0) {
                mod_rs.push(cfg);
            }
            if let Some(cfg) = cfg_condition_string(config.cfg_condition.as_ref(), false, 0) {
                mod_rs.push(cfg);
            }
            mod_rs.push(format!("pub use self::flags::{};", flags.name));
            generate_flags(env, w, flags, config, flags_analysis)?;
        }

        Ok(())
    });
}

#[allow(clippy::write_literal)]
fn generate_flags(
    env: &Env,
    w: &mut dyn Write,
    flags: &Bitfield,
    config: &GObject,
    analysis: &Info,
) -> Result<()> {
    let sys_crate_name = env.main_sys_crate_name();
    cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
    version_condition_no_doc(w, env, None, flags.version, false, 0)?;
    writeln!(w, "bitflags! {{")?;
    cfg_condition_doc(w, config.cfg_condition.as_ref(), false, 1)?;
    version_condition_doc(w, env, flags.version, false, 1)?;
    cfg_deprecated(
        w,
        env,
        Some(analysis.type_id),
        flags.deprecated_version,
        false,
        1,
    )?;
    if config.must_use {
        writeln!(w, "    #[must_use]")?;
    }

    if let Some(ref d) = config.derives {
        derives(w, d, 1)?;
    }

    doc_alias(w, &flags.c_type, "", 1)?;
    writeln!(w, "    {} struct {}: u32 {{", config.visibility, flags.name)?;
    for member in &flags.members {
        let member_config = config.members.matched(&member.name);
        if member.status.ignored() {
            continue;
        }

        let name = bitfield_member_name(&member.name);
        let deprecated_version = member_config
            .iter()
            .find_map(|m| m.deprecated_version)
            .or(member.deprecated_version);
        let version = member_config
            .iter()
            .find_map(|m| m.version)
            .or(member.version);
        let cfg_cond = member_config.iter().find_map(|m| m.cfg_condition.as_ref());
        cfg_deprecated(w, env, Some(analysis.type_id), deprecated_version, false, 2)?;
        version_condition(w, env, None, version, false, 2)?;
        cfg_condition(w, cfg_cond, false, 2)?;
        if member.c_identifier != member.name {
            doc_alias(w, &member.c_identifier, "", 2)?;
        }
        writeln!(
            w,
            "\t\tconst {} = {}::{} as u32;",
            name, sys_crate_name, member.c_identifier,
        )?;
    }

    writeln!(
        w,
        "    }}
}}"
    )?;

    let functions = analysis
        .functions
        .iter()
        .filter(|f| f.status.need_generate())
        .collect::<Vec<_>>();

    if !functions.is_empty() {
        writeln!(w)?;
        version_condition(w, env, None, flags.version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        write!(w, "impl {} {{", analysis.name)?;
        for func_analysis in functions {
            function::generate(
                w,
                env,
                func_analysis,
                Some(&analysis.specials),
                flags.version,
                false,
                false,
                1,
            )?;
        }
        writeln!(w, "}}")?;
    }

    trait_impls::generate(
        w,
        env,
        &analysis.name,
        &analysis.functions,
        &analysis.specials,
        None,
        None,
        config.cfg_condition.as_ref(),
    )?;

    writeln!(w)?;

    if config.generate_display_trait && !analysis.specials.has_trait(Type::Display) {
        // Generate Display trait implementation.
        version_condition(w, env, None, flags.version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        writeln!(
            w,
            "impl fmt::Display for {0} {{\n\
            \tfn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {{\n\
            \t\t<Self as fmt::Debug>::fmt(self, f)\n\
            \t}}\n\
            }}\n",
            flags.name
        )?;
    }

    version_condition(w, env, None, flags.version, false, 0)?;
    cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
    writeln!(
        w,
        "#[doc(hidden)]
impl IntoGlib for {name} {{
    type GlibType = {sys_crate_name}::{ffi_name};

    fn into_glib(self) -> {sys_crate_name}::{ffi_name} {{
        self.bits()
    }}
}}
",
        sys_crate_name = sys_crate_name,
        name = flags.name,
        ffi_name = flags.c_type
    )?;

    let assert = if env.config.generate_safety_asserts {
        "skip_assert_initialized!();\n\t\t"
    } else {
        ""
    };

    version_condition(w, env, None, flags.version, false, 0)?;
    cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
    writeln!(
        w,
        "#[doc(hidden)]
impl FromGlib<{sys_crate_name}::{ffi_name}> for {name} {{
    unsafe fn from_glib(value: {sys_crate_name}::{ffi_name}) -> Self {{
        {assert}Self::from_bits_truncate(value)
    }}
}}
",
        sys_crate_name = sys_crate_name,
        name = flags.name,
        ffi_name = flags.c_type,
        assert = assert
    )?;

    if let Some(ref get_type) = flags.glib_get_type {
        let configured_functions = config.functions.matched("get_type");
        let version = std::iter::once(flags.version)
            .chain(configured_functions.iter().map(|f| f.version))
            .max()
            .flatten();

        version_condition(w, env, None, version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        writeln!(
            w,
            "impl StaticType for {name} {{
    fn static_type() -> Type {{
        unsafe {{ from_glib({sys_crate_name}::{get_type}()) }}
    }}
}}",
            sys_crate_name = sys_crate_name,
            name = flags.name,
            get_type = get_type
        )?;
        writeln!(w)?;

        version_condition(w, env, None, version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        writeln!(
            w,
            "impl {valuetype} for {name} {{
    type Type = Self;
}}",
            name = flags.name,
            valuetype = use_glib_type(env, "value::ValueType"),
        )?;
        writeln!(w)?;

        version_condition(w, env, None, version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        writeln!(
            w,
            "unsafe impl<'a> FromValue<'a> for {name} {{
    type Checker = {genericwrongvaluetypechecker}<Self>;

    unsafe fn from_value(value: &'a {gvalue}) -> Self {{
        {assert}from_glib({glib}(value.to_glib_none().0))
    }}
}}",
            name = flags.name,
            glib = use_glib_type(env, "gobject_ffi::g_value_get_flags"),
            gvalue = use_glib_type(env, "Value"),
            genericwrongvaluetypechecker = use_glib_type(env, "value::GenericValueTypeChecker"),
            assert = assert,
        )?;
        writeln!(w)?;

        version_condition(w, env, None, version, false, 0)?;
        cfg_condition_no_doc(w, config.cfg_condition.as_ref(), false, 0)?;
        writeln!(
            w,
            "impl ToValue for {name} {{
    fn to_value(&self) -> {gvalue} {{
        let mut value = {gvalue}::for_value_type::<Self>();
        unsafe {{
            {glib}(value.to_glib_none_mut().0, self.into_glib());
        }}
        value
    }}

    fn value_type(&self) -> {gtype} {{
        Self::static_type()
    }}
}}",
            name = flags.name,
            glib = use_glib_type(env, "gobject_ffi::g_value_set_flags"),
            gvalue = use_glib_type(env, "Value"),
            gtype = use_glib_type(env, "Type"),
        )?;
        writeln!(w)?;
    }

    Ok(())
}
<|endoftext|>