<commit_before>// Copyright (c) 2016-2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::path::Path;
use std::time::Duration;

use hab_core::util::sys;
use hyper::client::{Client as HyperClient, IntoUrl, RequestBuilder};
use hyper::client::pool::{Config, Pool};
use hyper::header::UserAgent;
use hyper::http::h1::Http11Protocol;
use hyper::net::HttpsConnector;
use hyper_openssl::OpensslClient;
use openssl::ssl::{SslConnectorBuilder, SslConnector, SslMethod, SslOption, SSL_OP_NO_SSLV2,
                   SSL_OP_NO_SSLV3, SSL_OP_NO_COMPRESSION};
use url::Url;

use error::{Error, Result};
use net::ProxyHttpsConnector;
use proxy::{ProxyInfo, proxy_unless_domain_exempted};
use ssl;

// Read and write TCP socket timeout for Hyper/HTTP client calls.
const CLIENT_SOCKET_RW_TIMEOUT: u64 = 60;

header! { (ProxyAuthorization, "Proxy-Authorization") => [String] }

/// A generic wrapper around a Hyper HTTP client intended for API-like usage.
///
/// When an `ApiClient` is created, it has a constant URL base which is assumed to be some API
/// endpoint. This allows the underlying Hyper client to load and use any relevant HTTP proxy
/// support and to provide reasonable User-Agent HTTP headers, etc.
#[derive(Debug)]
pub struct ApiClient {
    /// The base URL for the client.
    endpoint: Url,
    /// An instance of a `hyper::Client` which is configured with an SSL context and optionally
    /// using an HTTP proxy.
    inner: HyperClient,
    /// Proxy information, if a proxy is being used.
    proxy: Option<ProxyInfo>,
    /// The URL scheme of the endpoint.
    target_scheme: String,
    /// The `User-Agent` header string to use for HTTP calls.
    user_agent_header: UserAgent,
}

impl ApiClient {
    /// Creates and returns a new `ApiClient` instance.
    ///
    /// # Errors
    ///
    /// * If the underlying Hyper client cannot be created
    /// * If a suitable SSL context cannot be established
    /// * If an HTTP proxy cannot be correctly setup
    /// * If a `User-Agent` HTTP header string cannot be constructed
    pub fn new<T>(
        endpoint: T,
        product: &str,
        version: &str,
        fs_root_path: Option<&Path>,
    ) -> Result<Self>
    where
        T: IntoUrl,
    {
        let endpoint = endpoint.into_url().map_err(Error::UrlParseError)?;
        Ok(ApiClient {
            inner: new_hyper_client(&endpoint, fs_root_path)?,
            proxy: proxy_unless_domain_exempted(Some(&endpoint))?,
            target_scheme: endpoint.scheme().to_string(),
            endpoint: endpoint,
            user_agent_header: user_agent(product, version)?,
        })
    }

    /// Builds an HTTP GET request for a given path.
    pub fn get(&self, path: &str) -> RequestBuilder {
        self.get_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP GET request for a given path with the ability to customize the target URL.
    pub fn get_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("GET {} with {:?}", &url, &self);
        self.add_headers(self.inner.get(url))
    }

    /// Builds an HTTP HEAD request for a given path.
    pub fn head(&self, path: &str) -> RequestBuilder {
        self.head_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP HEAD request for a given path with the ability to customize the target URL.
    pub fn head_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("HEAD {} with {:?}", &url, &self);
        self.add_headers(self.inner.head(url))
    }

    /// Builds an HTTP PATCH request for a given path.
    pub fn patch(&self, path: &str) -> RequestBuilder {
        self.patch_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP PATCH request for a given path with the ability to customize the target URL.
    pub fn patch_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("PATH {} with {:?}", &url, &self);
        self.add_headers(self.inner.patch(url))
    }

    /// Builds an HTTP POST request for a given path.
    pub fn post(&self, path: &str) -> RequestBuilder {
        self.post_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP POST request for a given path with the ability to customize the target URL.
    pub fn post_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("POST {} with {:?}", &url, &self);
        self.add_headers(self.inner.post(url))
    }

    /// Builds an HTTP PUT request for a given path.
    pub fn put(&self, path: &str) -> RequestBuilder {
        self.put_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP PUT request for a given path with the ability to customize the target URL.
    pub fn put_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("PUT {} with {:?}", &url, &self);
        self.add_headers(self.inner.put(url))
    }

    /// Builds an HTTP DELETE request for a given path.
    pub fn delete(&self, path: &str) -> RequestBuilder {
        self.delete_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP DELETE request for a given path with the ability to customize the target URL.
    pub fn delete_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("DELETE {} with {:?}", &url, &self);
        self.add_headers(self.inner.delete(url))
    }

    fn add_headers<'a>(&'a self, rb: RequestBuilder<'a>) -> RequestBuilder {
        let mut rb = rb.header(self.user_agent_header.clone());
        // If the target URL is an `"http"` scheme and we're using a proxy server, then add the
        // proxy authorization header if appropriate. Note that for `"https"` targets, the proxy
        // server will be operating in TCP tunneling mode and will be authenticated on connection to
        // the proxy server which is why we should not add an additional header in this latter
        // case.
        if self.target_scheme == "http" {
            if let Some(ref info) = self.proxy {
                if let Some(header_value) = info.authorization_header_value() {
                    rb = rb.header(ProxyAuthorization(header_value));
                }
            }
        }
        rb
    }

    fn url_for(&self, path: &str) -> Url {
        let mut url = self.endpoint.clone();

        if path.is_empty() {
            return url;
        }

        if url.path().ends_with("/") {
            url.set_path(&format!("{}{}", self.endpoint.path(), path));
        } else {
            url.set_path(&format!("{}/{}", self.endpoint.path(), path));
        }
        url
    }
}

/// Builds a new hyper HTTP client with appropriate SSL configuration and HTTP/HTTPS proxy support.
///
/// ## Linux Platforms
///
/// We need a set of root certificates when connected to SSL/TLS web endpoints and this usually
/// boiled down to using an all-in-one certificate file (such as a `cacert.pem` file) or a directory
/// of files which are certificates. The strategy to location or use a reasonable set of
/// certificates is as follows:
///
/// 1. If the `SSL_CERT_FILE` environment variable is set, then use its value for the certificates.
///    Internally this is triggering default OpenSSL behavior for this environment variable.
/// 2. If the `SSL_CERT_DIR` environment variable is set, then use its value for the directory
///    containing certificates. Like the `SSL_CERT_FILE` case above, this triggers default OpenSSL
///    behavior for this environment variable.
/// 3. If the `core/cacerts` Habitat package is installed locally, then use the latest release's
///    `cacert.pem` file.
/// 4. If none of the conditions above are met, then a `cacert.pem` will be written in an SSL cache
///    directory (by default `/hab/cache/ssl` for a root user and `$HOME/.hab/cache/ssl` for a
///    non-root user) and this will be used. The contents of this file will be inlined in this
///    crate at build time as a fallback insurance policy, meaning that if the a program using this
///    code is operating in a minimal environment which may not contain system certificates, it can
///    still operate. Once a `core/cacerts` Habitat package is present, the logic would fall back
///    to preferring the package version to the cached/inline file version.
///
/// ## Mac Platforms
///
/// The Mac platform uses a Security Framework to store and find root certificates and the hyper
/// library will default to using this on the Mac. Therefore the behavior on the Mac remains
/// unchanged and will use the system's certificates.
///
fn new_hyper_client(url: &Url, fs_root_path: Option<&Path>) -> Result<HyperClient> {
    let connector = ssl_connector(fs_root_path)?;
    let ssl_client = OpensslClient::from(connector);
    let timeout = Some(Duration::from_secs(CLIENT_SOCKET_RW_TIMEOUT));

    match proxy_unless_domain_exempted(Some(url))? {
        Some(proxy) => {
            debug!("Using proxy {}:{}...", proxy.host(), proxy.port());
            let connector = ProxyHttpsConnector::new(proxy, ssl_client)?;
            let pool = Pool::with_connector(Config::default(), connector);
            let mut client = HyperClient::with_protocol(Http11Protocol::with_connector(pool));
            client.set_read_timeout(timeout);
            client.set_write_timeout(timeout);
            Ok(client)
        }
        None => {
            let connector = HttpsConnector::new(ssl_client);
            let pool = Pool::with_connector(Config::default(), connector);
            let mut client = HyperClient::with_protocol(Http11Protocol::with_connector(pool));
            client.set_read_timeout(timeout);
            client.set_write_timeout(timeout);
            Ok(client)
        }
    }
}

/// Returns an HTTP User-Agent string type for use by Hyper when making HTTP requests.
///
/// The general form for Habitat-related clients are of the following form:
///
/// ```text
/// <PRODUCT>/<VERSION> (<TARGET>; <KERNEL_RELEASE>)
/// ```
///
/// where:
///
/// * `<PRODUCT>`: is the provided product name
/// * `<VERSION>`: is the provided version string which may also include a release number
/// * `<TARGET>`: is the machine architecture and the kernel separated by a dash in lower case
/// * `<KERNEL_RELEASE>`: is the kernel release string from `uname`
///
/// For example:
///
/// ```text
/// hab/0.6.0/20160606153031 (x86_64-darwin; 14.5.0)
/// ```
///
/// # Errors
///
/// * If system information cannot be obtained via `uname`
fn user_agent(product: &str, version: &str) -> Result<UserAgent> {
    let uname = sys::uname()?;
    let ua = format!(
        "{}/{} ({}-{}; {})",
        product.trim(),
        version.trim(),
        uname.machine.trim().to_lowercase(),
        uname.sys_name.trim().to_lowercase(),
        uname.release.trim().to_lowercase()
    );
    debug!("User-Agent: {}", &ua);
    Ok(UserAgent(ua))
}

fn ssl_connector(fs_root_path: Option<&Path>) -> Result<SslConnector> {
    let mut conn = SslConnectorBuilder::new(SslMethod::tls())?;
    let mut options = SslOption::empty();
    options.toggle(SSL_OP_NO_SSLV2);
    options.toggle(SSL_OP_NO_SSLV3);
    options.toggle(SSL_OP_NO_COMPRESSION);
    ssl::set_ca(conn.builder_mut(), fs_root_path)?;
    conn.builder_mut().set_options(options);
    conn.builder_mut().set_cipher_list(
        "ALL!EXPORT!EXPORT40!EXPORT56!aNULL!LOW!RC4@STRENGTH",
    )?;
    Ok(conn.build())
}
<commit_msg>Convert github-api-client to use http-client, which respects proxies<commit_after>// Copyright (c) 2016-2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::path::Path;
use std::time::Duration;

use hab_core::util::sys;
use hyper::client::{Client as HyperClient, IntoUrl, RequestBuilder};
use hyper::client::pool::{Config, Pool};
use hyper::header::UserAgent;
use hyper::http::h1::Http11Protocol;
use hyper::net::HttpsConnector;
use hyper_openssl::OpensslClient;
use openssl::ssl::{SslConnectorBuilder, SslConnector, SslMethod, SslOption, SSL_OP_NO_SSLV2,
                   SSL_OP_NO_SSLV3, SSL_OP_NO_COMPRESSION};
use url::Url;

use error::{Error, Result};
use net::ProxyHttpsConnector;
use proxy::{ProxyInfo, proxy_unless_domain_exempted};
use ssl;

// Read and write TCP socket timeout for Hyper/HTTP client calls.
const CLIENT_SOCKET_RW_TIMEOUT: u64 = 60;

header! { (ProxyAuthorization, "Proxy-Authorization") => [String] }

/// A generic wrapper around a Hyper HTTP client intended for API-like usage.
///
/// When an `ApiClient` is created, it has a constant URL base which is assumed to be some API
/// endpoint. This allows the underlying Hyper client to load and use any relevant HTTP proxy
/// support and to provide reasonable User-Agent HTTP headers, etc.
#[derive(Debug)]
pub struct ApiClient {
    /// The base URL for the client.
    endpoint: Url,
    /// An instance of a `hyper::Client` which is configured with an SSL context and optionally
    /// using an HTTP proxy.
    inner: HyperClient,
    /// Proxy information, if a proxy is being used.
    proxy: Option<ProxyInfo>,
    /// The URL scheme of the endpoint.
    target_scheme: String,
    /// The `User-Agent` header string to use for HTTP calls.
    user_agent_header: UserAgent,
}

impl ApiClient {
    /// Creates and returns a new `ApiClient` instance.
    ///
    /// # Errors
    ///
    /// * If the underlying Hyper client cannot be created
    /// * If a suitable SSL context cannot be established
    /// * If an HTTP proxy cannot be correctly setup
    /// * If a `User-Agent` HTTP header string cannot be constructed
    pub fn new<T>(
        endpoint: T,
        product: &str,
        version: &str,
        fs_root_path: Option<&Path>,
    ) -> Result<Self>
    where
        T: IntoUrl,
    {
        let endpoint = endpoint.into_url().map_err(Error::UrlParseError)?;
        Ok(ApiClient {
            inner: new_hyper_client(&endpoint, fs_root_path)?,
            proxy: proxy_unless_domain_exempted(Some(&endpoint))?,
            target_scheme: endpoint.scheme().to_string(),
            endpoint: endpoint,
            user_agent_header: user_agent(product, version)?,
        })
    }

    /// Builds an HTTP GET request for a given path.
    pub fn get(&self, path: &str) -> RequestBuilder {
        self.get_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP GET request for a given path with the ability to customize the target URL.
    pub fn get_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("GET {} with {:?}", &url, &self);
        self.add_headers(self.inner.get(url))
    }

    /// Builds an HTTP HEAD request for a given path.
    pub fn head(&self, path: &str) -> RequestBuilder {
        self.head_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP HEAD request for a given path with the ability to customize the target URL.
    pub fn head_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("HEAD {} with {:?}", &url, &self);
        self.add_headers(self.inner.head(url))
    }

    /// Builds an HTTP PATCH request for a given path.
    pub fn patch(&self, path: &str) -> RequestBuilder {
        self.patch_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP PATCH request for a given path with the ability to customize the target URL.
    pub fn patch_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("PATH {} with {:?}", &url, &self);
        self.add_headers(self.inner.patch(url))
    }

    /// Builds an HTTP POST request for a given path.
    pub fn post(&self, path: &str) -> RequestBuilder {
        self.post_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP POST request for a given path with the ability to customize the target URL.
    pub fn post_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("POST {} with {:?}", &url, &self);
        self.add_headers(self.inner.post(url))
    }

    /// Builds an HTTP PUT request for a given path.
    pub fn put(&self, path: &str) -> RequestBuilder {
        self.put_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP PUT request for a given path with the ability to customize the target URL.
    pub fn put_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("PUT {} with {:?}", &url, &self);
        self.add_headers(self.inner.put(url))
    }

    /// Builds an HTTP DELETE request for a given path.
    pub fn delete(&self, path: &str) -> RequestBuilder {
        self.delete_with_custom_url(path, |_| {})
    }

    /// Builds an HTTP DELETE request for a given path with the ability to customize the target URL.
    pub fn delete_with_custom_url<F>(&self, path: &str, mut customize_url: F) -> RequestBuilder
    where
        F: FnMut(&mut Url),
    {
        let mut url = self.url_for(path);
        customize_url(&mut url);
        debug!("DELETE {} with {:?}", &url, &self);
        self.add_headers(self.inner.delete(url))
    }

    fn add_headers<'a>(&'a self, rb: RequestBuilder<'a>) -> RequestBuilder {
        let mut rb = rb.header(self.user_agent_header.clone());
        // If the target URL is an `"http"` scheme and we're using a proxy server, then add the
        // proxy authorization header if appropriate. Note that for `"https"` targets, the proxy
        // server will be operating in TCP tunneling mode and will be authenticated on connection to
        // the proxy server which is why we should not add an additional header in this latter
        // case.
        if self.target_scheme == "http" {
            if let Some(ref info) = self.proxy {
                if let Some(header_value) = info.authorization_header_value() {
                    rb = rb.header(ProxyAuthorization(header_value));
                }
            }
        }
        rb
    }

    fn url_for(&self, path: &str) -> Url {
        let mut url = self.endpoint.clone();

        if path.is_empty() {
            return url;
        }

        if url.path().ends_with("/") || path.starts_with("/") {
            url.set_path(&format!("{}{}", self.endpoint.path(), path));
        } else {
            url.set_path(&format!("{}/{}", self.endpoint.path(), path));
        }

        url
    }
}

/// Builds a new hyper HTTP client with appropriate SSL configuration and HTTP/HTTPS proxy support.
///
/// ## Linux Platforms
///
/// We need a set of root certificates when connected to SSL/TLS web endpoints and this usually
/// boiled down to using an all-in-one certificate file (such as a `cacert.pem` file) or a directory
/// of files which are certificates. The strategy to location or use a reasonable set of
/// certificates is as follows:
///
/// 1. If the `SSL_CERT_FILE` environment variable is set, then use its value for the certificates.
///    Internally this is triggering default OpenSSL behavior for this environment variable.
/// 2. If the `SSL_CERT_DIR` environment variable is set, then use its value for the directory
///    containing certificates. Like the `SSL_CERT_FILE` case above, this triggers default OpenSSL
///    behavior for this environment variable.
/// 3. If the `core/cacerts` Habitat package is installed locally, then use the latest release's
///    `cacert.pem` file.
/// 4. If none of the conditions above are met, then a `cacert.pem` will be written in an SSL cache
///    directory (by default `/hab/cache/ssl` for a root user and `$HOME/.hab/cache/ssl` for a
///    non-root user) and this will be used. The contents of this file will be inlined in this
///    crate at build time as a fallback insurance policy, meaning that if the a program using this
///    code is operating in a minimal environment which may not contain system certificates, it can
///    still operate. Once a `core/cacerts` Habitat package is present, the logic would fall back
///    to preferring the package version to the cached/inline file version.
///
/// ## Mac Platforms
///
/// The Mac platform uses a Security Framework to store and find root certificates and the hyper
/// library will default to using this on the Mac. Therefore the behavior on the Mac remains
/// unchanged and will use the system's certificates.
///
fn new_hyper_client(url: &Url, fs_root_path: Option<&Path>) -> Result<HyperClient> {
    let connector = ssl_connector(fs_root_path)?;
    let ssl_client = OpensslClient::from(connector);
    let timeout = Some(Duration::from_secs(CLIENT_SOCKET_RW_TIMEOUT));

    match proxy_unless_domain_exempted(Some(url))? {
        Some(proxy) => {
            debug!("Using proxy {}:{}...", proxy.host(), proxy.port());
            let connector = ProxyHttpsConnector::new(proxy, ssl_client)?;
            let pool = Pool::with_connector(Config::default(), connector);
            let mut client = HyperClient::with_protocol(Http11Protocol::with_connector(pool));
            client.set_read_timeout(timeout);
            client.set_write_timeout(timeout);
            Ok(client)
        }
        None => {
            let connector = HttpsConnector::new(ssl_client);
            let pool = Pool::with_connector(Config::default(), connector);
            let mut client = HyperClient::with_protocol(Http11Protocol::with_connector(pool));
            client.set_read_timeout(timeout);
            client.set_write_timeout(timeout);
            Ok(client)
        }
    }
}

/// Returns an HTTP User-Agent string type for use by Hyper when making HTTP requests.
///
/// The general form for Habitat-related clients are of the following form:
///
/// ```text
/// <PRODUCT>/<VERSION> (<TARGET>; <KERNEL_RELEASE>)
/// ```
///
/// where:
///
/// * `<PRODUCT>`: is the provided product name
/// * `<VERSION>`: is the provided version string which may also include a release number
/// * `<TARGET>`: is the machine architecture and the kernel separated by a dash in lower case
/// * `<KERNEL_RELEASE>`: is the kernel release string from `uname`
///
/// For example:
///
/// ```text
/// hab/0.6.0/20160606153031 (x86_64-darwin; 14.5.0)
/// ```
///
/// # Errors
///
/// * If system information cannot be obtained via `uname`
fn user_agent(product: &str, version: &str) -> Result<UserAgent> {
    let uname = sys::uname()?;
    let ua = format!(
        "{}/{} ({}-{}; {})",
        product.trim(),
        version.trim(),
        uname.machine.trim().to_lowercase(),
        uname.sys_name.trim().to_lowercase(),
        uname.release.trim().to_lowercase()
    );
    debug!("User-Agent: {}", &ua);
    Ok(UserAgent(ua))
}

fn ssl_connector(fs_root_path: Option<&Path>) -> Result<SslConnector> {
    let mut conn = SslConnectorBuilder::new(SslMethod::tls())?;
    let mut options = SslOption::empty();
    options.toggle(SSL_OP_NO_SSLV2);
    options.toggle(SSL_OP_NO_SSLV3);
    options.toggle(SSL_OP_NO_COMPRESSION);
    ssl::set_ca(conn.builder_mut(), fs_root_path)?;
    conn.builder_mut().set_options(options);
    conn.builder_mut().set_cipher_list(
        "ALL!EXPORT!EXPORT40!EXPORT56!aNULL!LOW!RC4@STRENGTH",
    )?;
    Ok(conn.build())
}
<|endoftext|>


<commit_before>// Copyright (C) 2018 O.S. Systems Sofware LTDA
//
// SPDX-License-Identifier: Apache-2.0

use crate::{firmware::Metadata, runtime_settings::RuntimeSettings, update_package::UpdatePackage};

use failure::bail;
use log::debug;
use reqwest::{
    header::{HeaderMap, HeaderName, CONTENT_TYPE, RANGE, USER_AGENT},
    Client, StatusCode,
};
use serde_derive::Serialize;
use std::{path::Path, time::Duration};

#[cfg(test)]
pub(crate) mod tests;

pub(crate) struct Api<'a> {
    server: &'a str,
}

#[derive(Debug)]
pub(crate) enum ProbeResponse {
    NoUpdate,
    Update(UpdatePackage),
    ExtraPoll(i64),
}

impl<'a> Api<'a> {
    pub(crate) fn new(server: &'a str) -> Api<'a> {
        Api { server }
    }

    fn client(&self) -> Result<Client, failure::Error> {
        let mut headers = HeaderMap::new();

        headers.insert(USER_AGENT, "updatehub/next".parse()?);
        headers.insert(CONTENT_TYPE, "application/json".parse()?);
        headers.insert(
            HeaderName::from_static("api-content-type"),
            "application/vnd.updatehub-v1+json".parse()?,
        );

        Ok(Client::builder()
            .timeout(Duration::from_secs(10))
            .default_headers(headers)
            .build()?)
    }

    pub fn probe(
        &self,
        runtime_settings: &RuntimeSettings,
        firmware: &Metadata,
    ) -> Result<ProbeResponse, failure::Error> {
        let mut response = self
            .client()?
            .post(&format!("{}/upgrades", &self.server))
            .header(
                HeaderName::from_static("api-retries"),
                runtime_settings.retries(),
            )
            .json(firmware)
            .send()?;

        match response.status() {
            StatusCode::NOT_FOUND => Ok(ProbeResponse::NoUpdate),
            StatusCode::OK => {
                match response
                    .headers()
                    .get("add-extra-poll")
                    .and_then(|extra_poll| extra_poll.to_str().ok())
                    .and_then(|extra_poll| extra_poll.parse().ok())
                {
                    Some(extra_poll) => Ok(ProbeResponse::ExtraPoll(extra_poll)),
                    None => Ok(ProbeResponse::Update(UpdatePackage::parse(
                        &response.text()?,
                    )?)),
                }
            }
            _ => bail!("Invalid response. Status: {}", response.status()),
        }
    }

    pub fn download_object(
        &self,
        product_uid: &str,
        package_uid: &str,
        download_dir: &Path,
        object: &str,
    ) -> Result<(), failure::Error> {
        use std::fs::{create_dir_all, OpenOptions};

        // FIXME: Discuss the need of packages inside the route
        let mut client = self.client()?.get(&format!(
            "{}/products/{}/packages/{}/objects/{}",
            &self.server, product_uid, package_uid, object
        ));

        if !download_dir.exists() {
            debug!("Creating directory to store the downloads.");
            create_dir_all(download_dir)?;
        }

        let file = download_dir.join(object);
        if file.exists() {
            client = client.header(RANGE, format!("bytes={}-", file.metadata()?.len() - 1));
        }

        let mut file = OpenOptions::new().create(true).append(true).open(&file)?;
        let mut response = client.send()?;
        if response.status().is_success() {
            response.copy_to(&mut file)?;
            return Ok(());
        }

        bail!("Couldn't download the object {}", object)
    }

    pub fn report(
        &self,
        state: &str,
        firmware: &'a Metadata,
        package_uid: &str,
        previous_state: Option<&str>,
        error_message: Option<String>,
    ) -> Result<(), failure::Error> {
        #[derive(Serialize)]
        #[serde(rename_all = "kebab-case")]
        struct Payload<'a> {
            #[serde(rename = "status")]
            state: &'a str,
            #[serde(flatten)]
            firmware: &'a Metadata,
            package_uid: &'a str,
            #[serde(skip_serializing_if = "Option::is_none")]
            previous_state: Option<&'a str>,
            #[serde(skip_serializing_if = "Option::is_none")]
            error_message: Option<String>,
        }

        let payload = Payload {
            state,
            firmware,
            package_uid,
            previous_state,
            error_message,
        };

        self.client()?
            .post(&format!("{}/report", &self.server))
            .json(&payload)
            .send()?;
        Ok(())
    }
}
<commit_msg>client: Remove redundant lifetime specifiers<commit_after>// Copyright (C) 2018 O.S. Systems Sofware LTDA
//
// SPDX-License-Identifier: Apache-2.0

use crate::{firmware::Metadata, runtime_settings::RuntimeSettings, update_package::UpdatePackage};

use failure::bail;
use log::debug;
use reqwest::{
    header::{HeaderMap, HeaderName, CONTENT_TYPE, RANGE, USER_AGENT},
    Client, StatusCode,
};
use serde_derive::Serialize;
use std::{path::Path, time::Duration};

#[cfg(test)]
pub(crate) mod tests;

pub(crate) struct Api<'a> {
    server: &'a str,
}

#[derive(Debug)]
pub(crate) enum ProbeResponse {
    NoUpdate,
    Update(UpdatePackage),
    ExtraPoll(i64),
}

impl<'a> Api<'a> {
    pub(crate) fn new(server: &'a str) -> Self {
        Self { server }
    }

    fn client(&self) -> Result<Client, failure::Error> {
        let mut headers = HeaderMap::new();

        headers.insert(USER_AGENT, "updatehub/next".parse()?);
        headers.insert(CONTENT_TYPE, "application/json".parse()?);
        headers.insert(
            HeaderName::from_static("api-content-type"),
            "application/vnd.updatehub-v1+json".parse()?,
        );

        Ok(Client::builder()
            .timeout(Duration::from_secs(10))
            .default_headers(headers)
            .build()?)
    }

    pub fn probe(
        &self,
        runtime_settings: &RuntimeSettings,
        firmware: &Metadata,
    ) -> Result<ProbeResponse, failure::Error> {
        let mut response = self
            .client()?
            .post(&format!("{}/upgrades", &self.server))
            .header(
                HeaderName::from_static("api-retries"),
                runtime_settings.retries(),
            )
            .json(firmware)
            .send()?;

        match response.status() {
            StatusCode::NOT_FOUND => Ok(ProbeResponse::NoUpdate),
            StatusCode::OK => {
                match response
                    .headers()
                    .get("add-extra-poll")
                    .and_then(|extra_poll| extra_poll.to_str().ok())
                    .and_then(|extra_poll| extra_poll.parse().ok())
                {
                    Some(extra_poll) => Ok(ProbeResponse::ExtraPoll(extra_poll)),
                    None => Ok(ProbeResponse::Update(UpdatePackage::parse(
                        &response.text()?,
                    )?)),
                }
            }
            _ => bail!("Invalid response. Status: {}", response.status()),
        }
    }

    pub fn download_object(
        &self,
        product_uid: &str,
        package_uid: &str,
        download_dir: &Path,
        object: &str,
    ) -> Result<(), failure::Error> {
        use std::fs::{create_dir_all, OpenOptions};

        // FIXME: Discuss the need of packages inside the route
        let mut client = self.client()?.get(&format!(
            "{}/products/{}/packages/{}/objects/{}",
            &self.server, product_uid, package_uid, object
        ));

        if !download_dir.exists() {
            debug!("Creating directory to store the downloads.");
            create_dir_all(download_dir)?;
        }

        let file = download_dir.join(object);
        if file.exists() {
            client = client.header(RANGE, format!("bytes={}-", file.metadata()?.len() - 1));
        }

        let mut file = OpenOptions::new().create(true).append(true).open(&file)?;
        let mut response = client.send()?;
        if response.status().is_success() {
            response.copy_to(&mut file)?;
            return Ok(());
        }

        bail!("Couldn't download the object {}", object)
    }

    pub fn report(
        &self,
        state: &str,
        firmware: &Metadata,
        package_uid: &str,
        previous_state: Option<&str>,
        error_message: Option<String>,
    ) -> Result<(), failure::Error> {
        #[derive(Serialize)]
        #[serde(rename_all = "kebab-case")]
        struct Payload<'a> {
            #[serde(rename = "status")]
            state: &'a str,
            #[serde(flatten)]
            firmware: &'a Metadata,
            package_uid: &'a str,
            #[serde(skip_serializing_if = "Option::is_none")]
            previous_state: Option<&'a str>,
            #[serde(skip_serializing_if = "Option::is_none")]
            error_message: Option<String>,
        }

        let payload = Payload {
            state,
            firmware,
            package_uid,
            previous_state,
            error_message,
        };

        self.client()?
            .post(&format!("{}/report", &self.server))
            .json(&payload)
            .send()?;
        Ok(())
    }
}
<|endoftext|>


<commit_before>#[macro_use] extern crate log;
#[macro_use] extern crate matches;
extern crate env_logger;
extern crate argparse;
extern crate quire;
extern crate lithos;
extern crate time;
extern crate scan_dir;
extern crate serde_json;

use std::env;
use std::rc::Rc;
use std::io::{self, BufReader, BufRead};
use std::fs::{File, remove_dir_all, remove_file};
use std::path::{PathBuf, Path};
use std::process::exit;
use std::collections::HashSet;
use std::collections::BTreeMap;
use std::collections::VecDeque;

use time::{Tm, Duration, now_utc};
use quire::{parse_config, Options};
use argparse::{ArgumentParser, Parse, ParseOption, StoreTrue, StoreConst};
use argparse::{Print};

use lithos::child_config::ChildConfig;
use lithos::master_config::MasterConfig;
use lithos::MAX_CONFIG_LOGS;
use lithos::sandbox_config::SandboxConfig;

#[derive(Clone, Copy, Debug)]
enum Action {
    Used,
    Unused,
    DeleteUnused,
}

enum Candidate {
    Config(Tm, BTreeMap<String, ChildConfig>),
    BrokenLine(Rc<PathBuf>, usize),
}

struct LogFiles<'a> {
    base: &'a Path,
    name: String,
    idx: u32,
}

struct ScanResult {
    images: HashSet<PathBuf>,
    image_dirs: HashSet<PathBuf>,
    unused_logs: Vec<PathBuf>,
}

fn main() {

    if env::var("RUST_LOG").is_err() {
        env::set_var("RUST_LOG", "warn");
    }
    env_logger::init().unwrap();
    let mut config_file = PathBuf::from("/etc/lithos/master.yaml");
    let mut verbose = false;
    let mut ver_min = 0;
    let mut ver_max = 1000;
    let mut action = Action::Used;
    let mut clean_logs = false;
    let mut days = None::<u32>;
    {
        let mut ap = ArgumentParser::new();
        ap.set_description("Show used/unused images and clean if needed");
        ap.refer(&mut config_file)
          .add_option(&["-C", "--config"], Parse,
            "Name of the global configuration file \
             (default /etc/lithos/master.yaml)")
          .metavar("FILE");
        ap.refer(&mut days)
          .add_option(&["-D", "--history-days"], ParseOption,
            r"Keep images that used no more than DAYS ago.
              There is no reasonable default, so you should specify this
              argument or --versions-min to get sane behavior.")
          .metavar("DAYS");
        ap.refer(&mut ver_min)
          .add_option(&["--vmin", "--versions-min"], Parse,
            r"Keep minimum NUM versions (even if they are older than DAYS).
              Default is 0 which means keep only current version.")
          .metavar("NUM");
        ap.refer(&mut ver_max)
          .add_option(&["--vmax", "--versions-max"], Parse,
            r"Keep maximum NUM versions
              (even if need to delete images more recent than DAYS).
              Default is 1000.")
          .metavar("NUM");
        ap.refer(&mut verbose)
          .add_option(&["-v", "--verbose"], StoreTrue,
            "Verbose output");
        ap.refer(&mut action)
          .add_option(&["--used"], StoreConst(Action::Used),
            "Show used images")
          .add_option(&["--unused"], StoreConst(Action::Unused),
            "Show unused images")
          .add_option(&["--delete-unused"], StoreConst(Action::DeleteUnused),
            "Delete unused images");
        ap.refer(&mut clean_logs)
          .add_option(&["--clean-logs"], StoreConst(true),
            "In combination with `--unused` shows unused logs, \
             in combination with `--delete-unused` deletes them.");
        ap.add_option(&["--version"],
            Print(env!("CARGO_PKG_VERSION").to_string()),
            "Show version of the lithos");
        ap.parse_args_or_exit();
    }
    let master: MasterConfig = match parse_config(&config_file,
        &MasterConfig::validator(), &Options::default()) {
        Ok(cfg) => cfg,
        Err(e) => {
            error!("Can't parse config: {}", e);
            exit(1);
        }
    };
    let tm = days.map(|days| now_utc() - Duration::days(days as i64));
    let scan_result = match find_used_images(&master, &config_file,
        tm, ver_min, ver_max)
    {
        Ok(scan_result) => scan_result,
        Err(e) => {
            error!("Error finding out used images: {}", e);
            exit(1);
        }
    };
    match action {
        Action::Used => {
            for i in &scan_result.images {
                println!("{:?}", i);
            }
        }
        Action::Unused => {
            let unused = find_unused(&scan_result.images,
                                     &scan_result.image_dirs)
                .map_err(|e| {
                    error!("Error finding unused images: {:?}", e);
                    exit(2);
                })
                .unwrap();
            for i in unused {
                println!("{:?}", i);
            }
            if clean_logs {
                for log in scan_result.unused_logs {
                    if log.exists() {
                        println!("{:?}", log);
                    }
                }
            }
        }
        Action::DeleteUnused => {
            let unused = find_unused(&scan_result.images,
                                     &scan_result.image_dirs)
                .map_err(|e| {
                    error!("Error finding unused images: {:?}", e);
                    exit(2);
                })
                .unwrap();
            for i in unused {
                if verbose {
                    println!("Deleting {:?}", i);
                }
                remove_dir_all(&i)
                    .map_err(|e| error!("Error removing {:?}: {}", i, e)).ok();
            }
            if clean_logs {
                for log in scan_result.unused_logs {
                    if verbose {
                        println!("Deleting log {:?}", log);
                    }
                    match remove_file(&log) {
                        Ok(()) => {}
                        Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
                        Err(ref e) => {
                            error!("Can't remove log {:?}: {}", log, e);
                        }
                    }
                }
            }
        }
    }
}

impl<'a> LogFiles<'a> {
    fn new(path: &'a Path, name: &'a str) -> LogFiles<'a> {
        LogFiles {
            base: path,
            name: format!("{}.log", name),
            idx: 0,
        }
    }
}

impl<'a> Iterator for LogFiles<'a> {
    type Item = PathBuf;
    fn next(&mut self) -> Option<PathBuf> {
        if self.idx == 0 {
            let result = self.base.join(&self.name);
            self.idx += 1;
            debug!("Trying {:?}: {}", result, result.exists());
            if result.exists() {
                return Some(result);
            }
        }
        while self.idx < MAX_CONFIG_LOGS {
            let result = self.base.join(format!("{}.{}", self.name, self.idx));
            self.idx += 1;
            debug!("Trying {:?}: {}", result, result.exists());
            if result.exists() {
                return Some(result);
            }
        }
        return None;
    }
}

fn find_unused(used: &HashSet<PathBuf>, dirs: &HashSet<PathBuf>)
    -> Result<Vec<PathBuf>, scan_dir::Error>
{
    let mut unused = Vec::new();
    for dir in dirs.iter() {
        if !dir.exists() {
            warn!("Directory {:?} does not exists", dir);
            continue;
        }
        try!(scan_dir::ScanDir::dirs().skip_symlinks(true).read(dir, |iter| {
            for (entry, _) in iter {
                let path = entry.path().to_path_buf();
                if !used.contains(&path) {
                    unused.push(path);
                }
            }
        }));
    }
    Ok(unused)
}

impl PartialEq for Candidate {
    fn eq(&self, other: &Candidate) -> bool {
        use Candidate::*;
        match (self, other) {
            (&BrokenLine(..), &BrokenLine(..)) => true,
            (&Config(_, ref a), &Config(_, ref b)) => a == b,
            _ => false,
        }
    }
}

fn parse_line(line: &str) -> Result<(Tm, BTreeMap<String, ChildConfig>), ()> {
    let mut iter = line.splitn(2, " ");
    let date = iter.next().ok_or(())?;
    let config = iter.next().ok_or(())?;
    Ok((
        time::strptime(date, "%Y-%m-%dT%H:%M:%SZ").map_err(|_| ())?,
        serde_json::from_str(config).map_err(|_| ())?,
    ))
}

fn find_used_images(master: &MasterConfig, master_file: &Path,
    min_time: Option<Tm>, ver_min: u32, ver_max: u32)
    -> Result<ScanResult, String>
{
    let config_dir = master_file.parent().unwrap().join(&master.sandboxes_dir);
    let mut bad_dirs = HashSet::new();
    let mut images = HashSet::new();
    let mut image_dirs = HashSet::new();
    let mut unused_logs = Vec::new();
    let childval = ChildConfig::mapping_validator();
    scan_dir::ScanDir::files().read(&config_dir, |iter| -> Result<(), String> {
        let yamls = iter.filter(|&(_, ref name)| name.ends_with(".yaml"));
        for (entry, sandbox_fname) in yamls {
            let sandbox_name = &sandbox_fname[..sandbox_fname.len()-5];  // strip .yaml
            let sandbox_config: SandboxConfig = parse_config(&entry.path(),
                &SandboxConfig::validator(), &Options::default())
                .map_err(|e| e.to_string())?;
            image_dirs.insert(sandbox_config.image_dir.clone());

            let cfg = master_file.parent().unwrap()
                .join(&master.processes_dir)
                .join(sandbox_config.config_file.as_ref().unwrap_or(
                    &PathBuf::from(&(sandbox_name.to_string() + ".yaml"))));
            if cfg.exists() {
                let all_children: BTreeMap<String, ChildConfig>;
                all_children =
                    parse_config(&cfg, &childval, &Options::default())
                    .map_err(|e| format!("Can't read child config {:?}: {}",
                                         sandbox_config.config_file, e))?;
                for child in all_children.values() {
                    // Current are always added
                    images.insert(sandbox_config.image_dir.join(&child.image));
                }
            } else {
                info!("No current processes for {}", sandbox_name);
            }

            let mut configs = VecDeque::<Candidate>::new();
            let mut log_iter = LogFiles::new(
                &master.config_log_dir, &sandbox_name);
            for logname in log_iter.by_ref() {
                let logname = Rc::new(logname);
                let log = try!(File::open(&*logname)
                    .map_err(|e| format!("Can't read log file {:?}: {}", logname, e)));
                for (line_no, line) in BufReader::new(log).lines().enumerate() {
                    let line = try!(line
                        .map_err(|e| format!("Readline error: {}", e)));
                    if line.trim() == "" {
                        continue; // Probably last line
                    }
                    let cand = match parse_line(&line) {
                        Ok((tm, cfg)) => Candidate::Config(tm, cfg),
                        Err(()) => {
                            warn!("Broken line {}: {}",
                                logname.display(), line_no);
                            Candidate::BrokenLine(logname.clone(), line_no)
                        }
                    };
                    if configs.back() == Some(&cand) {
                        continue;
                    }
                    configs.push_back(cand);
                    if configs.len() as u32 >= ver_max {
                        configs.pop_front();
                    }
                }
                if configs.len() as u32 >= ver_max ||
                    matches!((configs.front(), min_time),
                        (Some(&Candidate::Config(tm, _)), Some(min_time))
                                              if tm > min_time)
                {
                    break;
                }
            }
            min_time.map(|min_time| {
                while configs.len() > ver_min as usize {
                    match configs.front() {
                        Some(&Candidate::Config(tm, _)) if tm > min_time
                        => {
                            break;
                        }
                        _ => {}
                    }
                    configs.pop_front();
                }
            });
            for cand in configs.iter() {
                match *cand {
                    Candidate::Config(_, ref cfg) => {
                        for child in cfg.values() {
                            images.insert(
                                sandbox_config.image_dir
                                .join(&child.image));
                        }
                    }
                    Candidate::BrokenLine(..) => {
                        bad_dirs.insert(sandbox_config.image_dir.clone());
                    }
                }
            }
            for logname in log_iter {
                unused_logs.push(logname);
            }
        }
        Ok(())
    }).map_err(|e| format!("Read dir error: {}", e))??;

    for dir in &bad_dirs {
        error!("Can't reliably find out used images in the directory {:?}",
            dir);
        image_dirs.remove(dir);
    }
    Ok(ScanResult { images, image_dirs, unused_logs })
}
<commit_msg>Fixes top limit of the number of versions to keep (`--vmax`)<commit_after>#[macro_use] extern crate log;
#[macro_use] extern crate matches;
extern crate env_logger;
extern crate argparse;
extern crate quire;
extern crate lithos;
extern crate time;
extern crate scan_dir;
extern crate serde_json;

use std::env;
use std::rc::Rc;
use std::io::{self, BufReader, BufRead};
use std::fs::{File, remove_dir_all, remove_file};
use std::path::{PathBuf, Path};
use std::process::exit;
use std::collections::HashSet;
use std::collections::BTreeMap;
use std::collections::VecDeque;

use time::{Tm, Duration, now_utc};
use quire::{parse_config, Options};
use argparse::{ArgumentParser, Parse, ParseOption, StoreTrue, StoreConst};
use argparse::{Print};

use lithos::child_config::ChildConfig;
use lithos::master_config::MasterConfig;
use lithos::MAX_CONFIG_LOGS;
use lithos::sandbox_config::SandboxConfig;

#[derive(Clone, Copy, Debug)]
enum Action {
    Used,
    Unused,
    DeleteUnused,
}

enum Candidate {
    Config(Tm, BTreeMap<String, ChildConfig>),
    BrokenLine(Rc<PathBuf>, usize),
}

struct LogFiles<'a> {
    base: &'a Path,
    name: String,
    idx: u32,
}

struct ScanResult {
    images: HashSet<PathBuf>,
    image_dirs: HashSet<PathBuf>,
    unused_logs: Vec<PathBuf>,
}

fn main() {

    if env::var("RUST_LOG").is_err() {
        env::set_var("RUST_LOG", "warn");
    }
    env_logger::init().unwrap();
    let mut config_file = PathBuf::from("/etc/lithos/master.yaml");
    let mut verbose = false;
    let mut ver_min = 0;
    let mut ver_max = 1000;
    let mut action = Action::Used;
    let mut clean_logs = false;
    let mut days = None::<u32>;
    {
        let mut ap = ArgumentParser::new();
        ap.set_description("Show used/unused images and clean if needed");
        ap.refer(&mut config_file)
          .add_option(&["-C", "--config"], Parse,
            "Name of the global configuration file \
             (default /etc/lithos/master.yaml)")
          .metavar("FILE");
        ap.refer(&mut days)
          .add_option(&["-D", "--history-days"], ParseOption,
            r"Keep images that used no more than DAYS ago.
              There is no reasonable default, so you should specify this
              argument or --versions-min to get sane behavior.")
          .metavar("DAYS");
        ap.refer(&mut ver_min)
          .add_option(&["--vmin", "--versions-min"], Parse,
            r"Keep minimum NUM versions (even if they are older than DAYS).
              Default is 0 which means keep only current version.")
          .metavar("NUM");
        ap.refer(&mut ver_max)
          .add_option(&["--vmax", "--versions-max"], Parse,
            r"Keep maximum NUM versions
              (even if need to delete images more recent than DAYS).
              Default is 1000.")
          .metavar("NUM");
        ap.refer(&mut verbose)
          .add_option(&["-v", "--verbose"], StoreTrue,
            "Verbose output");
        ap.refer(&mut action)
          .add_option(&["--used"], StoreConst(Action::Used),
            "Show used images")
          .add_option(&["--unused"], StoreConst(Action::Unused),
            "Show unused images")
          .add_option(&["--delete-unused"], StoreConst(Action::DeleteUnused),
            "Delete unused images");
        ap.refer(&mut clean_logs)
          .add_option(&["--clean-logs"], StoreConst(true),
            "In combination with `--unused` shows unused logs, \
             in combination with `--delete-unused` deletes them.");
        ap.add_option(&["--version"],
            Print(env!("CARGO_PKG_VERSION").to_string()),
            "Show version of the lithos");
        ap.parse_args_or_exit();
    }
    let master: MasterConfig = match parse_config(&config_file,
        &MasterConfig::validator(), &Options::default()) {
        Ok(cfg) => cfg,
        Err(e) => {
            error!("Can't parse config: {}", e);
            exit(1);
        }
    };
    let tm = days.map(|days| now_utc() - Duration::days(days as i64));
    let scan_result = match find_used_images(&master, &config_file,
        tm, ver_min, ver_max)
    {
        Ok(scan_result) => scan_result,
        Err(e) => {
            error!("Error finding out used images: {}", e);
            exit(1);
        }
    };
    match action {
        Action::Used => {
            for i in &scan_result.images {
                println!("{:?}", i);
            }
        }
        Action::Unused => {
            let unused = find_unused(&scan_result.images,
                                     &scan_result.image_dirs)
                .map_err(|e| {
                    error!("Error finding unused images: {:?}", e);
                    exit(2);
                })
                .unwrap();
            for i in unused {
                println!("{:?}", i);
            }
            if clean_logs {
                for log in scan_result.unused_logs {
                    if log.exists() {
                        println!("{:?}", log);
                    }
                }
            }
        }
        Action::DeleteUnused => {
            let unused = find_unused(&scan_result.images,
                                     &scan_result.image_dirs)
                .map_err(|e| {
                    error!("Error finding unused images: {:?}", e);
                    exit(2);
                })
                .unwrap();
            for i in unused {
                if verbose {
                    println!("Deleting {:?}", i);
                }
                remove_dir_all(&i)
                    .map_err(|e| error!("Error removing {:?}: {}", i, e)).ok();
            }
            if clean_logs {
                for log in scan_result.unused_logs {
                    if verbose {
                        println!("Deleting log {:?}", log);
                    }
                    match remove_file(&log) {
                        Ok(()) => {}
                        Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
                        Err(ref e) => {
                            error!("Can't remove log {:?}: {}", log, e);
                        }
                    }
                }
            }
        }
    }
}

impl<'a> LogFiles<'a> {
    fn new(path: &'a Path, name: &'a str) -> LogFiles<'a> {
        LogFiles {
            base: path,
            name: format!("{}.log", name),
            idx: 0,
        }
    }
}

impl<'a> Iterator for LogFiles<'a> {
    type Item = PathBuf;
    fn next(&mut self) -> Option<PathBuf> {
        if self.idx == 0 {
            let result = self.base.join(&self.name);
            self.idx += 1;
            debug!("Trying {:?}: {}", result, result.exists());
            if result.exists() {
                return Some(result);
            }
        }
        while self.idx < MAX_CONFIG_LOGS {
            let result = self.base.join(format!("{}.{}", self.name, self.idx));
            self.idx += 1;
            debug!("Trying {:?}: {}", result, result.exists());
            if result.exists() {
                return Some(result);
            }
        }
        return None;
    }
}

fn find_unused(used: &HashSet<PathBuf>, dirs: &HashSet<PathBuf>)
    -> Result<Vec<PathBuf>, scan_dir::Error>
{
    let mut unused = Vec::new();
    for dir in dirs.iter() {
        if !dir.exists() {
            warn!("Directory {:?} does not exists", dir);
            continue;
        }
        try!(scan_dir::ScanDir::dirs().skip_symlinks(true).read(dir, |iter| {
            for (entry, _) in iter {
                let path = entry.path().to_path_buf();
                if !used.contains(&path) {
                    unused.push(path);
                }
            }
        }));
    }
    Ok(unused)
}

impl PartialEq for Candidate {
    fn eq(&self, other: &Candidate) -> bool {
        use Candidate::*;
        match (self, other) {
            (&BrokenLine(..), &BrokenLine(..)) => true,
            (&Config(_, ref a), &Config(_, ref b)) => a == b,
            _ => false,
        }
    }
}

fn parse_line(line: &str) -> Result<(Tm, BTreeMap<String, ChildConfig>), ()> {
    let mut iter = line.splitn(2, " ");
    let date = iter.next().ok_or(())?;
    let config = iter.next().ok_or(())?;
    Ok((
        time::strptime(date, "%Y-%m-%dT%H:%M:%SZ").map_err(|_| ())?,
        serde_json::from_str(config).map_err(|_| ())?,
    ))
}

fn find_used_images(master: &MasterConfig, master_file: &Path,
    min_time: Option<Tm>, ver_min: u32, ver_max: u32)
    -> Result<ScanResult, String>
{
    let config_dir = master_file.parent().unwrap().join(&master.sandboxes_dir);
    let mut bad_dirs = HashSet::new();
    let mut images = HashSet::new();
    let mut image_dirs = HashSet::new();
    let mut unused_logs = Vec::new();
    let childval = ChildConfig::mapping_validator();
    scan_dir::ScanDir::files().read(&config_dir, |iter| -> Result<(), String> {
        let yamls = iter.filter(|&(_, ref name)| name.ends_with(".yaml"));
        for (entry, sandbox_fname) in yamls {
            let sandbox_name = &sandbox_fname[..sandbox_fname.len()-5];  // strip .yaml
            let sandbox_config: SandboxConfig = parse_config(&entry.path(),
                &SandboxConfig::validator(), &Options::default())
                .map_err(|e| e.to_string())?;
            image_dirs.insert(sandbox_config.image_dir.clone());

            let cfg = master_file.parent().unwrap()
                .join(&master.processes_dir)
                .join(sandbox_config.config_file.as_ref().unwrap_or(
                    &PathBuf::from(&(sandbox_name.to_string() + ".yaml"))));
            if cfg.exists() {
                let all_children: BTreeMap<String, ChildConfig>;
                all_children =
                    parse_config(&cfg, &childval, &Options::default())
                    .map_err(|e| format!("Can't read child config {:?}: {}",
                                         sandbox_config.config_file, e))?;
                for child in all_children.values() {
                    // Current are always added
                    images.insert(sandbox_config.image_dir.join(&child.image));
                }
            } else {
                info!("No current processes for {}", sandbox_name);
            }

            let mut configs = VecDeque::<Candidate>::new();
            let mut log_iter = LogFiles::new(
                &master.config_log_dir, &sandbox_name);
            for logname in log_iter.by_ref() {
                let logname = Rc::new(logname);
                let log = try!(File::open(&*logname)
                    .map_err(|e| format!("Can't read log file {:?}: {}", logname, e)));
                for (line_no, line) in BufReader::new(log).lines().enumerate() {
                    let line = try!(line
                        .map_err(|e| format!("Readline error: {}", e)));
                    if line.trim() == "" {
                        continue; // Probably last line
                    }
                    let cand = match parse_line(&line) {
                        Ok((tm, cfg)) => Candidate::Config(tm, cfg),
                        Err(()) => {
                            warn!("Broken line {}: {}",
                                logname.display(), line_no);
                            Candidate::BrokenLine(logname.clone(), line_no)
                        }
                    };
                    if configs.back() == Some(&cand) {
                        continue;
                    }
                    configs.push_back(cand);
                    if configs.len() as u32 > ver_max {
                        configs.pop_front();
                    }
                }
                if configs.len() as u32 >= ver_max ||
                    matches!((configs.front(), min_time),
                        (Some(&Candidate::Config(tm, _)), Some(min_time))
                                              if tm > min_time)
                {
                    break;
                }
            }
            min_time.map(|min_time| {
                while configs.len() > ver_min as usize {
                    match configs.front() {
                        Some(&Candidate::Config(tm, _)) if tm > min_time
                        => {
                            break;
                        }
                        _ => {}
                    }
                    configs.pop_front();
                }
            });
            for cand in configs.iter() {
                match *cand {
                    Candidate::Config(_, ref cfg) => {
                        for child in cfg.values() {
                            images.insert(
                                sandbox_config.image_dir
                                .join(&child.image));
                        }
                    }
                    Candidate::BrokenLine(..) => {
                        bad_dirs.insert(sandbox_config.image_dir.clone());
                    }
                }
            }
            for logname in log_iter {
                unused_logs.push(logname);
            }
        }
        Ok(())
    }).map_err(|e| format!("Read dir error: {}", e))??;

    for dir in &bad_dirs {
        error!("Can't reliably find out used images in the directory {:?}",
            dir);
        image_dirs.remove(dir);
    }
    Ok(ScanResult { images, image_dirs, unused_logs })
}
<|endoftext|>


<commit_before>/* Copyright 2015 Jordan Miner
 *
 * Licensed under the MIT license <LICENSE or
 * http://opensource.org/licenses/MIT>. This file may not be copied,
 * modified, or distributed except according to those terms.
 */

use std::borrow::Cow;
use std::ffi::CStr;
use libc::{c_char, c_int};
use iup_sys::*;
use smallvec::SmallVec;
use super::Control;

pub fn str_to_c_vec<'a: 'b, 'b, A: ::smallvec::Array<Item=u8>>(s: &'a str, buf: &'b mut SmallVec<A>) -> *const c_char {
    // `CString` in the std library doesn't check if the &str already ends in a null terminator
    // It allocates and pushes a 0 unconditionally. However, I can add the null to string literals
    // and avoid many allocations.
    if s.as_bytes().last() == Some(&0) && !s.as_bytes()[..s.len() - 1].contains(&b'\0') {
        s.as_bytes().as_ptr() as *const c_char
    } else {
        buf.grow(s.len() + 1);
        buf.extend(s.as_bytes().iter().map(|c| if *c == b'\0' { b'?' } else { *c }));
        buf.push(0);
        (&buf[..]).as_ptr() as *const c_char
    }
}

pub fn set_str_attribute(handle: *mut Ihandle, name: &str, value: &str) {
    unsafe {
        let mut name_buf = SmallVec::<[u8; 32]>::new();
        let c_name = str_to_c_vec(name, &mut name_buf);
        let mut value_buf = SmallVec::<[u8; 32]>::new(); // TODO: change to 64 after upgrading smallvec
        let c_value = str_to_c_vec(value, &mut value_buf);
        //println!("setting {:?} to {:?}", CStr::from_ptr(c_name).to_string_lossy(), CStr::from_ptr(c_value).to_string_lossy());
        IupSetStrAttribute(handle, c_name, c_value);
    }
}

// Unfortunately, the return value has to be copied because its lifetime isn't guaranteed.
// IUP's docs state:
//     "The returned pointer can be used safely even if IupGetGlobal or IupGetAttribute are called
//     several times. But not too many times, because it is an internal buffer and after IUP may
//     reuse it after around 50 calls."
pub fn get_str_attribute(handle: *mut Ihandle, name: &str) -> String {
    unsafe {
        get_str_attribute_slice(handle, name).into_owned()
    }
}

// This function isn't very error prone (see above), but isn't completely safe either.
pub unsafe fn get_str_attribute_slice(handle: *mut Ihandle, name: &str) -> Cow<str> {
    let mut name_buf = SmallVec::<[u8; 32]>::new();
    let c_name = str_to_c_vec(name, &mut name_buf);
    let value = IupGetAttribute(handle as *mut Ihandle, c_name);
    //println!("getting {:?}: {:?}", name, CStr::from_ptr(value).to_string_lossy());
    CStr::from_ptr(value).to_string_lossy()
}

pub fn get_int_int_attribute(handle: *mut Ihandle, name: &str) -> (i32, i32) {
    unsafe {
        let mut name_buf = SmallVec::<[u8; 32]>::new();
        let c_name = str_to_c_vec(name, &mut name_buf);
        let mut x: i32 = 0;
        let mut y: i32 = 0;
        assert!(IupGetIntInt(handle as *mut Ihandle,
                             c_name,
                             &mut x as *mut c_int,
                             &mut y as *mut c_int) == 2);
        (x, y)
    }
}

pub trait CommonAttributes : Control {
    fn active(&self) -> bool {
        get_str_attribute(self.handle(), "ACTIVE") == "YES"
    }

    fn set_active(&self, active: bool) {
        set_str_attribute(self.handle(), "ACTIVE", if active { "YES" } else { "NO" });
    }

    fn tip(&self) -> String {
        get_str_attribute(self.handle(), "TIP")
    }
    unsafe fn tip_slice(&self) -> Cow<str> {
        get_str_attribute_slice(self.handle(), "TIP")
    }

    fn set_tip(&self, tip: &str) {
        set_str_attribute(self.handle(), "TIP", tip);
    }

    fn min_size(&self) -> (i32, i32) {
        get_int_int_attribute(self.handle(), "MINSIZE")
    }

    fn set_min_size(&self, x: i32, y: i32) {
        let s = format!("{}x{}", x, y);
        set_str_attribute(self.handle(), "MINSIZE", &s);
    }

    fn max_size(&self) -> (i32, i32) {
        get_int_int_attribute(self.handle(), "MAXSIZE")
    }

    fn set_max_size(&self, x: i32, y: i32) {
        let s = format!("{}x{}", x, y);
        set_str_attribute(self.handle(), "MAXSIZE", &s);
    }

    fn show(&self) -> Result<(), ()> {
        unsafe {
            if IupShow(self.handle()) == IUP_NOERROR {
                Ok(())
            } else {
                Err(())
            }
        }
    }

    fn hide(&self) -> Result<(), ()> {
        unsafe {
            if IupHide(self.handle()) == IUP_NOERROR {
                Ok(())
            } else {
                Err(())
            }
        }
    }

    fn set_visible(&self, visible: bool) -> Result<(), ()> {
        if visible { self.show() } else { self.hide() }
    }
}

pub trait TitleAttribute : Control {
    fn title(&self) -> String {
        get_str_attribute(self.handle(), "TITLE")
    }

    fn set_title(&self, title: &str) {
        set_str_attribute(self.handle(), "TITLE", title);
    }
}

pub trait OrientationAttribute : Control {
    fn orientation(&self) -> ::Orientation {
        unsafe {
            let s = get_str_attribute_slice(self.handle(), "ORIENTATION");
            ::Orientation::from_str(s.as_bytes())
        }
    }

    fn set_orientation(&self, orientation: ::Orientation) {
        set_str_attribute(self.handle(), "ORIENTATION", orientation.to_str());
    }
}
<commit_msg>Add a get_attribute_ptr method<commit_after>/* Copyright 2015 Jordan Miner
 *
 * Licensed under the MIT license <LICENSE or
 * http://opensource.org/licenses/MIT>. This file may not be copied,
 * modified, or distributed except according to those terms.
 */

use std::borrow::Cow;
use std::ffi::CStr;
use libc::{c_char, c_int};
use iup_sys::*;
use smallvec::SmallVec;
use super::Control;

pub fn str_to_c_vec<'a: 'b, 'b, A: ::smallvec::Array<Item=u8>>(s: &'a str, buf: &'b mut SmallVec<A>) -> *const c_char {
    // `CString` in the std library doesn't check if the &str already ends in a null terminator
    // It allocates and pushes a 0 unconditionally. However, I can add the null to string literals
    // and avoid many allocations.
    if s.as_bytes().last() == Some(&0) && !s.as_bytes()[..s.len() - 1].contains(&b'\0') {
        s.as_bytes().as_ptr() as *const c_char
    } else {
        buf.grow(s.len() + 1);
        buf.extend(s.as_bytes().iter().map(|c| if *c == b'\0' { b'?' } else { *c }));
        buf.push(0);
        (&buf[..]).as_ptr() as *const c_char
    }
}

pub fn set_str_attribute(handle: *mut Ihandle, name: &str, value: &str) {
    unsafe {
        let mut name_buf = SmallVec::<[u8; 32]>::new();
        let c_name = str_to_c_vec(name, &mut name_buf);
        let mut value_buf = SmallVec::<[u8; 32]>::new(); // TODO: change to 64 after upgrading smallvec
        let c_value = str_to_c_vec(value, &mut value_buf);
        //println!("setting {:?} to {:?}", CStr::from_ptr(c_name).to_string_lossy(), CStr::from_ptr(c_value).to_string_lossy());
        IupSetStrAttribute(handle, c_name, c_value);
    }
}

pub fn get_attribute_ptr(handle: *mut Ihandle, name: &str) -> *mut c_char {
    unsafe {
        let mut name_buf = SmallVec::<[u8; 32]>::new();
        let c_name = str_to_c_vec(name, &mut name_buf);
        IupGetAttribute(handle as *mut Ihandle, c_name)
    }
}

// Unfortunately, the return value has to be copied because its lifetime isn't guaranteed.
// IUP's docs state:
//     "The returned pointer can be used safely even if IupGetGlobal or IupGetAttribute are called
//     several times. But not too many times, because it is an internal buffer and after IUP may
//     reuse it after around 50 calls."
pub fn get_str_attribute(handle: *mut Ihandle, name: &str) -> String {
    unsafe {
        get_str_attribute_slice(handle, name).into_owned()
    }
}

// This function isn't very error prone (see above), but isn't completely safe either.
pub unsafe fn get_str_attribute_slice(handle: *mut Ihandle, name: &str) -> Cow<str> {
    let value = get_attribute_ptr(handle, name);
    //println!("getting {:?}: {:?}", name, CStr::from_ptr(value).to_string_lossy());
    CStr::from_ptr(value).to_string_lossy()
}

pub fn get_int_int_attribute(handle: *mut Ihandle, name: &str) -> (i32, i32) {
    unsafe {
        let mut name_buf = SmallVec::<[u8; 32]>::new();
        let c_name = str_to_c_vec(name, &mut name_buf);
        let mut x: i32 = 0;
        let mut y: i32 = 0;
        assert!(IupGetIntInt(handle as *mut Ihandle,
                             c_name,
                             &mut x as *mut c_int,
                             &mut y as *mut c_int) == 2);
        (x, y)
    }
}

pub trait CommonAttributes : Control {
    fn active(&self) -> bool {
        get_str_attribute(self.handle(), "ACTIVE") == "YES"
    }

    fn set_active(&self, active: bool) {
        set_str_attribute(self.handle(), "ACTIVE", if active { "YES" } else { "NO" });
    }

    fn tip(&self) -> String {
        get_str_attribute(self.handle(), "TIP")
    }
    unsafe fn tip_slice(&self) -> Cow<str> {
        get_str_attribute_slice(self.handle(), "TIP")
    }

    fn set_tip(&self, tip: &str) {
        set_str_attribute(self.handle(), "TIP", tip);
    }

    fn min_size(&self) -> (i32, i32) {
        get_int_int_attribute(self.handle(), "MINSIZE")
    }

    fn set_min_size(&self, x: i32, y: i32) {
        let s = format!("{}x{}", x, y);
        set_str_attribute(self.handle(), "MINSIZE", &s);
    }

    fn max_size(&self) -> (i32, i32) {
        get_int_int_attribute(self.handle(), "MAXSIZE")
    }

    fn set_max_size(&self, x: i32, y: i32) {
        let s = format!("{}x{}", x, y);
        set_str_attribute(self.handle(), "MAXSIZE", &s);
    }

    fn show(&self) -> Result<(), ()> {
        unsafe {
            if IupShow(self.handle()) == IUP_NOERROR {
                Ok(())
            } else {
                Err(())
            }
        }
    }

    fn hide(&self) -> Result<(), ()> {
        unsafe {
            if IupHide(self.handle()) == IUP_NOERROR {
                Ok(())
            } else {
                Err(())
            }
        }
    }

    fn set_visible(&self, visible: bool) -> Result<(), ()> {
        if visible { self.show() } else { self.hide() }
    }
}

pub trait TitleAttribute : Control {
    fn title(&self) -> String {
        get_str_attribute(self.handle(), "TITLE")
    }

    fn set_title(&self, title: &str) {
        set_str_attribute(self.handle(), "TITLE", title);
    }
}

pub trait OrientationAttribute : Control {
    fn orientation(&self) -> ::Orientation {
        unsafe {
            let s = get_str_attribute_slice(self.handle(), "ORIENTATION");
            ::Orientation::from_str(s.as_bytes())
        }
    }

    fn set_orientation(&self, orientation: ::Orientation) {
        set_str_attribute(self.handle(), "ORIENTATION", orientation.to_str());
    }
}
<|endoftext|>


<commit_before>extern crate mdbook;
#[macro_use]
extern crate clap;
extern crate log;
extern crate env_logger;
extern crate open;

use std::env;
use std::ffi::OsStr;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use clap::{App, ArgMatches, AppSettings};

pub mod build;
pub mod init;
pub mod test;
#[cfg(feature = "serve")]
pub mod serve;
#[cfg(feature = "watch")]
pub mod watch;

const NAME: &'static str = "mdbook";

fn main() {
    env_logger::init().unwrap();

    // Create a list of valid arguments and sub-commands
    let app = App::new(NAME)
                .about("Create a book in form of a static website from markdown files")
                .author("Mathieu David <mathieudavid@mathieudavid.org>")
                // Get the version from our Cargo.toml using clap's crate_version!() macro
                .version(concat!("v",crate_version!()))
                .setting(AppSettings::SubcommandRequired)
                .after_help("For more information about a specific command, try `mdbook <command> --help`\nSource code for mdbook available at: https://github.com/azerupi/mdBook")
                .subcommand(init::make_subcommand())
                .subcommand(build::make_subcommand())
                .subcommand(test::make_subcommand());

    #[cfg(feature = "watch")]
    let app = app.subcommand(watch::make_subcommand());
    #[cfg(feature = "serve")]
    let app = app.subcommand(serve::make_subcommand());

    // Check which subcomamnd the user ran...
    let res = match app.get_matches().subcommand() {
        ("init", Some(sub_matches)) => init::execute(sub_matches),
        ("build", Some(sub_matches)) => build::execute(sub_matches),
        #[cfg(feature = "watch")]
        ("watch", Some(sub_matches)) => watch::execute(sub_matches),
        #[cfg(feature = "serve")]
        ("serve", Some(sub_matches)) => serve::execute(sub_matches),
        ("test", Some(sub_matches)) => test::execute(sub_matches),
        (_, _) => unreachable!(),
    };

    if let Err(e) = res {
        writeln!(&mut io::stderr(), "An error occured:\n{}", e).ok();
        ::std::process::exit(101);
    }
}

fn get_book_dir(args: &ArgMatches) -> PathBuf {
    if let Some(dir) = args.value_of("dir") {
        // Check if path is relative from current dir, or absolute...
        let p = Path::new(dir);
        if p.is_relative() {
            env::current_dir().unwrap().join(dir)
        } else {
            p.to_path_buf()
        }
    } else {
        env::current_dir().unwrap()
    }
}

fn open<P: AsRef<OsStr>>(path: P) {
    if let Err(e) = open::that(path) {
        println!("Error opening web browser: {}", e);
    }
}
<commit_msg>Increased default logging level to info unless RUST_LOG is set<commit_after>extern crate mdbook;
#[macro_use]
extern crate clap;
extern crate log;
extern crate env_logger;
extern crate open;

use std::env;
use std::ffi::OsStr;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use clap::{App, ArgMatches, AppSettings};
use log::{LogRecord, LogLevelFilter};
use env_logger::LogBuilder;

pub mod build;
pub mod init;
pub mod test;
#[cfg(feature = "serve")]
pub mod serve;
#[cfg(feature = "watch")]
pub mod watch;

const NAME: &'static str = "mdbook";

fn main() {
    init_logger();

    // Create a list of valid arguments and sub-commands
    let app = App::new(NAME)
                .about("Create a book in form of a static website from markdown files")
                .author("Mathieu David <mathieudavid@mathieudavid.org>")
                // Get the version from our Cargo.toml using clap's crate_version!() macro
                .version(concat!("v",crate_version!()))
                .setting(AppSettings::SubcommandRequired)
                .after_help("For more information about a specific command, try `mdbook <command> --help`\nSource code for mdbook available at: https://github.com/azerupi/mdBook")
                .subcommand(init::make_subcommand())
                .subcommand(build::make_subcommand())
                .subcommand(test::make_subcommand());

    #[cfg(feature = "watch")]
    let app = app.subcommand(watch::make_subcommand());
    #[cfg(feature = "serve")]
    let app = app.subcommand(serve::make_subcommand());

    // Check which subcomamnd the user ran...
    let res = match app.get_matches().subcommand() {
        ("init", Some(sub_matches)) => init::execute(sub_matches),
        ("build", Some(sub_matches)) => build::execute(sub_matches),
        #[cfg(feature = "watch")]
        ("watch", Some(sub_matches)) => watch::execute(sub_matches),
        #[cfg(feature = "serve")]
        ("serve", Some(sub_matches)) => serve::execute(sub_matches),
        ("test", Some(sub_matches)) => test::execute(sub_matches),
        (_, _) => unreachable!(),
    };

    if let Err(e) = res {
        writeln!(&mut io::stderr(), "An error occured:\n{}", e).ok();
        ::std::process::exit(101);
    }
}

fn init_logger() {
    let format = |record: &LogRecord| {
        let module_path = record.location().module_path();
        format!("{}:{}: {}", record.level(), module_path, record.args())
    };

    let mut builder = LogBuilder::new();
    builder.format(format).filter(None, LogLevelFilter::Info);

    if let Ok(var) = env::var("RUST_LOG") {
       builder.parse(&var);
    }

    builder.init().unwrap();
}

fn get_book_dir(args: &ArgMatches) -> PathBuf {
    if let Some(dir) = args.value_of("dir") {
        // Check if path is relative from current dir, or absolute...
        let p = Path::new(dir);
        if p.is_relative() {
            env::current_dir().unwrap().join(dir)
        } else {
            p.to_path_buf()
        }
    } else {
        env::current_dir().unwrap()
    }
}

fn open<P: AsRef<OsStr>>(path: P) {
    if let Err(e) = open::that(path) {
        println!("Error opening web browser: {}", e);
    }
}
<|endoftext|>


<commit_before>#![warn(clippy::all, rust_2018_idioms)]

use cargo_registry::{metrics::LogEncoder, util::errors::AppResult, App, Env};
use std::{fs::File, process::Command, sync::Arc, time::Duration};

use conduit_hyper::Service;
use futures_util::future::FutureExt;
use prometheus::Encoder;
use reqwest::blocking::Client;
use std::io::Write;
use tokio::io::AsyncWriteExt;
use tokio::signal::unix::{signal, SignalKind};

const CORE_THREADS: usize = 4;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let _sentry = cargo_registry::sentry::init();

    // Initialize logging
    tracing_subscriber::fmt::init();

    let config = cargo_registry::config::Server::default();
    let env = config.env();
    let client = Client::new();
    let app = Arc::new(App::new(config, Some(client)));

    // Start the background thread periodically persisting download counts to the database.
    downloads_counter_thread(app.clone());

    // Start the background thread periodically logging instance metrics.
    log_instance_metrics_thread(app.clone());

    let handler = cargo_registry::build_handler(app.clone());

    let heroku = dotenv::var("HEROKU").is_ok();
    let fastboot = dotenv::var("USE_FASTBOOT").is_ok();
    let dev_docker = dotenv::var("DEV_DOCKER").is_ok();

    let ip = if dev_docker {
        [0, 0, 0, 0]
    } else {
        [127, 0, 0, 1]
    };
    let port = if heroku {
        8888
    } else {
        dotenv::var("PORT")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(8888)
    };
    let threads = dotenv::var("SERVER_THREADS")
        .map(|s| s.parse().expect("SERVER_THREADS was not a valid number"))
        .unwrap_or_else(|_| {
            if env == Env::Development {
                5
            } else {
                // A large default because this can be easily changed via env and in production we
                // want the logging middleware to accurately record the start time.
                500
            }
        });

    println!("Booting with a hyper based server");

    let rt = tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .worker_threads(CORE_THREADS)
        .max_blocking_threads(threads as usize)
        .build()
        .unwrap();

    let handler = Arc::new(conduit_hyper::BlockingHandler::new(handler));
    let make_service =
        hyper::service::make_service_fn(move |socket: &hyper::server::conn::AddrStream| {
            let addr = socket.remote_addr();
            let handler = handler.clone();
            async move { Service::from_blocking(handler, addr) }
        });

    let addr = (ip, port).into();
    #[allow(clippy::async_yields_async)]
    let server = rt.block_on(async { hyper::Server::bind(&addr).serve(make_service) });

    let mut sig_int = rt.block_on(async { signal(SignalKind::interrupt()) })?;
    let mut sig_term = rt.block_on(async { signal(SignalKind::terminate()) })?;

    // When the user configures PORT=0 the operative system will allocate a random unused port.
    // This fetches that random port and uses it to display the "listening on port" message later.
    let actual_port = server.local_addr().port();

    let server = server.with_graceful_shutdown(async move {
        // Wait for either signal
        futures_util::select! {
            _ = sig_int.recv().fuse() => {},
            _ = sig_term.recv().fuse() => {},
        };
        let mut stdout = tokio::io::stdout();
        stdout.write_all(b"Starting graceful shutdown\n").await.ok();
    });

    let server = rt.spawn(async { server.await.unwrap() });

    // Do not change this line! Removing the line or changing its contents in any way will break
    // the test suite :)
    println!("listening on port {}", actual_port);

    // Creating this file tells heroku to tell nginx that the application is ready
    // to receive traffic.
    if heroku {
        let path = if fastboot {
            "/tmp/backend-initialized"
        } else {
            "/tmp/app-initialized"
        };
        println!("Writing to {}", path);
        File::create(path).unwrap();

        // Launch nginx via the Heroku nginx buildpack
        // `wait()` is never called on the child process, but it should be okay to leave a zombie
        // process around on shutdown when Heroku is tearing down the entire container anyway.
        Command::new("./script/start-web.sh")
            .spawn()
            .expect("Couldn't spawn nginx");
    }

    // Block the main thread until the server has shutdown
    rt.block_on(async { server.await.unwrap() });

    println!("Persisting remaining downloads counters");
    match app.downloads_counter.persist_all_shards(&app) {
        Ok(stats) => stats.log(),
        Err(err) => println!("downloads_counter error: {}", err),
    }

    println!("Server has gracefully shutdown!");
    Ok(())
}

fn downloads_counter_thread(app: Arc<App>) {
    let interval = Duration::from_millis(
        (app.config.downloads_persist_interval_ms / app.downloads_counter.shards_count()) as u64,
    );

    std::thread::spawn(move || loop {
        std::thread::sleep(interval);

        match app.downloads_counter.persist_next_shard(&app) {
            Ok(stats) => stats.log(),
            Err(err) => println!("downloads_counter error: {}", err),
        }
    });
}

fn log_instance_metrics_thread(app: Arc<App>) {
    // Only run the thread if the configuration is provided
    let interval = if let Some(secs) = app.config.instance_metrics_log_every_seconds {
        Duration::from_secs(secs)
    } else {
        return;
    };

    std::thread::spawn(move || loop {
        if let Err(err) = log_instance_metrics_inner(&app) {
            eprintln!("log_instance_metrics error: {}", err);
        }
        std::thread::sleep(interval);
    });
}

fn log_instance_metrics_inner(app: &App) -> AppResult<()> {
    let families = app.instance_metrics.gather(app)?;

    let mut stdout = std::io::stdout();
    LogEncoder::new().encode(&families, &mut stdout)?;
    stdout.flush()?;

    Ok(())
}
<commit_msg>tracing: Use per-layer event filtering<commit_after>#![warn(clippy::all, rust_2018_idioms)]

use cargo_registry::{metrics::LogEncoder, util::errors::AppResult, App, Env};
use std::{env, fs::File, process::Command, sync::Arc, time::Duration};

use conduit_hyper::Service;
use futures_util::future::FutureExt;
use prometheus::Encoder;
use reqwest::blocking::Client;
use std::io::Write;
use tokio::io::AsyncWriteExt;
use tokio::signal::unix::{signal, SignalKind};
use tracing::Level;
use tracing_subscriber::{filter, prelude::*};

const CORE_THREADS: usize = 4;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let _sentry = cargo_registry::sentry::init();

    // Initialize logging

    let log_filter = env::var("RUST_LOG")
        .unwrap_or_default()
        .parse::<filter::Targets>()
        .expect("Invalid RUST_LOG value");

    tracing_subscriber::registry()
        .with(tracing_subscriber::fmt::layer().with_filter(log_filter))
        .init();

    let config = cargo_registry::config::Server::default();
    let env = config.env();
    let client = Client::new();
    let app = Arc::new(App::new(config, Some(client)));

    // Start the background thread periodically persisting download counts to the database.
    downloads_counter_thread(app.clone());

    // Start the background thread periodically logging instance metrics.
    log_instance_metrics_thread(app.clone());

    let handler = cargo_registry::build_handler(app.clone());

    let heroku = dotenv::var("HEROKU").is_ok();
    let fastboot = dotenv::var("USE_FASTBOOT").is_ok();
    let dev_docker = dotenv::var("DEV_DOCKER").is_ok();

    let ip = if dev_docker {
        [0, 0, 0, 0]
    } else {
        [127, 0, 0, 1]
    };
    let port = if heroku {
        8888
    } else {
        dotenv::var("PORT")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(8888)
    };
    let threads = dotenv::var("SERVER_THREADS")
        .map(|s| s.parse().expect("SERVER_THREADS was not a valid number"))
        .unwrap_or_else(|_| {
            if env == Env::Development {
                5
            } else {
                // A large default because this can be easily changed via env and in production we
                // want the logging middleware to accurately record the start time.
                500
            }
        });

    println!("Booting with a hyper based server");

    let rt = tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .worker_threads(CORE_THREADS)
        .max_blocking_threads(threads as usize)
        .build()
        .unwrap();

    let handler = Arc::new(conduit_hyper::BlockingHandler::new(handler));
    let make_service =
        hyper::service::make_service_fn(move |socket: &hyper::server::conn::AddrStream| {
            let addr = socket.remote_addr();
            let handler = handler.clone();
            async move { Service::from_blocking(handler, addr) }
        });

    let addr = (ip, port).into();
    #[allow(clippy::async_yields_async)]
    let server = rt.block_on(async { hyper::Server::bind(&addr).serve(make_service) });

    let mut sig_int = rt.block_on(async { signal(SignalKind::interrupt()) })?;
    let mut sig_term = rt.block_on(async { signal(SignalKind::terminate()) })?;

    // When the user configures PORT=0 the operative system will allocate a random unused port.
    // This fetches that random port and uses it to display the "listening on port" message later.
    let actual_port = server.local_addr().port();

    let server = server.with_graceful_shutdown(async move {
        // Wait for either signal
        futures_util::select! {
            _ = sig_int.recv().fuse() => {},
            _ = sig_term.recv().fuse() => {},
        };
        let mut stdout = tokio::io::stdout();
        stdout.write_all(b"Starting graceful shutdown\n").await.ok();
    });

    let server = rt.spawn(async { server.await.unwrap() });

    // Do not change this line! Removing the line or changing its contents in any way will break
    // the test suite :)
    println!("listening on port {}", actual_port);

    // Creating this file tells heroku to tell nginx that the application is ready
    // to receive traffic.
    if heroku {
        let path = if fastboot {
            "/tmp/backend-initialized"
        } else {
            "/tmp/app-initialized"
        };
        println!("Writing to {}", path);
        File::create(path).unwrap();

        // Launch nginx via the Heroku nginx buildpack
        // `wait()` is never called on the child process, but it should be okay to leave a zombie
        // process around on shutdown when Heroku is tearing down the entire container anyway.
        Command::new("./script/start-web.sh")
            .spawn()
            .expect("Couldn't spawn nginx");
    }

    // Block the main thread until the server has shutdown
    rt.block_on(async { server.await.unwrap() });

    println!("Persisting remaining downloads counters");
    match app.downloads_counter.persist_all_shards(&app) {
        Ok(stats) => stats.log(),
        Err(err) => println!("downloads_counter error: {}", err),
    }

    println!("Server has gracefully shutdown!");
    Ok(())
}

fn downloads_counter_thread(app: Arc<App>) {
    let interval = Duration::from_millis(
        (app.config.downloads_persist_interval_ms / app.downloads_counter.shards_count()) as u64,
    );

    std::thread::spawn(move || loop {
        std::thread::sleep(interval);

        match app.downloads_counter.persist_next_shard(&app) {
            Ok(stats) => stats.log(),
            Err(err) => println!("downloads_counter error: {}", err),
        }
    });
}

fn log_instance_metrics_thread(app: Arc<App>) {
    // Only run the thread if the configuration is provided
    let interval = if let Some(secs) = app.config.instance_metrics_log_every_seconds {
        Duration::from_secs(secs)
    } else {
        return;
    };

    std::thread::spawn(move || loop {
        if let Err(err) = log_instance_metrics_inner(&app) {
            eprintln!("log_instance_metrics error: {}", err);
        }
        std::thread::sleep(interval);
    });
}

fn log_instance_metrics_inner(app: &App) -> AppResult<()> {
    let families = app.instance_metrics.gather(app)?;

    let mut stdout = std::io::stdout();
    LogEncoder::new().encode(&families, &mut stdout)?;
    stdout.flush()?;

    Ok(())
}
<|endoftext|>