<commit_before>#![allow(unused_variables)]
use std::str::FromStr;

use hyper::{self, Uri};

use serde::{Serialize, Deserialize};
use serde_json;

use client::{self, ApiClient};

static BIGQUERY_ROOT: &str = "https://www.googleapis.com/bigquery/v2/projects";
static QUERY_RESOURCE_KIND: &str = "bigquery#queryResults";

pub struct BigQueryService {}
pub type Hub<'a> = client::Hub<'a, BigQueryService>;

#[derive(Serialize, Default, Debug)]
pub struct ListDatasetsRequest {
    pub all: bool,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub filter: Option<String>,

    #[serde(rename="maxResults")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub max_results: Option<usize>,

    #[serde(rename="pageToken")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub page_token: Option<String>,
}

impl ListDatasetsRequest {
    fn to_query(&self) -> String {
        let mut params = vec![("all", self.all.to_string())];
        if let Some(ref filter) = self.filter {
            params.push(("filter", filter.clone()));
        }
        if let Some(ref max_results) = self.max_results {
            params.push(("maxResults", max_results.to_string()));
        }
        if let Some(ref page_token) = self.page_token {
            params.push(("pageToken", page_token.clone()));
        }
        client::encode_query_params(params)
    }
}

#[derive(Deserialize, Default, Debug)]
pub struct ListDatasetsResponse {
    #[serde(rename="nextPageToken")]
    pub next_page_token: Option<String>,

    pub datasets: Vec<DatasetMeta>,
}

#[derive(Deserialize, Default, Debug)]
pub struct DatasetMeta {
    pub id: String,

    #[serde(rename="datasetReference")]
    pub dataset_reference: DatasetReference,

    #[serde(rename="friendlyName")]
    pub friendly_name: Option<String>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
pub struct DatasetReference {
    #[serde(rename="projectId")]
    pub project_id: String,

    #[serde(rename="datasetId")]
    pub dataset_id: String,
}

#[derive(Serialize, Default, Debug)]
pub struct ListTablesRequest {
    #[serde(rename="maxResults")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub max_results: Option<usize>,

    #[serde(rename="pageToken")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub page_token: Option<String>,
}

impl ListTablesRequest {
    fn to_query(&self) -> String {
        let mut params = vec![];
        if let Some(ref max_results) = self.max_results {
            params.push(("maxResults", max_results.to_string()));
        }
        if let Some(ref page_token) = self.page_token {
            params.push(("pageToken", page_token.clone()));
        }
        client::encode_query_params(params)
    }
}

#[derive(Deserialize, Default, Debug)]
pub struct ListTablesResponse {
    #[serde(rename="nextPageToken")]
    pub next_page_token: Option<String>,

    pub tables: Vec<TableMeta>,

    #[serde(rename="totalItems")]
    pub total_items: usize,
}

#[derive(Deserialize, Default, Debug)]
pub struct TableMeta {
    pub id: String,

    #[serde(rename="tableReference")]
    pub table_reference: TableReference,

    #[serde(rename="friendlyName")]
    pub friendly_name: Option<String>,

    #[serde(rename="type")]
    pub type0: String,
}

#[derive(Deserialize, Default, Debug)]
pub struct TableReference {
    #[serde(rename="projectId")]
    pub project_id: String,

    #[serde(rename="datasetId")]
    pub dataset_id: String,

    #[serde(rename="tableId")]
    pub table_id: String,
}

#[derive(Deserialize, Default, Debug)]
pub struct DescribeTableResponse {
    pub id: String,
    pub schema: TableFieldSchema,

    #[serde(rename="type")]
    pub type0: String,

    #[serde(rename="tableReference")]
    pub table_reference: TableReference,

    #[serde(rename="friendlyName")]
    pub friendly_name: Option<String>,
}

#[derive(Deserialize, Default, Debug)]
pub struct TableFieldSchema {
    pub fields: Vec<TableField>,
}

#[derive(Deserialize, Default, Debug)]
pub struct TableField {
    pub name: String,
    pub mode: String,

    #[serde(rename="type")]
    pub type0: String,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub fields: Option<Vec<TableField>>,
}

#[derive(Serialize, Debug)]
pub struct SubmitQueryRequest {
    pub kind: String,
    pub query: String,

    #[serde(rename="timeoutMs")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub timeout_ms: Option<usize>,

    #[serde(rename="useLegacySql")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub use_legacy_sql: Option<bool>,

    #[serde(rename="useQueryCache")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub use_query_cache: Option<bool>,

    #[serde(rename="dryRun")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub dry_run: Option<bool>,

    #[serde(rename="maxResults")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub max_results: Option<usize>,
}

impl Default for SubmitQueryRequest {
    fn default() -> Self {
        SubmitQueryRequest {
            kind: QUERY_RESOURCE_KIND.to_string(),
            query: Default::default(),
            timeout_ms: None,
            use_legacy_sql: None,
            use_query_cache: None,
            dry_run: None,
            max_results: None,
        }
    }
}

#[derive(Deserialize, Default, Debug)]
pub struct SubmitQueryResponse {
    pub schema: Option<TableFieldSchema>,

    #[serde(rename="jobReference")]
    pub job_reference: JobReference,

    #[serde(rename="totalRows")]
    pub total_rows: Option<String>,

    pub rows: Option<Vec<TableRow>>,

    #[serde(rename="jobComplete")]
    pub job_complete: bool,

    #[serde(rename="pageToken")]
    pub page_token: Option<String>,

    pub errors: Option<Vec<QueryError>>,
}

#[derive(Deserialize, Default, Debug)]
pub struct JobReference {
    #[serde(rename="projectId")]
    pub project_id: String,

    #[serde(rename="jobId")]
    pub job_id: String,
}

#[derive(Deserialize, Default, Debug)]
pub struct QueryError {
    pub reason: Option<String>,
    pub location: Option<String>,
    pub message: Option<String>,

    #[serde(rename="debugInfo")]
    pub debug_info: Option<String>,
}

#[derive(Deserialize, Default, Debug)]
pub struct CancelQueryResponse {}

#[derive(Serialize, Default, Debug)]
pub struct AwaitQueryRequest {
    #[serde(rename="timeoutMs")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub timeout_ms: Option<usize>,

    #[serde(rename="maxResults")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub max_results: Option<usize>,

    #[serde(rename="startIndex")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub start_index: Option<usize>,

    #[serde(rename="pageToken")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub page_token: Option<String>,
}

impl AwaitQueryRequest {
    fn to_query(&self) -> String {
        let mut params = Vec::new();
        if let Some(ref timeout_ms) = self.timeout_ms {
            params.push(("timeoutMs", timeout_ms.to_string()));
        }
        if let Some(ref max_results) = self.max_results {
            params.push(("maxResults", max_results.to_string()));
        }
        if let Some(ref page_token) = self.page_token {
            params.push(("pageToken", page_token.clone()));
        }
        if let Some(ref start_index) = self.start_index {
            params.push(("startIndex", start_index.to_string()));
        }
        client::encode_query_params(params)
    }
}

#[derive(Deserialize, Default, Debug)]
pub struct AwaitQueryResponse {
    pub schema: Option<TableFieldSchema>,

    #[serde(rename="jobReference")]
    pub job_reference: Option<JobReference>,

    #[serde(rename="totalRows")]
    pub total_rows: Option<String>,

    pub rows: Option<Vec<TableRow>>,

    #[serde(rename="jobComplete")]
    pub job_complete: bool,

    #[serde(rename="pageToken")]
    pub page_token: Option<String>,

    pub errors: Option<Vec<QueryError>>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
pub struct TableRow {
    pub f: Vec<TableCell>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
pub struct TableCell {
    pub v: Cell,
}

#[serde(untagged)]
#[derive(Serialize, Deserialize, Debug)]
pub enum Cell {
    Value(Option<String>),
    Repeat(Vec<TableCell>),
    Row(TableRow),
}

impl Default for Cell {
    fn default() -> Self {
        Cell::Value(None)
    }
}

impl<'a> Hub<'a> {
    pub fn list_datasets(&self,
                         token: &str,
                         project_id: &str,
                         req: &ListDatasetsRequest)
                         -> client::Result<ListDatasetsResponse> {
        let path = format!("{}/{}/datasets?{}",
                           BIGQUERY_ROOT,
                           project_id,
                           req.to_query());

        let uri = Uri::from_str(&path).expect("uri to be valid");
        self.get_bq::<ListDatasetsResponse>(&uri, token.to_string())
    }

    pub fn list_tables(&self,
                       token: &str,
                       project_id: &str,
                       dataset_id: &str,
                       req: &ListTablesRequest)
                       -> client::Result<ListTablesResponse> {
        let path = format!("{}/{}/datasets/{}/tables?{}",
                           BIGQUERY_ROOT,
                           project_id,
                           dataset_id,
                           req.to_query());
        let uri = Uri::from_str(&path).expect("uri to be valid");
        self.get_bq::<ListTablesResponse>(&uri, token.to_string())
    }

    pub fn describe_table(&self,
                          token: &str,
                          project_id: &str,
                          dataset_id: &str,
                          table_id: &str)
                          -> client::Result<DescribeTableResponse> {
        let path = format!("{}/{}/datasets/{}/tables/{}",
                           BIGQUERY_ROOT,
                           project_id,
                           dataset_id,
                           table_id);
        let uri = Uri::from_str(&path).expect("uri to be valid");
        self.get_bq::<DescribeTableResponse>(&uri, token.to_string())
    }

    pub fn submit_query(&self,
                        token: &str,
                        project_id: &str,
                        req: &SubmitQueryRequest)
                        -> client::Result<SubmitQueryResponse> {
        let path = format!("{}/{}/queries", BIGQUERY_ROOT, project_id);
        let uri = Uri::from_str(&path).expect("uri to be valid");
        self.post_bq::<_, SubmitQueryResponse>(&uri, req, token.to_string())
    }

    pub fn await_query(&self,
                       token: &str,
                       project_id: &str,
                       job_id: &str,
                       req: &AwaitQueryRequest)
                       -> client::Result<AwaitQueryResponse> {
        let path = format!("{}/{}/queries/{}?{}",
                           BIGQUERY_ROOT,
                           project_id,
                           job_id,
                           req.to_query());
        let uri = Uri::from_str(&path).expect("uri to be valid");
        self.get_bq::<AwaitQueryResponse>(&uri, token.to_string())
    }

    pub fn cancel_query(&self,
                        token: &str,
                        project_id: &str,
                        job_id: &str)
                        -> client::Result<CancelQueryResponse> {
        let path = format!("{}/{}/jobs/{}/cancel", BIGQUERY_ROOT, project_id, job_id);
        let uri = Uri::from_str(&path).expect("uri to be valid");

        let mut req = hyper::Request::new(hyper::Method::Post, uri);
        req.headers_mut().set(hyper::header::ContentType::json());

        let auth = hyper::header::Authorization(hyper::header::Bearer { token: token.to_string() });
        req.headers_mut().set(auth);

        self.request::<CancelQueryResponse>(req)
    }

    // helper method for making a GET request
    fn get_bq<D>(&self, uri: &hyper::Uri, token: String) -> client::Result<D>
        where for<'de> D: 'static + Send + Deserialize<'de>
    {
        let mut req = hyper::Request::new(hyper::Method::Get, uri.clone());
        let auth = hyper::header::Authorization(hyper::header::Bearer { token });
        req.headers_mut().set(auth);
        self.request(req)
    }

    // helper method for making a POST request with a JSON body
    fn post_bq<B: Serialize, D>(&self,
                                uri: &hyper::Uri,
                                body: B,
                                token: String)
                                -> client::Result<D>
        where for<'de> D: 'static + Send + Deserialize<'de>
    {
        let mut req = hyper::Request::new(hyper::Method::Post, uri.clone());
        req.headers_mut().set(hyper::header::ContentType::json());

        let auth = hyper::header::Authorization(hyper::header::Bearer { token });
        req.headers_mut().set(auth);

        let body = serde_json::to_string(&body).unwrap();
        req.set_body(body);

        self.request(req)
    }

    /*
     */
}
<commit_msg>address comments<commit_after>#![allow(unused_variables)]
use std::str::FromStr;

use hyper::{self, Uri};

use serde::{Serialize, Deserialize};
use serde_json;

use client::{self, ApiClient};

static BIGQUERY_ROOT: &str = "https://www.googleapis.com/bigquery/v2/projects";
static QUERY_RESOURCE_KIND: &str = "bigquery#queryResults";

pub struct BigQueryService {}
pub type Hub<'a> = client::Hub<'a, BigQueryService>;

#[derive(Default, Debug)]
pub struct ListDatasetsRequest {
    pub all: bool,
    pub filter: Option<String>,
    pub max_results: Option<usize>,
    pub page_token: Option<String>,
}

impl ListDatasetsRequest {
    fn to_query(&self) -> String {
        let mut params = vec![("all", self.all.to_string())];
        if let Some(ref filter) = self.filter {
            params.push(("filter", filter.clone()));
        }
        if let Some(ref max_results) = self.max_results {
            params.push(("maxResults", max_results.to_string()));
        }
        if let Some(ref page_token) = self.page_token {
            params.push(("pageToken", page_token.clone()));
        }
        client::encode_query_params(params)
    }
}

#[derive(Deserialize, Default, Debug)]
pub struct ListDatasetsResponse {
    #[serde(rename="nextPageToken")]
    pub next_page_token: Option<String>,

    pub datasets: Vec<DatasetMeta>,
}

#[derive(Deserialize, Default, Debug)]
pub struct DatasetMeta {
    pub id: String,

    #[serde(rename="datasetReference")]
    pub dataset_reference: DatasetReference,

    #[serde(rename="friendlyName")]
    pub friendly_name: Option<String>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
pub struct DatasetReference {
    #[serde(rename="projectId")]
    pub project_id: String,

    #[serde(rename="datasetId")]
    pub dataset_id: String,
}

#[derive(Default, Debug)]
pub struct ListTablesRequest {
    pub max_results: Option<usize>,
    pub page_token: Option<String>,
}

impl ListTablesRequest {
    fn to_query(&self) -> String {
        let mut params = vec![];
        if let Some(ref max_results) = self.max_results {
            params.push(("maxResults", max_results.to_string()));
        }
        if let Some(ref page_token) = self.page_token {
            params.push(("pageToken", page_token.clone()));
        }
        client::encode_query_params(params)
    }
}

#[derive(Deserialize, Default, Debug)]
pub struct ListTablesResponse {
    #[serde(rename="nextPageToken")]
    pub next_page_token: Option<String>,

    pub tables: Vec<TableMeta>,

    #[serde(rename="totalItems")]
    pub total_items: usize,
}

#[derive(Deserialize, Default, Debug)]
pub struct TableMeta {
    pub id: String,

    #[serde(rename="tableReference")]
    pub table_reference: TableReference,

    #[serde(rename="friendlyName")]
    pub friendly_name: Option<String>,

    #[serde(rename="type")]
    pub type0: String,
}

#[derive(Deserialize, Default, Debug)]
pub struct TableReference {
    #[serde(rename="projectId")]
    pub project_id: String,

    #[serde(rename="datasetId")]
    pub dataset_id: String,

    #[serde(rename="tableId")]
    pub table_id: String,
}

#[derive(Deserialize, Default, Debug)]
pub struct DescribeTableResponse {
    pub id: String,
    pub schema: TableFieldSchema,

    #[serde(rename="type")]
    pub type0: String,

    #[serde(rename="tableReference")]
    pub table_reference: TableReference,

    #[serde(rename="friendlyName")]
    pub friendly_name: Option<String>,
}

#[derive(Deserialize, Default, Debug)]
pub struct TableFieldSchema {
    pub fields: Vec<TableField>,
}

#[derive(Deserialize, Default, Debug)]
pub struct TableField {
    pub name: String,
    pub mode: String,

    #[serde(rename="type")]
    pub type0: String,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub fields: Option<Vec<TableField>>,
}

#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct SubmitQueryRequest {
    pub kind: String,
    pub query: String,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub timeout_ms: Option<usize>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub use_legacy_sql: Option<bool>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub use_query_cache: Option<bool>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub dry_run: Option<bool>,

    #[serde(skip_serializing_if = "Option::is_none")]
    pub max_results: Option<usize>,
}

impl Default for SubmitQueryRequest {
    fn default() -> Self {
        SubmitQueryRequest {
            kind: QUERY_RESOURCE_KIND.to_string(),
            query: Default::default(),
            timeout_ms: None,
            use_legacy_sql: None,
            use_query_cache: None,
            dry_run: None,
            max_results: None,
        }
    }
}

#[derive(Deserialize, Default, Debug)]
pub struct SubmitQueryResponse {
    pub schema: Option<TableFieldSchema>,

    #[serde(rename="jobReference")]
    pub job_reference: JobReference,

    #[serde(rename="totalRows")]
    pub total_rows: Option<String>,

    pub rows: Option<Vec<TableRow>>,

    #[serde(rename="jobComplete")]
    pub job_complete: bool,

    #[serde(rename="pageToken")]
    pub page_token: Option<String>,

    pub errors: Option<Vec<QueryError>>,
}

#[derive(Deserialize, Default, Debug)]
pub struct JobReference {
    #[serde(rename="projectId")]
    pub project_id: String,

    #[serde(rename="jobId")]
    pub job_id: String,
}

#[derive(Deserialize, Default, Debug)]
pub struct QueryError {
    pub reason: Option<String>,
    pub location: Option<String>,
    pub message: Option<String>,

    #[serde(rename="debugInfo")]
    pub debug_info: Option<String>,
}

#[derive(Deserialize, Default, Debug)]
pub struct CancelQueryResponse {}

#[derive(Serialize, Default, Debug)]
pub struct AwaitQueryRequest {
    #[serde(rename="timeoutMs")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub timeout_ms: Option<usize>,

    #[serde(rename="maxResults")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub max_results: Option<usize>,

    #[serde(rename="startIndex")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub start_index: Option<usize>,

    #[serde(rename="pageToken")]
    #[serde(skip_serializing_if = "Option::is_none")]
    pub page_token: Option<String>,
}

impl AwaitQueryRequest {
    fn to_query(&self) -> String {
        let mut params = Vec::new();
        if let Some(ref timeout_ms) = self.timeout_ms {
            params.push(("timeoutMs", timeout_ms.to_string()));
        }
        if let Some(ref max_results) = self.max_results {
            params.push(("maxResults", max_results.to_string()));
        }
        if let Some(ref page_token) = self.page_token {
            params.push(("pageToken", page_token.clone()));
        }
        if let Some(ref start_index) = self.start_index {
            params.push(("startIndex", start_index.to_string()));
        }
        client::encode_query_params(params)
    }
}

#[derive(Deserialize, Default, Debug)]
pub struct AwaitQueryResponse {
    pub schema: Option<TableFieldSchema>,

    #[serde(rename="jobReference")]
    pub job_reference: Option<JobReference>,

    #[serde(rename="totalRows")]
    pub total_rows: Option<String>,

    pub rows: Option<Vec<TableRow>>,

    #[serde(rename="jobComplete")]
    pub job_complete: bool,

    #[serde(rename="pageToken")]
    pub page_token: Option<String>,

    pub errors: Option<Vec<QueryError>>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
pub struct TableRow {
    pub f: Vec<TableCell>,
}

#[derive(Serialize, Deserialize, Default, Debug)]
pub struct TableCell {
    pub v: Cell,
}

#[serde(untagged)]
#[derive(Serialize, Deserialize, Debug)]
pub enum Cell {
    Value(Option<String>),
    Repeat(Vec<TableCell>),
    Row(TableRow),
}

impl Default for Cell {
    fn default() -> Self {
        Cell::Value(None)
    }
}

impl<'a> Hub<'a> {
    pub fn list_datasets(&self,
                         token: &str,
                         project_id: &str,
                         req: &ListDatasetsRequest)
                         -> client::Result<ListDatasetsResponse> {
        let path = format!("{}/{}/datasets?{}",
                           BIGQUERY_ROOT,
                           project_id,
                           req.to_query());

        let uri = Uri::from_str(&path).expect("uri to be valid");
        self.get_bq::<ListDatasetsResponse>(&uri, token.to_string())
    }

    pub fn list_tables(&self,
                       token: &str,
                       project_id: &str,
                       dataset_id: &str,
                       req: &ListTablesRequest)
                       -> client::Result<ListTablesResponse> {
        let path = format!("{}/{}/datasets/{}/tables?{}",
                           BIGQUERY_ROOT,
                           project_id,
                           dataset_id,
                           req.to_query());
        let uri = Uri::from_str(&path).expect("uri to be valid");
        self.get_bq::<ListTablesResponse>(&uri, token.to_string())
    }

    pub fn describe_table(&self,
                          token: &str,
                          project_id: &str,
                          dataset_id: &str,
                          table_id: &str)
                          -> client::Result<DescribeTableResponse> {
        let path = format!("{}/{}/datasets/{}/tables/{}",
                           BIGQUERY_ROOT,
                           project_id,
                           dataset_id,
                           table_id);
        let uri = Uri::from_str(&path).expect("uri to be valid");
        self.get_bq::<DescribeTableResponse>(&uri, token.to_string())
    }

    pub fn submit_query(&self,
                        token: &str,
                        project_id: &str,
                        req: &SubmitQueryRequest)
                        -> client::Result<SubmitQueryResponse> {
        let path = format!("{}/{}/queries", BIGQUERY_ROOT, project_id);
        let uri = Uri::from_str(&path).expect("uri to be valid");
        self.post_bq::<_, SubmitQueryResponse>(&uri, req, token.to_string())
    }

    pub fn await_query(&self,
                       token: &str,
                       project_id: &str,
                       job_id: &str,
                       req: &AwaitQueryRequest)
                       -> client::Result<AwaitQueryResponse> {
        let path = format!("{}/{}/queries/{}?{}",
                           BIGQUERY_ROOT,
                           project_id,
                           job_id,
                           req.to_query());
        let uri = Uri::from_str(&path).expect("uri to be valid");
        self.get_bq::<AwaitQueryResponse>(&uri, token.to_string())
    }

    pub fn cancel_query(&self,
                        token: &str,
                        project_id: &str,
                        job_id: &str)
                        -> client::Result<CancelQueryResponse> {
        let path = format!("{}/{}/jobs/{}/cancel", BIGQUERY_ROOT, project_id, job_id);
        let uri = Uri::from_str(&path).expect("uri to be valid");

        let mut req = hyper::Request::new(hyper::Method::Post, uri);
        req.headers_mut().set(hyper::header::ContentType::json());

        let auth = hyper::header::Authorization(hyper::header::Bearer { token: token.to_string() });
        req.headers_mut().set(auth);

        self.request::<CancelQueryResponse>(req)
    }

    // helper method for making a GET request
    fn get_bq<D>(&self, uri: &hyper::Uri, token: String) -> client::Result<D>
        where for<'de> D: 'static + Send + Deserialize<'de>
    {
        let mut req = hyper::Request::new(hyper::Method::Get, uri.clone());
        let auth = hyper::header::Authorization(hyper::header::Bearer { token });
        req.headers_mut().set(auth);
        self.request(req)
    }

    // helper method for making a POST request with a JSON body
    fn post_bq<B: Serialize, D>(&self,
                                uri: &hyper::Uri,
                                body: B,
                                token: String)
                                -> client::Result<D>
        where for<'de> D: 'static + Send + Deserialize<'de>
    {
        let mut req = hyper::Request::new(hyper::Method::Post, uri.clone());
        req.headers_mut().set(hyper::header::ContentType::json());

        let auth = hyper::header::Authorization(hyper::header::Bearer { token });
        req.headers_mut().set(auth);

        let body = serde_json::to_string(&body).unwrap();
        req.set_body(body);

        self.request(req)
    }
}
<|endoftext|>


<commit_before>use super::{Null, Value, ValueRef};
#[cfg(feature = "array")]
use crate::vtab::array::Array;
use crate::Result;
use std::borrow::Cow;

/// `ToSqlOutput` represents the possible output types for implementors of the
/// `ToSql` trait.
#[derive(Clone, Debug, PartialEq)]
#[non_exhaustive]
pub enum ToSqlOutput<'a> {
    /// A borrowed SQLite-representable value.
    Borrowed(ValueRef<'a>),

    /// An owned SQLite-representable value.
    Owned(Value),

    /// `feature = "blob"` A BLOB of the given length that is filled with
    /// zeroes.
    #[cfg(feature = "blob")]
    ZeroBlob(i32),

    /// `feature = "array"`
    #[cfg(feature = "array")]
    Array(Array),
}

// Generically allow any type that can be converted into a ValueRef
// to be converted into a ToSqlOutput as well.
impl<'a, T: ?Sized> From<&'a T> for ToSqlOutput<'a>
where
    &'a T: Into<ValueRef<'a>>,
{
    fn from(t: &'a T) -> Self {
        ToSqlOutput::Borrowed(t.into())
    }
}

// We cannot also generically allow any type that can be converted
// into a Value to be converted into a ToSqlOutput because of
// coherence rules (https://github.com/rust-lang/rust/pull/46192),
// so we'll manually implement it for all the types we know can
// be converted into Values.
macro_rules! from_value(
    ($t:ty) => (
        impl From<$t> for ToSqlOutput<'_> {
            fn from(t: $t) -> Self { ToSqlOutput::Owned(t.into())}
        }
    )
);
from_value!(String);
from_value!(Null);
from_value!(bool);
from_value!(i8);
from_value!(i16);
from_value!(i32);
from_value!(i64);
from_value!(isize);
from_value!(u8);
from_value!(u16);
from_value!(u32);
from_value!(f64);
from_value!(Vec<u8>);

// It would be nice if we could avoid the heap allocation (of the `Vec`) that
// `i128` needs in `Into<Value>`, but it's probably fine for the moment, and not
// worth adding another case to Value.
#[cfg(feature = "i128_blob")]
from_value!(i128);

#[cfg(feature = "uuid")]
from_value!(uuid::Uuid);

impl ToSql for ToSqlOutput<'_> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(match *self {
            ToSqlOutput::Borrowed(v) => ToSqlOutput::Borrowed(v),
            ToSqlOutput::Owned(ref v) => ToSqlOutput::Borrowed(ValueRef::from(v)),

            #[cfg(feature = "blob")]
            ToSqlOutput::ZeroBlob(i) => ToSqlOutput::ZeroBlob(i),
            #[cfg(feature = "array")]
            ToSqlOutput::Array(ref a) => ToSqlOutput::Array(a.clone()),
        })
    }
}

/// A trait for types that can be converted into SQLite values.
pub trait ToSql {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>>;
}

impl<T: ToSql + Clone + ?Sized> ToSql for Cow<'_, T> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        self.as_ref().to_sql()
    }
}

impl<T: ToSql + ?Sized> ToSql for Box<T> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        self.as_ref().to_sql()
    }
}

impl<T: ToSql + ?Sized> ToSql for std::rc::Rc<T> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        self.as_ref().to_sql()
    }
}

impl<T: ToSql + ?Sized> ToSql for std::sync::Arc<T> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        self.as_ref().to_sql()
    }
}

// We should be able to use a generic impl like this:
//
// impl<T: Copy> ToSql for T where T: Into<Value> {
//     fn to_sql(&self) -> Result<ToSqlOutput> {
//         Ok(ToSqlOutput::from((*self).into()))
//     }
// }
//
// instead of the following macro, but this runs afoul of
// https://github.com/rust-lang/rust/issues/30191 and reports conflicting
// implementations even when there aren't any.

macro_rules! to_sql_self(
    ($t:ty) => (
        impl ToSql for $t {
            fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
                Ok(ToSqlOutput::from(*self))
            }
        }
    )
);

to_sql_self!(Null);
to_sql_self!(bool);
to_sql_self!(i8);
to_sql_self!(i16);
to_sql_self!(i32);
to_sql_self!(i64);
to_sql_self!(isize);
to_sql_self!(u8);
to_sql_self!(u16);
to_sql_self!(u32);
to_sql_self!(f64);

#[cfg(feature = "i128_blob")]
to_sql_self!(i128);

#[cfg(feature = "uuid")]
to_sql_self!(uuid::Uuid);

impl<T: ?Sized> ToSql for &'_ T
where
    T: ToSql,
{
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        (*self).to_sql()
    }
}

impl ToSql for String {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self.as_str()))
    }
}

impl ToSql for str {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self))
    }
}

impl ToSql for Vec<u8> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self.as_slice()))
    }
}

impl ToSql for [u8] {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self))
    }
}

impl ToSql for Value {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self))
    }
}

impl<T: ToSql> ToSql for Option<T> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        match *self {
            None => Ok(ToSqlOutput::from(Null)),
            Some(ref t) => t.to_sql(),
        }
    }
}

#[cfg(test)]
mod test {
    use super::ToSql;

    fn is_to_sql<T: ToSql>() {}

    #[test]
    fn test_integral_types() {
        is_to_sql::<i8>();
        is_to_sql::<i16>();
        is_to_sql::<i32>();
        is_to_sql::<i64>();
        is_to_sql::<u8>();
        is_to_sql::<u16>();
        is_to_sql::<u32>();
    }

    #[test]
    fn test_cow_str() {
        use std::borrow::Cow;
        let s = "str";
        let cow = Cow::Borrowed(s);
        let r = cow.to_sql();
        assert!(r.is_ok());
        let cow = Cow::Owned::<str>(String::from(s));
        let r = cow.to_sql();
        assert!(r.is_ok());
    }

    #[test]
    fn test_box_dyn() {
        let s: Box<dyn ToSql> = Box::new("Hello world!");
        let r = ToSql::to_sql(&s);

        assert!(r.is_ok());
    }

    #[test]
    fn test_box_deref() {
        let s: Box<str> = "Hello world!".into();
        let r = s.to_sql();

        assert!(r.is_ok());
    }

    #[test]
    fn test_box_direct() {
        let s: Box<str> = "Hello world!".into();
        let r = ToSql::to_sql(&s);

        assert!(r.is_ok());
    }

    #[test]
    fn test_cells() {
        use std::{rc::Rc, sync::Arc};

        let source_str: Box<str> = "Hello world!".into();

        let s: Rc<_> = Rc::new(source_str.clone());
        let r = s.to_sql();
        assert!(r.is_ok());

        let s: Arc<_> = Arc::new(source_str);
        let r = s.to_sql();
        assert!(r.is_ok());
    }

    #[cfg(feature = "i128_blob")]
    #[test]
    fn test_i128() {
        use crate::{Connection, NO_PARAMS};
        use std::i128;
        let db = Connection::open_in_memory().unwrap();
        db.execute_batch("CREATE TABLE foo (i128 BLOB, desc TEXT)")
            .unwrap();
        db.execute(
            "
            INSERT INTO foo(i128, desc) VALUES
                (?, 'zero'),
                (?, 'neg one'), (?, 'neg two'),
                (?, 'pos one'), (?, 'pos two'),
                (?, 'min'), (?, 'max')",
            &[0i128, -1i128, -2i128, 1i128, 2i128, i128::MIN, i128::MAX],
        )
        .unwrap();

        let mut stmt = db
            .prepare("SELECT i128, desc FROM foo ORDER BY i128 ASC")
            .unwrap();

        let res = stmt
            .query_map(NO_PARAMS, |row| {
                Ok((row.get::<_, i128>(0)?, row.get::<_, String>(1)?))
            })
            .unwrap()
            .collect::<Result<Vec<_>, _>>()
            .unwrap();

        assert_eq!(
            res,
            &[
                (i128::MIN, "min".to_owned()),
                (-2, "neg two".to_owned()),
                (-1, "neg one".to_owned()),
                (0, "zero".to_owned()),
                (1, "pos one".to_owned()),
                (2, "pos two".to_owned()),
                (i128::MAX, "max".to_owned()),
            ]
        );
    }

    #[cfg(feature = "uuid")]
    #[test]
    fn test_uuid() {
        use crate::{params, Connection};
        use uuid::Uuid;

        let db = Connection::open_in_memory().unwrap();
        db.execute_batch("CREATE TABLE foo (id BLOB CHECK(length(id) = 16), label TEXT);")
            .unwrap();

        let id = Uuid::new_v4();

        db.execute(
            "INSERT INTO foo (id, label) VALUES (?, ?)",
            params![id, "target"],
        )
        .unwrap();

        let mut stmt = db
            .prepare("SELECT id, label FROM foo WHERE id = ?")
            .unwrap();

        let mut rows = stmt.query(params![id]).unwrap();
        let row = rows.next().unwrap().unwrap();

        let found_id: Uuid = row.get_unwrap(0);
        let found_label: String = row.get_unwrap(1);

        assert_eq!(found_id, id);
        assert_eq!(found_label, "target");
    }
}
<commit_msg>Fix regression with <Cow<'_, _> as ToSql> (#681)<commit_after>use super::{Null, Value, ValueRef};
#[cfg(feature = "array")]
use crate::vtab::array::Array;
use crate::Result;
use std::borrow::Cow;

/// `ToSqlOutput` represents the possible output types for implementors of the
/// `ToSql` trait.
#[derive(Clone, Debug, PartialEq)]
#[non_exhaustive]
pub enum ToSqlOutput<'a> {
    /// A borrowed SQLite-representable value.
    Borrowed(ValueRef<'a>),

    /// An owned SQLite-representable value.
    Owned(Value),

    /// `feature = "blob"` A BLOB of the given length that is filled with
    /// zeroes.
    #[cfg(feature = "blob")]
    ZeroBlob(i32),

    /// `feature = "array"`
    #[cfg(feature = "array")]
    Array(Array),
}

// Generically allow any type that can be converted into a ValueRef
// to be converted into a ToSqlOutput as well.
impl<'a, T: ?Sized> From<&'a T> for ToSqlOutput<'a>
where
    &'a T: Into<ValueRef<'a>>,
{
    fn from(t: &'a T) -> Self {
        ToSqlOutput::Borrowed(t.into())
    }
}

// We cannot also generically allow any type that can be converted
// into a Value to be converted into a ToSqlOutput because of
// coherence rules (https://github.com/rust-lang/rust/pull/46192),
// so we'll manually implement it for all the types we know can
// be converted into Values.
macro_rules! from_value(
    ($t:ty) => (
        impl From<$t> for ToSqlOutput<'_> {
            fn from(t: $t) -> Self { ToSqlOutput::Owned(t.into())}
        }
    )
);
from_value!(String);
from_value!(Null);
from_value!(bool);
from_value!(i8);
from_value!(i16);
from_value!(i32);
from_value!(i64);
from_value!(isize);
from_value!(u8);
from_value!(u16);
from_value!(u32);
from_value!(f64);
from_value!(Vec<u8>);

// It would be nice if we could avoid the heap allocation (of the `Vec`) that
// `i128` needs in `Into<Value>`, but it's probably fine for the moment, and not
// worth adding another case to Value.
#[cfg(feature = "i128_blob")]
from_value!(i128);

#[cfg(feature = "uuid")]
from_value!(uuid::Uuid);

impl ToSql for ToSqlOutput<'_> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(match *self {
            ToSqlOutput::Borrowed(v) => ToSqlOutput::Borrowed(v),
            ToSqlOutput::Owned(ref v) => ToSqlOutput::Borrowed(ValueRef::from(v)),

            #[cfg(feature = "blob")]
            ToSqlOutput::ZeroBlob(i) => ToSqlOutput::ZeroBlob(i),
            #[cfg(feature = "array")]
            ToSqlOutput::Array(ref a) => ToSqlOutput::Array(a.clone()),
        })
    }
}

/// A trait for types that can be converted into SQLite values.
pub trait ToSql {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>>;
}

impl<T: ToSql + ToOwned + ?Sized> ToSql for Cow<'_, T> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        self.as_ref().to_sql()
    }
}

impl<T: ToSql + ?Sized> ToSql for Box<T> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        self.as_ref().to_sql()
    }
}

impl<T: ToSql + ?Sized> ToSql for std::rc::Rc<T> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        self.as_ref().to_sql()
    }
}

impl<T: ToSql + ?Sized> ToSql for std::sync::Arc<T> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        self.as_ref().to_sql()
    }
}

// We should be able to use a generic impl like this:
//
// impl<T: Copy> ToSql for T where T: Into<Value> {
//     fn to_sql(&self) -> Result<ToSqlOutput> {
//         Ok(ToSqlOutput::from((*self).into()))
//     }
// }
//
// instead of the following macro, but this runs afoul of
// https://github.com/rust-lang/rust/issues/30191 and reports conflicting
// implementations even when there aren't any.

macro_rules! to_sql_self(
    ($t:ty) => (
        impl ToSql for $t {
            fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
                Ok(ToSqlOutput::from(*self))
            }
        }
    )
);

to_sql_self!(Null);
to_sql_self!(bool);
to_sql_self!(i8);
to_sql_self!(i16);
to_sql_self!(i32);
to_sql_self!(i64);
to_sql_self!(isize);
to_sql_self!(u8);
to_sql_self!(u16);
to_sql_self!(u32);
to_sql_self!(f64);

#[cfg(feature = "i128_blob")]
to_sql_self!(i128);

#[cfg(feature = "uuid")]
to_sql_self!(uuid::Uuid);

impl<T: ?Sized> ToSql for &'_ T
where
    T: ToSql,
{
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        (*self).to_sql()
    }
}

impl ToSql for String {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self.as_str()))
    }
}

impl ToSql for str {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self))
    }
}

impl ToSql for Vec<u8> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self.as_slice()))
    }
}

impl ToSql for [u8] {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self))
    }
}

impl ToSql for Value {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        Ok(ToSqlOutput::from(self))
    }
}

impl<T: ToSql> ToSql for Option<T> {
    fn to_sql(&self) -> Result<ToSqlOutput<'_>> {
        match *self {
            None => Ok(ToSqlOutput::from(Null)),
            Some(ref t) => t.to_sql(),
        }
    }
}

#[cfg(test)]
mod test {
    use super::ToSql;

    fn is_to_sql<T: ToSql>() {}

    #[test]
    fn test_integral_types() {
        is_to_sql::<i8>();
        is_to_sql::<i16>();
        is_to_sql::<i32>();
        is_to_sql::<i64>();
        is_to_sql::<u8>();
        is_to_sql::<u16>();
        is_to_sql::<u32>();
    }

    #[test]
    fn test_cow_str() {
        use std::borrow::Cow;
        let s = "str";
        let cow: Cow<str> = Cow::Borrowed(s);
        let r = cow.to_sql();
        assert!(r.is_ok());
        let cow: Cow<str> = Cow::Owned::<str>(String::from(s));
        let r = cow.to_sql();
        assert!(r.is_ok());
        // Ensure this compiles.
        let _p: &[&dyn ToSql] = crate::params![cow];
    }

    #[test]
    fn test_box_dyn() {
        let s: Box<dyn ToSql> = Box::new("Hello world!");
        let _s: &[&dyn ToSql] = crate::params![s];
        let r = ToSql::to_sql(&s);

        assert!(r.is_ok());
    }

    #[test]
    fn test_box_deref() {
        let s: Box<str> = "Hello world!".into();
        let _s: &[&dyn ToSql] = crate::params![s];
        let r = s.to_sql();

        assert!(r.is_ok());
    }

    #[test]
    fn test_box_direct() {
        let s: Box<str> = "Hello world!".into();
        let _s: &[&dyn ToSql] = crate::params![s];
        let r = ToSql::to_sql(&s);

        assert!(r.is_ok());
    }

    #[test]
    fn test_cells() {
        use std::{rc::Rc, sync::Arc};

        let source_str: Box<str> = "Hello world!".into();

        let s: Rc<Box<str>> = Rc::new(source_str.clone());
        let _s: &[&dyn ToSql] = crate::params![s];
        let r = s.to_sql();
        assert!(r.is_ok());

        let s: Arc<Box<str>> = Arc::new(source_str.clone());
        let _s: &[&dyn ToSql] = crate::params![s];
        let r = s.to_sql();
        assert!(r.is_ok());

        let s: Arc<str> = Arc::from(&*source_str);
        let _s: &[&dyn ToSql] = crate::params![s];
        let r = s.to_sql();
        assert!(r.is_ok());

        let s: Arc<dyn ToSql> = Arc::new(source_str.clone());
        let _s: &[&dyn ToSql] = crate::params![s];
        let r = s.to_sql();
        assert!(r.is_ok());

        let s: Rc<str> = Rc::from(&*source_str);
        let _s: &[&dyn ToSql] = crate::params![s];
        let r = s.to_sql();
        assert!(r.is_ok());

        let s: Rc<dyn ToSql> = Rc::new(source_str);
        let _s: &[&dyn ToSql] = crate::params![s];
        let r = s.to_sql();
        assert!(r.is_ok());
    }

    #[cfg(feature = "i128_blob")]
    #[test]
    fn test_i128() {
        use crate::{Connection, NO_PARAMS};
        use std::i128;
        let db = Connection::open_in_memory().unwrap();
        db.execute_batch("CREATE TABLE foo (i128 BLOB, desc TEXT)")
            .unwrap();
        db.execute(
            "
            INSERT INTO foo(i128, desc) VALUES
                (?, 'zero'),
                (?, 'neg one'), (?, 'neg two'),
                (?, 'pos one'), (?, 'pos two'),
                (?, 'min'), (?, 'max')",
            &[0i128, -1i128, -2i128, 1i128, 2i128, i128::MIN, i128::MAX],
        )
        .unwrap();

        let mut stmt = db
            .prepare("SELECT i128, desc FROM foo ORDER BY i128 ASC")
            .unwrap();

        let res = stmt
            .query_map(NO_PARAMS, |row| {
                Ok((row.get::<_, i128>(0)?, row.get::<_, String>(1)?))
            })
            .unwrap()
            .collect::<Result<Vec<_>, _>>()
            .unwrap();

        assert_eq!(
            res,
            &[
                (i128::MIN, "min".to_owned()),
                (-2, "neg two".to_owned()),
                (-1, "neg one".to_owned()),
                (0, "zero".to_owned()),
                (1, "pos one".to_owned()),
                (2, "pos two".to_owned()),
                (i128::MAX, "max".to_owned()),
            ]
        );
    }

    #[cfg(feature = "uuid")]
    #[test]
    fn test_uuid() {
        use crate::{params, Connection};
        use uuid::Uuid;

        let db = Connection::open_in_memory().unwrap();
        db.execute_batch("CREATE TABLE foo (id BLOB CHECK(length(id) = 16), label TEXT);")
            .unwrap();

        let id = Uuid::new_v4();

        db.execute(
            "INSERT INTO foo (id, label) VALUES (?, ?)",
            params![id, "target"],
        )
        .unwrap();

        let mut stmt = db
            .prepare("SELECT id, label FROM foo WHERE id = ?")
            .unwrap();

        let mut rows = stmt.query(params![id]).unwrap();
        let row = rows.next().unwrap().unwrap();

        let found_id: Uuid = row.get_unwrap(0);
        let found_label: String = row.get_unwrap(1);

        assert_eq!(found_id, id);
        assert_eq!(found_label, "target");
    }
}
<|endoftext|>


<commit_before>//! Determine which URLs are allowed for loading.

use std::fmt;
use std::io;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use url::Url;

use crate::error::AllowedUrlError;

/// Currently only contains the base URL.
///
/// The plan is to add:
/// base_only:    Only allow to load content from the same base URL. By default
//                this restriction is enabled and requires to provide base_url.
/// include_xml:  Allows to use xi:include with XML. Enabled by default.
/// include_text: Allows to use xi:include with text. Enabled by default.
/// local_only:   Only allow to load content from the local filesystem.
///               Enabled by default.
#[derive(Clone)]
pub struct UrlResolver {
    /// Base URL; all relative references will be resolved with respect to this.
    pub base_url: Option<Url>,
}

impl UrlResolver {
    /// Creates a `UrlResolver` with defaults, and sets the `base_url`.
    pub fn new(base_url: Option<Url>) -> Self {
        UrlResolver { base_url }
    }

    pub fn resolve_href(&self, href: &str) -> Result<AllowedUrl, AllowedUrlError> {
        let url = Url::options()
            .base_url(self.base_url.as_ref())
            .parse(href)
            .map_err(AllowedUrlError::UrlParseError)?;

        // Allow loads of data: from any location
        if url.scheme() == "data" {
            return Ok(AllowedUrl(url));
        }

        // All other sources require a base url
        if self.base_url.is_none() {
            return Err(AllowedUrlError::BaseRequired);
        }

        let base_url = self.base_url.as_ref().unwrap();

        // Deny loads from differing URI schemes
        if url.scheme() != base_url.scheme() {
            return Err(AllowedUrlError::DifferentUriSchemes);
        }

        // resource: is allowed to load anything from other resources
        if url.scheme() == "resource" {
            return Ok(AllowedUrl(url));
        }

        // Non-file: isn't allowed to load anything
        if url.scheme() != "file" {
            return Err(AllowedUrlError::DisallowedScheme);
        }

        // We have two file: URIs.  Now canonicalize them (remove .. and symlinks, etc.)
        // and see if the directories match

        let url_path = url
            .to_file_path()
            .map_err(|_| AllowedUrlError::InvalidPath)?;
        let base_path = base_url
            .to_file_path()
            .map_err(|_| AllowedUrlError::InvalidPath)?;

        let base_parent = base_path.parent();
        if base_parent.is_none() {
            return Err(AllowedUrlError::BaseIsRoot);
        }

        let base_parent = base_parent.unwrap();

        let url_canon =
            canonicalize(&url_path).map_err(|_| AllowedUrlError::CanonicalizationError)?;
        let parent_canon =
            canonicalize(&base_parent).map_err(|_| AllowedUrlError::CanonicalizationError)?;

        if url_canon.starts_with(parent_canon) {
            Ok(AllowedUrl(url))
        } else {
            Err(AllowedUrlError::NotSiblingOrChildOfBaseFile)
        }
    }
}

/// Wrapper for URLs which are allowed to be loaded
///
/// SVG files can reference other files (PNG/JPEG images, other SVGs,
/// CSS files, etc.).  This object is constructed by checking whether
/// a specified `href` (a possibly-relative filename, for example)
/// should be allowed to be loaded, given the base URL of the SVG
/// being loaded.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct AllowedUrl(Url);

impl Deref for AllowedUrl {
    type Target = Url;

    fn deref(&self) -> &Url {
        &self.0
    }
}

impl fmt::Display for AllowedUrl {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        self.0.fmt(f)
    }
}

// For tests, we don't want to touch the filesystem.  In that case,
// assume that we are being passed canonical file names.
#[cfg(not(test))]
fn canonicalize<P: AsRef<Path>>(path: P) -> Result<PathBuf, io::Error> {
    path.as_ref().canonicalize()
}
#[cfg(test)]
fn canonicalize<P: AsRef<Path>>(path: P) -> Result<PathBuf, io::Error> {
    Ok(path.as_ref().to_path_buf())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn disallows_relative_file_with_no_base_file() {
        let url_resolver = UrlResolver::new(None);
        assert!(matches!(
            url_resolver.resolve_href("foo.svg"),
            Err(AllowedUrlError::UrlParseError(
                url::ParseError::RelativeUrlWithoutBase
            ))
        ));
    }

    #[test]
    fn disallows_different_schemes() {
        let url_resolver = UrlResolver::new(Some(
            Url::parse("http://example.com/malicious.svg").unwrap(),
        ));
        assert!(matches!(
            url_resolver.resolve_href("file:///etc/passwd"),
            Err(AllowedUrlError::DifferentUriSchemes)
        ));
    }

    fn make_file_uri(p: &str) -> String {
        if cfg!(windows) {
            format!("file:///c:{}", p)
        } else {
            format!("file://{}", p)
        }
    }

    #[test]
    fn disallows_base_is_root() {
        let url_resolver = UrlResolver::new(Some(Url::parse(&make_file_uri("/")).unwrap()));
        assert!(matches!(
            url_resolver.resolve_href("foo.svg"),
            Err(AllowedUrlError::BaseIsRoot)
        ));
    }

    #[test]
    fn disallows_non_file_scheme() {
        let url_resolver = UrlResolver::new(Some(Url::parse("http://foo.bar/baz.svg").unwrap()));
        assert!(matches!(
            url_resolver.resolve_href("foo.svg"),
            Err(AllowedUrlError::DisallowedScheme)
        ));
    }

    #[test]
    fn allows_data_url_with_no_base_file() {
        let url_resolver = UrlResolver::new(None);
        assert_eq!(
            url_resolver
                .resolve_href("data:image/jpeg;base64,xxyyzz")
                .unwrap()
                .as_ref(),
            "data:image/jpeg;base64,xxyyzz",
        );
    }

    #[test]
    fn allows_relative() {
        let url_resolver = UrlResolver::new(Some(
            Url::parse(&make_file_uri("/example/bar.svg")).unwrap(),
        ));
        let resolved = url_resolver.resolve_href("foo.svg").unwrap();
        let expected = make_file_uri("/example/foo.svg");
        assert_eq!(resolved.as_ref(), expected);
    }

    #[test]
    fn allows_sibling() {
        let url_resolver = UrlResolver::new(Some(
            Url::parse(&make_file_uri("/example/bar.svg")).unwrap(),
        ));
        let resolved = url_resolver
            .resolve_href(&make_file_uri("/example/foo.svg"))
            .unwrap();
        let expected = make_file_uri("/example/foo.svg");
        assert_eq!(resolved.as_ref(), expected);
    }

    #[test]
    fn allows_child_of_sibling() {
        let url_resolver = UrlResolver::new(Some(
            Url::parse(&make_file_uri("/example/bar.svg")).unwrap(),
        ));
        let resolved = url_resolver
            .resolve_href(&make_file_uri("/example/subdir/foo.svg"))
            .unwrap();
        let expected = make_file_uri("/example/subdir/foo.svg");
        assert_eq!(resolved.as_ref(), expected);
    }

    #[test]
    fn disallows_non_sibling() {
        let url_resolver = UrlResolver::new(Some(
            Url::parse(&make_file_uri("/example/bar.svg")).unwrap(),
        ));
        assert!(matches!(
            url_resolver.resolve_href(&make_file_uri("/etc/passwd")),
            Err(AllowedUrlError::NotSiblingOrChildOfBaseFile)
        ));
    }

    #[cfg(windows)]
    #[test]
    fn invalid_url_from_test_suite() {
        // This is required for Url to panic.
        let resolver = UrlResolver::new(Some(Url::parse("file:///c:/foo.svg").expect("initial url")));
        // With this, it doesn't panic:
        //   let resolver = UrlResolver::new(None);
        match resolver.resolve_href("file://invalid.css") {
            Ok(_) => println!("yay!"),
            Err(e) => println!("err: {}", e),
        }
    }

}
<commit_msg>Make the Windows test for UrlResolver not panic<commit_after>//! Determine which URLs are allowed for loading.

use std::fmt;
use std::io;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use url::Url;

use crate::error::AllowedUrlError;

/// Currently only contains the base URL.
///
/// The plan is to add:
/// base_only:    Only allow to load content from the same base URL. By default
//                this restriction is enabled and requires to provide base_url.
/// include_xml:  Allows to use xi:include with XML. Enabled by default.
/// include_text: Allows to use xi:include with text. Enabled by default.
/// local_only:   Only allow to load content from the local filesystem.
///               Enabled by default.
#[derive(Clone)]
pub struct UrlResolver {
    /// Base URL; all relative references will be resolved with respect to this.
    pub base_url: Option<Url>,
}

impl UrlResolver {
    /// Creates a `UrlResolver` with defaults, and sets the `base_url`.
    pub fn new(base_url: Option<Url>) -> Self {
        UrlResolver { base_url }
    }

    pub fn resolve_href(&self, href: &str) -> Result<AllowedUrl, AllowedUrlError> {
        let url = Url::options()
            .base_url(self.base_url.as_ref())
            .parse(href)
            .map_err(AllowedUrlError::UrlParseError)?;

        // Allow loads of data: from any location
        if url.scheme() == "data" {
            return Ok(AllowedUrl(url));
        }

        // All other sources require a base url
        if self.base_url.is_none() {
            return Err(AllowedUrlError::BaseRequired);
        }

        let base_url = self.base_url.as_ref().unwrap();

        // Deny loads from differing URI schemes
        if url.scheme() != base_url.scheme() {
            return Err(AllowedUrlError::DifferentUriSchemes);
        }

        // resource: is allowed to load anything from other resources
        if url.scheme() == "resource" {
            return Ok(AllowedUrl(url));
        }

        // Non-file: isn't allowed to load anything
        if url.scheme() != "file" {
            return Err(AllowedUrlError::DisallowedScheme);
        }

        // We have two file: URIs.  Now canonicalize them (remove .. and symlinks, etc.)
        // and see if the directories match

        let url_path = url
            .to_file_path()
            .map_err(|_| AllowedUrlError::InvalidPath)?;
        let base_path = base_url
            .to_file_path()
            .map_err(|_| AllowedUrlError::InvalidPath)?;

        let base_parent = base_path.parent();
        if base_parent.is_none() {
            return Err(AllowedUrlError::BaseIsRoot);
        }

        let base_parent = base_parent.unwrap();

        let url_canon =
            canonicalize(&url_path).map_err(|_| AllowedUrlError::CanonicalizationError)?;
        let parent_canon =
            canonicalize(&base_parent).map_err(|_| AllowedUrlError::CanonicalizationError)?;

        if url_canon.starts_with(parent_canon) {
            Ok(AllowedUrl(url))
        } else {
            Err(AllowedUrlError::NotSiblingOrChildOfBaseFile)
        }
    }
}

/// Wrapper for URLs which are allowed to be loaded
///
/// SVG files can reference other files (PNG/JPEG images, other SVGs,
/// CSS files, etc.).  This object is constructed by checking whether
/// a specified `href` (a possibly-relative filename, for example)
/// should be allowed to be loaded, given the base URL of the SVG
/// being loaded.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct AllowedUrl(Url);

impl Deref for AllowedUrl {
    type Target = Url;

    fn deref(&self) -> &Url {
        &self.0
    }
}

impl fmt::Display for AllowedUrl {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        self.0.fmt(f)
    }
}

// For tests, we don't want to touch the filesystem.  In that case,
// assume that we are being passed canonical file names.
#[cfg(not(test))]
fn canonicalize<P: AsRef<Path>>(path: P) -> Result<PathBuf, io::Error> {
    path.as_ref().canonicalize()
}
#[cfg(test)]
fn canonicalize<P: AsRef<Path>>(path: P) -> Result<PathBuf, io::Error> {
    Ok(path.as_ref().to_path_buf())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn disallows_relative_file_with_no_base_file() {
        let url_resolver = UrlResolver::new(None);
        assert!(matches!(
            url_resolver.resolve_href("foo.svg"),
            Err(AllowedUrlError::UrlParseError(
                url::ParseError::RelativeUrlWithoutBase
            ))
        ));
    }

    #[test]
    fn disallows_different_schemes() {
        let url_resolver = UrlResolver::new(Some(
            Url::parse("http://example.com/malicious.svg").unwrap(),
        ));
        assert!(matches!(
            url_resolver.resolve_href("file:///etc/passwd"),
            Err(AllowedUrlError::DifferentUriSchemes)
        ));
    }

    fn make_file_uri(p: &str) -> String {
        if cfg!(windows) {
            format!("file:///c:{}", p)
        } else {
            format!("file://{}", p)
        }
    }

    #[test]
    fn disallows_base_is_root() {
        let url_resolver = UrlResolver::new(Some(Url::parse(&make_file_uri("/")).unwrap()));
        assert!(matches!(
            url_resolver.resolve_href("foo.svg"),
            Err(AllowedUrlError::BaseIsRoot)
        ));
    }

    #[test]
    fn disallows_non_file_scheme() {
        let url_resolver = UrlResolver::new(Some(Url::parse("http://foo.bar/baz.svg").unwrap()));
        assert!(matches!(
            url_resolver.resolve_href("foo.svg"),
            Err(AllowedUrlError::DisallowedScheme)
        ));
    }

    #[test]
    fn allows_data_url_with_no_base_file() {
        let url_resolver = UrlResolver::new(None);
        assert_eq!(
            url_resolver
                .resolve_href("data:image/jpeg;base64,xxyyzz")
                .unwrap()
                .as_ref(),
            "data:image/jpeg;base64,xxyyzz",
        );
    }

    #[test]
    fn allows_relative() {
        let url_resolver = UrlResolver::new(Some(
            Url::parse(&make_file_uri("/example/bar.svg")).unwrap(),
        ));
        let resolved = url_resolver.resolve_href("foo.svg").unwrap();
        let expected = make_file_uri("/example/foo.svg");
        assert_eq!(resolved.as_ref(), expected);
    }

    #[test]
    fn allows_sibling() {
        let url_resolver = UrlResolver::new(Some(
            Url::parse(&make_file_uri("/example/bar.svg")).unwrap(),
        ));
        let resolved = url_resolver
            .resolve_href(&make_file_uri("/example/foo.svg"))
            .unwrap();
        let expected = make_file_uri("/example/foo.svg");
        assert_eq!(resolved.as_ref(), expected);
    }

    #[test]
    fn allows_child_of_sibling() {
        let url_resolver = UrlResolver::new(Some(
            Url::parse(&make_file_uri("/example/bar.svg")).unwrap(),
        ));
        let resolved = url_resolver
            .resolve_href(&make_file_uri("/example/subdir/foo.svg"))
            .unwrap();
        let expected = make_file_uri("/example/subdir/foo.svg");
        assert_eq!(resolved.as_ref(), expected);
    }

    #[test]
    fn disallows_non_sibling() {
        let url_resolver = UrlResolver::new(Some(
            Url::parse(&make_file_uri("/example/bar.svg")).unwrap(),
        ));
        assert!(matches!(
            url_resolver.resolve_href(&make_file_uri("/etc/passwd")),
            Err(AllowedUrlError::NotSiblingOrChildOfBaseFile)
        ));
    }

    #[cfg(windows)]
    #[test]
    fn invalid_url_from_test_suite() {
        // This is required for Url to panic.
        let resolver =
            UrlResolver::new(Some(Url::parse("file:///c:/foo.svg").expect("initial url")));
        // With this, it doesn't panic:
        //   let resolver = UrlResolver::new(None);

        // The following panics, when using a base URL
        //   match resolver.resolve_href("file://invalid.css") {
        // so, use a less problematic case, hopefully
        match resolver.resolve_href("file://") {
            Ok(_) => println!("yay!"),
            Err(e) => println!("err: {}", e),
        }
    }
}
<|endoftext|>


<commit_before>//! Simple daemon
//!
//! This daemon will start web server, track new packages and build them

use std::{env, thread};
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::time::Duration;
use std::path::PathBuf;
use time;
use crate::{
    docbuilder::RustwideBuilder,
    DocBuilderOptions,
    DocBuilder,
    utils::{update_release_activity, github_updater, pubsubhubbub},
    db::{connect_db, update_search_index},
};

#[cfg(not(target_os = "windows"))]
use ::{
    libc::fork,
    std::process::exit,
    std::fs::File,
    std::io::Write
};

pub fn start_daemon(background: bool) {
    // first check required environment variables
    for v in ["CRATESFYI_PREFIX",
              "CRATESFYI_PREFIX",
              "CRATESFYI_GITHUB_USERNAME",
              "CRATESFYI_GITHUB_ACCESSTOKEN"]
        .iter() {
        env::var(v).expect(&format!("Environment variable {} not found", v));
    }

    let dbopts = opts();

    // check paths once
    dbopts.check_paths().unwrap();

    if background {
        #[cfg(target_os = "windows")]
        {
            panic!("running in background not supported on windows");
        }
        #[cfg(not(target_os = "windows"))]
        {
            // fork the process
            let pid = unsafe { fork() };
            if pid > 0 {
                let mut file = File::create(dbopts.prefix.join("cratesfyi.pid"))
                    .expect("Failed to create pid file");
                writeln!(&mut file, "{}", pid).expect("Failed to write pid");

                info!("cratesfyi {} daemon started on: {}", ::BUILD_VERSION, pid);
                exit(0);
            }
        }
    }

    // check new crates every minute
    thread::Builder::new().name("crates.io reader".to_string()).spawn(move || {
        // space this out to prevent it from clashing against the queue-builder thread on launch
        thread::sleep(Duration::from_secs(30));
        loop {
            let opts = opts();
            let mut doc_builder = DocBuilder::new(opts);

            if doc_builder.is_locked() {
                debug!("Lock file exists, skipping checking new crates");
            } else {
                debug!("Checking new crates");
                match doc_builder.get_new_crates() {
                    Ok(n) => debug!("{} crates added to queue", n),
                    Err(e) => error!("Failed to get new crates: {}", e),
                }
            }

            thread::sleep(Duration::from_secs(60));
        }
    }).unwrap();

    // build new crates every minute
    thread::Builder::new().name("build queue reader".to_string()).spawn(move || {
        let opts = opts();
        let mut doc_builder = DocBuilder::new(opts);

        /// Represents the current state of the builder thread.
        enum BuilderState {
            /// The builder thread has just started, and hasn't built any crates yet.
            Fresh,
            /// The builder has just seen an empty build queue.
            EmptyQueue,
            /// The builder has just seen the lock file.
            Locked,
            /// The builder has just finished building a crate. The enclosed count is the number of
            /// crates built since the caches have been refreshed.
            QueueInProgress(usize),
        }

        let mut builder = RustwideBuilder::init().unwrap();

        let mut status = BuilderState::Fresh;

        loop {
            if !status.is_in_progress() {
                thread::sleep(Duration::from_secs(60));
            }

            // check lock file
            if doc_builder.is_locked() {
                warn!("Lock file exits, skipping building new crates");
                status = BuilderState::Locked;
                continue;
            }

            if status.count() >= 10 {
                // periodically, we need to flush our caches and ping the hubs
                debug!("10 builds in a row; flushing caches");
                status = BuilderState::QueueInProgress(0);

                match pubsubhubbub::ping_hubs() {
                    Err(e) => error!("Failed to ping hub: {}", e),
                    Ok(n) => debug!("Succesfully pinged {} hubs", n)
                }

                if let Err(e) = doc_builder.load_cache() {
                    error!("Failed to load cache: {}", e);
                }

                if let Err(e) = doc_builder.save_cache() {
                    error!("Failed to save cache: {}", e);
                }
            }

            // Only build crates if there are any to build
            debug!("Checking build queue");
            match doc_builder.get_queue_count() {
                Err(e) => {
                    error!("Failed to read the number of crates in the queue: {}", e);
                    continue;
                }
                Ok(0) => {
                    if status.count() > 0 {
                        // ping the hubs before continuing
                        match pubsubhubbub::ping_hubs() {
                            Err(e) => error!("Failed to ping hub: {}", e),
                            Ok(n) => debug!("Succesfully pinged {} hubs", n)
                        }

                        if let Err(e) = doc_builder.save_cache() {
                            error!("Failed to save cache: {}", e);
                        }
                    }
                    debug!("Queue is empty, going back to sleep");
                    status = BuilderState::EmptyQueue;
                    continue;
                }
                Ok(queue_count) => {
                    info!("Starting build with {} crates in queue (currently on a {} crate streak)",
                          queue_count, status.count());
                }
            }

            // if we're starting a new batch, reload our caches and sources
            if !status.is_in_progress() {
                if let Err(e) = doc_builder.load_cache() {
                    error!("Failed to load cache: {}", e);
                    continue;
                }
            }

            // Run build_packages_queue under `catch_unwind` to catch panics
            // This only panicked twice in the last 6 months but its just a better
            // idea to do this.
            let res = catch_unwind(AssertUnwindSafe(|| {
                match doc_builder.build_next_queue_package(&mut builder) {
                    Err(e) => error!("Failed to build crate from queue: {}", e),
                    Ok(crate_built) => if crate_built {
                        status.increment();
                    }

                }
            }));

            if let Err(e) = res {
                error!("GRAVE ERROR Building new crates panicked: {:?}", e);
            }
        }

        impl BuilderState {
            fn count(&self) -> usize {
                match *self {
                    BuilderState::QueueInProgress(n) => n,
                    _ => 0,
                }
            }

            fn is_in_progress(&self) -> bool {
                match *self {
                    BuilderState::QueueInProgress(_) => true,
                    _ => false,
                }
            }

            fn increment(&mut self) {
                *self = BuilderState::QueueInProgress(self.count() + 1);
            }
        }
    }).unwrap();

    // update release activity everyday at 23:55
    thread::Builder::new().name("release activity updater".to_string()).spawn(move || {
        loop {
            thread::sleep(Duration::from_secs(60));
            let now = time::now();
            if now.tm_hour == 23 && now.tm_min == 55 {
                info!("Updating release activity");
                if let Err(e) = update_release_activity() {
                    error!("Failed to update release activity: {}", e);
                }
            }
        }
    }).unwrap();

    // update search index every 3 hours
    thread::Builder::new().name("search index updater".to_string()).spawn(move || {
        loop {
            thread::sleep(Duration::from_secs(60 * 60 * 3));
            let conn = connect_db().expect("Failed to connect database");
            if let Err(e) = update_search_index(&conn) {
                error!("Failed to update search index: {}", e);
            }
        }
    }).unwrap();

    // update github stats every 6 hours
    thread::Builder::new().name("github stat updater".to_string()).spawn(move || {
        loop {
            thread::sleep(Duration::from_secs(60 * 60 * 6));
            if let Err(e) = github_updater() {
                error!("Failed to update github fields: {}", e);
            }
        }
    }).unwrap();

    // TODO: update ssl certificate every 3 months

    // at least start web server
    info!("Starting web server");
    crate::Server::start(None);
}

fn opts() -> DocBuilderOptions {
    let prefix = PathBuf::from(env::var("CRATESFYI_PREFIX")
        .expect("CRATESFYI_PREFIX environment variable not found"));
    DocBuilderOptions::from_prefix(prefix)
}
<commit_msg>Missed one<commit_after>//! Simple daemon
//!
//! This daemon will start web server, track new packages and build them

use std::{env, thread};
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::time::Duration;
use std::path::PathBuf;
use time;
use crate::{
    docbuilder::RustwideBuilder,
    DocBuilderOptions,
    DocBuilder,
    utils::{update_release_activity, github_updater, pubsubhubbub},
    db::{connect_db, update_search_index},
};

#[cfg(not(target_os = "windows"))]
use ::{
    libc::fork,
    std::process::exit,
    std::fs::File,
    std::io::Write
};

pub fn start_daemon(background: bool) {
    // first check required environment variables
    for v in ["CRATESFYI_PREFIX",
              "CRATESFYI_PREFIX",
              "CRATESFYI_GITHUB_USERNAME",
              "CRATESFYI_GITHUB_ACCESSTOKEN"]
        .iter() {
        env::var(v).expect(&format!("Environment variable {} not found", v));
    }

    let dbopts = opts();

    // check paths once
    dbopts.check_paths().unwrap();

    if background {
        #[cfg(target_os = "windows")]
        {
            panic!("running in background not supported on windows");
        }
        #[cfg(not(target_os = "windows"))]
        {
            // fork the process
            let pid = unsafe { fork() };
            if pid > 0 {
                let mut file = File::create(dbopts.prefix.join("cratesfyi.pid"))
                    .expect("Failed to create pid file");
                writeln!(&mut file, "{}", pid).expect("Failed to write pid");

                info!("cratesfyi {} daemon started on: {}", crate::BUILD_VERSION, pid);
                exit(0);
            }
        }
    }

    // check new crates every minute
    thread::Builder::new().name("crates.io reader".to_string()).spawn(move || {
        // space this out to prevent it from clashing against the queue-builder thread on launch
        thread::sleep(Duration::from_secs(30));
        loop {
            let opts = opts();
            let mut doc_builder = DocBuilder::new(opts);

            if doc_builder.is_locked() {
                debug!("Lock file exists, skipping checking new crates");
            } else {
                debug!("Checking new crates");
                match doc_builder.get_new_crates() {
                    Ok(n) => debug!("{} crates added to queue", n),
                    Err(e) => error!("Failed to get new crates: {}", e),
                }
            }

            thread::sleep(Duration::from_secs(60));
        }
    }).unwrap();

    // build new crates every minute
    thread::Builder::new().name("build queue reader".to_string()).spawn(move || {
        let opts = opts();
        let mut doc_builder = DocBuilder::new(opts);

        /// Represents the current state of the builder thread.
        enum BuilderState {
            /// The builder thread has just started, and hasn't built any crates yet.
            Fresh,
            /// The builder has just seen an empty build queue.
            EmptyQueue,
            /// The builder has just seen the lock file.
            Locked,
            /// The builder has just finished building a crate. The enclosed count is the number of
            /// crates built since the caches have been refreshed.
            QueueInProgress(usize),
        }

        let mut builder = RustwideBuilder::init().unwrap();

        let mut status = BuilderState::Fresh;

        loop {
            if !status.is_in_progress() {
                thread::sleep(Duration::from_secs(60));
            }

            // check lock file
            if doc_builder.is_locked() {
                warn!("Lock file exits, skipping building new crates");
                status = BuilderState::Locked;
                continue;
            }

            if status.count() >= 10 {
                // periodically, we need to flush our caches and ping the hubs
                debug!("10 builds in a row; flushing caches");
                status = BuilderState::QueueInProgress(0);

                match pubsubhubbub::ping_hubs() {
                    Err(e) => error!("Failed to ping hub: {}", e),
                    Ok(n) => debug!("Succesfully pinged {} hubs", n)
                }

                if let Err(e) = doc_builder.load_cache() {
                    error!("Failed to load cache: {}", e);
                }

                if let Err(e) = doc_builder.save_cache() {
                    error!("Failed to save cache: {}", e);
                }
            }

            // Only build crates if there are any to build
            debug!("Checking build queue");
            match doc_builder.get_queue_count() {
                Err(e) => {
                    error!("Failed to read the number of crates in the queue: {}", e);
                    continue;
                }
                Ok(0) => {
                    if status.count() > 0 {
                        // ping the hubs before continuing
                        match pubsubhubbub::ping_hubs() {
                            Err(e) => error!("Failed to ping hub: {}", e),
                            Ok(n) => debug!("Succesfully pinged {} hubs", n)
                        }

                        if let Err(e) = doc_builder.save_cache() {
                            error!("Failed to save cache: {}", e);
                        }
                    }
                    debug!("Queue is empty, going back to sleep");
                    status = BuilderState::EmptyQueue;
                    continue;
                }
                Ok(queue_count) => {
                    info!("Starting build with {} crates in queue (currently on a {} crate streak)",
                          queue_count, status.count());
                }
            }

            // if we're starting a new batch, reload our caches and sources
            if !status.is_in_progress() {
                if let Err(e) = doc_builder.load_cache() {
                    error!("Failed to load cache: {}", e);
                    continue;
                }
            }

            // Run build_packages_queue under `catch_unwind` to catch panics
            // This only panicked twice in the last 6 months but its just a better
            // idea to do this.
            let res = catch_unwind(AssertUnwindSafe(|| {
                match doc_builder.build_next_queue_package(&mut builder) {
                    Err(e) => error!("Failed to build crate from queue: {}", e),
                    Ok(crate_built) => if crate_built {
                        status.increment();
                    }

                }
            }));

            if let Err(e) = res {
                error!("GRAVE ERROR Building new crates panicked: {:?}", e);
            }
        }

        impl BuilderState {
            fn count(&self) -> usize {
                match *self {
                    BuilderState::QueueInProgress(n) => n,
                    _ => 0,
                }
            }

            fn is_in_progress(&self) -> bool {
                match *self {
                    BuilderState::QueueInProgress(_) => true,
                    _ => false,
                }
            }

            fn increment(&mut self) {
                *self = BuilderState::QueueInProgress(self.count() + 1);
            }
        }
    }).unwrap();

    // update release activity everyday at 23:55
    thread::Builder::new().name("release activity updater".to_string()).spawn(move || {
        loop {
            thread::sleep(Duration::from_secs(60));
            let now = time::now();
            if now.tm_hour == 23 && now.tm_min == 55 {
                info!("Updating release activity");
                if let Err(e) = update_release_activity() {
                    error!("Failed to update release activity: {}", e);
                }
            }
        }
    }).unwrap();

    // update search index every 3 hours
    thread::Builder::new().name("search index updater".to_string()).spawn(move || {
        loop {
            thread::sleep(Duration::from_secs(60 * 60 * 3));
            let conn = connect_db().expect("Failed to connect database");
            if let Err(e) = update_search_index(&conn) {
                error!("Failed to update search index: {}", e);
            }
        }
    }).unwrap();

    // update github stats every 6 hours
    thread::Builder::new().name("github stat updater".to_string()).spawn(move || {
        loop {
            thread::sleep(Duration::from_secs(60 * 60 * 6));
            if let Err(e) = github_updater() {
                error!("Failed to update github fields: {}", e);
            }
        }
    }).unwrap();

    // TODO: update ssl certificate every 3 months

    // at least start web server
    info!("Starting web server");
    crate::Server::start(None);
}

fn opts() -> DocBuilderOptions {
    let prefix = PathBuf::from(env::var("CRATESFYI_PREFIX")
        .expect("CRATESFYI_PREFIX environment variable not found"));
    DocBuilderOptions::from_prefix(prefix)
}
<|endoftext|>


<commit_before>//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use std::path::PathBuf;
use std::result::Result as RResult;
use std::ops::Deref;

use toml::{Parser, Value};

generate_error_module!(
    generate_error_types!(ConfigError, ConfigErrorKind,
        NoConfigFileFound   => "No config file found",

        ConfigOverrideError => "Config override error",
        ConfigOverrideKeyNotAvailable => "Key not available",
        ConfigOverrideTypeNotMatching => "Configuration Type not matching"

    );
);

use self::error::{ConfigError, ConfigErrorKind};

/**
 * Result type of this module. Either `T` or `ConfigError`
 */
pub type Result<T> = RResult<T, ConfigError>;

/// `Configuration` object
///
/// Holds all config variables which are globally available plus the configuration object from the
/// config parser, which can be accessed.
#[derive(Debug)]
pub struct Configuration {

    /// The plain configuration object for direct access if necessary
    config: Value,

    /// The verbosity the program should run with
    verbosity: bool,

    /// The editor which should be used
    editor: Option<String>,

    ///The options the editor should get when opening some file
    editor_opts: String,
}

impl Configuration {

    /// Get a new configuration object.
    ///
    /// The passed runtimepath is used for searching the configuration file, whereas several file
    /// names are tested. If that does not work, the home directory and the XDG basedir are tested
    /// with all variants.
    ///
    /// If that doesn't work either, an error is returned.
    pub fn new(rtp: &PathBuf) -> Result<Configuration> {
        fetch_config(&rtp).map(|cfg| {
            let verbosity   = get_verbosity(&cfg);
            let editor      = get_editor(&cfg);
            let editor_opts = get_editor_opts(&cfg);

            debug!("Building configuration");
            debug!("  - verbosity  : {:?}", verbosity);
            debug!("  - editor     : {:?}", editor);
            debug!("  - editor-opts: {}", editor_opts);

            Configuration {
                config: cfg,
                verbosity: verbosity,
                editor: editor,
                editor_opts: editor_opts,
            }
        })
    }

    pub fn editor(&self) -> Option<&String> {
        self.editor.as_ref()
    }

    #[allow(dead_code)] // Why do I actually need this annotation on a pub function?
    pub fn config(&self) -> &Value {
        &self.config
    }

    pub fn store_config(&self) -> Option<&Value> {
        match self.config {
            Value::Table(ref tabl) => tabl.get("store"),
            _ => None,
        }
    }

    /// Override the configuration.
    /// The `v` parameter is expected to contain 'key=value' pairs where the key is a path in the
    /// TOML tree, the value to be an appropriate value.
    ///
    /// The override fails if the configuration which is about to be overridden does not exist or
    /// the `value` part cannot be converted to the type of the configuration value.
    ///
    /// If `v` is empty, this is considered to be a successful `override_config()` call.
    pub fn override_config(&mut self, v: Vec<String>) -> Result<()> {
        use libimagutil::key_value_split::*;
        use libimagutil::iter::*;
        use self::error::ConfigErrorKind as CEK;
        use libimagerror::into::IntoError;

        v.into_iter()
            .map(|s| { debug!("Trying to process '{}'", s); s })
            .filter_map(|s| {
                match s.into_kv() {
                    Some(kv) => Some(kv.into()),
                    None => {
                        warn!("Could split at '=' - will be ignore override");
                        None
                    }
                }
            })
            .map(|(k, v)| {
                match self.config.lookup_mut(&k[..]) {
                    Some(value) => {
                        match into_value(value, v) {
                            Some(v) => {
                                *value = v;
                                info!("Successfully overridden: {} = {}", k, value);
                                Ok(())
                            },
                            None => Err(CEK::ConfigOverrideTypeNotMatching.into_error()),
                        }
                    },
                    None => Err(CEK::ConfigOverrideKeyNotAvailable.into_error()),
                }
            })
            .fold_defresult(|i| i)
            .map_err(Box::new)
            .map_err(|e| CEK::ConfigOverrideError.into_error_with_cause(e))
    }
}

/// Tries to convert the String `s` into the same type as `value`.
///
/// Returns None if string cannot be converted.
///
/// Arrays and Tables are not supported and will yield `None`.
fn into_value(value: &Value, s: String) -> Option<Value> {
    use std::str::FromStr;

    match value {
        &Value::String(_) => Some(Value::String(s)),
        &Value::Integer(_) => FromStr::from_str(&s[..]).ok().map(|i| Value::Integer(i)),
        &Value::Float(_) => FromStr::from_str(&s[..]).ok().map(|f| Value::Float(f)),
        &Value::Boolean(_) => {
            if s == "true" { Some(Value::Boolean(true)) }
            else if s == "false" { Some(Value::Boolean(false)) }
            else { None }
        }
        &Value::Datetime(_) => Some(Value::Datetime(s)),
        _ => None,
    }
}

impl Deref for Configuration {
    type Target = Value;

    fn deref(&self) -> &Value {
        &self.config
    }

}

fn get_verbosity(v: &Value) -> bool {
    match *v {
        Value::Table(ref t) => t.get("verbose")
                .map_or(false, |v| match *v { Value::Boolean(b) => b, _ => false, }),
        _ => false,
    }
}

fn get_editor(v: &Value) -> Option<String> {
    match *v {
        Value::Table(ref t) => t.get("editor")
                .and_then(|v| match *v { Value::String(ref s) => Some(s.clone()), _ => None, }),
        _ => None,
    }
}

fn get_editor_opts(v: &Value) -> String {
    match *v {
        Value::Table(ref t) => t.get("editor-opts")
                .and_then(|v| match *v { Value::String(ref s) => Some(s.clone()), _ => None, })
                .unwrap_or_default(),
        _ => String::new(),
    }
}

/**
 * Helper to fetch the config file
 *
 * Tests several variants for the config file path and uses the first one which works.
 */
fn fetch_config(rtp: &PathBuf) -> Result<Value> {
    use std::env;
    use std::fs::File;
    use std::io::Read;
    use std::io::Write;
    use std::io::stderr;

    use xdg_basedir;
    use itertools::Itertools;

    use libimagutil::variants::generate_variants as gen_vars;

    let variants = vec!["config", "config.toml", "imagrc", "imagrc.toml"];
    let modifier = |base: &PathBuf, v: &'static str| {
        let mut base = base.clone();
        base.push(String::from(v));
        base
    };

    vec![
        vec![rtp.clone()],
        gen_vars(rtp.clone(), variants.clone(), &modifier),

        env::var("HOME").map(|home| gen_vars(PathBuf::from(home), variants.clone(), &modifier))
                        .unwrap_or(vec![]),

        xdg_basedir::get_data_home().map(|data_dir| gen_vars(data_dir, variants.clone(), &modifier))
                                    .unwrap_or(vec![]),
    ].iter()
        .flatten()
        .filter(|path| path.exists() && path.is_file())
        .map(|path| {
            let content = {
                let mut s = String::new();
                let f = File::open(path);
                if f.is_err() {
                    return None
                }
                let mut f = f.unwrap();
                f.read_to_string(&mut s).ok();
                s
            };

            let mut parser = Parser::new(&content[..]);
            let res = parser.parse();

            if res.is_none() {
                write!(stderr(), "Config file parser error:").ok();
                for error in parser.errors {
                    write!(stderr(), "At [{}][{}] <> {}", error.lo, error.hi, error).ok();
                    write!(stderr(), "in: '{}'", &content[error.lo..error.hi]).ok();
                }
                None
            } else {
                res
            }
        })
        .filter(|loaded| loaded.is_some())
        .nth(0)
        .map(|inner| Value::Table(inner.unwrap()))
        .ok_or(ConfigErrorKind::NoConfigFileFound.into())
}

<commit_msg>Make error types public<commit_after>//
// imag - the personal information management suite for the commandline
// Copyright (C) 2015, 2016 Matthias Beyer <mail@beyermatthias.de> and contributors
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; version
// 2.1 of the License.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//

use std::path::PathBuf;
use std::result::Result as RResult;
use std::ops::Deref;

use toml::{Parser, Value};

generate_error_module!(
    generate_error_types!(ConfigError, ConfigErrorKind,
        NoConfigFileFound   => "No config file found",

        ConfigOverrideError => "Config override error",
        ConfigOverrideKeyNotAvailable => "Key not available",
        ConfigOverrideTypeNotMatching => "Configuration Type not matching"

    );
);

pub use self::error::{ConfigError, ConfigErrorKind};

/**
 * Result type of this module. Either `T` or `ConfigError`
 */
pub type Result<T> = RResult<T, ConfigError>;

/// `Configuration` object
///
/// Holds all config variables which are globally available plus the configuration object from the
/// config parser, which can be accessed.
#[derive(Debug)]
pub struct Configuration {

    /// The plain configuration object for direct access if necessary
    config: Value,

    /// The verbosity the program should run with
    verbosity: bool,

    /// The editor which should be used
    editor: Option<String>,

    ///The options the editor should get when opening some file
    editor_opts: String,
}

impl Configuration {

    /// Get a new configuration object.
    ///
    /// The passed runtimepath is used for searching the configuration file, whereas several file
    /// names are tested. If that does not work, the home directory and the XDG basedir are tested
    /// with all variants.
    ///
    /// If that doesn't work either, an error is returned.
    pub fn new(rtp: &PathBuf) -> Result<Configuration> {
        fetch_config(&rtp).map(|cfg| {
            let verbosity   = get_verbosity(&cfg);
            let editor      = get_editor(&cfg);
            let editor_opts = get_editor_opts(&cfg);

            debug!("Building configuration");
            debug!("  - verbosity  : {:?}", verbosity);
            debug!("  - editor     : {:?}", editor);
            debug!("  - editor-opts: {}", editor_opts);

            Configuration {
                config: cfg,
                verbosity: verbosity,
                editor: editor,
                editor_opts: editor_opts,
            }
        })
    }

    pub fn editor(&self) -> Option<&String> {
        self.editor.as_ref()
    }

    #[allow(dead_code)] // Why do I actually need this annotation on a pub function?
    pub fn config(&self) -> &Value {
        &self.config
    }

    pub fn store_config(&self) -> Option<&Value> {
        match self.config {
            Value::Table(ref tabl) => tabl.get("store"),
            _ => None,
        }
    }

    /// Override the configuration.
    /// The `v` parameter is expected to contain 'key=value' pairs where the key is a path in the
    /// TOML tree, the value to be an appropriate value.
    ///
    /// The override fails if the configuration which is about to be overridden does not exist or
    /// the `value` part cannot be converted to the type of the configuration value.
    ///
    /// If `v` is empty, this is considered to be a successful `override_config()` call.
    pub fn override_config(&mut self, v: Vec<String>) -> Result<()> {
        use libimagutil::key_value_split::*;
        use libimagutil::iter::*;
        use self::error::ConfigErrorKind as CEK;
        use libimagerror::into::IntoError;

        v.into_iter()
            .map(|s| { debug!("Trying to process '{}'", s); s })
            .filter_map(|s| {
                match s.into_kv() {
                    Some(kv) => Some(kv.into()),
                    None => {
                        warn!("Could split at '=' - will be ignore override");
                        None
                    }
                }
            })
            .map(|(k, v)| {
                match self.config.lookup_mut(&k[..]) {
                    Some(value) => {
                        match into_value(value, v) {
                            Some(v) => {
                                *value = v;
                                info!("Successfully overridden: {} = {}", k, value);
                                Ok(())
                            },
                            None => Err(CEK::ConfigOverrideTypeNotMatching.into_error()),
                        }
                    },
                    None => Err(CEK::ConfigOverrideKeyNotAvailable.into_error()),
                }
            })
            .fold_defresult(|i| i)
            .map_err(Box::new)
            .map_err(|e| CEK::ConfigOverrideError.into_error_with_cause(e))
    }
}

/// Tries to convert the String `s` into the same type as `value`.
///
/// Returns None if string cannot be converted.
///
/// Arrays and Tables are not supported and will yield `None`.
fn into_value(value: &Value, s: String) -> Option<Value> {
    use std::str::FromStr;

    match value {
        &Value::String(_) => Some(Value::String(s)),
        &Value::Integer(_) => FromStr::from_str(&s[..]).ok().map(|i| Value::Integer(i)),
        &Value::Float(_) => FromStr::from_str(&s[..]).ok().map(|f| Value::Float(f)),
        &Value::Boolean(_) => {
            if s == "true" { Some(Value::Boolean(true)) }
            else if s == "false" { Some(Value::Boolean(false)) }
            else { None }
        }
        &Value::Datetime(_) => Some(Value::Datetime(s)),
        _ => None,
    }
}

impl Deref for Configuration {
    type Target = Value;

    fn deref(&self) -> &Value {
        &self.config
    }

}

fn get_verbosity(v: &Value) -> bool {
    match *v {
        Value::Table(ref t) => t.get("verbose")
                .map_or(false, |v| match *v { Value::Boolean(b) => b, _ => false, }),
        _ => false,
    }
}

fn get_editor(v: &Value) -> Option<String> {
    match *v {
        Value::Table(ref t) => t.get("editor")
                .and_then(|v| match *v { Value::String(ref s) => Some(s.clone()), _ => None, }),
        _ => None,
    }
}

fn get_editor_opts(v: &Value) -> String {
    match *v {
        Value::Table(ref t) => t.get("editor-opts")
                .and_then(|v| match *v { Value::String(ref s) => Some(s.clone()), _ => None, })
                .unwrap_or_default(),
        _ => String::new(),
    }
}

/**
 * Helper to fetch the config file
 *
 * Tests several variants for the config file path and uses the first one which works.
 */
fn fetch_config(rtp: &PathBuf) -> Result<Value> {
    use std::env;
    use std::fs::File;
    use std::io::Read;
    use std::io::Write;
    use std::io::stderr;

    use xdg_basedir;
    use itertools::Itertools;

    use libimagutil::variants::generate_variants as gen_vars;

    let variants = vec!["config", "config.toml", "imagrc", "imagrc.toml"];
    let modifier = |base: &PathBuf, v: &'static str| {
        let mut base = base.clone();
        base.push(String::from(v));
        base
    };

    vec![
        vec![rtp.clone()],
        gen_vars(rtp.clone(), variants.clone(), &modifier),

        env::var("HOME").map(|home| gen_vars(PathBuf::from(home), variants.clone(), &modifier))
                        .unwrap_or(vec![]),

        xdg_basedir::get_data_home().map(|data_dir| gen_vars(data_dir, variants.clone(), &modifier))
                                    .unwrap_or(vec![]),
    ].iter()
        .flatten()
        .filter(|path| path.exists() && path.is_file())
        .map(|path| {
            let content = {
                let mut s = String::new();
                let f = File::open(path);
                if f.is_err() {
                    return None
                }
                let mut f = f.unwrap();
                f.read_to_string(&mut s).ok();
                s
            };

            let mut parser = Parser::new(&content[..]);
            let res = parser.parse();

            if res.is_none() {
                write!(stderr(), "Config file parser error:").ok();
                for error in parser.errors {
                    write!(stderr(), "At [{}][{}] <> {}", error.lo, error.hi, error).ok();
                    write!(stderr(), "in: '{}'", &content[error.lo..error.hi]).ok();
                }
                None
            } else {
                res
            }
        })
        .filter(|loaded| loaded.is_some())
        .nth(0)
        .map(|inner| Value::Table(inner.unwrap()))
        .ok_or(ConfigErrorKind::NoConfigFileFound.into())
}

<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    access_path::{AccessPath, Accesses},
    account_config,
    language_storage::StructTag,
};
use move_core_types::identifier::{IdentStr, Identifier};
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};

static LIBRA_TIMESTAMP_MODULE_NAME: Lazy<Identifier> =
    Lazy::new(|| Identifier::new("LibraTimestamp").unwrap());
static LIBRA_TIMESTAMP_STRUCT_NAME: Lazy<Identifier> =
    Lazy::new(|| Identifier::new("CurrentTimeMicroseconds").unwrap());

pub fn libra_timestamp_module_name() -> &'static IdentStr {
    &*LIBRA_TIMESTAMP_MODULE_NAME
}

pub fn libra_timestamp_struct_name() -> &'static IdentStr {
    &*LIBRA_TIMESTAMP_STRUCT_NAME
}

pub fn libra_timestamp_tag() -> StructTag {
    StructTag {
        address: account_config::CORE_CODE_ADDRESS,
        name: libra_timestamp_struct_name().to_owned(),
        module: libra_timestamp_module_name().to_owned(),
        type_params: vec![],
    }
}

/// The access path where the Validator Set resource is stored.
pub static LIBRA_TIMESTAMP_RESOURCE_PATH: Lazy<Vec<u8>> =
    Lazy::new(|| AccessPath::resource_access_vec(&libra_timestamp_tag(), &Accesses::empty()));

#[derive(Debug, Deserialize, Serialize)]
pub struct LibraTimestampResource {
    pub libra_timestamp: LibraTimestamp,
}

#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct LibraTimestamp {
    pub microseconds: u64,
}
<commit_msg>[move-resources] impl MoveResource for LibraTimestamp<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::{
    access_path::{AccessPath, Accesses},
    move_resource::MoveResource,
};
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};

/// The access path where the Libra Timestamp resource is stored.
pub static LIBRA_TIMESTAMP_RESOURCE_PATH: Lazy<Vec<u8>> = Lazy::new(|| {
    AccessPath::resource_access_vec(&LibraTimestampResource::struct_tag(), &Accesses::empty())
});

#[derive(Debug, Deserialize, Serialize)]
pub struct LibraTimestampResource {
    pub libra_timestamp: LibraTimestamp,
}

impl MoveResource for LibraTimestampResource {
    const MODULE_NAME: &'static str = "LibraTimestamp";
    const STRUCT_NAME: &'static str = "CurrentTimeMicroseconds";
}

#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct LibraTimestamp {
    pub microseconds: u64,
}
<|endoftext|>