<commit_before>/*
 * Copyright 2015-2016 Ben Ashford
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//! Implementation of the Bulk API

use std::collections::BTreeMap;

use hyper::status::StatusCode;

use rustc_serialize::json;
use rustc_serialize::json::{Json, ToJson};

use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde::de::{Error, MapVisitor, Visitor};
use serde_json;

use ::{Client, EsResponse};
use ::do_req;
use ::error::EsError;
use ::json::{FieldBased, NoOuter, ShouldSkip};
use ::units::Duration;

use super::ShardCountResult;
use super::common::{Options, OptionVal, VersionType};

// TODO - make Serialize
#[derive(Default)]
pub struct ActionSource {
    doc:           Option<Json>,
    upsert:        Option<Json>,
    doc_as_upsert: Option<bool>,
    script:        Option<String>,
    params:        Option<Json>,
    lang:          Option<String>
}

impl ActionSource {
    pub fn new() -> Self {
        Default::default()
    }

    add_field!(with_doc, doc, Json);
    add_field!(with_upsert, upsert, Json);
    add_field!(with_doc_as_upsert, doc_as_upsert, bool);
    add_field!(with_script, script, String);
    add_field!(with_params, params, Json);
    add_field!(with_lang, lang, String);
}

impl ToJson for ActionSource {
    fn to_json(&self) -> Json {
        let mut d = BTreeMap::new();

        optional_add!(self, d, doc);
        optional_add!(self, d, upsert);
        optional_add!(self, d, doc_as_upsert);
        optional_add!(self, d, script);
        optional_add!(self, d, params);
        optional_add!(self, d, lang);

        Json::Object(d)
    }
}

pub enum ActionType {
    Index,
    Create,
    Delete,
    Update
}

impl Serialize for ActionType {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: Serializer {

        self.to_string().serialize(serializer)
    }
}

// TODO - deprecated
impl<'a> From<&'a String> for ActionType {
    fn from(from: &'a String) -> ActionType {
        if from == "index" {
            ActionType::Index
        } else if from == "create" {
            ActionType::Create
        } else if from == "delete" {
            ActionType::Delete
        } else if from == "update" {
            ActionType::Update
        } else {
            panic!("Unknown action type: {}", from)
        }
    }
}

impl ToString for ActionType {
    fn to_string(&self) -> String {
        match *self {
            ActionType::Index => "index",
            ActionType::Create => "create",
            ActionType::Delete => "delete",
            ActionType::Update => "update"
        }.to_owned()
    }
}

#[derive(Default, Serialize)]
pub struct ActionOptions {
    #[serde(rename="_index", skip_serializing_if="ShouldSkip::should_skip")]
    index:             Option<String>,
    #[serde(rename="_type", skip_serializing_if="ShouldSkip::should_skip")]
    doc_type:          Option<String>,
    #[serde(rename="_id", skip_serializing_if="ShouldSkip::should_skip")]
    id:                Option<String>,
    #[serde(rename="_version", skip_serializing_if="ShouldSkip::should_skip")]
    version:           Option<u64>,
    // TODO - re-enable
    //#[serde(rename="_version_type", skip_serializing_if="ShouldSkip::should_skip")]
    //version_type:      Option<VersionType>,
    #[serde(rename="_routing", skip_serializing_if="ShouldSkip::should_skip")]
    routing:           Option<String>,
    #[serde(rename="_parent", skip_serializing_if="ShouldSkip::should_skip")]
    parent:            Option<String>,
    #[serde(rename="_timestamp", skip_serializing_if="ShouldSkip::should_skip")]
    timestamp:         Option<String>,
    #[serde(rename="_ttl", skip_serializing_if="ShouldSkip::should_skip")]
    ttl:               Option<Duration>,
    #[serde(rename="_retry_on_conflict", skip_serializing_if="ShouldSkip::should_skip")]
    retry_on_conflict: Option<u64>,
}

#[derive(Serialize)]
pub struct Action<X>(FieldBased<ActionType, ActionOptions, NoOuter>, Option<X>);

impl<S> Action<S>
    where S: Serialize {
    /// An index action.
    ///
    /// Takes the document to be indexed, other parameters can be set as
    /// optional on the `Action` struct returned.
    ///
    /// # Example
    ///
    /// ```
    /// use rs_es::operations::bulk::Action;
    ///
    /// let delete_action = Action::delete("doc_id");
    /// ```
    pub fn index(document: S) -> Self {
        Action(FieldBased::new(ActionType::Index,
                               Default::default(),
                               NoOuter),
               Some(document))
    }

    /// Create action
    pub fn create(document: S) -> Self {
        Action(FieldBased::new(ActionType::Create,
                               Default::default(),
                               NoOuter),
               Some(document))
    }

    /// Add the serialized version of this action to the bulk `String`.
    fn add(&self, actstr: &mut String) -> Result<(), EsError> {
        let command_str = try!(serde_json::to_string(&self.0));

        actstr.push_str(&command_str);
        actstr.push_str("\n");

        match self.1 {
            Some(ref source) => {
                let payload_str = try!(serde_json::to_string(source));
                actstr.push_str(&payload_str);
                actstr.push_str("\n");
            },
            None             => ()
        }
        Ok(())
    }
}

impl<S> Action<S> {
    pub fn delete<A: Into<String>>(id: A) -> Self {
        Action(FieldBased::new(ActionType::Delete,
                               ActionOptions {
                                   id: Some(id.into()),
                                   ..Default::default()
                               },
                               NoOuter),
               None)
    }

    pub fn update<A: Into<String>>(id: A, update: ActionSource) -> Self {
        unimplemented!()
        // Action(FieldBased::new(ActionType::Update,
        //                        ActionOptions {
        //                            id: Some(id.into()),
        //                            ..Default::default()
        //                        },
        //                        NoOuter),
        //        update)
    }

    add_inner_field!(with_index, index, String);
    add_inner_field!(with_doc_type, doc_type, String);
    add_inner_field!(with_id, id, String);
    add_inner_field!(with_version, version, u64);
    // TODO - re-enable
    //add_inner_field!(with_version_type, version_type, VersionType);
    add_inner_field!(with_routing, routing, String);
    add_inner_field!(with_parent, parent, String);
    add_inner_field!(with_timestamp, timestamp, String);
    add_inner_field!(with_ttl, ttl, Duration);
    add_inner_field!(with_retry_on_conflict, retry_on_conflict, u64);
}

pub struct BulkOperation<'a, 'b, S: 'b> {
    client:   &'a mut Client,
    index:    Option<&'b str>,
    doc_type: Option<&'b str>,
    actions:  &'b [Action<S>],
    options:  Options<'b>
}

impl<'a, 'b, S> BulkOperation<'a, 'b, S>
    where S: Serialize {

    pub fn new(client: &'a mut Client, actions: &'b [Action<S>]) -> Self {
        BulkOperation {
            client:   client,
            index:    None,
            doc_type: None,
            actions:  actions,
            options:  Options::new()
        }
    }

    pub fn with_index(&'b mut self, index: &'b str) -> &'b mut Self {
        self.index = Some(index);
        self
    }

    pub fn with_doc_type(&'b mut self, doc_type: &'b str) -> &'b mut Self {
        self.doc_type = Some(doc_type);
        self
    }

    add_option!(with_consistency, "consistency");
    add_option!(with_refresh, "refresh");

    fn format_url(&self) -> String {
        let mut url = String::new();
        url.push_str("/");
        match self.index {
            Some(index) => {
                url.push_str(index);
                url.push_str("/");
            },
            None        => ()
        }
        match self.doc_type {
            Some(doc_type) => {
                url.push_str(doc_type);
                url.push_str("/");
            },
            None           => ()
        }
        url.push_str("_bulk");
        url.push_str(&self.options.to_string());
        url
    }

    fn format_actions(&self) -> String {
        let mut actstr = String::new();
        for action in self.actions {
            action.add(&mut actstr).unwrap();
        }
        actstr
    }

    pub fn send(&'b mut self) -> Result<BulkResult, EsError> {
        //
        // This function does not use the standard GET/POST/DELETE functions of
        // the client, as they serve the happy path of JSON-in/JSON-out, this
        // function does send send JSON in.
        //
        // Various parts of the client are reused where it makes sense.
        //
        let full_url = {
            let url = self.format_url();
            self.client.full_url(&url)
        };
        let body = self.format_actions();
        println!("Sending: {}", body);
        // Doesn't use the standard macros as it's not standard JSON
        let mut result = try!(self.client.http_client
                              .post(&full_url)
                              .body(&body)
                              .send());

        let response = try!(do_req(result));

        match response.status_code() {
            &StatusCode::Ok => Ok(try!(response.read_response())),
            _              => Err(EsError::EsError(format!("Unexpected status: {}", response.status_code())))
        }
    }
}

/// The result of specific actions
pub struct ActionResult {
    pub action: ActionType,
    pub inner: ActionResultInner
}

impl Deserialize for ActionResult {
    fn deserialize<D>(deserializer: &mut D) -> Result<ActionResult, D::Error>
        where D: Deserializer {

        struct ActionResultVisitor;

        impl Visitor for ActionResultVisitor {
            type Value = ActionResult;

            fn visit_map<V>(&mut self, mut visitor: V) -> Result<ActionResult, V::Error>
                where V: MapVisitor {

                let visited:Option<(String, ActionResultInner)> = try!(visitor.visit());
                let (key, value) = match visited {
                    Some((key, value)) => (key, value),
                    None               => return Err(V::Error::custom("expecting at least one field"))
                };
                println!("Found key: {:?}, value: {:?}", key, value);
                try!(visitor.end());

                let result = ActionResult {
                    action: match key.as_ref() {
                        "index" => ActionType::Index,
                        "create" => ActionType::Create,
                        "delete" => ActionType::Delete,
                        "update" => ActionType::Update,
                        _ => {
                            return Err(V::Error::custom(format!("Unrecognised key: {}", key)))
                        }
                    },
                    inner:  value
                };

                Ok(result)
            }
        }

        deserializer.deserialize(ActionResultVisitor)
    }
}

#[derive(Debug, Deserialize)]
pub struct ActionResultInner {
    #[serde(rename="_index")]
    pub index:    String,
    #[serde(rename="_type")]
    pub doc_type: String,
    #[serde(rename="_version")]
    pub version:  u64,
    pub status:   u64,
    #[serde(rename="_shards")]
    pub shards:   ShardCountResult,
    pub found:    bool
}

// TODO - deprecated
// impl<'a> From<&'a Json> for ActionResult {
//     fn from(from: &'a Json) -> ActionResult {
//         info!("ActionResult from: {:?}", from);

//         let d = from.as_object().expect("Not a Json object");
//         assert_eq!(1, d.len());
//         let (key, inner) = d.iter().next().expect("No content");

//         ActionResult {
//             action:   ActionType::from(key),
//             index:    get_json_string!(inner, "_index"),
//             doc_type: get_json_string!(inner, "_type"),
//             version:  get_json_u64!(inner, "_version"),
//             status:   get_json_u64!(inner, "status")
//         }
//     }
// }

/// The result of a bulk operation
#[derive(Deserialize)]
pub struct BulkResult {
    pub errors: bool,
    pub items:  Vec<ActionResult>,
    pub took:   u64
}

// TODO - Deprecated
// impl<'a> From<&'a Json> for BulkResult {
//     fn from(from: &'a Json) -> BulkResult {
//         info!("Bulk result, result: {:?}", from);
//         BulkResult {
//             errors: get_json_bool!(from, "errors"),
//             items:  from.find("items")
//                 .expect("No field called 'items'")
//                 .as_array()
//                 .expect("Field 'items' not expected array")
//                 .iter()
//                 .map(|item| {
//                     ActionResult::from(item)
//                 })
//                 .collect(),
//             took:   get_json_u64!(from, "took")
//         }
//     }
// }
<commit_msg>Doctests for Delete etc.<commit_after>/*
 * Copyright 2015-2016 Ben Ashford
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//! Implementation of the Bulk API

use std::collections::BTreeMap;

use hyper::status::StatusCode;

use rustc_serialize::json;
use rustc_serialize::json::{Json, ToJson};

use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde::de::{Error, MapVisitor, Visitor};
use serde_json;

use ::{Client, EsResponse};
use ::do_req;
use ::error::EsError;
use ::json::{FieldBased, NoOuter, ShouldSkip};
use ::units::Duration;

use super::ShardCountResult;
use super::common::{Options, OptionVal, VersionType};

// TODO - make Serialize
#[derive(Default)]
pub struct ActionSource {
    doc:           Option<Json>,
    upsert:        Option<Json>,
    doc_as_upsert: Option<bool>,
    script:        Option<String>,
    params:        Option<Json>,
    lang:          Option<String>
}

impl ActionSource {
    pub fn new() -> Self {
        Default::default()
    }

    add_field!(with_doc, doc, Json);
    add_field!(with_upsert, upsert, Json);
    add_field!(with_doc_as_upsert, doc_as_upsert, bool);
    add_field!(with_script, script, String);
    add_field!(with_params, params, Json);
    add_field!(with_lang, lang, String);
}

impl ToJson for ActionSource {
    fn to_json(&self) -> Json {
        let mut d = BTreeMap::new();

        optional_add!(self, d, doc);
        optional_add!(self, d, upsert);
        optional_add!(self, d, doc_as_upsert);
        optional_add!(self, d, script);
        optional_add!(self, d, params);
        optional_add!(self, d, lang);

        Json::Object(d)
    }
}

pub enum ActionType {
    Index,
    Create,
    Delete,
    Update
}

impl Serialize for ActionType {
    fn serialize<S>(&self, serializer: &mut S) -> Result<(), S::Error>
        where S: Serializer {

        self.to_string().serialize(serializer)
    }
}

// TODO - deprecated
impl<'a> From<&'a String> for ActionType {
    fn from(from: &'a String) -> ActionType {
        if from == "index" {
            ActionType::Index
        } else if from == "create" {
            ActionType::Create
        } else if from == "delete" {
            ActionType::Delete
        } else if from == "update" {
            ActionType::Update
        } else {
            panic!("Unknown action type: {}", from)
        }
    }
}

impl ToString for ActionType {
    fn to_string(&self) -> String {
        match *self {
            ActionType::Index => "index",
            ActionType::Create => "create",
            ActionType::Delete => "delete",
            ActionType::Update => "update"
        }.to_owned()
    }
}

#[derive(Default, Serialize)]
pub struct ActionOptions {
    #[serde(rename="_index", skip_serializing_if="ShouldSkip::should_skip")]
    index:             Option<String>,
    #[serde(rename="_type", skip_serializing_if="ShouldSkip::should_skip")]
    doc_type:          Option<String>,
    #[serde(rename="_id", skip_serializing_if="ShouldSkip::should_skip")]
    id:                Option<String>,
    #[serde(rename="_version", skip_serializing_if="ShouldSkip::should_skip")]
    version:           Option<u64>,
    // TODO - re-enable
    //#[serde(rename="_version_type", skip_serializing_if="ShouldSkip::should_skip")]
    //version_type:      Option<VersionType>,
    #[serde(rename="_routing", skip_serializing_if="ShouldSkip::should_skip")]
    routing:           Option<String>,
    #[serde(rename="_parent", skip_serializing_if="ShouldSkip::should_skip")]
    parent:            Option<String>,
    #[serde(rename="_timestamp", skip_serializing_if="ShouldSkip::should_skip")]
    timestamp:         Option<String>,
    #[serde(rename="_ttl", skip_serializing_if="ShouldSkip::should_skip")]
    ttl:               Option<Duration>,
    #[serde(rename="_retry_on_conflict", skip_serializing_if="ShouldSkip::should_skip")]
    retry_on_conflict: Option<u64>,
}

#[derive(Serialize)]
pub struct Action<X>(FieldBased<ActionType, ActionOptions, NoOuter>, Option<X>);

impl<S> Action<S>
    where S: Serialize {
    /// An index action.
    ///
    /// Takes the document to be indexed, other parameters can be set as
    /// optional on the `Action` struct returned.
    pub fn index(document: S) -> Self {
        Action(FieldBased::new(ActionType::Index,
                               Default::default(),
                               NoOuter),
               Some(document))
    }

    /// Create action
    pub fn create(document: S) -> Self {
        Action(FieldBased::new(ActionType::Create,
                               Default::default(),
                               NoOuter),
               Some(document))
    }

    /// Add the serialized version of this action to the bulk `String`.
    fn add(&self, actstr: &mut String) -> Result<(), EsError> {
        let command_str = try!(serde_json::to_string(&self.0));

        actstr.push_str(&command_str);
        actstr.push_str("\n");

        match self.1 {
            Some(ref source) => {
                let payload_str = try!(serde_json::to_string(source));
                actstr.push_str(&payload_str);
                actstr.push_str("\n");
            },
            None             => ()
        }
        Ok(())
    }
}

impl<S> Action<S> {
    /// Delete a document based on ID.
    ///
    /// # Example
    ///
    /// ```
    /// use rs_es::operations::bulk::Action;
    ///
    /// let delete_action:Action<()> = Action::delete("doc_id");
    /// let delete_with_index:Action<()> = Action::delete("doc_id").with_index("index_name");
    /// ```
    pub fn delete<A: Into<String>>(id: A) -> Self {
        Action(FieldBased::new(ActionType::Delete,
                               ActionOptions {
                                   id: Some(id.into()),
                                   ..Default::default()
                               },
                               NoOuter),
               None)
    }

    pub fn update<A: Into<String>>(id: A, update: ActionSource) -> Self {
        unimplemented!()
        // Action(FieldBased::new(ActionType::Update,
        //                        ActionOptions {
        //                            id: Some(id.into()),
        //                            ..Default::default()
        //                        },
        //                        NoOuter),
        //        update)
    }

    add_inner_field!(with_index, index, String);
    add_inner_field!(with_doc_type, doc_type, String);
    add_inner_field!(with_id, id, String);
    add_inner_field!(with_version, version, u64);
    // TODO - re-enable
    //add_inner_field!(with_version_type, version_type, VersionType);
    add_inner_field!(with_routing, routing, String);
    add_inner_field!(with_parent, parent, String);
    add_inner_field!(with_timestamp, timestamp, String);
    add_inner_field!(with_ttl, ttl, Duration);
    add_inner_field!(with_retry_on_conflict, retry_on_conflict, u64);
}

pub struct BulkOperation<'a, 'b, S: 'b> {
    client:   &'a mut Client,
    index:    Option<&'b str>,
    doc_type: Option<&'b str>,
    actions:  &'b [Action<S>],
    options:  Options<'b>
}

impl<'a, 'b, S> BulkOperation<'a, 'b, S>
    where S: Serialize {

    pub fn new(client: &'a mut Client, actions: &'b [Action<S>]) -> Self {
        BulkOperation {
            client:   client,
            index:    None,
            doc_type: None,
            actions:  actions,
            options:  Options::new()
        }
    }

    pub fn with_index(&'b mut self, index: &'b str) -> &'b mut Self {
        self.index = Some(index);
        self
    }

    pub fn with_doc_type(&'b mut self, doc_type: &'b str) -> &'b mut Self {
        self.doc_type = Some(doc_type);
        self
    }

    add_option!(with_consistency, "consistency");
    add_option!(with_refresh, "refresh");

    fn format_url(&self) -> String {
        let mut url = String::new();
        url.push_str("/");
        match self.index {
            Some(index) => {
                url.push_str(index);
                url.push_str("/");
            },
            None        => ()
        }
        match self.doc_type {
            Some(doc_type) => {
                url.push_str(doc_type);
                url.push_str("/");
            },
            None           => ()
        }
        url.push_str("_bulk");
        url.push_str(&self.options.to_string());
        url
    }

    fn format_actions(&self) -> String {
        let mut actstr = String::new();
        for action in self.actions {
            action.add(&mut actstr).unwrap();
        }
        actstr
    }

    pub fn send(&'b mut self) -> Result<BulkResult, EsError> {
        //
        // This function does not use the standard GET/POST/DELETE functions of
        // the client, as they serve the happy path of JSON-in/JSON-out, this
        // function does send send JSON in.
        //
        // Various parts of the client are reused where it makes sense.
        //
        let full_url = {
            let url = self.format_url();
            self.client.full_url(&url)
        };
        let body = self.format_actions();
        println!("Sending: {}", body);
        // Doesn't use the standard macros as it's not standard JSON
        let mut result = try!(self.client.http_client
                              .post(&full_url)
                              .body(&body)
                              .send());

        let response = try!(do_req(result));

        match response.status_code() {
            &StatusCode::Ok => Ok(try!(response.read_response())),
            _              => Err(EsError::EsError(format!("Unexpected status: {}", response.status_code())))
        }
    }
}

/// The result of specific actions
pub struct ActionResult {
    pub action: ActionType,
    pub inner: ActionResultInner
}

impl Deserialize for ActionResult {
    fn deserialize<D>(deserializer: &mut D) -> Result<ActionResult, D::Error>
        where D: Deserializer {

        struct ActionResultVisitor;

        impl Visitor for ActionResultVisitor {
            type Value = ActionResult;

            fn visit_map<V>(&mut self, mut visitor: V) -> Result<ActionResult, V::Error>
                where V: MapVisitor {

                let visited:Option<(String, ActionResultInner)> = try!(visitor.visit());
                let (key, value) = match visited {
                    Some((key, value)) => (key, value),
                    None               => return Err(V::Error::custom("expecting at least one field"))
                };
                println!("Found key: {:?}, value: {:?}", key, value);
                try!(visitor.end());

                let result = ActionResult {
                    action: match key.as_ref() {
                        "index" => ActionType::Index,
                        "create" => ActionType::Create,
                        "delete" => ActionType::Delete,
                        "update" => ActionType::Update,
                        _ => {
                            return Err(V::Error::custom(format!("Unrecognised key: {}", key)))
                        }
                    },
                    inner:  value
                };

                Ok(result)
            }
        }

        deserializer.deserialize(ActionResultVisitor)
    }
}

#[derive(Debug, Deserialize)]
pub struct ActionResultInner {
    #[serde(rename="_index")]
    pub index:    String,
    #[serde(rename="_type")]
    pub doc_type: String,
    #[serde(rename="_version")]
    pub version:  u64,
    pub status:   u64,
    #[serde(rename="_shards")]
    pub shards:   ShardCountResult,
    pub found:    bool
}

// TODO - deprecated
// impl<'a> From<&'a Json> for ActionResult {
//     fn from(from: &'a Json) -> ActionResult {
//         info!("ActionResult from: {:?}", from);

//         let d = from.as_object().expect("Not a Json object");
//         assert_eq!(1, d.len());
//         let (key, inner) = d.iter().next().expect("No content");

//         ActionResult {
//             action:   ActionType::from(key),
//             index:    get_json_string!(inner, "_index"),
//             doc_type: get_json_string!(inner, "_type"),
//             version:  get_json_u64!(inner, "_version"),
//             status:   get_json_u64!(inner, "status")
//         }
//     }
// }

/// The result of a bulk operation
#[derive(Deserialize)]
pub struct BulkResult {
    pub errors: bool,
    pub items:  Vec<ActionResult>,
    pub took:   u64
}

// TODO - Deprecated
// impl<'a> From<&'a Json> for BulkResult {
//     fn from(from: &'a Json) -> BulkResult {
//         info!("Bulk result, result: {:?}", from);
//         BulkResult {
//             errors: get_json_bool!(from, "errors"),
//             items:  from.find("items")
//                 .expect("No field called 'items'")
//                 .as_array()
//                 .expect("Field 'items' not expected array")
//                 .iter()
//                 .map(|item| {
//                     ActionResult::from(item)
//                 })
//                 .collect(),
//             took:   get_json_u64!(from, "took")
//         }
//     }
// }
<|endoftext|>


<commit_before>//
// Copyright (c) Pirmin Kalberer. All rights reserved.
// Licensed under the MIT License. See LICENSE file in the project root for full license information.
//

use gdal_ds::GdalDatasource;
use datasource::DatasourceInput;
use core::feature::FeatureAttrValType;
use core::grid::Extent;
use core::grid::Grid;
use core::layer::Layer;

#[test]
fn test_gdal_api() {
    use std::path::Path;
    use gdal::vector::Dataset;

    let mut dataset = Dataset::open(Path::new("natural_earth.gpkg")).unwrap();
    let layer = dataset.layer_by_name("ne_10m_populated_places").unwrap();
    let feature = layer.features().next().unwrap();
    let name_field = feature.field("NAME").unwrap();
    let geometry = feature.geometry();
    assert_eq!(name_field.to_string(),
               Some("Colonia del Sacramento".to_string()));
    assert_eq!(geometry.wkt().unwrap(),
               "POINT (-6438719.62282072 -4093437.71441017)".to_string());
}

#[test]
fn test_gdal_retrieve_points() {
    let mut layer = Layer::new("points");
    layer.table_name = Some(String::from("ne_10m_populated_places"));
    layer.geometry_field = Some(String::from("geom"));
    layer.srid = Some(3857);
    layer.fid_field = Some(String::from("SCALERANK"));
    layer.geometry_type = Some(String::from("POINT"));
    let grid = Grid::web_mercator();
    let extent = Extent {
        minx: 821850.9,
        miny: 5909499.5,
        maxx: 860986.7,
        maxy: 5948635.3,
    };

    let ds = GdalDatasource::new("natural_earth.gpkg");
    let mut reccnt = 0;
    ds.retrieve_features(&layer, &extent, 10, &grid, |feat| {
        if reccnt == 0 {
            assert_eq!("Ok(Point(Point { x: -6438719.622820721, y: -4093437.7144101723, srid: Some(3857) }))",
                       &*format!("{:?}", feat.geometry()));
        }
        assert_eq!(3, feat.attributes().len());
        assert_eq!(feat.attributes()[0].key, "SCALERANK");
        assert_eq!(feat.attributes()[1].key, "NAME");
        assert_eq!(feat.attributes()[2].key, "POP_MAX");
        if reccnt == 0 {
            assert_eq!(feat.attributes()[0].value, FeatureAttrValType::Int(10));
            assert_eq!(feat.attributes()[1].value,
                       FeatureAttrValType::String("Colonia del Sacramento".to_string()));
            assert_eq!(feat.attributes()[2].value, FeatureAttrValType::Int(21714));
            assert_eq!(Some(10), feat.fid());
        }
        reccnt += 1;
    });
    assert_eq!(14644, reccnt);
}

#[test]
fn test_gdal_retrieve_multilines() {
    let mut layer = Layer::new("multilines");
    layer.table_name = Some(String::from("ne_10m_rivers_lake_centerlines"));
    layer.geometry_field = Some(String::from("geom"));
    layer.srid = Some(3857);
    layer.geometry_type = Some(String::from("MULTILINE"));
    let grid = Grid::web_mercator();
    let extent = Extent {
        minx: 821850.9,
        miny: 5909499.5,
        maxx: 860986.7,
        maxy: 5948635.3,
    };

    let ds = GdalDatasource::new("natural_earth.gpkg");
    let mut reccnt = 0;
    ds.retrieve_features(&layer, &extent, 10, &grid, |feat| {
        if reccnt == 1398 {
            assert_eq!("Ok(MultiLineString(MultiLineStringT { lines: [LineStringT { points: [Point { x: 524057.9769470511, y: 6406083.8740046825, srid: Some(3857) }, Point { x: 523360.4182889238, y: 6406083.8740046825, srid: Some(3857) }], srid: Some(3857) }], srid: Some(3857) }))",
                       &*format!("{:?}", feat.geometry()));
        }
        assert_eq!(2, feat.attributes().len());
        assert_eq!(feat.attributes()[0].key, "scalerank");
        assert_eq!(feat.attributes()[1].key, "name");
        if reccnt == 1398 {
            assert_eq!(feat.attributes()[0].value, FeatureAttrValType::Double(8.0));
            assert_eq!(feat.attributes()[1].value,
                       FeatureAttrValType::String("Meuse".to_string()));
            assert_eq!(None, feat.fid());
        }
        reccnt += 1;
    });
    assert_eq!(1404, reccnt);
}

#[test]
fn test_gdal_retrieve_multipolys() {
    let mut layer = Layer::new("multipolys");
    layer.table_name = Some(String::from("ne_110m_admin_0_countries"));
    layer.geometry_field = Some(String::from("geom"));
    layer.srid = Some(3857);
    layer.geometry_type = Some(String::from("MULTIPOLYGON"));
    let grid = Grid::web_mercator();
    let extent = Extent {
        minx: 821850.9,
        miny: 5909499.5,
        maxx: 860986.7,
        maxy: 5948635.3,
    };

    let ds = GdalDatasource::new("natural_earth.gpkg");
    let mut reccnt = 0;
    ds.retrieve_features(&layer, &extent, 10, &grid, |feat| {
        if reccnt == 97 {
            assert_eq!("Ok(MultiPolygon(MultiPolygonT { polygons: [PolygonT { rings: [LineStringT { points: [Point { x: 672711.8490145913, y: 6468481.737381289, srid: Some(3857) }, Point { x: 694939.8727280691, y: 6429360.233285289, srid: Some(3857) }, Point { x: 688658.0399394701, y: 6353928.606103209, srid: Some(3857) }, Point { x: 656535.5543245667, y: 6350309.278097487, srid: Some(3857) }, Point { x: 631632.5743412259, y: 6365185.930146941, srid: Some(3857) }, Point { x: 643695.764229205, y: 6461933.756740372, srid: Some(3857) }, Point { x: 672711.8490145913, y: 6468481.737381289, srid: Some(3857) }], srid: Some(3857) }], srid: Some(3857) }], srid: Some(3857) }))",
                       &*format!("{:?}", feat.geometry()));
        }
        assert_eq!(2, feat.attributes().len());
        assert_eq!(feat.attributes()[0].key, "name");
        assert_eq!(feat.attributes()[1].key, "iso_a3");
        if reccnt == 97 {
            assert_eq!(feat.attributes()[0].value, FeatureAttrValType::String("Luxembourg".to_string()));
            assert_eq!(feat.attributes()[1].value,
                       FeatureAttrValType::String("LUX".to_string()));
            assert_eq!(None, feat.fid());
        }
        reccnt += 1;
    });
    assert_eq!(177, reccnt);
}
<commit_msg>Handle floating point difference on x86_64-apple-darwin<commit_after>//
// Copyright (c) Pirmin Kalberer. All rights reserved.
// Licensed under the MIT License. See LICENSE file in the project root for full license information.
//

use gdal_ds::GdalDatasource;
use datasource::DatasourceInput;
use core::feature::FeatureAttrValType;
use core::grid::Extent;
use core::grid::Grid;
use core::layer::Layer;

#[test]
fn test_gdal_api() {
    use std::path::Path;
    use gdal::vector::Dataset;

    let mut dataset = Dataset::open(Path::new("natural_earth.gpkg")).unwrap();
    let layer = dataset.layer_by_name("ne_10m_populated_places").unwrap();
    let feature = layer.features().next().unwrap();
    let name_field = feature.field("NAME").unwrap();
    let geometry = feature.geometry();
    assert_eq!(name_field.to_string(),
               Some("Colonia del Sacramento".to_string()));
    #[cfg(not(target_os = "macos"))]
    assert_eq!(geometry.wkt().unwrap(),
               "POINT (-6438719.62282072 -4093437.71441017)".to_string());
    #[cfg(target_os = "macos")]
    assert_eq!(geometry.wkt().unwrap(),
               "POINT (-6438719.622820721007884 -4093437.714410172309726)".to_string());
}

#[cfg(not(target_os = "macos"))] // ? ERROR 1: pragma integrity_check on 'natural_earth.gpkg' failed
#[test]
fn test_gdal_retrieve_points() {
    let mut layer = Layer::new("points");
    layer.table_name = Some(String::from("ne_10m_populated_places"));
    layer.geometry_field = Some(String::from("geom"));
    layer.srid = Some(3857);
    layer.fid_field = Some(String::from("SCALERANK"));
    layer.geometry_type = Some(String::from("POINT"));
    let grid = Grid::web_mercator();
    let extent = Extent {
        minx: 821850.9,
        miny: 5909499.5,
        maxx: 860986.7,
        maxy: 5948635.3,
    };

    let ds = GdalDatasource::new("natural_earth.gpkg");
    let mut reccnt = 0;
    ds.retrieve_features(&layer, &extent, 10, &grid, |feat| {
        if reccnt == 0 {
            #[cfg(not(target_os = "macos"))]
            assert_eq!("Ok(Point(Point { x: -6438719.622820721, y: -4093437.7144101723, srid: Some(3857) }))",
                       &*format!("{:?}", feat.geometry()));
            #[cfg(target_os = "macos")]
            assert_eq!("Ok(Point(Point { x: -6438719.622820721007884, y: -4093437.714410172309726, srid: Some(3857) }))",
                       &*format!("{:?}", feat.geometry()));
        }
        assert_eq!(3, feat.attributes().len());
        assert_eq!(feat.attributes()[0].key, "SCALERANK");
        assert_eq!(feat.attributes()[1].key, "NAME");
        assert_eq!(feat.attributes()[2].key, "POP_MAX");
        if reccnt == 0 {
            assert_eq!(feat.attributes()[0].value, FeatureAttrValType::Int(10));
            assert_eq!(feat.attributes()[1].value,
                       FeatureAttrValType::String("Colonia del Sacramento".to_string()));
            assert_eq!(feat.attributes()[2].value, FeatureAttrValType::Int(21714));
            assert_eq!(Some(10), feat.fid());
        }
        reccnt += 1;
    });
    assert_eq!(14644, reccnt);
}

#[test]
fn test_gdal_retrieve_multilines() {
    let mut layer = Layer::new("multilines");
    layer.table_name = Some(String::from("ne_10m_rivers_lake_centerlines"));
    layer.geometry_field = Some(String::from("geom"));
    layer.srid = Some(3857);
    layer.geometry_type = Some(String::from("MULTILINE"));
    let grid = Grid::web_mercator();
    let extent = Extent {
        minx: 821850.9,
        miny: 5909499.5,
        maxx: 860986.7,
        maxy: 5948635.3,
    };

    let ds = GdalDatasource::new("natural_earth.gpkg");
    let mut reccnt = 0;
    ds.retrieve_features(&layer, &extent, 10, &grid, |feat| {
        if reccnt == 1398 {
            assert_eq!("Ok(MultiLineString(MultiLineStringT { lines: [LineStringT { points: [Point { x: 524057.9769470511, y: 6406083.8740046825, srid: Some(3857) }, Point { x: 523360.4182889238, y: 6406083.8740046825, srid: Some(3857) }], srid: Some(3857) }], srid: Some(3857) }))",
                       &*format!("{:?}", feat.geometry()));
        }
        assert_eq!(2, feat.attributes().len());
        assert_eq!(feat.attributes()[0].key, "scalerank");
        assert_eq!(feat.attributes()[1].key, "name");
        if reccnt == 1398 {
            assert_eq!(feat.attributes()[0].value, FeatureAttrValType::Double(8.0));
            assert_eq!(feat.attributes()[1].value,
                       FeatureAttrValType::String("Meuse".to_string()));
            assert_eq!(None, feat.fid());
        }
        reccnt += 1;
    });
    assert_eq!(1404, reccnt);
}

#[test]
fn test_gdal_retrieve_multipolys() {
    let mut layer = Layer::new("multipolys");
    layer.table_name = Some(String::from("ne_110m_admin_0_countries"));
    layer.geometry_field = Some(String::from("geom"));
    layer.srid = Some(3857);
    layer.geometry_type = Some(String::from("MULTIPOLYGON"));
    let grid = Grid::web_mercator();
    let extent = Extent {
        minx: 821850.9,
        miny: 5909499.5,
        maxx: 860986.7,
        maxy: 5948635.3,
    };

    let ds = GdalDatasource::new("natural_earth.gpkg");
    let mut reccnt = 0;
    ds.retrieve_features(&layer, &extent, 10, &grid, |feat| {
        if reccnt == 97 {
            assert_eq!("Ok(MultiPolygon(MultiPolygonT { polygons: [PolygonT { rings: [LineStringT { points: [Point { x: 672711.8490145913, y: 6468481.737381289, srid: Some(3857) }, Point { x: 694939.8727280691, y: 6429360.233285289, srid: Some(3857) }, Point { x: 688658.0399394701, y: 6353928.606103209, srid: Some(3857) }, Point { x: 656535.5543245667, y: 6350309.278097487, srid: Some(3857) }, Point { x: 631632.5743412259, y: 6365185.930146941, srid: Some(3857) }, Point { x: 643695.764229205, y: 6461933.756740372, srid: Some(3857) }, Point { x: 672711.8490145913, y: 6468481.737381289, srid: Some(3857) }], srid: Some(3857) }], srid: Some(3857) }], srid: Some(3857) }))",
                       &*format!("{:?}", feat.geometry()));
        }
        assert_eq!(2, feat.attributes().len());
        assert_eq!(feat.attributes()[0].key, "name");
        assert_eq!(feat.attributes()[1].key, "iso_a3");
        if reccnt == 97 {
            assert_eq!(feat.attributes()[0].value, FeatureAttrValType::String("Luxembourg".to_string()));
            assert_eq!(feat.attributes()[1].value,
                       FeatureAttrValType::String("LUX".to_string()));
            assert_eq!(None, feat.fid());
        }
        reccnt += 1;
    });
    assert_eq!(177, reccnt);
}
<|endoftext|>


<commit_before>use parse;
use serde::json;
use std::collections::btree_map::BTreeMap;
use std::fs::File;
use std::io::Read;

impl parse::Value {
    fn to_json(&self) -> json::Value {
        let maybe_s = match *self {
            parse::Value::DataFile(ref path)    => slurp(path),
            parse::Value::DataString(ref s)     => Some(s.to_string()),
            parse::Value::FormFile(ref path)    => slurp(path),
            parse::Value::HttpHeader(ref s)     => Some(s.to_string()),
            parse::Value::RawJsonString(ref s)  => Some(s.to_string()),
            parse::Value::RawJsonFile(ref path) => slurp(path).and_then(|s| json::from_str(&s).ok()),
            parse::Value::UrlParam(ref s)       => Some(s.to_string()),
        };
        match maybe_s {
            Some(s) => json::Value::String(s),
            None    => json::Value::Null,
        }
    }
}

pub fn to_json(pairs: &[parse::Pair]) -> String {
    let mut map = BTreeMap::new();
    for pair in pairs.iter() {
        let k = pair.0.clone();
        let v = pair.1.clone().map(|v| v.to_json()).unwrap_or(json::Value::Null);
        map.insert(k, v);
    }
    let root = json::Value::Object(map);
    json::ser::to_string(&root).unwrap_or("".to_string())
}

fn slurp(path: &str) -> Option<String> {
    File::open(path).ok().as_mut().and_then(|f| {
        let mut s = String::new();
        f.read_to_string(&mut s).ok().and_then(|_| Some(s))
    })
}
<commit_msg>slurp() returns io::Result<commit_after>use parse;
use serde::json;
use std::collections::btree_map::BTreeMap;
use std::fs::File;
use std::io::{self, Read};

impl parse::Value {
    fn to_json(&self) -> json::Value {
        let maybe_s = match *self {
            parse::Value::DataFile(ref path)    => slurp(path).ok(),
            parse::Value::DataString(ref s)     => Some(s.to_string()),
            parse::Value::FormFile(ref path)    => slurp(path).ok(),
            parse::Value::HttpHeader(ref s)     => Some(s.to_string()),
            parse::Value::RawJsonString(ref s)  => Some(s.to_string()),
            parse::Value::RawJsonFile(ref path) => slurp(path).ok().and_then(|s| json::from_str(&s).ok()),
            parse::Value::UrlParam(ref s)       => Some(s.to_string()),
        };
        match maybe_s {
            Some(s) => json::Value::String(s),
            None    => json::Value::Null,
        }
    }
}

pub fn to_json(pairs: &[parse::Pair]) -> String {
    let mut map = BTreeMap::new();
    for pair in pairs.iter() {
        let k = pair.0.clone();
        let v = pair.1.clone().map(|v| v.to_json()).unwrap_or(json::Value::Null);
        map.insert(k, v);
    }
    let root = json::Value::Object(map);
    json::ser::to_string(&root).unwrap_or("".to_string())
}

fn slurp(path: &str) -> io::Result<String> {
    let mut f = try!(File::open(path));
    let mut s = String::new();
    try!(f.read_to_string(&mut s));
    Ok(s)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#[cfg(target_arch="x86_64")]
#[path="linux-x86_64/mod.rs"]
pub mod linux;

#[cfg(target_arch="x86")]
#[path="linux-x86/mod.rs"]
pub mod linux;

#[cfg(target_arch="aarch64")]
#[path="linux-aarch64/mod.rs"]
pub mod linux;

#[cfg(target_arch="arm")]
#[path="linux-arm/mod.rs"]
pub mod linux;<commit_msg>syscall_defines: fix style by running cargo fmt<commit_after>// Copyright 2017 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#[cfg(target_arch = "x86_64")]
#[path = "linux-x86_64/mod.rs"]
pub mod linux;

#[cfg(target_arch = "x86")]
#[path = "linux-x86/mod.rs"]
pub mod linux;

#[cfg(target_arch = "aarch64")]
#[path = "linux-aarch64/mod.rs"]
pub mod linux;

#[cfg(target_arch = "arm")]
#[path = "linux-arm/mod.rs"]
pub mod linux;
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

use std::convert::TryFrom;

use crate::session;
use crate::action;

/// Untyped request to execute an action.
///
/// It is called "demand" rather than "request" here, as requests should be more
/// structured. The demand is only a slightly more structured variant of the
/// raw Protocol Buffers message with actual request still being in a serialized
/// form.
///
/// The request has to stay serialized because, until dispatched to a specific
/// handler, the type into which it should be deserialized remains unknown.
#[derive(Clone, Debug)]
pub struct Demand {
    /// A name of the action to execute.
    pub action: String,
    /// A demand metadata.
    pub header: Header,
    /// Serialized action request.
    pub payload: Payload,
}

/// Metadata about the demand issued by the server.
#[derive(Clone, Debug)]
pub struct Header {
    /// A server-issued session identifier (usually corresponds to a flow).
    pub session_id: String,
    /// A server-issued request identifier.
    pub request_id: u64,
}

/// Serialized request data for the action handler.
#[derive(Clone, Debug)]
pub struct Payload {
    /// Raw bytes of the serialized request.
    pub data: Option<Vec<u8>>,
}

impl Payload {

    /// Parses the serialized data to a specific request type.
    ///
    /// If the payload contains no data, a default request instance is created.
    ///
    /// If the data is malformed and fails to correctly parse to a specific
    /// request instance, an error is returned.
    pub fn parse<R>(&self) -> Result<R, session::ParseError>
    where
        R: action::Request,
    {
        let proto = match &self.data {
            Some(ref bytes) => prost::Message::decode(&bytes[..])?,
            None => Default::default(),
        };

        Ok(R::from_proto(proto))
    }
}

impl TryFrom<rrg_proto::GrrMessage> for Demand {

    type Error = session::ParseError;

    fn try_from(message: rrg_proto::GrrMessage) -> Result<Demand, Self::Error> {
        use session::ParseError;

        let header = Header {
            session_id: message.session_id.ok_or(ParseError::missing_field("session id"))?,
            request_id: message.request_id.ok_or(ParseError::missing_field("request id"))?,
        };

        Ok(Demand {
            action: message.name.ok_or(ParseError::missing_field("action name"))?,
            header: header,
            payload: Payload {
                data: message.args,
            },
        })
    }
}

<commit_msg>Make demand parsing fit into the 80 column limit.<commit_after>// Copyright 2020 Google LLC
//
// Use of this source code is governed by an MIT-style license that can be found
// in the LICENSE file or at https://opensource.org/licenses/MIT.

use std::convert::TryFrom;

use crate::session;
use crate::action;

/// Untyped request to execute an action.
///
/// It is called "demand" rather than "request" here, as requests should be more
/// structured. The demand is only a slightly more structured variant of the
/// raw Protocol Buffers message with actual request still being in a serialized
/// form.
///
/// The request has to stay serialized because, until dispatched to a specific
/// handler, the type into which it should be deserialized remains unknown.
#[derive(Clone, Debug)]
pub struct Demand {
    /// A name of the action to execute.
    pub action: String,
    /// A demand metadata.
    pub header: Header,
    /// Serialized action request.
    pub payload: Payload,
}

/// Metadata about the demand issued by the server.
#[derive(Clone, Debug)]
pub struct Header {
    /// A server-issued session identifier (usually corresponds to a flow).
    pub session_id: String,
    /// A server-issued request identifier.
    pub request_id: u64,
}

/// Serialized request data for the action handler.
#[derive(Clone, Debug)]
pub struct Payload {
    /// Raw bytes of the serialized request.
    pub data: Option<Vec<u8>>,
}

impl Payload {

    /// Parses the serialized data to a specific request type.
    ///
    /// If the payload contains no data, a default request instance is created.
    ///
    /// If the data is malformed and fails to correctly parse to a specific
    /// request instance, an error is returned.
    pub fn parse<R>(&self) -> Result<R, session::ParseError>
    where
        R: action::Request,
    {
        let proto = match &self.data {
            Some(ref bytes) => prost::Message::decode(&bytes[..])?,
            None => Default::default(),
        };

        Ok(R::from_proto(proto))
    }
}

impl TryFrom<rrg_proto::GrrMessage> for Demand {

    type Error = session::ParseError;

    fn try_from(message: rrg_proto::GrrMessage) -> Result<Demand, Self::Error> {
        let missing = session::ParseError::missing_field;

        let header = Header {
            session_id: message.session_id.ok_or(missing("session id"))?,
            request_id: message.request_id.ok_or(missing("request id"))?,
        };

        Ok(Demand {
            action: message.name.ok_or(missing("action name"))?,
            header: header,
            payload: Payload {
                data: message.args,
            },
        })
    }
}

<|endoftext|>


<commit_before>/**
Constructs a list of css style rules from a token stream
*/

// TODO: fail according to the css spec instead of failing when things
// are not as expected

use values::*;
use util::{DataStream, DataStreamFactory};
use std::cell::Cell;
use netsurfcss::stylesheet::{CssStylesheetParams, CssStylesheetParamsVersion1, css_stylesheet_create};
use netsurfcss::types::CssLevel21;
use netsurfcss::CssResult;
use wapcaplet::LwcString;

// This takes a DataStreamFactory instead of a DataStream because
// servo's DataStream contains a comm::Port, which is not sendable,
// so DataStream is an @fn which can't be sent to the lexer task.
// So the DataStreamFactory gives the caller an opportunity to create
// the data stream from inside the lexer task.
pub fn parse_stylesheet(url: Url, input_factory: DataStreamFactory) -> Stylesheet {
    let params: CssStylesheetParams = CssStylesheetParams {
        params_version: CssStylesheetParamsVersion1,
        level: CssLevel21,
        charset: ~"UTF-8",
        url: url.to_str(),
        title: ~"FIXME-css-title",
        allow_quirks: false,
        inline_style: false,
        resolve: Some(resolve_url),
        import: None,
        color: None,
        font: None,
    };
    let sheet = css_stylesheet_create(move params);

    let input = input_factory();
    loop {
        match input() {
            Some(move data) => {
                sheet.append_data(data);
            }
            None => break
        }
    }
    sheet.data_done();
    return move sheet;
}

fn resolve_url(_base: &str, _rel: &LwcString) -> CssResult<LwcString> {
    fail ~"resolving url";
}
<commit_msg>Fix build<commit_after>/**
Constructs a list of css style rules from a token stream
*/

// TODO: fail according to the css spec instead of failing when things
// are not as expected

use values::*;
use util::{DataStream, DataStreamFactory};
use std::cell::Cell;
use netsurfcss::stylesheet::{CssStylesheetParams, CssStylesheetParamsVersion1, css_stylesheet_create};
use netsurfcss::types::CssLevel21;
use netsurfcss::CssResult;
use wapcaplet::LwcString;

// This takes a DataStreamFactory instead of a DataStream because
// servo's DataStream contains a comm::Port, which is not sendable,
// so DataStream is an @fn which can't be sent to the lexer task.
// So the DataStreamFactory gives the caller an opportunity to create
// the data stream from inside the lexer task.
pub fn parse_stylesheet(url: Url, input_factory: DataStreamFactory) -> Stylesheet {
    let params: CssStylesheetParams = CssStylesheetParams {
        params_version: CssStylesheetParamsVersion1,
        level: CssLevel21,
        charset: ~"UTF-8",
        url: url.to_str(),
        title: ~"FIXME-css-title",
        allow_quirks: false,
        inline_style: false,
        resolve: Some(resolve_url),
        import: None,
        color: None,
        font: None,
    };
    let mut sheet = css_stylesheet_create(&params);

    let input = input_factory();
    loop {
        match input() {
            Some(move data) => {
                sheet.append_data(data);
            }
            None => break
        }
    }
    sheet.data_done();
    return move sheet;
}

fn resolve_url(_base: &str, _rel: &LwcString) -> CssResult<LwcString> {
    fail ~"resolving url";
}
<|endoftext|>


<commit_before>use std::env;
use std::path::PathBuf;

use cratesfyi::db::{self, add_path_into_database, connect_db};
use cratesfyi::utils::{add_crate_to_queue, remove_crate_priority, set_crate_priority};
use cratesfyi::Server;
use cratesfyi::{DocBuilder, DocBuilderOptions, RustwideBuilder};
use structopt::StructOpt;

pub fn main() {
    logger_init();

    CommandLine::from_args().handle_args();
}

fn logger_init() {
    use std::io::Write;

    let mut builder = env_logger::Builder::new();
    builder.format(|buf, record| {
        writeln!(
            buf,
            "{} [{}] {}: {}",
            time::now().strftime("%Y/%m/%d %H:%M:%S").unwrap(),
            record.level(),
            record.target(),
            record.args()
        )
    });
    builder.parse_filters(
        env::var("RUST_LOG")
            .ok()
            .as_deref()
            .unwrap_or("cratesfyi=info"),
    );

    rustwide::logging::init_with(builder.build());
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
#[structopt(
    name = "cratesfyi",
    about = env!("CARGO_PKG_DESCRIPTION"),
    version = cratesfyi::BUILD_VERSION,
    rename_all = "kebab-case",
)]
enum CommandLine {
    Build(Build),

    /// Starts web server
    StartWebServer {
        #[structopt(name = "SOCKET_ADDR", default_value = "0.0.0.0:3000")]
        socket_addr: String,
    },

    /// Starts cratesfyi daemon
    Daemon {
        /// Run the server in the foreground instead of detaching a child
        #[structopt(name = "FOREGROUND", short = "f", long = "foreground")]
        foreground: bool,
    },

    /// Database operations
    Database {
        #[structopt(subcommand)]
        subcommand: DatabaseSubcommand,
    },

    /// Interactions with the build queue
    Queue {
        #[structopt(subcommand)]
        subcommand: QueueSubcommand,
    },
}

impl CommandLine {
    pub fn handle_args(self) {
        match self {
            Self::Build(build) => build.handle_args(),
            Self::StartWebServer { socket_addr } => {
                Server::start(Some(&socket_addr));
            }
            Self::Daemon { foreground } => cratesfyi::utils::start_daemon(!foreground),
            Self::Database { subcommand } => subcommand.handle_args(),
            Self::Queue { subcommand } => subcommand.handle_args(),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
enum QueueSubcommand {
    /// Add a crate to the build queue
    Add {
        /// Name of crate to build
        #[structopt(name = "CRATE_NAME")]
        crate_name: String,
        /// Version of crate to build
        #[structopt(name = "CRATE_VERSION")]
        crate_version: String,
        /// Priority of build (new crate builds get priority 0)
        #[structopt(
            name = "BUILD_PRIORITY",
            short = "p",
            long = "priority",
            default_value = "5"
        )]
        build_priority: i32,
    },

    /// Interactions with build queue priorities
    DefaultPriority {
        #[structopt(subcommand)]
        subcommand: PrioritySubcommand,
    },
}

impl QueueSubcommand {
    pub fn handle_args(self) {
        match self {
            Self::Add {
                crate_name,
                crate_version,
                build_priority,
            } => {
                let conn = connect_db().expect("Could not connect to database");
                add_crate_to_queue(&conn, &crate_name, &crate_version, build_priority)
                    .expect("Could not add crate to queue");
            }

            Self::DefaultPriority { subcommand } => subcommand.handle_args(),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
enum PrioritySubcommand {
    /// Set all crates matching a pattern to a priority level
    Set {
        /// See https://www.postgresql.org/docs/current/functions-matching.html for pattern syntax
        #[structopt(name = "PATTERN")]
        pattern: String,
        /// The priority to give crates matching the given `PATTERN`
        priority: i32,
    },

    /// Remove the prioritization of crates for a pattern
    Remove {
        /// See https://www.postgresql.org/docs/current/functions-matching.html for pattern syntax
        #[structopt(name = "PATTERN")]
        pattern: String,
    },
}

impl PrioritySubcommand {
    pub fn handle_args(self) {
        match self {
            Self::Set { pattern, priority } => {
                let conn = connect_db().expect("Could not connect to the database");

                set_crate_priority(&conn, &pattern, priority)
                    .expect("Could not set pattern's priority");
            }

            Self::Remove { pattern } => {
                let conn = connect_db().expect("Could not connect to the database");

                if let Some(priority) = remove_crate_priority(&conn, &pattern)
                    .expect("Could not remove pattern's priority")
                {
                    println!("Removed pattern with priority {}", priority);
                } else {
                    println!("Pattern did not exist and so was not removed");
                }
            }
        }
    }
}

/// Builds documentation in a chroot environment
#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
#[structopt(rename_all = "kebab-case")]
struct Build {
    #[structopt(
        name = "PREFIX",
        short = "P",
        long = "prefix",
        env = "CRATESFYI_PREFIX"
    )]
    prefix: PathBuf,

    /// Sets crates.io-index path
    #[structopt(name = "CRATES_IO_INDEX_PATH", long = "crates-io-index-path")]
    crates_io_index_path: Option<PathBuf>,

    /// Skips building documentation if documentation exists
    #[structopt(name = "SKIP_IF_EXISTS", short = "s", long = "skip")]
    skip_if_exists: bool,

    /// Skips building documentation if build log exists
    #[structopt(name = "SKIP_IF_LOG_EXISTS", long = "skip-if-log-exists")]
    skip_if_log_exists: bool,

    /// Keeps build directory after build.
    #[structopt(
        name = "KEEP_BUILD_DIRECTORY",
        short = "k",
        long = "keep-build-directory"
    )]
    keep_build_directory: bool,

    #[structopt(subcommand)]
    subcommand: BuildSubcommand,
}

impl Build {
    pub fn handle_args(self) {
        let docbuilder = {
            let mut doc_options = DocBuilderOptions::from_prefix(self.prefix);

            if let Some(crates_io_index_path) = self.crates_io_index_path {
                doc_options.crates_io_index_path = crates_io_index_path;
            }

            doc_options.skip_if_exists = self.skip_if_exists;
            doc_options.skip_if_log_exists = self.skip_if_log_exists;
            doc_options.keep_build_directory = self.keep_build_directory;

            doc_options
                .check_paths()
                .expect("The given paths were invalid");

            DocBuilder::new(doc_options)
        };

        self.subcommand.handle_args(docbuilder);
    }
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
enum BuildSubcommand {
    /// Builds documentation of every crate
    World,

    /// Builds documentation for a crate
    Crate {
        /// Crate name
        #[structopt(
            name = "CRATE_NAME",
            required_unless("LOCAL"),
            requires("CRATE_VERSION")
        )]
        crate_name: String,

        /// Version of crate
        #[structopt(name = "CRATE_VERSION")]
        crate_version: String,

        /// Build a crate at a specific path
        #[structopt(short = "l", long = "local", conflicts_with_all(&["CRATE_NAME", "CRATE_VERSION"]))]
        local: Option<PathBuf>,
    },

    /// update the currently installed rustup toolchain
    UpdateToolchain {
        /// Update the toolchain only if no toolchain is currently installed
        #[structopt(name = "ONLY_FIRST_TIME", long = "only-first-time")]
        only_first_time: bool,
    },

    /// Adds essential files for the installed version of rustc
    AddEssentialFiles,

    /// Locks cratesfyi daemon to stop building new crates
    Lock,

    /// Unlocks cratesfyi daemon to continue building new crates
    Unlock,

    PrintOptions,
}

impl BuildSubcommand {
    pub fn handle_args(self, mut docbuilder: DocBuilder) {
        match self {
            Self::World => {
                docbuilder.load_cache().expect("Failed to load cache");

                let mut builder = RustwideBuilder::init().unwrap();
                builder
                    .build_world(&mut docbuilder)
                    .expect("Failed to build world");

                docbuilder.save_cache().expect("Failed to save cache");
            }

            Self::Crate {
                crate_name,
                crate_version,
                local,
            } => {
                docbuilder.load_cache().expect("Failed to load cache");
                let mut builder = RustwideBuilder::init().expect("failed to initialize rustwide");

                if let Some(path) = local {
                    builder
                        .build_local_package(&mut docbuilder, &path)
                        .expect("Building documentation failed");
                } else {
                    builder
                        .build_package(&mut docbuilder, &crate_name, &crate_version, None)
                        .expect("Building documentation failed");
                }

                docbuilder.save_cache().expect("Failed to save cache");
            }

            Self::UpdateToolchain { only_first_time } => {
                if only_first_time {
                    let conn = db::connect_db().unwrap();
                    let res = conn
                        .query("SELECT * FROM config WHERE name = 'rustc_version';", &[])
                        .unwrap();

                    if !res.is_empty() {
                        println!("update-toolchain was already called in the past, exiting");
                        return;
                    }
                }

                let mut builder = RustwideBuilder::init().unwrap();
                builder
                    .update_toolchain()
                    .expect("failed to update toolchain");
            }

            Self::AddEssentialFiles => {
                let mut builder = RustwideBuilder::init().unwrap();
                builder
                    .add_essential_files()
                    .expect("failed to add essential files");
            }

            Self::Lock => docbuilder.lock().expect("Failed to lock"),
            Self::Unlock => docbuilder.unlock().expect("Failed to unlock"),
            Self::PrintOptions => println!("{:?}", docbuilder.options()),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
enum DatabaseSubcommand {
    /// Run database migrations
    Migrate {
        /// The database version to migrate to
        #[structopt(name = "VERSION")]
        version: Option<i64>,
    },

    /// Updates github stats for crates.
    UpdateGithubFields,

    AddDirectory {
        /// Path of file or directory
        #[structopt(name = "DIRECTORY")]
        directory: PathBuf,
        /// Prefix of files in database
        #[structopt(name = "PREFIX", env = "CRATESFYI_PREFIX")]
        prefix: String,
    },

    /// Updates monthly release activity chart
    UpdateReleaseActivity,

    /// Updates search index
    UpdateSearchIndex,

    /// Removes a whole crate from the database
    DeleteCrate {
        /// Name of the crate to delete
        #[structopt(name = "CRATE_NAME")]
        crate_name: String,
    },

    /// Blacklist operations
    Blacklist {
        #[structopt(subcommand)]
        command: BlacklistSubcommand,
    },
}

impl DatabaseSubcommand {
    pub fn handle_args(self) {
        match self {
            Self::Migrate { version } => {
                let conn = connect_db().expect("failed to connect to the database");
                db::migrate(version, &conn).expect("Failed to run database migrations");
            }

            Self::UpdateGithubFields => {
                cratesfyi::utils::github_updater().expect("Failed to update github fields");
            }

            Self::AddDirectory { directory, prefix } => {
                let conn = db::connect_db().expect("failed to connect to the database");
                add_path_into_database(&conn, &prefix, directory)
                    .expect("Failed to add directory into database");
            }

            // FIXME: This is actually util command not database
            Self::UpdateReleaseActivity => cratesfyi::utils::update_release_activity()
                .expect("Failed to update release activity"),

            Self::UpdateSearchIndex => {
                let conn = db::connect_db().expect("failed to connect to the database");
                db::update_search_index(&conn).expect("Failed to update search index");
            }

            Self::DeleteCrate { crate_name } => {
                let conn = db::connect_db().expect("failed to connect to the database");
                db::delete_crate(&conn, &crate_name).expect("failed to delete the crate");
            }

            Self::Blacklist { command } => command.handle_args(),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
enum BlacklistSubcommand {
    /// List all crates on the blacklist
    List,

    /// Add a crate to the blacklist
    Add {
        /// Crate name
        #[structopt(name = "CRATE_NAME")]
        crate_name: String,
    },

    /// Remove a crate from the blacklist
    Remove {
        /// Crate name
        #[structopt(name = "CRATE_NAME")]
        crate_name: String,
    },
}

impl BlacklistSubcommand {
    fn handle_args(self) {
        let conn = db::connect_db().expect("failed to connect to the database");

        match self {
            Self::List => {
                let crates =
                    db::blacklist::list_crates(&conn).expect("failed to list crates on blacklist");

                println!("{}", crates.join("\n"));
            }

            Self::Add { crate_name } => db::blacklist::add_crate(&conn, &crate_name)
                .expect("failed to add crate to blacklist"),

            Self::Remove { crate_name } => db::blacklist::remove_crate(&conn, &crate_name)
                .expect("failed to remove crate from blacklist"),
        }
    }
}
<commit_msg>Lost dotenv along the way<commit_after>use std::env;
use std::path::PathBuf;

use cratesfyi::db::{self, add_path_into_database, connect_db};
use cratesfyi::utils::{add_crate_to_queue, remove_crate_priority, set_crate_priority};
use cratesfyi::Server;
use cratesfyi::{DocBuilder, DocBuilderOptions, RustwideBuilder};
use structopt::StructOpt;

pub fn main() {
    let _ = dotenv::dotenv();
    logger_init();

    CommandLine::from_args().handle_args();
}

fn logger_init() {
    use std::io::Write;

    let mut builder = env_logger::Builder::new();
    builder.format(|buf, record| {
        writeln!(
            buf,
            "{} [{}] {}: {}",
            time::now().strftime("%Y/%m/%d %H:%M:%S").unwrap(),
            record.level(),
            record.target(),
            record.args()
        )
    });
    builder.parse_filters(
        env::var("RUST_LOG")
            .ok()
            .as_deref()
            .unwrap_or("cratesfyi=info"),
    );

    rustwide::logging::init_with(builder.build());
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
#[structopt(
    name = "cratesfyi",
    about = env!("CARGO_PKG_DESCRIPTION"),
    version = cratesfyi::BUILD_VERSION,
    rename_all = "kebab-case",
)]
enum CommandLine {
    Build(Build),

    /// Starts web server
    StartWebServer {
        #[structopt(name = "SOCKET_ADDR", default_value = "0.0.0.0:3000")]
        socket_addr: String,
    },

    /// Starts cratesfyi daemon
    Daemon {
        /// Run the server in the foreground instead of detaching a child
        #[structopt(name = "FOREGROUND", short = "f", long = "foreground")]
        foreground: bool,
    },

    /// Database operations
    Database {
        #[structopt(subcommand)]
        subcommand: DatabaseSubcommand,
    },

    /// Interactions with the build queue
    Queue {
        #[structopt(subcommand)]
        subcommand: QueueSubcommand,
    },
}

impl CommandLine {
    pub fn handle_args(self) {
        match self {
            Self::Build(build) => build.handle_args(),
            Self::StartWebServer { socket_addr } => {
                Server::start(Some(&socket_addr));
            }
            Self::Daemon { foreground } => cratesfyi::utils::start_daemon(!foreground),
            Self::Database { subcommand } => subcommand.handle_args(),
            Self::Queue { subcommand } => subcommand.handle_args(),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
enum QueueSubcommand {
    /// Add a crate to the build queue
    Add {
        /// Name of crate to build
        #[structopt(name = "CRATE_NAME")]
        crate_name: String,
        /// Version of crate to build
        #[structopt(name = "CRATE_VERSION")]
        crate_version: String,
        /// Priority of build (new crate builds get priority 0)
        #[structopt(
            name = "BUILD_PRIORITY",
            short = "p",
            long = "priority",
            default_value = "5"
        )]
        build_priority: i32,
    },

    /// Interactions with build queue priorities
    DefaultPriority {
        #[structopt(subcommand)]
        subcommand: PrioritySubcommand,
    },
}

impl QueueSubcommand {
    pub fn handle_args(self) {
        match self {
            Self::Add {
                crate_name,
                crate_version,
                build_priority,
            } => {
                let conn = connect_db().expect("Could not connect to database");
                add_crate_to_queue(&conn, &crate_name, &crate_version, build_priority)
                    .expect("Could not add crate to queue");
            }

            Self::DefaultPriority { subcommand } => subcommand.handle_args(),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
enum PrioritySubcommand {
    /// Set all crates matching a pattern to a priority level
    Set {
        /// See https://www.postgresql.org/docs/current/functions-matching.html for pattern syntax
        #[structopt(name = "PATTERN")]
        pattern: String,
        /// The priority to give crates matching the given `PATTERN`
        priority: i32,
    },

    /// Remove the prioritization of crates for a pattern
    Remove {
        /// See https://www.postgresql.org/docs/current/functions-matching.html for pattern syntax
        #[structopt(name = "PATTERN")]
        pattern: String,
    },
}

impl PrioritySubcommand {
    pub fn handle_args(self) {
        match self {
            Self::Set { pattern, priority } => {
                let conn = connect_db().expect("Could not connect to the database");

                set_crate_priority(&conn, &pattern, priority)
                    .expect("Could not set pattern's priority");
            }

            Self::Remove { pattern } => {
                let conn = connect_db().expect("Could not connect to the database");

                if let Some(priority) = remove_crate_priority(&conn, &pattern)
                    .expect("Could not remove pattern's priority")
                {
                    println!("Removed pattern with priority {}", priority);
                } else {
                    println!("Pattern did not exist and so was not removed");
                }
            }
        }
    }
}

/// Builds documentation in a chroot environment
#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
#[structopt(rename_all = "kebab-case")]
struct Build {
    #[structopt(
        name = "PREFIX",
        short = "P",
        long = "prefix",
        env = "CRATESFYI_PREFIX"
    )]
    prefix: PathBuf,

    /// Sets crates.io-index path
    #[structopt(name = "CRATES_IO_INDEX_PATH", long = "crates-io-index-path")]
    crates_io_index_path: Option<PathBuf>,

    /// Skips building documentation if documentation exists
    #[structopt(name = "SKIP_IF_EXISTS", short = "s", long = "skip")]
    skip_if_exists: bool,

    /// Skips building documentation if build log exists
    #[structopt(name = "SKIP_IF_LOG_EXISTS", long = "skip-if-log-exists")]
    skip_if_log_exists: bool,

    /// Keeps build directory after build.
    #[structopt(
        name = "KEEP_BUILD_DIRECTORY",
        short = "k",
        long = "keep-build-directory"
    )]
    keep_build_directory: bool,

    #[structopt(subcommand)]
    subcommand: BuildSubcommand,
}

impl Build {
    pub fn handle_args(self) {
        let docbuilder = {
            let mut doc_options = DocBuilderOptions::from_prefix(self.prefix);

            if let Some(crates_io_index_path) = self.crates_io_index_path {
                doc_options.crates_io_index_path = crates_io_index_path;
            }

            doc_options.skip_if_exists = self.skip_if_exists;
            doc_options.skip_if_log_exists = self.skip_if_log_exists;
            doc_options.keep_build_directory = self.keep_build_directory;

            doc_options
                .check_paths()
                .expect("The given paths were invalid");

            DocBuilder::new(doc_options)
        };

        self.subcommand.handle_args(docbuilder);
    }
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
enum BuildSubcommand {
    /// Builds documentation of every crate
    World,

    /// Builds documentation for a crate
    Crate {
        /// Crate name
        #[structopt(
            name = "CRATE_NAME",
            required_unless("LOCAL"),
            requires("CRATE_VERSION")
        )]
        crate_name: String,

        /// Version of crate
        #[structopt(name = "CRATE_VERSION")]
        crate_version: String,

        /// Build a crate at a specific path
        #[structopt(short = "l", long = "local", conflicts_with_all(&["CRATE_NAME", "CRATE_VERSION"]))]
        local: Option<PathBuf>,
    },

    /// update the currently installed rustup toolchain
    UpdateToolchain {
        /// Update the toolchain only if no toolchain is currently installed
        #[structopt(name = "ONLY_FIRST_TIME", long = "only-first-time")]
        only_first_time: bool,
    },

    /// Adds essential files for the installed version of rustc
    AddEssentialFiles,

    /// Locks cratesfyi daemon to stop building new crates
    Lock,

    /// Unlocks cratesfyi daemon to continue building new crates
    Unlock,

    PrintOptions,
}

impl BuildSubcommand {
    pub fn handle_args(self, mut docbuilder: DocBuilder) {
        match self {
            Self::World => {
                docbuilder.load_cache().expect("Failed to load cache");

                let mut builder = RustwideBuilder::init().unwrap();
                builder
                    .build_world(&mut docbuilder)
                    .expect("Failed to build world");

                docbuilder.save_cache().expect("Failed to save cache");
            }

            Self::Crate {
                crate_name,
                crate_version,
                local,
            } => {
                docbuilder.load_cache().expect("Failed to load cache");
                let mut builder = RustwideBuilder::init().expect("failed to initialize rustwide");

                if let Some(path) = local {
                    builder
                        .build_local_package(&mut docbuilder, &path)
                        .expect("Building documentation failed");
                } else {
                    builder
                        .build_package(&mut docbuilder, &crate_name, &crate_version, None)
                        .expect("Building documentation failed");
                }

                docbuilder.save_cache().expect("Failed to save cache");
            }

            Self::UpdateToolchain { only_first_time } => {
                if only_first_time {
                    let conn = db::connect_db().unwrap();
                    let res = conn
                        .query("SELECT * FROM config WHERE name = 'rustc_version';", &[])
                        .unwrap();

                    if !res.is_empty() {
                        println!("update-toolchain was already called in the past, exiting");
                        return;
                    }
                }

                let mut builder = RustwideBuilder::init().unwrap();
                builder
                    .update_toolchain()
                    .expect("failed to update toolchain");
            }

            Self::AddEssentialFiles => {
                let mut builder = RustwideBuilder::init().unwrap();
                builder
                    .add_essential_files()
                    .expect("failed to add essential files");
            }

            Self::Lock => docbuilder.lock().expect("Failed to lock"),
            Self::Unlock => docbuilder.unlock().expect("Failed to unlock"),
            Self::PrintOptions => println!("{:?}", docbuilder.options()),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
enum DatabaseSubcommand {
    /// Run database migrations
    Migrate {
        /// The database version to migrate to
        #[structopt(name = "VERSION")]
        version: Option<i64>,
    },

    /// Updates github stats for crates.
    UpdateGithubFields,

    AddDirectory {
        /// Path of file or directory
        #[structopt(name = "DIRECTORY")]
        directory: PathBuf,
        /// Prefix of files in database
        #[structopt(name = "PREFIX", env = "CRATESFYI_PREFIX")]
        prefix: String,
    },

    /// Updates monthly release activity chart
    UpdateReleaseActivity,

    /// Updates search index
    UpdateSearchIndex,

    /// Removes a whole crate from the database
    DeleteCrate {
        /// Name of the crate to delete
        #[structopt(name = "CRATE_NAME")]
        crate_name: String,
    },

    /// Blacklist operations
    Blacklist {
        #[structopt(subcommand)]
        command: BlacklistSubcommand,
    },
}

impl DatabaseSubcommand {
    pub fn handle_args(self) {
        match self {
            Self::Migrate { version } => {
                let conn = connect_db().expect("failed to connect to the database");
                db::migrate(version, &conn).expect("Failed to run database migrations");
            }

            Self::UpdateGithubFields => {
                cratesfyi::utils::github_updater().expect("Failed to update github fields");
            }

            Self::AddDirectory { directory, prefix } => {
                let conn = db::connect_db().expect("failed to connect to the database");
                add_path_into_database(&conn, &prefix, directory)
                    .expect("Failed to add directory into database");
            }

            // FIXME: This is actually util command not database
            Self::UpdateReleaseActivity => cratesfyi::utils::update_release_activity()
                .expect("Failed to update release activity"),

            Self::UpdateSearchIndex => {
                let conn = db::connect_db().expect("failed to connect to the database");
                db::update_search_index(&conn).expect("Failed to update search index");
            }

            Self::DeleteCrate { crate_name } => {
                let conn = db::connect_db().expect("failed to connect to the database");
                db::delete_crate(&conn, &crate_name).expect("failed to delete the crate");
            }

            Self::Blacklist { command } => command.handle_args(),
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
enum BlacklistSubcommand {
    /// List all crates on the blacklist
    List,

    /// Add a crate to the blacklist
    Add {
        /// Crate name
        #[structopt(name = "CRATE_NAME")]
        crate_name: String,
    },

    /// Remove a crate from the blacklist
    Remove {
        /// Crate name
        #[structopt(name = "CRATE_NAME")]
        crate_name: String,
    },
}

impl BlacklistSubcommand {
    fn handle_args(self) {
        let conn = db::connect_db().expect("failed to connect to the database");

        match self {
            Self::List => {
                let crates =
                    db::blacklist::list_crates(&conn).expect("failed to list crates on blacklist");

                println!("{}", crates.join("\n"));
            }

            Self::Add { crate_name } => db::blacklist::add_crate(&conn, &crate_name)
                .expect("failed to add crate to blacklist"),

            Self::Remove { crate_name } => db::blacklist::remove_crate(&conn, &crate_name)
                .expect("failed to remove crate from blacklist"),
        }
    }
}
<|endoftext|>


<commit_before>use std::collections::BTreeMap;

use toml;

use cargo_edit::Manifest;
use list_error::ListError;

type PkgName = String;
type PkgVersion = String;
type Package = (PkgName, PkgVersion);
type Dependency = Package;
type Dependencies = Vec<Dependency>;

/// A package as declared in the lock file
pub type Packages = BTreeMap<Package, Dependencies>;

/// Parse stuff like `"docopt 0.6.67 (registry+https://github.com/rust-lang/crates.io-index)"`
/// by splitting at whitespace and taking the first two things.
fn parse_dep_from_str(input: &str) -> Option<Dependency> {
    let pkg = input.split(' ').collect::<Vec<&str>>();
    if pkg.len() != 3 {
        return None;
    }

    let (name, version) = (pkg[0], pkg[1]);

    Some((String::from(name), String::from(version)))
}

fn get_root_deps(lock_file: &toml::Table) -> Result<Vec<Dependency>, ListError> {
    let root_deps = try!(lock_file.get("root")
                                  .and_then(|field| field.lookup("dependencies"))
                                  .ok_or(ListError::SectionMissing("dependencies".to_owned())));

    let output = root_deps.as_slice()
                          .unwrap_or(&vec![])
                          .iter()
                          .filter_map(|dep| {
                              let dep = dep.clone();
                              if let toml::Value::String(pkg_desc) = dep {
                                  parse_dep_from_str(&pkg_desc)
                              } else {
                                  None
                              }
                          })
                          .collect::<Vec<Dependency>>();

    Ok(output)
}

fn get_packages(lock_file: &toml::Table) -> Result<Packages, ListError> {
    let packages: &toml::Value = try!(lock_file.get("package").ok_or(ListError::PackagesMissing));

    let mut output = BTreeMap::new();

    for pkg in packages.as_slice().unwrap_or(&vec![]) {
        let package = try!(pkg.as_table().ok_or(ListError::PackageInvalid));

        let name = try!(package.get("name")
                               .and_then(|item| item.as_str())
                               .ok_or(ListError::PackageFieldMissing("name")));

        let version = try!(package.get("version")
                                  .and_then(|item| item.as_str())
                                  .ok_or(ListError::PackageFieldMissing("version")));

        let deps: Dependencies = package.get("dependencies")
                                        .and_then(|item| {
                                            let item = item.clone();
                                            if let toml::Value::Array(d) = item {
                                                Some(d)
                                            } else {
                                                None
                                            }
                                        })
                                        .and_then(|items| {
                                            Some(items.iter()
                                                      .filter_map(|i| i.as_str())
                                                      .filter_map(parse_dep_from_str)
                                                      .collect::<Dependencies>())
                                        })
                                        .unwrap_or(vec![]);

        output.insert((name.to_owned(), version.to_owned()), deps);
    }

    Ok(output)
}

fn list_deps_helper(pkgs: &Packages,
                    deps: &[Dependency],
                    levels: &mut Vec<bool>)
                    -> Result<String, ListError> {
    let mut output = String::new();
    for (i, dep) in deps.iter().enumerate() {
        // For any indent level where the parent is the last dependency in the list, we
        // don't want
        // to print out a branch.
        for is_last in levels.iter().cloned() {
            let indent = if is_last {
                "    "
            } else {
                "│   "
            };
            output.push_str(indent);
        }

        let is_last = i == deps.len() - 1;
        let branch = if !is_last {
            "├──"
        } else {
            "└──"
        };

        output.push_str(&format!("{} {} ({})\n", branch, dep.0, dep.1));

        if let Some(subdeps) = pkgs.get(dep) {
            levels.push(is_last);
            let sublist = try!(list_deps_helper(pkgs, subdeps, levels));
            output.push_str(&sublist);
            levels.pop();
        }
    }
    Ok(output)
}

fn list_deps(pkgs: &Packages, deps: &[Dependency]) -> Result<String, ListError> {
    list_deps_helper(pkgs, deps, &mut vec![])
}

/// Parse a `Cargo.lock` file and list its dependencies
pub fn parse_lock_file(manifest: &Manifest) -> Result<String, ListError> {
    let lock_file = &manifest.data;

    let root_deps = try!(get_root_deps(lock_file));
    let pkgs = try!(get_packages(lock_file));

    list_deps(&pkgs, &root_deps)
}

#[cfg(test)]
mod test {
    use cargo_edit::Manifest;
    use super::parse_lock_file;

    #[test]
    fn basic_tree() {
        let manifile = Manifest::open_lock_file(&Some("tests/fixtures/tree/Cargo.lock")).unwrap();

        assert_eq!(parse_lock_file(&manifile).unwrap(),
                   "\
├── clippy (0.0.5)
├── docopt (0.6.67)
│   ├── regex (0.1.38)
│   │   ├── aho-corasick (0.2.1)
│   │   │   └── memchr (0.1.3)
│   │   │       └── libc (0.1.8)
│   │   ├── memchr (0.1.3)
│   │   │   └── libc (0.1.8)
│   │   └── regex-syntax (0.1.2)
│   ├── rustc-serialize (0.3.15)
│   └── strsim (0.3.0)
├── pad (0.1.4)
│   └── unicode-width (0.1.1)
├── rustc-serialize (0.3.15)
├── semver (0.1.19)
└── toml (0.1.20)
    └── rustc-serialize (0.3.15)
");
    }
}
<commit_msg>Fix private_in_public Warnings<commit_after>use std::collections::BTreeMap;

use toml;

use cargo_edit::Manifest;
use list_error::ListError;

pub type PkgName = String;
pub type PkgVersion = String;
pub type Package = (PkgName, PkgVersion);
pub type Dependency = Package;
pub type Dependencies = Vec<Dependency>;

/// A package as declared in the lock file
pub type Packages = BTreeMap<Package, Dependencies>;

/// Parse stuff like `"docopt 0.6.67 (registry+https://github.com/rust-lang/crates.io-index)"`
/// by splitting at whitespace and taking the first two things.
fn parse_dep_from_str(input: &str) -> Option<Dependency> {
    let pkg = input.split(' ').collect::<Vec<&str>>();
    if pkg.len() != 3 {
        return None;
    }

    let (name, version) = (pkg[0], pkg[1]);

    Some((String::from(name), String::from(version)))
}

fn get_root_deps(lock_file: &toml::Table) -> Result<Vec<Dependency>, ListError> {
    let root_deps = try!(lock_file.get("root")
                                  .and_then(|field| field.lookup("dependencies"))
                                  .ok_or(ListError::SectionMissing("dependencies".to_owned())));

    let output = root_deps.as_slice()
                          .unwrap_or(&vec![])
                          .iter()
                          .filter_map(|dep| {
                              let dep = dep.clone();
                              if let toml::Value::String(pkg_desc) = dep {
                                  parse_dep_from_str(&pkg_desc)
                              } else {
                                  None
                              }
                          })
                          .collect::<Vec<Dependency>>();

    Ok(output)
}

fn get_packages(lock_file: &toml::Table) -> Result<Packages, ListError> {
    let packages: &toml::Value = try!(lock_file.get("package").ok_or(ListError::PackagesMissing));

    let mut output = BTreeMap::new();

    for pkg in packages.as_slice().unwrap_or(&vec![]) {
        let package = try!(pkg.as_table().ok_or(ListError::PackageInvalid));

        let name = try!(package.get("name")
                               .and_then(|item| item.as_str())
                               .ok_or(ListError::PackageFieldMissing("name")));

        let version = try!(package.get("version")
                                  .and_then(|item| item.as_str())
                                  .ok_or(ListError::PackageFieldMissing("version")));

        let deps: Dependencies = package.get("dependencies")
                                        .and_then(|item| {
                                            let item = item.clone();
                                            if let toml::Value::Array(d) = item {
                                                Some(d)
                                            } else {
                                                None
                                            }
                                        })
                                        .and_then(|items| {
                                            Some(items.iter()
                                                      .filter_map(|i| i.as_str())
                                                      .filter_map(parse_dep_from_str)
                                                      .collect::<Dependencies>())
                                        })
                                        .unwrap_or(vec![]);

        output.insert((name.to_owned(), version.to_owned()), deps);
    }

    Ok(output)
}

fn list_deps_helper(pkgs: &Packages,
                    deps: &[Dependency],
                    levels: &mut Vec<bool>)
                    -> Result<String, ListError> {
    let mut output = String::new();
    for (i, dep) in deps.iter().enumerate() {
        // For any indent level where the parent is the last dependency in the list, we
        // don't want
        // to print out a branch.
        for is_last in levels.iter().cloned() {
            let indent = if is_last {
                "    "
            } else {
                "│   "
            };
            output.push_str(indent);
        }

        let is_last = i == deps.len() - 1;
        let branch = if !is_last {
            "├──"
        } else {
            "└──"
        };

        output.push_str(&format!("{} {} ({})\n", branch, dep.0, dep.1));

        if let Some(subdeps) = pkgs.get(dep) {
            levels.push(is_last);
            let sublist = try!(list_deps_helper(pkgs, subdeps, levels));
            output.push_str(&sublist);
            levels.pop();
        }
    }
    Ok(output)
}

fn list_deps(pkgs: &Packages, deps: &[Dependency]) -> Result<String, ListError> {
    list_deps_helper(pkgs, deps, &mut vec![])
}

/// Parse a `Cargo.lock` file and list its dependencies
pub fn parse_lock_file(manifest: &Manifest) -> Result<String, ListError> {
    let lock_file = &manifest.data;

    let root_deps = try!(get_root_deps(lock_file));
    let pkgs = try!(get_packages(lock_file));

    list_deps(&pkgs, &root_deps)
}

#[cfg(test)]
mod test {
    use cargo_edit::Manifest;
    use super::parse_lock_file;

    #[test]
    fn basic_tree() {
        let manifile = Manifest::open_lock_file(&Some("tests/fixtures/tree/Cargo.lock")).unwrap();

        assert_eq!(parse_lock_file(&manifile).unwrap(),
                   "\
├── clippy (0.0.5)
├── docopt (0.6.67)
│   ├── regex (0.1.38)
│   │   ├── aho-corasick (0.2.1)
│   │   │   └── memchr (0.1.3)
│   │   │       └── libc (0.1.8)
│   │   ├── memchr (0.1.3)
│   │   │   └── libc (0.1.8)
│   │   └── regex-syntax (0.1.2)
│   ├── rustc-serialize (0.3.15)
│   └── strsim (0.3.0)
├── pad (0.1.4)
│   └── unicode-width (0.1.1)
├── rustc-serialize (0.3.15)
├── semver (0.1.19)
└── toml (0.1.20)
    └── rustc-serialize (0.3.15)
");
    }
}
<|endoftext|>


<commit_before>// xfail-stage0

// Since the bogus configuration isn't defined main will just be
// parsed, but nothing further will be done with it
#[cfg(bogus)]
fn main() { fail }

fn main() {}<commit_msg>Add more tests for conditional compilation. Issue #489<commit_after>// xfail-stage0

#[cfg(bogus)]
const bool b = false;

const bool b = true;

#[cfg(bogus)]
native "rust" mod rustrt {
  // This symbol doesn't exist and would be a link error if this
  // module was translated
  fn bogus();
}

native "rust" mod rustrt {
}

#[cfg(bogus)]
type t = int;

type t = bool;

#[cfg(bogus)]
tag tg {
  foo;
}

tag tg {
  bar;
}

#[cfg(bogus)]
obj o() {
  fn f() { ret bogus; }
}

obj o() {
}

#[cfg(bogus)]
resource r(int i) {}

resource r(int i) {}

#[cfg(bogus)]
mod m {
  // This needs to parse but would fail in typeck. Since it's not in
  // the current config it should not be typechecked.
  fn bogus() { ret 0; }
}

mod m {

  // Submodules have slightly different code paths than the top-level
  // module, so let's make sure this jazz works here as well
  #[cfg(bogus)]
  fn f() {}

  fn f() {}
}

// Since the bogus configuration isn't defined main will just be
// parsed, but nothing further will be done with it
#[cfg(bogus)]
fn main() { fail }

fn main() {
  // Exercise some of the configured items in ways that wouldn't be possible
  // if they had the bogus definition
  assert b;
  let t x = true;
  let tg y = bar;
}

<|endoftext|>


<commit_before>use cocoa::base::{id, nil};
use cocoa::foundation::NSString;
use std::borrow::Cow;
use std::convert::AsRef;
use std::error::Error;
use std::ffi::CStr;
use std::fmt::{self, Display, Formatter};
use sys::MTLCommandQueue;
use {CommandBuffer, FromRaw, FromRawError};

pub struct CommandQueue(id);

impl CommandQueue {
    pub fn command_buffer(&mut self) -> CommandBuffer {
        let cmd_buf = unsafe { self.0.commandBuffer() };
        FromRaw::from_raw(cmd_buf).unwrap()
    }

    pub fn command_buffer_with_unretained_references(&mut self) -> CommandBuffer {
        let cmd_buf = unsafe { self.0.commandBufferWithUnretainedReferences() };
        FromRaw::from_raw(cmd_buf).unwrap()
    }

    pub fn insert_debug_capture_boundary(&mut self) {
        unsafe {
            self.0.insertDebugCaptureBoundary();
        }
    }

    pub fn set_label<S: AsRef<str>>(&mut self, label: S) {
        unsafe { self.0.setLabel(NSString::alloc(nil).init_str(label.as_ref())) }
    }

    pub fn get_label(&self) -> Cow<str> {
        unsafe { CStr::from_ptr(self.0.label().UTF8String()).to_string_lossy() }
    }
}

impl_from_into_raw!(CommandQueue, "MTLCommandQueue");

#[derive(Clone, Debug)]
pub enum CommandQueueError {
    CouldNotCreate
}

impl Display for CommandQueueError {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        let descr = match *self {
            CommandQueueError::CouldNotCreate => "DeviceError::CouldNotCreate",
        };
        write!(f, "{}", descr)
    }
}

impl Error for CommandQueueError {
    fn description(&self) -> &str {
        match *self {
            CommandQueueError::CouldNotCreate => "Could not create command queue.",
        }
    }
}

impl From<FromRawError> for CommandQueueError {
    fn from(_: FromRawError) -> Self {
        CommandQueueError::CouldNotCreate
    }
}
<commit_msg>Impl Send + Sync for CommandQueue ; see https://developer.apple.com/library/ios/documentation/Miscellaneous/Conceptual/MetalProgrammingGuide/Cmd-Submiss/Cmd-Submiss.html<commit_after>use cocoa::base::{id, nil};
use cocoa::foundation::NSString;
use std::borrow::Cow;
use std::convert::AsRef;
use std::error::Error;
use std::ffi::CStr;
use std::fmt::{self, Display, Formatter};
use std::marker::{Send, Sync};
use sys::MTLCommandQueue;
use {CommandBuffer, FromRaw, FromRawError};

pub struct CommandQueue(id);

unsafe impl Send for CommandQueue {}
unsafe impl Sync for CommandQueue {}

impl CommandQueue {
    pub fn command_buffer(&mut self) -> CommandBuffer {
        let cmd_buf = unsafe { self.0.commandBuffer() };
        FromRaw::from_raw(cmd_buf).unwrap()
    }

    pub fn command_buffer_with_unretained_references(&mut self) -> CommandBuffer {
        let cmd_buf = unsafe { self.0.commandBufferWithUnretainedReferences() };
        FromRaw::from_raw(cmd_buf).unwrap()
    }

    pub fn insert_debug_capture_boundary(&mut self) {
        unsafe {
            self.0.insertDebugCaptureBoundary();
        }
    }

    pub fn set_label<S: AsRef<str>>(&mut self, label: S) {
        unsafe { self.0.setLabel(NSString::alloc(nil).init_str(label.as_ref())) }
    }

    pub fn get_label(&self) -> Cow<str> {
        unsafe { CStr::from_ptr(self.0.label().UTF8String()).to_string_lossy() }
    }
}

impl_from_into_raw!(CommandQueue, "MTLCommandQueue");

#[derive(Clone, Debug)]
pub enum CommandQueueError {
    CouldNotCreate
}

impl Display for CommandQueueError {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        let descr = match *self {
            CommandQueueError::CouldNotCreate => "DeviceError::CouldNotCreate",
        };
        write!(f, "{}", descr)
    }
}

impl Error for CommandQueueError {
    fn description(&self) -> &str {
        match *self {
            CommandQueueError::CouldNotCreate => "Could not create command queue.",
        }
    }
}

impl From<FromRawError> for CommandQueueError {
    fn from(_: FromRawError) -> Self {
        CommandQueueError::CouldNotCreate
    }
}
<|endoftext|>