<commit_before>// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

use std::result;

use futures::sync::oneshot;
use hyper::Method;
use serde_json::Value;

use request::{IntoParsedRequest, ParsedRequest};
use vmm::VmmAction;

// The names of the members from this enum must precisely correspond (as a string) to the possible
// values of "action_type" from the json request body. This is useful to get a strongly typed
// struct from the Serde deserialization process.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub enum ActionType {
    BlockDeviceRescan,
    InstanceStart,
}

#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub enum DeviceType {
    Drive,
}

// The model of the json body from a sync request. We use Serde to transform each associated
// json body into this.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ActionBody {
    pub action_type: ActionType,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub payload: Option<Value>,
}

fn validate_payload(action_body: &ActionBody) -> Result<(), String> {
    match action_body.action_type {
        ActionType::BlockDeviceRescan => {
            match action_body.payload {
                Some(ref payload) => {
                    // Expecting to have drive_id as a String in the payload.
                    if !payload.is_string() {
                        return Err(
                            "Invalid payload type. Expected a string representing the drive_id"
                                .to_string(),
                        );
                    }
                    Ok(())
                }
                None => return Err("Payload is required for block device rescan.".to_string()),
            }
        }
        ActionType::InstanceStart => {
            // InstanceStart does not have a payload
            if !action_body.payload.is_none() {
                return Err("InstanceStart does not support a payload.".to_string());
            }
            Ok(())
        }
    }
}

impl IntoParsedRequest for ActionBody {
    fn into_parsed_request(
        self,
        _: Option<String>,
        _: Method,
    ) -> result::Result<ParsedRequest, String> {
        validate_payload(&self)?;
        match self.action_type {
            ActionType::BlockDeviceRescan => {
                // Safe to unwrap because we validated the payload in the validate_payload func.
                let block_device_id = self.payload.unwrap().as_str().unwrap().to_string();
                let (sync_sender, sync_receiver) = oneshot::channel();
                Ok(ParsedRequest::Sync(
                    VmmAction::RescanBlockDevice(block_device_id, sync_sender),
                    sync_receiver,
                ))
            }
            ActionType::InstanceStart => {
                let (sync_sender, sync_receiver) = oneshot::channel();
                Ok(ParsedRequest::Sync(
                    VmmAction::StartMicroVm(sync_sender),
                    sync_receiver,
                ))
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json;

    #[test]
    fn test_validate_payload() {
        // Test InstanceStart.
        let action_body = ActionBody {
            action_type: ActionType::InstanceStart,
            payload: None,
        };
        assert!(validate_payload(&action_body).is_ok());
        // Error case: InstanceStart with payload.
        let action_body = ActionBody {
            action_type: ActionType::InstanceStart,
            payload: Some(Value::String("dummy-payload".to_string())),
        };
        assert!(validate_payload(&action_body).is_err());

        // Test BlockDeviceRescan
        let action_body = ActionBody {
            action_type: ActionType::BlockDeviceRescan,
            payload: Some(Value::String(String::from("dummy_id"))),
        };
        assert!(validate_payload(&action_body).is_ok());
        // Error case: no payload.
        let action_body = ActionBody {
            action_type: ActionType::BlockDeviceRescan,
            payload: None,
        };
        assert!(validate_payload(&action_body).is_err());
        // Error case: payload is not String.
        let action_body = ActionBody {
            action_type: ActionType::BlockDeviceRescan,
            payload: Some(Value::Bool(false)),
        };
        assert!(validate_payload(&action_body).is_err());
    }

    #[test]
    fn test_into_parsed_request() {
        {
            let json = r#"{
                "action_type": "BlockDeviceRescan",
                "payload": "dummy_id"
              }"#;
            let (sender, receiver) = oneshot::channel();
            let req = ParsedRequest::Sync(
                VmmAction::RescanBlockDevice("dummy_id".to_string(), sender),
                receiver,
            );

            let result: Result<ActionBody, serde_json::Error> = serde_json::from_str(json);
            assert!(result.is_ok());
            assert!(
                result
                    .unwrap()
                    .into_parsed_request(None, Method::Put)
                    .unwrap()
                    .eq(&req)
            );
        }

        {
            let json = r#"{
                "action_type": "InstanceStart"
            }"#;

            let (sender, receiver) = oneshot::channel();
            let req: ParsedRequest = ParsedRequest::Sync(VmmAction::StartMicroVm(sender), receiver);
            let result: Result<ActionBody, serde_json::Error> = serde_json::from_str(json);
            assert!(result.is_ok());
            assert!(
                result
                    .unwrap()
                    .into_parsed_request(None, Method::Put)
                    .unwrap()
                    .eq(&req)
            );
        }
    }
}
<commit_msg>api_server: removed unnecessary pubs & structs<commit_after>// Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

use std::result;

use futures::sync::oneshot;
use hyper::Method;
use serde_json::Value;

use request::{IntoParsedRequest, ParsedRequest};
use vmm::VmmAction;

// The names of the members from this enum must precisely correspond (as a string) to the possible
// values of "action_type" from the json request body. This is useful to get a strongly typed
// struct from the Serde deserialization process.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
enum ActionType {
    BlockDeviceRescan,
    InstanceStart,
}

// The model of the json body from a sync request. We use Serde to transform each associated
// json body into this.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ActionBody {
    action_type: ActionType,
    #[serde(skip_serializing_if = "Option::is_none")]
    payload: Option<Value>,
}

fn validate_payload(action_body: &ActionBody) -> Result<(), String> {
    match action_body.action_type {
        ActionType::BlockDeviceRescan => {
            match action_body.payload {
                Some(ref payload) => {
                    // Expecting to have drive_id as a String in the payload.
                    if !payload.is_string() {
                        return Err(
                            "Invalid payload type. Expected a string representing the drive_id"
                                .to_string(),
                        );
                    }
                    Ok(())
                }
                None => return Err("Payload is required for block device rescan.".to_string()),
            }
        }
        ActionType::InstanceStart => {
            // InstanceStart does not have a payload
            if !action_body.payload.is_none() {
                return Err("InstanceStart does not support a payload.".to_string());
            }
            Ok(())
        }
    }
}

impl IntoParsedRequest for ActionBody {
    fn into_parsed_request(
        self,
        _: Option<String>,
        _: Method,
    ) -> result::Result<ParsedRequest, String> {
        validate_payload(&self)?;
        match self.action_type {
            ActionType::BlockDeviceRescan => {
                // Safe to unwrap because we validated the payload in the validate_payload func.
                let block_device_id = self.payload.unwrap().as_str().unwrap().to_string();
                let (sync_sender, sync_receiver) = oneshot::channel();
                Ok(ParsedRequest::Sync(
                    VmmAction::RescanBlockDevice(block_device_id, sync_sender),
                    sync_receiver,
                ))
            }
            ActionType::InstanceStart => {
                let (sync_sender, sync_receiver) = oneshot::channel();
                Ok(ParsedRequest::Sync(
                    VmmAction::StartMicroVm(sync_sender),
                    sync_receiver,
                ))
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json;

    #[test]
    fn test_validate_payload() {
        // Test InstanceStart.
        let action_body = ActionBody {
            action_type: ActionType::InstanceStart,
            payload: None,
        };
        assert!(validate_payload(&action_body).is_ok());
        // Error case: InstanceStart with payload.
        let action_body = ActionBody {
            action_type: ActionType::InstanceStart,
            payload: Some(Value::String("dummy-payload".to_string())),
        };
        assert!(validate_payload(&action_body).is_err());

        // Test BlockDeviceRescan
        let action_body = ActionBody {
            action_type: ActionType::BlockDeviceRescan,
            payload: Some(Value::String(String::from("dummy_id"))),
        };
        assert!(validate_payload(&action_body).is_ok());
        // Error case: no payload.
        let action_body = ActionBody {
            action_type: ActionType::BlockDeviceRescan,
            payload: None,
        };
        assert!(validate_payload(&action_body).is_err());
        // Error case: payload is not String.
        let action_body = ActionBody {
            action_type: ActionType::BlockDeviceRescan,
            payload: Some(Value::Bool(false)),
        };
        assert!(validate_payload(&action_body).is_err());
    }

    #[test]
    fn test_into_parsed_request() {
        {
            let json = r#"{
                "action_type": "BlockDeviceRescan",
                "payload": "dummy_id"
              }"#;
            let (sender, receiver) = oneshot::channel();
            let req = ParsedRequest::Sync(
                VmmAction::RescanBlockDevice("dummy_id".to_string(), sender),
                receiver,
            );

            let result: Result<ActionBody, serde_json::Error> = serde_json::from_str(json);
            assert!(result.is_ok());
            assert!(
                result
                    .unwrap()
                    .into_parsed_request(None, Method::Put)
                    .unwrap()
                    .eq(&req)
            );
        }

        {
            let json = r#"{
                "action_type": "InstanceStart"
            }"#;

            let (sender, receiver) = oneshot::channel();
            let req: ParsedRequest = ParsedRequest::Sync(VmmAction::StartMicroVm(sender), receiver);
            let result: Result<ActionBody, serde_json::Error> = serde_json::from_str(json);
            assert!(result.is_ok());
            assert!(
                result
                    .unwrap()
                    .into_parsed_request(None, Method::Put)
                    .unwrap()
                    .eq(&req)
            );
        }
    }
}
<|endoftext|>


<commit_before>use cucumber::{t, Steps};
use mimir2::{
    adapters::primary::bragi::autocomplete::{build_query, Filters},
    adapters::secondary::elasticsearch,
    adapters::secondary::elasticsearch::{
        internal::{IndexConfiguration, IndexMappings, IndexParameters, IndexSettings},
        remote::{connection_test_pool, Error as PoolError},
    },
    domain::model::query::Query,
    domain::ports::remote::Error as ConnectionError,
    domain::ports::remote::Remote,
    domain::ports::search::SearchParameters,
    domain::usecases::search_documents::{SearchDocuments, SearchDocumentsParameters},
    domain::usecases::UseCase,
};
use places::{admin::Admin, MimirObject};
use snafu::{ResultExt, Snafu};

use std::path::PathBuf;
use tokio::io::AsyncWriteExt;
use url::Url;

pub fn steps() -> Steps<crate::MyWorld> {
    let mut steps: Steps<crate::MyWorld> = Steps::new();

    steps.given_regex_async(
        "osm file has been downloaded for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            world
                .processing_step
                .insert(download_osm(&region).await.unwrap());
            world
        }),
    );

    steps.given_regex_async(
        "osm file has been processed by cosmogony for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            let previous = world.processing_step.clone().unwrap(); // we must have done something before, file either skipped or downloaded.
            world
                .processing_step
                .insert(generate_cosmogony(&region, previous).await.unwrap());
            world
        }),
    );

    steps.given_regex_async(
        "cosmogony file has been indexed for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            let previous = world.processing_step.clone().unwrap(); // we must have done something before, file either skipped or downloaded.
            world
                .processing_step
                .insert(index_cosmogony(&region, previous).await.unwrap());
            world
        }),
    );

    steps.when_regex_async(
        "the user searches for \"(.*)\"",
        t!(|mut world, ctx| {
            let pool = elasticsearch::remote::connection_test_pool().await.unwrap();

            let client = pool.conn().await.unwrap();

            let search_documents = SearchDocuments::new(Box::new(client));

            let filters = Filters::default();

            let dsl = build_query(&ctx.matches[1], filters, &["fr"], &world.query_settings);

            let parameters = SearchDocumentsParameters {
                parameters: SearchParameters {
                    query: Query::QueryDSL(dsl),
                    doc_types: vec![String::from(Admin::doc_type())],
                },
            };
            world.search_result = search_documents.execute(parameters).await.unwrap();
            world
        }),
    );

    steps.then_regex("he finds \"(.*)\" as the first result", |world, ctx| {
        assert_eq!(
            rank(&ctx.matches[1], &world.search_result)
                .expect("the user must perform a search before checking results"),
            0
        );

        world
    });

    steps
}

#[derive(PartialEq, Clone)]
pub enum ProcessingStep {
    Downloaded,
    Generated,
    Indexed,
    Skipped,
}

#[derive(Debug, Snafu)]
pub enum Error {
    #[snafu(display("Invalid Download URL: {} ({})", source, details))]
    InvalidUrl {
        details: String,
        source: url::ParseError,
    },
    #[snafu(display("Invalid IO: {} ({})", source, details))]
    InvalidIO {
        details: String,
        source: std::io::Error,
    },
    #[snafu(display("Download Error: {} ({})", source, details))]
    DownloadError {
        details: String,
        source: reqwest::Error,
    },
    #[snafu(display("Elasticsearch Pool Error: {} ({})", source, details))]
    ElasticsearchPoolError { details: String, source: PoolError },
    #[snafu(display("Elasticsearch Connection Error: {} ({})", source, details))]
    ElasticsearchConnectionError {
        details: String,
        source: ConnectionError,
    },
    #[snafu(display("Indexing Error: {}", details))]
    IndexingError { details: String },

    #[snafu(display("JSON Error: {} ({})", details, source))]
    JsonError {
        details: String,
        source: serde_json::Error,
    },
}

// Given the name of a french region, it will download the matching OSM file
// If that file is already in the local filesystem, then we skip the download.
// This function makes several assumptions:
// 1. The name of the region is one found in http://download.geofabrik.de/europe/france.html
// 2. The file will be downloaded to `tests/data/osm` under the project's root (identified
//    by the CARGO_MANIFEST_DIR environment variable
// The function returns either `ProcessingStep::Skipped` or `ProcessingStep::Downloaded`,
// or an error.
async fn download_osm(region: &str) -> Result<ProcessingStep, Error> {
    // Here we make sure that there is a folder tests/data/osm at the root of the project.
    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let mut path = PathBuf::from(path);
    path.push("tests");
    path.push("data");
    if tokio::fs::metadata(&path).await.is_err() {
        tokio::fs::create_dir(&path).await.context(InvalidIO {
            details: format!("could no create directory {}", path.display()),
        })?;
    }
    path.push("osm");
    if tokio::fs::metadata(&path).await.is_err() {
        tokio::fs::create_dir(&path).await.context(InvalidIO {
            details: format!("could no create directory {}", path.display()),
        })?;
    }
    // Then we try to see if there is already a file with the expected name in that
    // folder, in which case we skip the actual download, to save time.
    let filename = format!("{}-latest.osm.pbf", region);
    path.push(&filename);
    if tokio::fs::metadata(&path).await.is_ok() {
        return Ok(ProcessingStep::Skipped);
    }

    // Ok, directory structure is fine, no file in 'cache', so we download
    let url = format!(
        "https://download.geofabrik.de/europe/france/{}-latest.osm.pbf",
        region
    );
    let url = Url::parse(&url).context(InvalidUrl { details: url })?;

    let resp = reqwest::get(url.clone()).await.context(DownloadError {
        details: format!("could not download url {}", url),
    })?;

    // If we got a response which is an error (eg 404), then turn it
    // into an Error.
    let mut resp = resp.error_for_status().context(DownloadError {
        details: format!("download response error {}", url),
    })?;

    let file = tokio::fs::File::create(&path).await.context(InvalidIO {
        details: format!("could no create file {}", path.display()),
    })?;

    // Do an asynchronous, buffered copy of the download to the output file
    let mut file = tokio::io::BufWriter::new(file);

    while let Some(chunk) = resp.chunk().await.context(DownloadError {
        details: String::from("read chunk"),
    })? {
        file.write(&chunk).await.context(InvalidIO {
            details: String::from("write chunk"),
        })?;
    }

    file.flush().await.context(InvalidIO {
        details: String::from("flush"),
    })?;

    Ok(ProcessingStep::Downloaded)
}

// Generate a cosmogony file given the region
// The function makes several assumptions:
// 1. The OSM file has previously been downloaded into the expected folder (tests/data/osm)
// 2. The output is a jsonl.gz
// 3. The output will be stored in tests/data/cosmogony
//
// The OSM file will be processed if:
// 1. The output file is not found
// 2. If the output file is found and the previous step is 'downloaded' (that is it's probably a
//    new OSM file and we need to generate a new cosmogony file.
async fn generate_cosmogony(
    region: &str,
    previous: ProcessingStep,
) -> Result<ProcessingStep, Error> {
    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let mut input_path: PathBuf = [path, "tests", "data", "osm"].iter().collect();
    let filename = format!("{}-latest.osm.pbf", region);
    input_path.push(&filename);
    let mut output_path: PathBuf = [path, "tests", "data", "cosmogony"].iter().collect();
    if tokio::fs::metadata(&output_path).await.is_err() {
        tokio::fs::create_dir(&output_path)
            .await
            .context(InvalidIO {
                details: format!("could no create directory {}", output_path.display()),
            })?;
    }
    let filename = format!("{}.jsonl.gz", region);
    output_path.push(&filename);
    // if the output already exists, and the input is not a new file, then skip the generation
    if (tokio::fs::metadata(&output_path).await.is_ok()) && (previous != ProcessingStep::Downloaded)
    {
        return Ok(ProcessingStep::Skipped);
    }
    let mut child =
        tokio::process::Command::new("/home/remi/code/cosmogony/target/release/cosmogony")
            .arg("--country-code")
            .arg("FR")
            .arg("--input")
            .arg(&input_path)
            .arg("--output")
            .arg(&output_path)
            .spawn()
            .expect("failed to spawn cosmogony");

    let _status = child.wait().await.context(InvalidIO {
        details: format!(
            "failed to generate cosmogony with input {} and output {}",
            input_path.display(),
            output_path.display()
        ),
    })?;

    // TODO Do something with the status?

    Ok(ProcessingStep::Generated)
}

async fn index_cosmogony(region: &str, previous: ProcessingStep) -> Result<ProcessingStep, Error> {
    // if the previous step is 'generated', then we need to index the cosmogony file.
    // Otherwise, we skip.
    // TODO: change this logic to check immutably what appends?
    if previous != ProcessingStep::Generated {
        return Ok(ProcessingStep::Skipped);
    }
    let pool = connection_test_pool()
        .await
        .context(ElasticsearchPoolError {
            details: String::from("Could not retrieve Elasticsearch test pool"),
        })?;

    let client = pool.conn().await.context(ElasticsearchConnectionError {
        details: String::from("Could not establish connection to Elasticsearch"),
    })?;

    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let config_path: PathBuf = [path, "config", "admin"].iter().collect();
    let mut settings_path = config_path.clone();
    settings_path.push("settings.json");
    let settings = tokio::fs::read_to_string(&settings_path)
        .await
        .context(InvalidIO {
            details: format!(
                "could not read settings file from '{}'",
                settings_path.display()
            ),
        })?;

    let settings = serde_json::from_str(&settings).context(JsonError {
        details: String::from("Could not deserialize settings"),
    })?;

    let mut mappings_path = config_path.clone();
    mappings_path.push("mappings.json");
    let mappings = tokio::fs::read_to_string(&mappings_path)
        .await
        .context(InvalidIO {
            details: format!(
                "could not read mappings file from '{}'",
                mappings_path.display()
            ),
        })?;

    let mappings = serde_json::from_str(&mappings).context(JsonError {
        details: String::from("Could not deserialize settings"),
    })?;

    let config = IndexConfiguration {
        name: String::from("test"),
        parameters: IndexParameters {
            timeout: String::from("10s"),
            wait_for_active_shards: String::from("1"), // only the primary shard
        },
        settings: IndexSettings { value: settings },
        mappings: IndexMappings { value: mappings },
    };

    let mut input_path: PathBuf = [path, "tests", "data", "cosmogony"].iter().collect();
    let filename = format!("{}.jsonl.gz", region);
    input_path.push(&filename);
    mimirsbrunn::admin::index_cosmogony(
        input_path.into_os_string().into_string().unwrap(),
        vec![String::from("fr")],
        config,
        client,
    )
    .await
    .map_err(|err| Error::IndexingError {
        details: format!("could not index cosmogony: {}", err.to_string(),),
    })?;
    Ok(ProcessingStep::Indexed)
}

pub fn rank(id: &str, list: &[serde_json::Value]) -> Option<usize> {
    list.iter()
        .enumerate()
        // .find(|(_i, v)| v.as_object().unwrap().get("id").unwrap().as_str().unwrap() == id)
        .find(|(_i, v)| {
            let idr = v.as_object().unwrap().get("id").unwrap().as_str().unwrap();
            idr == id
        })
        .map(|(i, _r)| i)
}
<commit_msg>use COSMOGONY_EXE env var in tests<commit_after>use cucumber::{t, Steps};
use mimir2::{
    adapters::primary::bragi::autocomplete::{build_query, Filters},
    adapters::secondary::elasticsearch,
    adapters::secondary::elasticsearch::{
        internal::{IndexConfiguration, IndexMappings, IndexParameters, IndexSettings},
        remote::{connection_test_pool, Error as PoolError},
    },
    domain::model::query::Query,
    domain::ports::remote::Error as ConnectionError,
    domain::ports::remote::Remote,
    domain::ports::search::SearchParameters,
    domain::usecases::search_documents::{SearchDocuments, SearchDocumentsParameters},
    domain::usecases::UseCase,
};
use places::{admin::Admin, MimirObject};
use snafu::{ResultExt, Snafu};

use std::path::PathBuf;
use tokio::io::AsyncWriteExt;
use url::Url;

pub fn steps() -> Steps<crate::MyWorld> {
    let mut steps: Steps<crate::MyWorld> = Steps::new();

    steps.given_regex_async(
        "osm file has been downloaded for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            world
                .processing_step
                .insert(download_osm(&region).await.unwrap());
            world
        }),
    );

    steps.given_regex_async(
        "osm file has been processed by cosmogony for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            let previous = world.processing_step.clone().unwrap(); // we must have done something before, file either skipped or downloaded.
            world
                .processing_step
                .insert(generate_cosmogony(&region, previous).await.unwrap());
            world
        }),
    );

    steps.given_regex_async(
        "cosmogony file has been indexed for (.*)",
        t!(|mut world, ctx| {
            let region = ctx.matches[1].clone();
            let previous = world.processing_step.clone().unwrap(); // we must have done something before, file either skipped or downloaded.
            world
                .processing_step
                .insert(index_cosmogony(&region, previous).await.unwrap());
            world
        }),
    );

    steps.when_regex_async(
        "the user searches for \"(.*)\"",
        t!(|mut world, ctx| {
            let pool = elasticsearch::remote::connection_test_pool().await.unwrap();

            let client = pool.conn().await.unwrap();

            let search_documents = SearchDocuments::new(Box::new(client));

            let filters = Filters::default();

            let dsl = build_query(&ctx.matches[1], filters, &["fr"], &world.query_settings);

            let parameters = SearchDocumentsParameters {
                parameters: SearchParameters {
                    query: Query::QueryDSL(dsl),
                    doc_types: vec![String::from(Admin::doc_type())],
                },
            };
            world.search_result = search_documents.execute(parameters).await.unwrap();
            world
        }),
    );

    steps.then_regex("he finds \"(.*)\" as the first result", |world, ctx| {
        assert_eq!(
            rank(&ctx.matches[1], &world.search_result)
                .expect("the user must perform a search before checking results"),
            0
        );

        world
    });

    steps
}

#[derive(PartialEq, Clone)]
pub enum ProcessingStep {
    Downloaded,
    Generated,
    Indexed,
    Skipped,
}

#[derive(Debug, Snafu)]
pub enum Error {
    #[snafu(display("Invalid Download URL: {} ({})", source, details))]
    InvalidUrl {
        details: String,
        source: url::ParseError,
    },
    #[snafu(display("Invalid IO: {} ({})", source, details))]
    InvalidIO {
        details: String,
        source: std::io::Error,
    },
    #[snafu(display("Download Error: {} ({})", source, details))]
    DownloadError {
        details: String,
        source: reqwest::Error,
    },
    #[snafu(display("Elasticsearch Pool Error: {} ({})", source, details))]
    ElasticsearchPoolError { details: String, source: PoolError },
    #[snafu(display("Elasticsearch Connection Error: {} ({})", source, details))]
    ElasticsearchConnectionError {
        details: String,
        source: ConnectionError,
    },
    #[snafu(display("Indexing Error: {}", details))]
    IndexingError { details: String },

    #[snafu(display("JSON Error: {} ({})", details, source))]
    JsonError {
        details: String,
        source: serde_json::Error,
    },

    #[snafu(display("Environment Variable Error: {} ({})", details, source))]
    EnvironmentVariableError {
        details: String,
        source: std::env::VarError,
    },
}

// Given the name of a french region, it will download the matching OSM file
// If that file is already in the local filesystem, then we skip the download.
// This function makes several assumptions:
// 1. The name of the region is one found in http://download.geofabrik.de/europe/france.html
// 2. The file will be downloaded to `tests/data/osm` under the project's root (identified
//    by the CARGO_MANIFEST_DIR environment variable
// The function returns either `ProcessingStep::Skipped` or `ProcessingStep::Downloaded`,
// or an error.
async fn download_osm(region: &str) -> Result<ProcessingStep, Error> {
    // Here we make sure that there is a folder tests/data/osm at the root of the project.
    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let mut path = PathBuf::from(path);
    path.push("tests");
    path.push("data");
    if tokio::fs::metadata(&path).await.is_err() {
        tokio::fs::create_dir(&path).await.context(InvalidIO {
            details: format!("could no create directory {}", path.display()),
        })?;
    }
    path.push("osm");
    if tokio::fs::metadata(&path).await.is_err() {
        tokio::fs::create_dir(&path).await.context(InvalidIO {
            details: format!("could no create directory {}", path.display()),
        })?;
    }
    // Then we try to see if there is already a file with the expected name in that
    // folder, in which case we skip the actual download, to save time.
    let filename = format!("{}-latest.osm.pbf", region);
    path.push(&filename);
    if tokio::fs::metadata(&path).await.is_ok() {
        return Ok(ProcessingStep::Skipped);
    }

    // Ok, directory structure is fine, no file in 'cache', so we download
    let url = format!(
        "https://download.geofabrik.de/europe/france/{}-latest.osm.pbf",
        region
    );
    let url = Url::parse(&url).context(InvalidUrl { details: url })?;

    let resp = reqwest::get(url.clone()).await.context(DownloadError {
        details: format!("could not download url {}", url),
    })?;

    // If we got a response which is an error (eg 404), then turn it
    // into an Error.
    let mut resp = resp.error_for_status().context(DownloadError {
        details: format!("download response error {}", url),
    })?;

    let file = tokio::fs::File::create(&path).await.context(InvalidIO {
        details: format!("could no create file {}", path.display()),
    })?;

    // Do an asynchronous, buffered copy of the download to the output file
    let mut file = tokio::io::BufWriter::new(file);

    while let Some(chunk) = resp.chunk().await.context(DownloadError {
        details: String::from("read chunk"),
    })? {
        file.write(&chunk).await.context(InvalidIO {
            details: String::from("write chunk"),
        })?;
    }

    file.flush().await.context(InvalidIO {
        details: String::from("flush"),
    })?;

    Ok(ProcessingStep::Downloaded)
}

// Generate a cosmogony file given the region
// The function makes several assumptions:
// 1. The OSM file has previously been downloaded into the expected folder (tests/data/osm)
// 2. The output is a jsonl.gz
// 3. The output will be stored in tests/data/cosmogony
//
// The OSM file will be processed if:
// 1. The output file is not found
// 2. If the output file is found and the previous step is 'downloaded' (that is it's probably a
//    new OSM file and we need to generate a new cosmogony file.
async fn generate_cosmogony(
    region: &str,
    previous: ProcessingStep,
) -> Result<ProcessingStep, Error> {
    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let mut input_path: PathBuf = [path, "tests", "data", "osm"].iter().collect();
    let filename = format!("{}-latest.osm.pbf", region);
    input_path.push(&filename);
    let mut output_path: PathBuf = [path, "tests", "data", "cosmogony"].iter().collect();
    if tokio::fs::metadata(&output_path).await.is_err() {
        tokio::fs::create_dir(&output_path)
            .await
            .context(InvalidIO {
                details: format!("could no create directory {}", output_path.display()),
            })?;
    }
    let filename = format!("{}.jsonl.gz", region);
    output_path.push(&filename);
    // if the output already exists, and the input is not a new file, then skip the generation
    if (tokio::fs::metadata(&output_path).await.is_ok()) && (previous != ProcessingStep::Downloaded)
    {
        return Ok(ProcessingStep::Skipped);
    }
    let cosmogony_path = std::env::var("COSMOGONY_EXE").context(EnvironmentVariableError {
        details: String::from("Could not get cosmogony executable"),
    })?;

    let mut child = tokio::process::Command::new(&cosmogony_path)
        .arg("--country-code")
        .arg("FR")
        .arg("--input")
        .arg(&input_path)
        .arg("--output")
        .arg(&output_path)
        .spawn()
        .expect("failed to spawn cosmogony");

    let _status = child.wait().await.context(InvalidIO {
        details: format!(
            "failed to generate cosmogony with input {} and output {}",
            input_path.display(),
            output_path.display()
        ),
    })?;

    // TODO Do something with the status?

    Ok(ProcessingStep::Generated)
}

async fn index_cosmogony(region: &str, previous: ProcessingStep) -> Result<ProcessingStep, Error> {
    // if the previous step is 'generated', then we need to index the cosmogony file.
    // Otherwise, we skip.
    // TODO: change this logic to check immutably what appends?
    if previous != ProcessingStep::Generated {
        return Ok(ProcessingStep::Skipped);
    }
    let pool = connection_test_pool()
        .await
        .context(ElasticsearchPoolError {
            details: String::from("Could not retrieve Elasticsearch test pool"),
        })?;

    let client = pool.conn().await.context(ElasticsearchConnectionError {
        details: String::from("Could not establish connection to Elasticsearch"),
    })?;

    let path: &'static str = env!("CARGO_MANIFEST_DIR");
    let config_path: PathBuf = [path, "config", "admin"].iter().collect();
    let mut settings_path = config_path.clone();
    settings_path.push("settings.json");
    let settings = tokio::fs::read_to_string(&settings_path)
        .await
        .context(InvalidIO {
            details: format!(
                "could not read settings file from '{}'",
                settings_path.display()
            ),
        })?;

    let settings = serde_json::from_str(&settings).context(JsonError {
        details: String::from("Could not deserialize settings"),
    })?;

    let mut mappings_path = config_path.clone();
    mappings_path.push("mappings.json");
    let mappings = tokio::fs::read_to_string(&mappings_path)
        .await
        .context(InvalidIO {
            details: format!(
                "could not read mappings file from '{}'",
                mappings_path.display()
            ),
        })?;

    let mappings = serde_json::from_str(&mappings).context(JsonError {
        details: String::from("Could not deserialize settings"),
    })?;

    let config = IndexConfiguration {
        name: String::from("test"),
        parameters: IndexParameters {
            timeout: String::from("10s"),
            wait_for_active_shards: String::from("1"), // only the primary shard
        },
        settings: IndexSettings { value: settings },
        mappings: IndexMappings { value: mappings },
    };

    let mut input_path: PathBuf = [path, "tests", "data", "cosmogony"].iter().collect();
    let filename = format!("{}.jsonl.gz", region);
    input_path.push(&filename);
    mimirsbrunn::admin::index_cosmogony(
        input_path.into_os_string().into_string().unwrap(),
        vec![String::from("fr")],
        config,
        client,
    )
    .await
    .map_err(|err| Error::IndexingError {
        details: format!("could not index cosmogony: {}", err.to_string(),),
    })?;
    Ok(ProcessingStep::Indexed)
}

pub fn rank(id: &str, list: &[serde_json::Value]) -> Option<usize> {
    list.iter()
        .enumerate()
        // .find(|(_i, v)| v.as_object().unwrap().get("id").unwrap().as_str().unwrap() == id)
        .find(|(_i, v)| {
            let idr = v.as_object().unwrap().get("id").unwrap().as_str().unwrap();
            idr == id
        })
        .map(|(i, _r)| i)
}
<|endoftext|>


<commit_before>use std::io;
use std::io::fs;
use std::io::process::{ProcessOutput};
use std::os;
use std::path::{Path,BytesContainer};
use std::str;
use std::vec::Vec;
use std::fmt::Show;
use ham = hamcrest;
use cargo::util::{process,ProcessBuilder};
use cargo::util::ProcessError;

use support::paths::PathExt;

pub mod paths;

/*
 *
 * ===== Builders =====
 *
 */

#[deriving(PartialEq,Clone)]
struct FileBuilder {
    path: Path,
    body: String
}

impl FileBuilder {
    pub fn new(path: Path, body: &str) -> FileBuilder {
        FileBuilder { path: path, body: body.to_string() }
    }

    fn mk(&self) -> Result<(), String> {
        try!(mkdir_recursive(&self.dirname()));

        let mut file = try!(
            fs::File::create(&self.path)
                .with_err_msg(format!("Could not create file; path={}",
                                      self.path.display())));

        file.write_str(self.body.as_slice())
            .with_err_msg(format!("Could not write to file; path={}",
                                  self.path.display()))
    }

    fn dirname(&self) -> Path {
        Path::new(self.path.dirname())
    }
}

#[deriving(PartialEq,Clone)]
struct SymlinkBuilder {
    dst: Path,
    src: Path
}

impl SymlinkBuilder {
    pub fn new(dst: Path, src: Path) -> SymlinkBuilder {
        SymlinkBuilder { dst: dst, src: src }
    }

    fn mk(&self) -> Result<(), String> {
        try!(mkdir_recursive(&self.dirname()));

        fs::symlink(&self.dst, &self.src)
            .with_err_msg(format!("Could not create symlink; dst={} src={}",
                                   self.dst.display(), self.src.display()))
    }

    fn dirname(&self) -> Path {
        Path::new(self.src.dirname())
    }
}

#[deriving(PartialEq,Clone)]
pub struct ProjectBuilder {
    name: String,
    root: Path,
    files: Vec<FileBuilder>,
    symlinks: Vec<SymlinkBuilder>
}

impl ProjectBuilder {
    pub fn new(name: &str, root: Path) -> ProjectBuilder {
        ProjectBuilder {
            name: name.to_string(),
            root: root,
            files: vec!(),
            symlinks: vec!()
        }
    }

    pub fn root(&self) -> Path {
        self.root.clone()
    }

    pub fn bin(&self, b: &str) -> Path {
        self.build_dir().join(format!("{}{}", b, os::consts::EXE_SUFFIX))
    }

    pub fn target_bin(&self, target: &str, b: &str) -> Path {
        self.build_dir().join(target).join(format!("{}{}", b,
                                                   os::consts::EXE_SUFFIX))
    }

    pub fn build_dir(&self) -> Path {
        self.root.join("target")
    }

    pub fn process<T: ToCStr>(&self, program: T) -> ProcessBuilder {
        process(program)
            .cwd(self.root())
            .env("HOME", Some(paths::home().display().to_string().as_slice()))
    }

    pub fn cargo_process(&self, program: &str) -> ProcessBuilder {
        self.build();
        self.process(cargo_dir().join(program))
    }

    pub fn file<B: BytesContainer, S: Str>(mut self, path: B,
                                           body: S) -> ProjectBuilder {
        self.files.push(FileBuilder::new(self.root.join(path), body.as_slice()));
        self
    }

    pub fn symlink<T: BytesContainer>(mut self, dst: T,
                                      src: T) -> ProjectBuilder {
        self.symlinks.push(SymlinkBuilder::new(self.root.join(dst),
                                               self.root.join(src)));
        self
    }

    // TODO: return something different than a ProjectBuilder
    pub fn build(&self) -> &ProjectBuilder {
        match self.build_with_result() {
            Err(e) => fail!(e),
            _ => return self
        }
    }

    pub fn build_with_result(&self) -> Result<(), String> {
        // First, clean the directory if it already exists
        try!(self.rm_root());

        // Create the empty directory
        try!(mkdir_recursive(&self.root));

        for file in self.files.iter() {
            try!(file.mk());
        }

        for symlink in self.symlinks.iter() {
            try!(symlink.mk());
        }

        Ok(())
    }

    fn rm_root(&self) -> Result<(), String> {
        if self.root.exists() {
            rmdir_recursive(&self.root)
        } else {
            Ok(())
        }
    }
}

// Generates a project layout
pub fn project(name: &str) -> ProjectBuilder {
    ProjectBuilder::new(name, paths::root().join(name))
}

// === Helpers ===

pub fn mkdir_recursive(path: &Path) -> Result<(), String> {
    fs::mkdir_recursive(path, io::UserDir)
        .with_err_msg(format!("could not create directory; path={}",
                              path.display()))
}

pub fn rmdir_recursive(path: &Path) -> Result<(), String> {
    path.rm_rf()
        .with_err_msg(format!("could not rm directory; path={}",
                              path.display()))
}

pub fn main_file<T: Str>(println: T, deps: &[&str]) -> String {
    let mut buf = String::new();

    for dep in deps.iter() {
        buf.push_str(format!("extern crate {};\n", dep).as_slice());
    }

    buf.push_str("fn main() { println!(");
    buf.push_str(println.as_slice());
    buf.push_str("); }\n");

    buf.to_string()
}

trait ErrMsg<T> {
    fn with_err_msg(self, val: String) -> Result<T, String>;
}

impl<T, E: Show> ErrMsg<T> for Result<T, E> {
    fn with_err_msg(self, val: String) -> Result<T, String> {
        match self {
            Ok(val) => Ok(val),
            Err(err) => Err(format!("{}; original={}", val, err))
        }
    }
}

// Path to cargo executables
pub fn cargo_dir() -> Path {
    os::getenv("CARGO_BIN_PATH").map(Path::new)
        .or_else(|| os::self_exe_path().map(|p| p.dir_path()))
        .unwrap_or_else(|| {
            fail!("CARGO_BIN_PATH wasn't set. Cannot continue running test")
        })
}

/// Returns an absolute path in the filesystem that `path` points to. The
/// returned path does not contain any symlinks in its hierarchy.
/*
 *
 * ===== Matchers =====
 *
 */

#[deriving(Clone)]
struct Execs {
    expect_stdout: Option<String>,
    expect_stdin: Option<String>,
    expect_stderr: Option<String>,
    expect_exit_code: Option<int>
}

impl Execs {

    pub fn with_stdout<S: ToString>(mut self, expected: S) -> Execs {
        self.expect_stdout = Some(expected.to_string());
        self
    }

    pub fn with_stderr<S: ToString>(mut self, expected: S) -> Execs {
        self.expect_stderr = Some(expected.to_string());
        self
    }

    pub fn with_status(mut self, expected: int) -> Execs {
        self.expect_exit_code = Some(expected);
        self
    }

    fn match_output(&self, actual: &ProcessOutput) -> ham::MatchResult {
        self.match_status(actual)
            .and(self.match_stdout(actual))
            .and(self.match_stderr(actual))
    }

    fn match_status(&self, actual: &ProcessOutput) -> ham::MatchResult {
        match self.expect_exit_code {
            None => ham::success(),
            Some(code) => {
                ham::expect(
                    actual.status.matches_exit_status(code),
                    format!("exited with {}\n--- stdout\n{}\n--- stderr\n{}",
                            actual.status,
                            String::from_utf8_lossy(actual.output.as_slice()),
                            String::from_utf8_lossy(actual.error.as_slice())))
            }
        }
    }

    fn match_stdout(&self, actual: &ProcessOutput) -> ham::MatchResult {
        self.match_std(self.expect_stdout.as_ref(), actual.output.as_slice(),
        "stdout", actual.error.as_slice())
    }

    fn match_stderr(&self, actual: &ProcessOutput) -> ham::MatchResult {
        self.match_std(self.expect_stderr.as_ref(), actual.error.as_slice(),
        "stderr", actual.output.as_slice())
    }

    fn match_std(&self, expected: Option<&String>, actual: &[u8],
                 description: &str, extra: &[u8]) -> ham::MatchResult {
        match expected.as_ref().map(|s| s.as_slice()) {
            None => ham::success(),
            Some(out) => {
                match str::from_utf8(actual) {
                    None => Err(format!("{} was not utf8 encoded", description)),
                    Some(actual) => {
                        // Let's not deal with \r\n vs \n on windows...
                        let actual = actual.replace("\r", "");
                        let actual = actual.replace("\t", "<tab>");

                        let a = actual.as_slice().lines();
                        let e = out.lines();

                        let diffs = zip_all(a, e).enumerate();
                        let mut diffs = diffs.filter_map(|(i, (a,e))| {
                            match (a, e) {
                                (Some(a), Some(e)) => {
                                    if e.as_slice().equiv(&a.as_slice()) {
                                        None
                                    } else {
                                        Some(format!("{:3} - |{}|\n    + |{}|\n", i, e, a))
                                    }
                                },
                                (Some(a), None) => {
                                    Some(format!("{:3} -\n    + |{}|\n", i, a))
                                },
                                (None, Some(e)) => {
                                    Some(format!("{:3} - |{}|\n    +\n", i, e))
                                },
                                (None, None) => fail!("Cannot get here")
                            }
                        });

                        let diffs = diffs.collect::<Vec<String>>().connect("\n");

                        ham::expect(actual.as_slice() == out,
                                    format!("differences:\n\
                                            {}\n\n\
                                            other output:\n\
                                            `{}`", diffs,
                                            String::from_utf8_lossy(extra)))
                    }
                }
            }
        }
    }
}

struct ZipAll<T, I1, I2> {
    first: I1,
    second: I2,
}

impl<T, I1: Iterator<T>, I2: Iterator<T>> Iterator<(Option<T>, Option<T>)> for ZipAll<T, I1, I2> {
    fn next(&mut self) -> Option<(Option<T>, Option<T>)> {
        let first = self.first.next();
        let second = self.second.next();

        match (first, second) {
            (None, None) => None,
            (a, b) => Some((a, b))
        }
    }
}

fn zip_all<T, I1: Iterator<T>, I2: Iterator<T>>(a: I1, b: I2) -> ZipAll<T, I1, I2> {
    ZipAll {
        first: a,
        second: b
    }
}

impl ham::SelfDescribing for Execs {
    fn describe(&self) -> String {
        "execs".to_string()
    }
}

impl ham::Matcher<ProcessBuilder> for Execs {
    fn matches(&self, process: ProcessBuilder) -> ham::MatchResult {
        let res = process.exec_with_output();

        match res {
            Ok(out) => self.match_output(&out),
            Err(ProcessError { output: Some(ref out), .. }) => {
                self.match_output(out)
            }
            Err(e) => Err(format!("could not exec process {}: {}", process, e))
        }
    }
}

pub fn execs() -> Execs {
    Execs {
        expect_stdout: None,
        expect_stderr: None,
        expect_stdin: None,
        expect_exit_code: None
    }
}

#[deriving(Clone)]
struct ShellWrites {
    expected: String
}

impl ham::SelfDescribing for ShellWrites {
    fn describe(&self) -> String {
        format!("`{}` written to the shell", self.expected)
    }
}

impl<'a> ham::Matcher<&'a [u8]> for ShellWrites {
    fn matches(&self, actual: &[u8])
        -> ham::MatchResult
    {
        println!("{}", actual);
        let actual = String::from_utf8_lossy(actual);
        let actual = actual.to_string();
        ham::expect(actual == self.expected, actual)
    }
}

pub fn shell_writes<T: Show>(string: T) -> ShellWrites {
    ShellWrites { expected: string.to_string() }
}

pub trait ResultTest<T,E> {
    fn assert(self) -> T;
}

impl<T,E: Show> ResultTest<T,E> for Result<T,E> {
    fn assert(self) -> T {
        match self {
            Ok(val) => val,
            Err(err) => fail!("Result was error: {}", err)
        }
    }
}

impl<T> ResultTest<T,()> for Option<T> {
    fn assert(self) -> T {
        match self {
            Some(val) => val,
            None => fail!("Option was None")
        }
    }
}

pub trait Tap {
    fn tap(mut self, callback: |&mut Self|) -> Self;
}

impl<T> Tap for T {
    fn tap(mut self, callback: |&mut T|) -> T {
        callback(&mut self);
        self
    }
}

pub fn basic_bin_manifest(name: &str) -> String {
    format!(r#"
        [package]

        name = "{}"
        version = "0.5.0"
        authors = ["wycats@example.com"]

        [[bin]]

        name = "{}"
    "#, name, name)
}

pub fn basic_lib_manifest(name: &str) -> String {
    format!(r#"
        [package]

        name = "{}"
        version = "0.5.0"
        authors = ["wycats@example.com"]

        [[lib]]

        name = "{}"
    "#, name, name)
}

pub static RUNNING:   &'static str = "     Running";
pub static COMPILING: &'static str = "   Compiling";
pub static FRESH:     &'static str = "       Fresh";
pub static UPDATING:  &'static str = "    Updating";
<commit_msg>Allow omitting part of the output of a process<commit_after>use std::io;
use std::io::fs;
use std::io::process::{ProcessOutput};
use std::os;
use std::path::{Path,BytesContainer};
use std::str;
use std::vec::Vec;
use std::fmt::Show;
use ham = hamcrest;
use cargo::util::{process,ProcessBuilder};
use cargo::util::ProcessError;

use support::paths::PathExt;

pub mod paths;

/*
 *
 * ===== Builders =====
 *
 */

#[deriving(PartialEq,Clone)]
struct FileBuilder {
    path: Path,
    body: String
}

impl FileBuilder {
    pub fn new(path: Path, body: &str) -> FileBuilder {
        FileBuilder { path: path, body: body.to_string() }
    }

    fn mk(&self) -> Result<(), String> {
        try!(mkdir_recursive(&self.dirname()));

        let mut file = try!(
            fs::File::create(&self.path)
                .with_err_msg(format!("Could not create file; path={}",
                                      self.path.display())));

        file.write_str(self.body.as_slice())
            .with_err_msg(format!("Could not write to file; path={}",
                                  self.path.display()))
    }

    fn dirname(&self) -> Path {
        Path::new(self.path.dirname())
    }
}

#[deriving(PartialEq,Clone)]
struct SymlinkBuilder {
    dst: Path,
    src: Path
}

impl SymlinkBuilder {
    pub fn new(dst: Path, src: Path) -> SymlinkBuilder {
        SymlinkBuilder { dst: dst, src: src }
    }

    fn mk(&self) -> Result<(), String> {
        try!(mkdir_recursive(&self.dirname()));

        fs::symlink(&self.dst, &self.src)
            .with_err_msg(format!("Could not create symlink; dst={} src={}",
                                   self.dst.display(), self.src.display()))
    }

    fn dirname(&self) -> Path {
        Path::new(self.src.dirname())
    }
}

#[deriving(PartialEq,Clone)]
pub struct ProjectBuilder {
    name: String,
    root: Path,
    files: Vec<FileBuilder>,
    symlinks: Vec<SymlinkBuilder>
}

impl ProjectBuilder {
    pub fn new(name: &str, root: Path) -> ProjectBuilder {
        ProjectBuilder {
            name: name.to_string(),
            root: root,
            files: vec!(),
            symlinks: vec!()
        }
    }

    pub fn root(&self) -> Path {
        self.root.clone()
    }

    pub fn bin(&self, b: &str) -> Path {
        self.build_dir().join(format!("{}{}", b, os::consts::EXE_SUFFIX))
    }

    pub fn target_bin(&self, target: &str, b: &str) -> Path {
        self.build_dir().join(target).join(format!("{}{}", b,
                                                   os::consts::EXE_SUFFIX))
    }

    pub fn build_dir(&self) -> Path {
        self.root.join("target")
    }

    pub fn process<T: ToCStr>(&self, program: T) -> ProcessBuilder {
        process(program)
            .cwd(self.root())
            .env("HOME", Some(paths::home().display().to_string().as_slice()))
    }

    pub fn cargo_process(&self, program: &str) -> ProcessBuilder {
        self.build();
        self.process(cargo_dir().join(program))
    }

    pub fn file<B: BytesContainer, S: Str>(mut self, path: B,
                                           body: S) -> ProjectBuilder {
        self.files.push(FileBuilder::new(self.root.join(path), body.as_slice()));
        self
    }

    pub fn symlink<T: BytesContainer>(mut self, dst: T,
                                      src: T) -> ProjectBuilder {
        self.symlinks.push(SymlinkBuilder::new(self.root.join(dst),
                                               self.root.join(src)));
        self
    }

    // TODO: return something different than a ProjectBuilder
    pub fn build(&self) -> &ProjectBuilder {
        match self.build_with_result() {
            Err(e) => fail!(e),
            _ => return self
        }
    }

    pub fn build_with_result(&self) -> Result<(), String> {
        // First, clean the directory if it already exists
        try!(self.rm_root());

        // Create the empty directory
        try!(mkdir_recursive(&self.root));

        for file in self.files.iter() {
            try!(file.mk());
        }

        for symlink in self.symlinks.iter() {
            try!(symlink.mk());
        }

        Ok(())
    }

    fn rm_root(&self) -> Result<(), String> {
        if self.root.exists() {
            rmdir_recursive(&self.root)
        } else {
            Ok(())
        }
    }
}

// Generates a project layout
pub fn project(name: &str) -> ProjectBuilder {
    ProjectBuilder::new(name, paths::root().join(name))
}

// === Helpers ===

pub fn mkdir_recursive(path: &Path) -> Result<(), String> {
    fs::mkdir_recursive(path, io::UserDir)
        .with_err_msg(format!("could not create directory; path={}",
                              path.display()))
}

pub fn rmdir_recursive(path: &Path) -> Result<(), String> {
    path.rm_rf()
        .with_err_msg(format!("could not rm directory; path={}",
                              path.display()))
}

pub fn main_file<T: Str>(println: T, deps: &[&str]) -> String {
    let mut buf = String::new();

    for dep in deps.iter() {
        buf.push_str(format!("extern crate {};\n", dep).as_slice());
    }

    buf.push_str("fn main() { println!(");
    buf.push_str(println.as_slice());
    buf.push_str("); }\n");

    buf.to_string()
}

trait ErrMsg<T> {
    fn with_err_msg(self, val: String) -> Result<T, String>;
}

impl<T, E: Show> ErrMsg<T> for Result<T, E> {
    fn with_err_msg(self, val: String) -> Result<T, String> {
        match self {
            Ok(val) => Ok(val),
            Err(err) => Err(format!("{}; original={}", val, err))
        }
    }
}

// Path to cargo executables
pub fn cargo_dir() -> Path {
    os::getenv("CARGO_BIN_PATH").map(Path::new)
        .or_else(|| os::self_exe_path().map(|p| p.dir_path()))
        .unwrap_or_else(|| {
            fail!("CARGO_BIN_PATH wasn't set. Cannot continue running test")
        })
}

/// Returns an absolute path in the filesystem that `path` points to. The
/// returned path does not contain any symlinks in its hierarchy.
/*
 *
 * ===== Matchers =====
 *
 */

#[deriving(Clone)]
struct Execs {
    expect_stdout: Option<String>,
    expect_stdin: Option<String>,
    expect_stderr: Option<String>,
    expect_exit_code: Option<int>
}

impl Execs {

    pub fn with_stdout<S: ToString>(mut self, expected: S) -> Execs {
        self.expect_stdout = Some(expected.to_string());
        self
    }

    pub fn with_stderr<S: ToString>(mut self, expected: S) -> Execs {
        self.expect_stderr = Some(expected.to_string());
        self
    }

    pub fn with_status(mut self, expected: int) -> Execs {
        self.expect_exit_code = Some(expected);
        self
    }

    fn match_output(&self, actual: &ProcessOutput) -> ham::MatchResult {
        self.match_status(actual)
            .and(self.match_stdout(actual))
            .and(self.match_stderr(actual))
    }

    fn match_status(&self, actual: &ProcessOutput) -> ham::MatchResult {
        match self.expect_exit_code {
            None => ham::success(),
            Some(code) => {
                ham::expect(
                    actual.status.matches_exit_status(code),
                    format!("exited with {}\n--- stdout\n{}\n--- stderr\n{}",
                            actual.status,
                            String::from_utf8_lossy(actual.output.as_slice()),
                            String::from_utf8_lossy(actual.error.as_slice())))
            }
        }
    }

    fn match_stdout(&self, actual: &ProcessOutput) -> ham::MatchResult {
        self.match_std(self.expect_stdout.as_ref(), actual.output.as_slice(),
        "stdout", actual.error.as_slice())
    }

    fn match_stderr(&self, actual: &ProcessOutput) -> ham::MatchResult {
        self.match_std(self.expect_stderr.as_ref(), actual.error.as_slice(),
        "stderr", actual.output.as_slice())
    }

    fn match_std(&self, expected: Option<&String>, actual: &[u8],
                 description: &str, extra: &[u8]) -> ham::MatchResult {
        match expected.as_ref().map(|s| s.as_slice()) {
            None => ham::success(),
            Some(out) => {
                let actual = match str::from_utf8(actual) {
                    None => return Err(format!("{} was not utf8 encoded",
                                               description)),
                    Some(actual) => actual,
                };
                // Let's not deal with \r\n vs \n on windows...
                let actual = actual.replace("\r", "");
                let actual = actual.replace("\t", "<tab>");

                let a = actual.as_slice().lines();
                let e = out.lines();

                let diffs = zip_all(a, e).enumerate();
                let mut diffs = diffs.filter_map(|(i, (a,e))| {
                    match (a, e) {
                        (Some(a), Some(e)) => {
                            if lines_match(e.as_slice(), a.as_slice()) {
                                None
                            } else {
                                Some(format!("{:3} - |{}|\n    + |{}|\n", i, e, a))
                            }
                        },
                        (Some(a), None) => {
                            Some(format!("{:3} -\n    + |{}|\n", i, a))
                        },
                        (None, Some(e)) => {
                            Some(format!("{:3} - |{}|\n    +\n", i, e))
                        },
                        (None, None) => fail!("Cannot get here")
                    }
                });

                let diffs = diffs.collect::<Vec<String>>().connect("\n");

                ham::expect(diffs.len() == 0,
                            format!("differences:\n\
                                    {}\n\n\
                                    other output:\n\
                                    `{}`", diffs,
                                    String::from_utf8_lossy(extra)))
            }
        }
    }
}

fn lines_match(expected: &str, mut actual: &str) -> bool {
    for part in expected.split_str("[..]") {
        match actual.find_str(part) {
            Some(i) => actual = actual.slice_from(i),
            None => {
                return false
            }
        }
    }
    return true;
}

struct ZipAll<T, I1, I2> {
    first: I1,
    second: I2,
}

impl<T, I1: Iterator<T>, I2: Iterator<T>> Iterator<(Option<T>, Option<T>)> for ZipAll<T, I1, I2> {
    fn next(&mut self) -> Option<(Option<T>, Option<T>)> {
        let first = self.first.next();
        let second = self.second.next();

        match (first, second) {
            (None, None) => None,
            (a, b) => Some((a, b))
        }
    }
}

fn zip_all<T, I1: Iterator<T>, I2: Iterator<T>>(a: I1, b: I2) -> ZipAll<T, I1, I2> {
    ZipAll {
        first: a,
        second: b
    }
}

impl ham::SelfDescribing for Execs {
    fn describe(&self) -> String {
        "execs".to_string()
    }
}

impl ham::Matcher<ProcessBuilder> for Execs {
    fn matches(&self, process: ProcessBuilder) -> ham::MatchResult {
        let res = process.exec_with_output();

        match res {
            Ok(out) => self.match_output(&out),
            Err(ProcessError { output: Some(ref out), .. }) => {
                self.match_output(out)
            }
            Err(e) => Err(format!("could not exec process {}: {}", process, e))
        }
    }
}

pub fn execs() -> Execs {
    Execs {
        expect_stdout: None,
        expect_stderr: None,
        expect_stdin: None,
        expect_exit_code: None
    }
}

#[deriving(Clone)]
struct ShellWrites {
    expected: String
}

impl ham::SelfDescribing for ShellWrites {
    fn describe(&self) -> String {
        format!("`{}` written to the shell", self.expected)
    }
}

impl<'a> ham::Matcher<&'a [u8]> for ShellWrites {
    fn matches(&self, actual: &[u8])
        -> ham::MatchResult
    {
        println!("{}", actual);
        let actual = String::from_utf8_lossy(actual);
        let actual = actual.to_string();
        ham::expect(actual == self.expected, actual)
    }
}

pub fn shell_writes<T: Show>(string: T) -> ShellWrites {
    ShellWrites { expected: string.to_string() }
}

pub trait ResultTest<T,E> {
    fn assert(self) -> T;
}

impl<T,E: Show> ResultTest<T,E> for Result<T,E> {
    fn assert(self) -> T {
        match self {
            Ok(val) => val,
            Err(err) => fail!("Result was error: {}", err)
        }
    }
}

impl<T> ResultTest<T,()> for Option<T> {
    fn assert(self) -> T {
        match self {
            Some(val) => val,
            None => fail!("Option was None")
        }
    }
}

pub trait Tap {
    fn tap(mut self, callback: |&mut Self|) -> Self;
}

impl<T> Tap for T {
    fn tap(mut self, callback: |&mut T|) -> T {
        callback(&mut self);
        self
    }
}

pub fn basic_bin_manifest(name: &str) -> String {
    format!(r#"
        [package]

        name = "{}"
        version = "0.5.0"
        authors = ["wycats@example.com"]

        [[bin]]

        name = "{}"
    "#, name, name)
}

pub fn basic_lib_manifest(name: &str) -> String {
    format!(r#"
        [package]

        name = "{}"
        version = "0.5.0"
        authors = ["wycats@example.com"]

        [[lib]]

        name = "{}"
    "#, name, name)
}

pub static RUNNING:   &'static str = "     Running";
pub static COMPILING: &'static str = "   Compiling";
pub static FRESH:     &'static str = "       Fresh";
pub static UPDATING:  &'static str = "    Updating";
<|endoftext|>


<commit_before>/*!
Test supports module.

*/

#![allow(dead_code)]

use glutin;
use glium::{mod, DisplayBuild};

use std::os;

/// Returns true if we are executed headless tests.
pub fn is_headless() -> bool {
    os::getenv("HEADLESS_TESTS").is_some()
}

/// Builds a headless display for tests.
#[cfg(feature = "headless")]
pub fn build_display() -> glium::Display {
    let display = if is_headless() {
        glutin::HeadlessRendererBuilder::new(1024, 768).build_glium().unwrap()
    } else {
        glutin::WindowBuilder::new().with_visibility(false).build_glium().unwrap()
    };

    unsafe {
        display.set_debug_callback_sync(|&mut: msg: String, _, _, severity: glium::debug::Severity| {
            if severity == glium::debug::Severity::Medium ||
               severity == glium::debug::Severity::High
            {
                panic!("{}", msg);
            }
        })
    };

    display
}

/// Builds a headless display for tests.
#[cfg(not(feature = "headless"))]
pub fn build_display() -> glium::Display {
    assert!(!is_headless());

    let display = glutin::WindowBuilder::new().with_visibility(false).build_glium().unwrap();

    unsafe {
        display.set_debug_callback_sync(|&mut: msg: String, _, _, severity: glium::debug::Severity| {
            if severity == glium::debug::Severity::Medium ||
               severity == glium::debug::Severity::High
            {
                panic!("{}", msg);
            }
        })
    };

    display
}

/// Builds a 2x2 unicolor texture.
pub fn build_unicolor_texture2d(display: &glium::Display, red: f32, green: f32, blue: f32)
    -> glium::Texture2d
{
    let color = ((red * 255.0) as u8, (green * 255.0) as u8, (blue * 255.0) as u8);

    glium::texture::Texture2d::new(display, vec![
        vec![color, color],
        vec![color, color],
    ])
}

#[vertex_format]
#[deriving(Copy)]
struct Vertex {
    position: [f32, ..2],
}

/// Builds a VB, IB and program that draw the red color `(1.0, 0.0, 0.0, 1.0)` on the whole screen.
pub fn build_fullscreen_red_pipeline(display: &glium::Display) -> (glium::VertexBuffer<Vertex>,
    glium::IndexBuffer, glium::Program)
{
    (
        glium::VertexBuffer::new(display, vec![
            Vertex { position: [-1.0,  1.0] }, Vertex { position: [1.0,  1.0] },
            Vertex { position: [-1.0, -1.0] }, Vertex { position: [1.0, -1.0] },
        ]),

        glium::IndexBuffer::new(display, glium::index_buffer::TriangleStrip(vec![0u8, 1, 2, 3])),

        glium::Program::new(display,
            "
                #version 110

                attribute vec2 position;

                void main() {
                    gl_Position = vec4(position, 0.0, 1.0);
                }
            ",
            "
                #version 110

                void main() {
                    gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);
                }
            ",
            None).unwrap()
    )
}

/// Builds a VB and an IB corresponding to a rectangle.
///
/// The VB has the "position" attribute of type "vec2".
pub fn build_rectangle_vb_ib(display: &glium::Display)
    -> (glium::VertexBuffer<Vertex>, glium::IndexBuffer)
{
    (
        glium::VertexBuffer::new(display, vec![
            Vertex { position: [-1.0,  1.0] }, Vertex { position: [1.0,  1.0] },
            Vertex { position: [-1.0, -1.0] }, Vertex { position: [1.0, -1.0] },
        ]),

        glium::IndexBuffer::new(display, glium::index_buffer::TriangleStrip(vec![0u8, 1, 2, 3])),
    )
}
<commit_msg>Use VertexBufferAny in the tests<commit_after>/*!
Test supports module.

*/

#![allow(dead_code)]

use glutin;
use glium::{mod, DisplayBuild};

use std::os;

/// Returns true if we are executed headless tests.
pub fn is_headless() -> bool {
    os::getenv("HEADLESS_TESTS").is_some()
}

/// Builds a headless display for tests.
#[cfg(feature = "headless")]
pub fn build_display() -> glium::Display {
    let display = if is_headless() {
        glutin::HeadlessRendererBuilder::new(1024, 768).build_glium().unwrap()
    } else {
        glutin::WindowBuilder::new().with_visibility(false).build_glium().unwrap()
    };

    unsafe {
        display.set_debug_callback_sync(|&mut: msg: String, _, _, severity: glium::debug::Severity| {
            if severity == glium::debug::Severity::Medium ||
               severity == glium::debug::Severity::High
            {
                panic!("{}", msg);
            }
        })
    };

    display
}

/// Builds a headless display for tests.
#[cfg(not(feature = "headless"))]
pub fn build_display() -> glium::Display {
    assert!(!is_headless());

    let display = glutin::WindowBuilder::new().with_visibility(false).build_glium().unwrap();

    unsafe {
        display.set_debug_callback_sync(|&mut: msg: String, _, _, severity: glium::debug::Severity| {
            if severity == glium::debug::Severity::Medium ||
               severity == glium::debug::Severity::High
            {
                panic!("{}", msg);
            }
        })
    };

    display
}

/// Builds a 2x2 unicolor texture.
pub fn build_unicolor_texture2d(display: &glium::Display, red: f32, green: f32, blue: f32)
    -> glium::Texture2d
{
    let color = ((red * 255.0) as u8, (green * 255.0) as u8, (blue * 255.0) as u8);

    glium::texture::Texture2d::new(display, vec![
        vec![color, color],
        vec![color, color],
    ])
}

/// Builds a VB, IB and program that draw the red color `(1.0, 0.0, 0.0, 1.0)` on the whole screen.
pub fn build_fullscreen_red_pipeline(display: &glium::Display) -> (glium::vertex_buffer::VertexBufferAny,
    glium::IndexBuffer, glium::Program)
{
    #[vertex_format]
    #[deriving(Copy)]
    struct Vertex {
        position: [f32, ..2],
    }

    (
        glium::VertexBuffer::new(display, vec![
            Vertex { position: [-1.0,  1.0] }, Vertex { position: [1.0,  1.0] },
            Vertex { position: [-1.0, -1.0] }, Vertex { position: [1.0, -1.0] },
        ]).into_vertex_buffer_any(),

        glium::IndexBuffer::new(display, glium::index_buffer::TriangleStrip(vec![0u8, 1, 2, 3])),

        glium::Program::new(display,
            "
                #version 110

                attribute vec2 position;

                void main() {
                    gl_Position = vec4(position, 0.0, 1.0);
                }
            ",
            "
                #version 110

                void main() {
                    gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);
                }
            ",
            None).unwrap()
    )
}

/// Builds a VB and an IB corresponding to a rectangle.
///
/// The VB has the "position" attribute of type "vec2".
pub fn build_rectangle_vb_ib(display: &glium::Display)
    -> (glium::vertex_buffer::VertexBufferAny, glium::IndexBuffer)
{
    #[vertex_format]
    #[deriving(Copy)]
    struct Vertex {
        position: [f32, ..2],
    }

    (
        glium::VertexBuffer::new(display, vec![
            Vertex { position: [-1.0,  1.0] }, Vertex { position: [1.0,  1.0] },
            Vertex { position: [-1.0, -1.0] }, Vertex { position: [1.0, -1.0] },
        ]).into_vertex_buffer_any(),

        glium::IndexBuffer::new(display, glium::index_buffer::TriangleStrip(vec![0u8, 1, 2, 3])),
    )
}
<|endoftext|>


<commit_before>extern crate x11 as libx11;
extern crate libc;

use self::libx11::xlib;
use std::ptr::{null, null_mut};
use std::mem::zeroed;

pub struct X11Window<'a> {
    pub width: u32,
    pub height: u32,
    display: &'a mut xlib::Display,
    screen: i32,
    window: xlib::Window,
    gc: xlib::GC,
}

impl<'a> X11Window<'a> {
    pub fn new(width: u32, height: u32) -> Result<X11Window<'a>, String> {
        unsafe {
            let disp = xlib::XOpenDisplay(null());
            if disp == null_mut() {
                return Err(String::from("Can't open display"))
            }
            let scr = xlib::XDefaultScreen(disp);
            let root_win = xlib::XRootWindow(disp,scr);
            let black = xlib::XBlackPixel(disp, scr);
            let white = xlib::XWhitePixel(disp, scr);
            let win = xlib::XCreateSimpleWindow(
                disp, root_win,
                0, 0, width, height,
                0, black, white
            );

            xlib::XMapWindow(disp, win);

            let event_mask = xlib::StructureNotifyMask;
            xlib::XSelectInput(disp, win, event_mask);
            let mut e: xlib::XEvent = zeroed();

            loop {
                xlib::XNextEvent(disp, &mut e);
                match e.get_type() {
                    xlib::MapNotify => {break;},
                    _ => {},
                }
            }
            let mut gcval: xlib::XGCValues = zeroed();
            let gc = xlib::XCreateGC(disp, win, 0, &mut gcval);

            Ok(X11Window {
                width: width,
                height: height,
                display: &mut *disp,
                screen: scr,
                window: win,
                gc: gc,
            })
        }
    }

    pub fn update(&mut self) {
        unsafe {
            let black = xlib::XBlackPixel(self.display, self.screen);
            xlib::XSetForeground(self.display, self.gc, black);
            xlib::XDrawLine(self.display, self.window, self.gc, 10, 10,190,190); //from-to
            xlib::XDrawLine(self.display, self.window, self.gc, 10,190,190, 10);

            let mouse_events = xlib::ButtonPressMask | xlib::ButtonReleaseMask;
            xlib::XSelectInput(self.display, self.window, mouse_events);
            let mut e: xlib::XEvent = zeroed();
            loop {
                xlib::XNextEvent(self.display, &mut e);
                match e.get_type() {
                    xlib::ButtonRelease => {break;},
                    _ => {},
                }
            }
        }
    }

    pub fn close(&mut self) {
        unsafe {
            xlib::XDestroyWindow(self.display, self.window);
            xlib::XCloseDisplay(self.display);
        }
    }
}
<commit_msg>Added Xlib bitmap display capabilities<commit_after>extern crate x11 as libx11;
extern crate libc;

use self::libx11::xlib;
use std::ptr::{null, null_mut};
use std::mem::zeroed;

pub struct X11Window<'a> {
    pub width: u32,
    pub height: u32,
    display: &'a mut xlib::Display,
    screen: i32,
    window: xlib::Window,
    visual: &'a mut xlib::Visual,
    gc: xlib::GC,
}

impl<'a> X11Window<'a> {
    pub fn new(width: u32, height: u32) -> Result<X11Window<'a>, String> {
        unsafe {
            let disp = xlib::XOpenDisplay(null());
            if disp == null_mut() {
                return Err(String::from("Can't open display"))
            }
            let scr = xlib::XDefaultScreen(disp);
            let vis = xlib::XDefaultVisual(disp, scr);
            let root_win = xlib::XRootWindow(disp,scr);
            let black = xlib::XBlackPixel(disp, scr);
            let white = xlib::XWhitePixel(disp, scr);
            let win = xlib::XCreateSimpleWindow(
                disp, root_win,
                0, 0, width, height,
                0, black, white
            );

            xlib::XMapWindow(disp, win);

            let event_mask = xlib::StructureNotifyMask;
            xlib::XSelectInput(disp, win, event_mask);
            let mut e: xlib::XEvent = zeroed();

            loop {
                xlib::XNextEvent(disp, &mut e);
                match e.get_type() {
                    xlib::MapNotify => {break;},
                    _ => {},
                }
            }
            let mut gcval: xlib::XGCValues = zeroed();
            let gc = xlib::XCreateGC(disp, win, 0, &mut gcval);

            Ok(X11Window {
                width: width,
                height: height,
                display: &mut *disp,
                screen: scr,
                visual: &mut *vis,
                window: win,
                gc: gc,
            })
        }
    }

    pub fn update(&mut self) {
        unsafe {
            let mut buffer = Image::new(self.display, self.visual, 640, 480);

            for x in 0..640 {
                for y in 0..480 {
                    let pad = 00;
                    let red = (x % 256);
                    let green = (y % 256);
                    let blue = 256 - (x % 256);
                    let color = (pad << 24) + (red << 16) + (green << 8) + blue;
                    xlib::XPutPixel(buffer.ximage, x, y, color as u64);
                };
            };

            self.blit(&mut buffer);

            let mouse_events = xlib::ButtonPressMask | xlib::ButtonReleaseMask;
            xlib::XSelectInput(self.display, self.window, mouse_events);
            let mut e: xlib::XEvent = zeroed();
            loop {
                xlib::XNextEvent(self.display, &mut e);
                match e.get_type() {
                    xlib::ButtonRelease => {break;},
                    _ => {},
                }
            }
        }
    }

    pub fn blit(&mut self, img: &mut Image) {
        unsafe {
            xlib::XPutImage(self.display,
                            self.window,
                            self.gc,
                            img.ximage,
                            0,0,
                            0,0,
                            img.width,
                            img.height,)
        };
    }

    pub fn close(&mut self) {
        unsafe {
            xlib::XDestroyWindow(self.display, self.window);
            xlib::XCloseDisplay(self.display);
        }
    }
}

pub struct Image<'a> {
    ximage: &'a mut xlib::XImage,
    data: Vec<i8>,
    width: u32,
    height: u32,
}

impl<'a> Image<'a> {
    pub fn new(disp: &mut xlib::Display, vis: &mut xlib::Visual, width: u32, height: u32) -> Image<'a> {
        let bitmap_pad: i32 = 32;
        let bytes_per_line: i32 = (width as i32)*bitmap_pad/4;
        let mut data: Vec<i8> = vec![0; (width*height*4) as usize];
        let mut ximg_ref = unsafe {
            let ximg_ptr = xlib::XCreateImage(disp, vis, 24, xlib::ZPixmap, 0,
                                              data.as_mut_ptr(), width, height, bitmap_pad, bytes_per_line);
            xlib::XInitImage(ximg_ptr);
            &mut *ximg_ptr
        };

        Image {
            ximage: ximg_ref,
            data: data,
            width: width,
            height: height,
        }
    }
}
<|endoftext|>


<commit_before>extern crate limonite;

use std::vec::MoveItems;

use limonite::syntax::lexer::Tokenizer;
use limonite::syntax::parser::Parser;
use limonite::syntax::core::tokens::{Token, EOF, Keyword, Identifier, Punctuation, Numeric};
use limonite::syntax::core::keywords::{Var};
use limonite::syntax::core::punctuation::{Assign};

struct MockLexer {
    tokens: MoveItems<Token>,
}

impl MockLexer {
    fn new(v: Vec<Token>) -> MockLexer {
        MockLexer {
            tokens: v.move_iter()
        }
    }
}

impl Tokenizer for MockLexer {
    fn get_tok(&mut self) -> Token {
        let next = self.tokens.next();
        match next {
            Some(tok) => tok,
            None => EOF,
        }
    }
}

#[test]
fn test_variable_int_declaration() {
    let mLexer = MockLexer::new(vec![Keyword(Var), Identifier("meow".to_string()),
                                     Punctuation(Assign), Numeric("3".to_string())]);
    let mut parser = Parser::new(mLexer);

    // parser.parse();
}
<commit_msg>Fixed /tests/test_parser.rs' use of old tokens.<commit_after>extern crate limonite;

use std::vec::MoveItems;

use limonite::syntax::lexer::Tokenizer;
use limonite::syntax::parser::Parser;
use limonite::syntax::core::tokens::{Token, EOF, Keyword, Identifier, Punctuation, Numeric};
use limonite::syntax::core::keywords::{Var};
use limonite::syntax::core::punctuation::{Equals};

struct MockLexer {
    tokens: MoveItems<Token>,
}

impl MockLexer {
    fn new(v: Vec<Token>) -> MockLexer {
        MockLexer {
            tokens: v.move_iter()
        }
    }
}

impl Tokenizer for MockLexer {
    fn get_tok(&mut self) -> Token {
        let next = self.tokens.next();
        match next {
            Some(tok) => tok,
            None => EOF,
        }
    }
}

#[test]
fn test_variable_int_declaration() {
    let mLexer = MockLexer::new(vec![Keyword(Var), Identifier("meow".to_string()),
                                     Punctuation(Equals), Numeric("3".to_string())]);
    let mut parser = Parser::new(mLexer);

    // parser.parse();
}
<|endoftext|>


<commit_before>use std::fs::{OpenOptions, File};
use std::io::Read;
use std::result::Result::{self, Ok, Err};

use libmodbus_rs::{Modbus, ModbusClient, ModbusRTU, SerialMode, Timeout};

use retry::{Retry, RetryError};

use hex_slice::AsHex;

use ::sunsaver::*;

pub trait SunSaverConnection {
    fn read_registers(&mut self) ->  [u16; 44];

    fn read_logged_data(&mut self) -> [u16; 32 * 16];

    fn read_response(&mut self) -> SunSaverResponse {
        SunSaverResponse::from_raw_bits(self.read_registers())
    }
}

pub struct ModbusSunSaverConnection {
    connection: Modbus,
}

impl ModbusSunSaverConnection {
    pub fn open(device: &str) -> ModbusSunSaverConnection {
        /* A Meterbus to Serial Converter (MSC) is required to adapt the Meter interface to an isolated RS-232 interface**.
           The SunSaver MPPT supports RTU mode only.
           16bit MODBUS® addresses (per the modbus.org spec)
           The serial communication parameters are:
             * BPS: 9600 baud
             * Parity: None
             * Data bits: 8
             * Stop bits: 2
             * Flow control: None
            All addresses listed are for the request PDU.
            The SunSaver MPPT default server address: 0x01. */
        debug!("Configuring device {}", device);
        let mut connection = Modbus::new_rtu(device, 9600, 'N', 8, 2).unwrap();
        assert!(connection.set_slave(0x01).is_ok());
        assert!(connection.rtu_set_serial_mode(SerialMode::MODBUS_RTU_RS232).is_ok());
        assert!(connection.set_response_timeout( Timeout { sec: 1, usec: 0 } ).is_ok());
        connection.set_debug(false).unwrap();

        let timeout = connection.get_response_timeout();
        info!("Timout {:?}", timeout);

        assert!(connection.connect().is_ok());
        debug!("Connected");

        ModbusSunSaverConnection {
            connection: connection,
        }
    }

    fn read_registers_retry(&self, address: i32, num_bit: i32, dest: &mut [u16]) -> Result<usize, RetryError> {
        match Retry::new(
            &mut || self.connection.read_registers(address, num_bit, dest),
            &mut |response| response.is_ok()
            ).try(3).wait(100).execute() {
            Ok(response) => Ok(response.unwrap() as usize),
            Err(error) => Err(error),
        }
    }
}

impl SunSaverConnection for ModbusSunSaverConnection {
    fn read_registers(&mut self) -> [u16; 44] {
        let mut response_register = [0u16; 44 as usize];
        let mut num_read_bytes = 0;
        num_read_bytes += self.read_registers_retry(0x08, 22, &mut response_register[0..22]).unwrap();
        num_read_bytes += self.read_registers_retry(0x1E, 22, &mut response_register[23..44]).unwrap();
        //if num_read_bytes != 44 {
        //    panic!("Failed to read all registers! Required 44 got {}", num_read_bytes);
        //}
        debug!("Read {} bytes", num_read_bytes);
        debug!("read reg 0x08 + 44: {:#x}", response_register.as_hex());

        response_register
    }

    fn read_logged_data(&mut self) -> [u16; 32 * 16] {
        let mut logged_data = [0u16; (32 * 16) as usize];

        self.read_registers_retry(0x8000, 32 * 16, &mut logged_data).unwrap();

        debug!("logged_data_start");
        for i in (0 as usize)..32 {
            debug!("{:#x}", logged_data[i..(i+16)].as_hex());
        }
        debug!("logged_data_end");

        logged_data
    }
}

#[derive(Debug)]
pub struct FileSunSaverConnection {
    file: File,
}

impl FileSunSaverConnection {
    pub fn open(filename: &str) -> FileSunSaverConnection {
        let file = OpenOptions::new().read(true).write(false).open(filename).unwrap();

        FileSunSaverConnection {
            file: file,
        }
    }
}

impl SunSaverConnection for FileSunSaverConnection {
    fn read_registers(&mut self) -> [u16; 44] {
        let mut response_register_u8 = [0u8; 88 as usize];
        assert!(self.file.read_exact(&mut response_register_u8).is_ok());

        let response_register_vec_u16: Vec<u16> = response_register_u8.chunks(2)
        .map(|items| {
            u16::from_le(((items[0] as u16) << 8) + (items[1] as u16))
        })
        .collect();

        let mut response_register_u16 = [0u16; 44 as usize];
        response_register_u16.clone_from_slice(&response_register_vec_u16);
        response_register_u16
    }

    fn read_logged_data(&mut self) -> [u16; 32 * 16] {
        unimplemented!();
    }
}<commit_msg>loop improvments<commit_after>use std::fs::{OpenOptions, File};
use std::io::Read;
use std::result::Result::{self, Ok, Err};

use libmodbus_rs::{Modbus, ModbusClient, ModbusRTU, SerialMode, Timeout};

use retry::{Retry, RetryError};

use hex_slice::AsHex;

use ::sunsaver::*;

pub trait SunSaverConnection {
    fn read_registers(&mut self) ->  [u16; 44];

    fn read_logged_data(&mut self) -> [u16; 32 * 16];

    fn read_response(&mut self) -> SunSaverResponse {
        SunSaverResponse::from_raw_bits(self.read_registers())
    }
}

pub struct ModbusSunSaverConnection {
    connection: Modbus,
}

impl ModbusSunSaverConnection {
    pub fn open(device: &str) -> ModbusSunSaverConnection {
        /* A Meterbus to Serial Converter (MSC) is required to adapt the Meter interface to an isolated RS-232 interface**.
           The SunSaver MPPT supports RTU mode only.
           16bit MODBUS® addresses (per the modbus.org spec)
           The serial communication parameters are:
             * BPS: 9600 baud
             * Parity: None
             * Data bits: 8
             * Stop bits: 2
             * Flow control: None
            All addresses listed are for the request PDU.
            The SunSaver MPPT default server address: 0x01. */
        debug!("Configuring device {}", device);
        let mut connection = Modbus::new_rtu(device, 9600, 'N', 8, 2).unwrap();
        assert!(connection.set_slave(0x01).is_ok());
        assert!(connection.rtu_set_serial_mode(SerialMode::MODBUS_RTU_RS232).is_ok());
        assert!(connection.set_response_timeout( Timeout { sec: 1, usec: 0 } ).is_ok());
        connection.set_debug(false).unwrap();

        let timeout = connection.get_response_timeout();
        info!("Timout {:?}", timeout);

        assert!(connection.connect().is_ok());
        debug!("Connected");

        ModbusSunSaverConnection {
            connection: connection,
        }
    }

    fn read_registers_retry(&self, address: i32, num_bit: i32, dest: &mut [u16]) -> Result<usize, RetryError> {
        match Retry::new(
            &mut || self.connection.read_registers(address, num_bit, dest),
            &mut |response| response.is_ok()
            ).try(3).wait(100).execute() {
            Ok(response) => Ok(response.unwrap() as usize),
            Err(error) => Err(error),
        }
    }
}

impl SunSaverConnection for ModbusSunSaverConnection {
    fn read_registers(&mut self) -> [u16; 44] {
        let mut response_register = [0u16; 44 as usize];
        let mut num_read_bytes = 0;
        num_read_bytes += self.read_registers_retry(0x08, 22, &mut response_register[0..22]).unwrap();
        num_read_bytes += self.read_registers_retry(0x1E, 22, &mut response_register[23..44]).unwrap();
        //if num_read_bytes != 44 {
        //    panic!("Failed to read all registers! Required 44 got {}", num_read_bytes);
        //}
        debug!("Read {} bytes", num_read_bytes);
        debug!("read reg 0x08 + 44: {:#x}", response_register.as_hex());

        response_register
    }

    fn read_logged_data(&mut self) -> [u16; 32 * 16] {
        let mut logged_data = [0u16; (32 * 16) as usize];

        for i in 0..32 {
            let offset: usize = i * 16;
            self.read_registers_retry((0x8000 + offset) as i32, 16, &mut logged_data[offset..offset + 16]).unwrap();
        }

        debug!("logged_data_start");
        for i in (0 as usize)..32 {
            let offset: usize = i * 16;
            debug!("{:#x}", logged_data[offset..(offset+16)].as_hex());
        }
        debug!("logged_data_end");

        logged_data
    }
}

#[derive(Debug)]
pub struct FileSunSaverConnection {
    file: File,
}

impl FileSunSaverConnection {
    pub fn open(filename: &str) -> FileSunSaverConnection {
        let file = OpenOptions::new().read(true).write(false).open(filename).unwrap();

        FileSunSaverConnection {
            file: file,
        }
    }
}

impl SunSaverConnection for FileSunSaverConnection {
    fn read_registers(&mut self) -> [u16; 44] {
        let mut response_register_u8 = [0u8; 88 as usize];
        assert!(self.file.read_exact(&mut response_register_u8).is_ok());

        let response_register_vec_u16: Vec<u16> = response_register_u8.chunks(2)
        .map(|items| {
            u16::from_le(((items[0] as u16) << 8) + (items[1] as u16))
        })
        .collect();

        let mut response_register_u16 = [0u16; 44 as usize];
        response_register_u16.clone_from_slice(&response_register_vec_u16);
        response_register_u16
    }

    fn read_logged_data(&mut self) -> [u16; 32 * 16] {
        unimplemented!();
    }
}<|endoftext|>


<commit_before>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::config::SafetyRulesConfig;
use libra_types::{account_address::AccountAddress, block_info::Round};
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, path::PathBuf};

#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct ConsensusConfig {
    pub contiguous_rounds: u32,
    pub max_block_size: u64,
    pub max_pruned_blocks_in_mem: usize,
    pub round_initial_timeout_ms: u64,
    pub proposer_type: ConsensusProposerType,
    pub safety_rules: SafetyRulesConfig,
    // Only sync committed transactions but not vote for any pending blocks. This is useful when
    // validators coordinate on the latest version to apply a manual transaction.
    pub sync_only: bool,
    // how many times to wait for txns from mempool when propose
    pub mempool_poll_count: u64,
}

impl Default for ConsensusConfig {
    fn default() -> ConsensusConfig {
        ConsensusConfig {
            contiguous_rounds: 2,
            max_block_size: 1000,
            max_pruned_blocks_in_mem: 10000,
            round_initial_timeout_ms: 1000,
            proposer_type: ConsensusProposerType::LeaderReputation(LeaderReputationConfig {
                active_weights: 99,
                inactive_weights: 1,
            }),
            safety_rules: SafetyRulesConfig::default(),
            sync_only: false,
            mempool_poll_count: 1,
        }
    }
}

impl ConsensusConfig {
    pub fn set_data_dir(&mut self, data_dir: PathBuf) {
        self.safety_rules.set_data_dir(data_dir);
    }
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case", tag = "type")]
pub enum ConsensusProposerType {
    // Choose the smallest PeerId as the proposer
    FixedProposer,
    // Round robin rotation of proposers
    RotatingProposer,
    // Committed history based proposer election
    LeaderReputation(LeaderReputationConfig),
    // Pre-specified proposers for each round,
    // or default proposer if round proposer not
    // specified
    RoundProposer(HashMap<Round, AccountAddress>),
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct LeaderReputationConfig {
    pub active_weights: u64,
    pub inactive_weights: u64,
}
<commit_msg>[consensus] reduce default max pruned block size<commit_after>// Copyright (c) The Libra Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::config::SafetyRulesConfig;
use libra_types::{account_address::AccountAddress, block_info::Round};
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, path::PathBuf};

#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct ConsensusConfig {
    pub contiguous_rounds: u32,
    pub max_block_size: u64,
    pub max_pruned_blocks_in_mem: usize,
    pub round_initial_timeout_ms: u64,
    pub proposer_type: ConsensusProposerType,
    pub safety_rules: SafetyRulesConfig,
    // Only sync committed transactions but not vote for any pending blocks. This is useful when
    // validators coordinate on the latest version to apply a manual transaction.
    pub sync_only: bool,
    // how many times to wait for txns from mempool when propose
    pub mempool_poll_count: u64,
}

impl Default for ConsensusConfig {
    fn default() -> ConsensusConfig {
        ConsensusConfig {
            contiguous_rounds: 2,
            max_block_size: 1000,
            max_pruned_blocks_in_mem: 100,
            round_initial_timeout_ms: 1000,
            proposer_type: ConsensusProposerType::LeaderReputation(LeaderReputationConfig {
                active_weights: 99,
                inactive_weights: 1,
            }),
            safety_rules: SafetyRulesConfig::default(),
            sync_only: false,
            mempool_poll_count: 1,
        }
    }
}

impl ConsensusConfig {
    pub fn set_data_dir(&mut self, data_dir: PathBuf) {
        self.safety_rules.set_data_dir(data_dir);
    }
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case", tag = "type")]
pub enum ConsensusProposerType {
    // Choose the smallest PeerId as the proposer
    FixedProposer,
    // Round robin rotation of proposers
    RotatingProposer,
    // Committed history based proposer election
    LeaderReputation(LeaderReputationConfig),
    // Pre-specified proposers for each round,
    // or default proposer if round proposer not
    // specified
    RoundProposer(HashMap<Round, AccountAddress>),
}

#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct LeaderReputationConfig {
    pub active_weights: u64,
    pub inactive_weights: u64,
}
<|endoftext|>