<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Code to handle a single block device.

use std::collections::HashMap;
use std::io::ErrorKind;
use std::fs::{OpenOptions, read_dir};
use std::os::linux::fs::MetadataExt;
use std::path::PathBuf;
use std::str::FromStr;

use nix::Errno;
use nix::sys::stat::{S_IFBLK, S_IFMT};
use serde_json;

use devicemapper::Device;

use super::super::errors::{EngineResult, EngineError, ErrorEnum};
use super::super::types::PoolUuid;

use super::blockdev::BlockDev;
use super::blockdevmgr::BlockDevMgr;
use super::device::blkdev_size;
use super::engine::DevOwnership;
use super::metadata::{BDA, StaticHeader};
use super::range_alloc::RangeAllocator;
use super::serde_structs::PoolSave;

/// Find all Stratis devices.
///
/// Returns a map of pool uuids to a vector of devices for each pool.
pub fn find_all() -> EngineResult<HashMap<PoolUuid, Vec<PathBuf>>> {

    let mut pool_map = HashMap::new();
    for dir_e in try!(read_dir("/dev")) {
        let dir_e = try!(dir_e);
        let mode = try!(dir_e.metadata()).st_mode();

        // Device node can't belong to Stratis if it is not a block device
        if mode & S_IFMT.bits() != S_IFBLK.bits() {
            continue;
        }

        let devnode = dir_e.path();

        let f = OpenOptions::new().read(true).open(&devnode);

        // There are some reasons for OpenOptions::open() to return an error
        // which are not reasons for this method to return an error.
        // Try to distinguish. Non-error conditions are:
        //
        // 1. ENXIO: The device does not exist anymore. This means that the device
        // was volatile for some reason; in that case it can not belong to
        // Stratis so it is safe to ignore it.
        //
        // 2. ENOMEDIUM: The device has no medium. An example of this case is an
        // empty optical drive.
        //
        // Note that it is better to be conservative and return with an
        // error in any case where failure to read the device could result
        // in bad data for Stratis. Additional exceptions may be added,
        // but only with a complete justification.
        if f.is_err() {
            let err = f.unwrap_err();
            match err.kind() {
                ErrorKind::NotFound => {
                    continue;
                }
                _ => {
                    if let Some(errno) = err.raw_os_error() {
                        match Errno::from_i32(errno) {
                            Errno::ENXIO | Errno::ENOMEDIUM => continue,
                            _ => return Err(EngineError::Io(err)),
                        };
                    } else {
                        return Err(EngineError::Io(err));
                    }
                }
            }
        }

        let mut f = f.expect("unreachable if f is err");
        if let DevOwnership::Ours(uuid) = try!(StaticHeader::determine_ownership(&mut f)) {
            pool_map
                .entry(uuid)
                .or_insert_with(Vec::new)
                .push(devnode)
        };
    }

    Ok(pool_map)
}

/// Get the most recent metadata from a set of Devices for a given pool UUID.
/// Returns None if no metadata found for this pool.
pub fn get_metadata(pool_uuid: &PoolUuid, devnodes: &[PathBuf]) -> EngineResult<Option<PoolSave>> {

    // No device nodes means no metadata
    if devnodes.is_empty() {
        return Ok(None);
    }

    // Get pairs of device nodes and matching BDAs
    // If no BDA, or BDA UUID does not match pool UUID, skip.
    // If there is an error reading the BDA, error. There could have been
    // vital information on that BDA, for example, it may have contained
    // the newest metadata.
    let mut bdas = Vec::new();
    for devnode in devnodes {
        let bda = try!(BDA::load(&mut try!(OpenOptions::new().read(true).open(devnode))));
        if bda.is_none() {
            continue;
        }
        let bda = bda.expect("unreachable if bda is None");

        if bda.pool_uuid() != pool_uuid {
            continue;
        }
        bdas.push((devnode, bda));
    }

    // We may have had no devices with BDAs for this pool, so return if no BDAs.
    if bdas.is_empty() {
        return Ok(None);
    }

    // Get a most recent BDA
    let &(_, ref most_recent_bda) = bdas.iter()
        .max_by_key(|p| p.1.last_update_time())
        .expect("bdas is not empty, must have a max");

    // Most recent time should never be None if this was a properly
    // created pool; this allows for the method to be called in other
    // circumstances.
    let most_recent_time = most_recent_bda.last_update_time();
    if most_recent_time.is_none() {
        return Ok(None);
    }

    // Try to read from all available devnodes that could contain most
    // recent metadata. In the event of errors, continue to try until all are
    // exhausted.
    for &(devnode, ref bda) in
        bdas.iter()
            .filter(|p| p.1.last_update_time() == most_recent_time) {

        let f = OpenOptions::new().read(true).open(devnode);
        if f.is_err() {
            continue;
        }
        let mut f = f.expect("f is not err");

        if let Ok(Some(data)) = bda.load_state(&mut f) {
            let json: serde_json::Result<PoolSave> = serde_json::from_slice(&data);
            if let Ok(pool) = json {
                return Ok(Some(pool));
            } else {
                continue;
            }
        } else {
            continue;
        }
    }

    // If no data has yet returned, we have an error. That is, we should have
    // some metadata, because we have a most recent time, but we failed to
    // get any.
    let err_str = "timestamp indicates data was written, but no data succesfully read";
    Err(EngineError::Engine(ErrorEnum::NotFound, err_str.into()))
}

/// Get the most recent metadata for each pool.
/// Since the metadata is written immediately after a pool is created, it
/// is considered an error for a pool to be w/out metadata.
pub fn get_pool_metadata(pool_table: &HashMap<PoolUuid, Vec<PathBuf>>)
                         -> EngineResult<HashMap<PoolUuid, PoolSave>> {
    let mut metadata = HashMap::new();
    for (pool_uuid, devices) in pool_table.iter() {
        if let Ok(Some(pool)) = get_metadata(pool_uuid, &devices) {
            metadata.insert(pool_uuid.clone(), pool);
        } else {
            return Err(EngineError::Engine(ErrorEnum::NotFound,
                                           format!("no metadata for pool {}", pool_uuid)));
        }
    }
    Ok(metadata)
}

/// Instantiate each pool's blockdevs.
/// Return a map from the pool uuid to the pools blockdevs.
pub fn get_pool_blockdevs(devnode_table: &HashMap<PoolUuid, Vec<PathBuf>>,
                          metadata_table: &HashMap<PoolUuid, PoolSave>)
                          -> EngineResult<HashMap<PoolUuid, BlockDevMgr>> {
    let mut result = HashMap::new();
    for (pool_uuid, pool_save) in metadata_table {
        let segments = pool_save
            .flex_devs
            .meta_dev
            .iter()
            .chain(pool_save.flex_devs.thin_meta_dev.iter())
            .chain(pool_save.flex_devs.thin_data_dev.iter());

        let mut segment_table = HashMap::new();
        for seg in segments {
            segment_table
                .entry(seg.0.clone())
                .or_insert(vec![])
                .push((seg.1, seg.2))
        }

        let devnodes = devnode_table
            .get(&pool_uuid)
            .expect("devnode_table.keys() == metadata_table.keys()");

        let mut blockdevs = vec![];
        for dev in devnodes {
            let bda = try!(BDA::load(&mut try!(OpenOptions::new().read(true).open(dev))));
            let bda = try!(bda.ok_or(EngineError::Engine(ErrorEnum::NotFound,
                                                         "no BDA found for Stratis device"
                                                             .into())));

            let actual_size = try!(blkdev_size(&try!(OpenOptions::new().read(true).open(dev))))
                .sectors();

            // If size of device has changed and is less, then it is possible
            // that the segments previously allocated for this blockdev no
            // longer exist. If that is the case, RangeAllocator::new() will
            // return an error.
            let allocator = match segment_table.get(bda.dev_uuid()) {
                Some(segments) => try!(RangeAllocator::new(actual_size, segments)),
                None => try!(RangeAllocator::new(actual_size, &vec![])),
            };

            let device = try!(Device::from_str(&dev.to_string_lossy()));
            blockdevs.push(BlockDev::new(device, dev.clone(), bda, allocator));
        }
        result.insert(pool_uuid.clone(), BlockDevMgr::new(blockdevs));
    }
    Ok(result)
}
<commit_msg>Make sure to skip devnodes referring to the same device<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Code to handle a single block device.

use std::collections::{HashMap, HashSet};
use std::io::ErrorKind;
use std::fs::{OpenOptions, read_dir};
use std::os::linux::fs::MetadataExt;
use std::path::PathBuf;
use std::str::FromStr;

use nix::Errno;
use nix::sys::stat::{S_IFBLK, S_IFMT};
use serde_json;

use devicemapper::Device;

use super::super::errors::{EngineResult, EngineError, ErrorEnum};
use super::super::types::PoolUuid;

use super::blockdev::BlockDev;
use super::blockdevmgr::BlockDevMgr;
use super::device::blkdev_size;
use super::engine::DevOwnership;
use super::metadata::{BDA, StaticHeader};
use super::range_alloc::RangeAllocator;
use super::serde_structs::PoolSave;

/// Find all Stratis devices.
///
/// Returns a map of pool uuids to a vector of devices for each pool.
pub fn find_all() -> EngineResult<HashMap<PoolUuid, Vec<PathBuf>>> {

    let mut pool_map = HashMap::new();
    for dir_e in try!(read_dir("/dev")) {
        let dir_e = try!(dir_e);
        let mode = try!(dir_e.metadata()).st_mode();

        // Device node can't belong to Stratis if it is not a block device
        if mode & S_IFMT.bits() != S_IFBLK.bits() {
            continue;
        }

        let devnode = dir_e.path();

        let f = OpenOptions::new().read(true).open(&devnode);

        // There are some reasons for OpenOptions::open() to return an error
        // which are not reasons for this method to return an error.
        // Try to distinguish. Non-error conditions are:
        //
        // 1. ENXIO: The device does not exist anymore. This means that the device
        // was volatile for some reason; in that case it can not belong to
        // Stratis so it is safe to ignore it.
        //
        // 2. ENOMEDIUM: The device has no medium. An example of this case is an
        // empty optical drive.
        //
        // Note that it is better to be conservative and return with an
        // error in any case where failure to read the device could result
        // in bad data for Stratis. Additional exceptions may be added,
        // but only with a complete justification.
        if f.is_err() {
            let err = f.unwrap_err();
            match err.kind() {
                ErrorKind::NotFound => {
                    continue;
                }
                _ => {
                    if let Some(errno) = err.raw_os_error() {
                        match Errno::from_i32(errno) {
                            Errno::ENXIO | Errno::ENOMEDIUM => continue,
                            _ => return Err(EngineError::Io(err)),
                        };
                    } else {
                        return Err(EngineError::Io(err));
                    }
                }
            }
        }

        let mut f = f.expect("unreachable if f is err");
        if let DevOwnership::Ours(uuid) = try!(StaticHeader::determine_ownership(&mut f)) {
            pool_map
                .entry(uuid)
                .or_insert_with(Vec::new)
                .push(devnode)
        };
    }

    Ok(pool_map)
}

/// Get the most recent metadata from a set of Devices for a given pool UUID.
/// Returns None if no metadata found for this pool.
pub fn get_metadata(pool_uuid: &PoolUuid, devnodes: &[PathBuf]) -> EngineResult<Option<PoolSave>> {

    // No device nodes means no metadata
    if devnodes.is_empty() {
        return Ok(None);
    }

    // Get pairs of device nodes and matching BDAs
    // If no BDA, or BDA UUID does not match pool UUID, skip.
    // If there is an error reading the BDA, error. There could have been
    // vital information on that BDA, for example, it may have contained
    // the newest metadata.
    let mut bdas = Vec::new();
    for devnode in devnodes {
        let bda = try!(BDA::load(&mut try!(OpenOptions::new().read(true).open(devnode))));
        if bda.is_none() {
            continue;
        }
        let bda = bda.expect("unreachable if bda is None");

        if bda.pool_uuid() != pool_uuid {
            continue;
        }
        bdas.push((devnode, bda));
    }

    // We may have had no devices with BDAs for this pool, so return if no BDAs.
    if bdas.is_empty() {
        return Ok(None);
    }

    // Get a most recent BDA
    let &(_, ref most_recent_bda) = bdas.iter()
        .max_by_key(|p| p.1.last_update_time())
        .expect("bdas is not empty, must have a max");

    // Most recent time should never be None if this was a properly
    // created pool; this allows for the method to be called in other
    // circumstances.
    let most_recent_time = most_recent_bda.last_update_time();
    if most_recent_time.is_none() {
        return Ok(None);
    }

    // Try to read from all available devnodes that could contain most
    // recent metadata. In the event of errors, continue to try until all are
    // exhausted.
    for &(devnode, ref bda) in
        bdas.iter()
            .filter(|p| p.1.last_update_time() == most_recent_time) {

        let f = OpenOptions::new().read(true).open(devnode);
        if f.is_err() {
            continue;
        }
        let mut f = f.expect("f is not err");

        if let Ok(Some(data)) = bda.load_state(&mut f) {
            let json: serde_json::Result<PoolSave> = serde_json::from_slice(&data);
            if let Ok(pool) = json {
                return Ok(Some(pool));
            } else {
                continue;
            }
        } else {
            continue;
        }
    }

    // If no data has yet returned, we have an error. That is, we should have
    // some metadata, because we have a most recent time, but we failed to
    // get any.
    let err_str = "timestamp indicates data was written, but no data succesfully read";
    Err(EngineError::Engine(ErrorEnum::NotFound, err_str.into()))
}

/// Get the most recent metadata for each pool.
/// Since the metadata is written immediately after a pool is created, it
/// is considered an error for a pool to be w/out metadata.
pub fn get_pool_metadata(pool_table: &HashMap<PoolUuid, Vec<PathBuf>>)
                         -> EngineResult<HashMap<PoolUuid, PoolSave>> {
    let mut metadata = HashMap::new();
    for (pool_uuid, devices) in pool_table.iter() {
        if let Ok(Some(pool)) = get_metadata(pool_uuid, &devices) {
            metadata.insert(pool_uuid.clone(), pool);
        } else {
            return Err(EngineError::Engine(ErrorEnum::NotFound,
                                           format!("no metadata for pool {}", pool_uuid)));
        }
    }
    Ok(metadata)
}

/// Instantiate each pool's blockdevs.
/// Return a map from the pool uuid to the pools blockdevs.
pub fn get_pool_blockdevs(devnode_table: &HashMap<PoolUuid, Vec<PathBuf>>,
                          metadata_table: &HashMap<PoolUuid, PoolSave>)
                          -> EngineResult<HashMap<PoolUuid, BlockDevMgr>> {
    let mut result = HashMap::new();
    for (pool_uuid, pool_save) in metadata_table {
        let segments = pool_save
            .flex_devs
            .meta_dev
            .iter()
            .chain(pool_save.flex_devs.thin_meta_dev.iter())
            .chain(pool_save.flex_devs.thin_data_dev.iter());

        let mut segment_table = HashMap::new();
        for seg in segments {
            segment_table
                .entry(seg.0.clone())
                .or_insert(vec![])
                .push((seg.1, seg.2))
        }

        let devnodes = devnode_table
            .get(&pool_uuid)
            .expect("devnode_table.keys() == metadata_table.keys()");

        let mut blockdevs = vec![];
        let mut devices = HashSet::new();
        for dev in devnodes {
            let device = try!(Device::from_str(&dev.to_string_lossy()));

            // If we've seen this device already, skip it.
            if devices.contains(&device) {
                continue;
            } else {
                devices.insert(device);
            }

            let bda = try!(BDA::load(&mut try!(OpenOptions::new().read(true).open(dev))));
            let bda = try!(bda.ok_or(EngineError::Engine(ErrorEnum::NotFound,
                                                         "no BDA found for Stratis device"
                                                             .into())));

            let actual_size = try!(blkdev_size(&try!(OpenOptions::new().read(true).open(dev))))
                .sectors();

            // If size of device has changed and is less, then it is possible
            // that the segments previously allocated for this blockdev no
            // longer exist. If that is the case, RangeAllocator::new() will
            // return an error.
            let allocator = match segment_table.get(bda.dev_uuid()) {
                Some(segments) => try!(RangeAllocator::new(actual_size, segments)),
                None => try!(RangeAllocator::new(actual_size, &vec![])),
            };

            blockdevs.push(BlockDev::new(device, dev.clone(), bda, allocator));
        }
        result.insert(pool_uuid.clone(), BlockDevMgr::new(blockdevs));
    }
    Ok(result)
}
<|endoftext|>


<commit_before>//! Runs enqueued background jobs
//!
//! This binary will loop until interrupted. It will run all jobs in the
//! background queue, sleeping for 1 second whenever the queue is empty. If we
//! are unable to spawn workers to run jobs (either because we couldn't connect
//! to the DB, an error occurred while loading, or we just never heard back from
//! the worker thread), we will rebuild the runner and try again up to 5 times.
//! After the 5th occurrence, we will panic.
//!
//! Usage:
//!      cargo run --bin background-worker

#![warn(clippy::all, rust_2018_idioms)]

use cargo_registry::config;
use cargo_registry::worker::cloudfront::CloudFront;
use cargo_registry::{background_jobs::*, db};
use cargo_registry_index::{Repository, RepositoryConfig};
use diesel::r2d2;
use reqwest::blocking::Client;
use std::sync::{Arc, Mutex};
use std::thread::sleep;
use std::time::Duration;

fn main() {
    println!("Booting runner");

    let _sentry = cargo_registry::sentry::init();

    // Initialize logging
    cargo_registry::util::tracing::init();

    let config = config::Server::default();
    let uploader = config.base.uploader();

    if config.db.are_all_read_only() {
        loop {
            println!(
                "Cannot run background jobs with a read-only pool. Please scale background_worker \
                to 0 processes until the leader database is available."
            );
            sleep(Duration::from_secs(60));
        }
    }

    let db_url = db::connection_url(&config.db, &config.db.primary.url);

    let job_start_timeout = dotenv::var("BACKGROUND_JOB_TIMEOUT")
        .unwrap_or_else(|_| "30".into())
        .parse()
        .expect("Invalid value for `BACKGROUND_JOB_TIMEOUT`");

    println!("Cloning index");

    let repository_config = RepositoryConfig::from_environment();
    let repository = Arc::new(Mutex::new(
        Repository::open(&repository_config).expect("Failed to clone index"),
    ));
    println!("Index cloned");

    let cloudfront = CloudFront::from_environment();

    let build_runner = || {
        let client = Client::builder()
            .timeout(Duration::from_secs(45))
            .build()
            .expect("Couldn't build client");
        let environment = Environment::new_shared(
            repository.clone(),
            uploader.clone(),
            client,
            cloudfront.clone(),
        );
        let db_config = r2d2::Pool::builder().min_idle(Some(0));
        swirl::Runner::builder(environment)
            .connection_pool_builder(&db_url, db_config)
            .job_start_timeout(Duration::from_secs(job_start_timeout))
            .build()
    };
    let mut runner = build_runner();

    println!("Runner booted, running jobs");

    let mut failure_count = 0;

    loop {
        if let Err(e) = runner.run_all_pending_jobs() {
            failure_count += 1;
            if failure_count < 5 {
                eprintln!(
                    "Error running jobs (n = {}) -- retrying: {:?}",
                    failure_count, e,
                );
                runner = build_runner();
            } else {
                panic!("Failed to begin running jobs 5 times. Restarting the process");
            }
        }
        sleep(Duration::from_secs(1));
    }
}
<commit_msg>bin/background-worker: Use `tracing` macros instead of `println!()`<commit_after>//! Runs enqueued background jobs
//!
//! This binary will loop until interrupted. It will run all jobs in the
//! background queue, sleeping for 1 second whenever the queue is empty. If we
//! are unable to spawn workers to run jobs (either because we couldn't connect
//! to the DB, an error occurred while loading, or we just never heard back from
//! the worker thread), we will rebuild the runner and try again up to 5 times.
//! After the 5th occurrence, we will panic.
//!
//! Usage:
//!      cargo run --bin background-worker

#![warn(clippy::all, rust_2018_idioms)]

#[macro_use]
extern crate tracing;

use cargo_registry::config;
use cargo_registry::worker::cloudfront::CloudFront;
use cargo_registry::{background_jobs::*, db};
use cargo_registry_index::{Repository, RepositoryConfig};
use diesel::r2d2;
use reqwest::blocking::Client;
use std::sync::{Arc, Mutex};
use std::thread::sleep;
use std::time::Duration;

fn main() {
    let _sentry = cargo_registry::sentry::init();

    // Initialize logging
    cargo_registry::util::tracing::init();

    info!("Booting runner");

    let config = config::Server::default();
    let uploader = config.base.uploader();

    if config.db.are_all_read_only() {
        loop {
            warn!(
                "Cannot run background jobs with a read-only pool. Please scale background_worker \
                to 0 processes until the leader database is available."
            );
            sleep(Duration::from_secs(60));
        }
    }

    let db_url = db::connection_url(&config.db, &config.db.primary.url);

    let job_start_timeout = dotenv::var("BACKGROUND_JOB_TIMEOUT")
        .unwrap_or_else(|_| "30".into())
        .parse()
        .expect("Invalid value for `BACKGROUND_JOB_TIMEOUT`");

    info!("Cloning index");

    let repository_config = RepositoryConfig::from_environment();
    let repository = Arc::new(Mutex::new(
        Repository::open(&repository_config).expect("Failed to clone index"),
    ));
    info!("Index cloned");

    let cloudfront = CloudFront::from_environment();

    let build_runner = || {
        let client = Client::builder()
            .timeout(Duration::from_secs(45))
            .build()
            .expect("Couldn't build client");
        let environment = Environment::new_shared(
            repository.clone(),
            uploader.clone(),
            client,
            cloudfront.clone(),
        );
        let db_config = r2d2::Pool::builder().min_idle(Some(0));
        swirl::Runner::builder(environment)
            .connection_pool_builder(&db_url, db_config)
            .job_start_timeout(Duration::from_secs(job_start_timeout))
            .build()
    };
    let mut runner = build_runner();

    info!("Runner booted, running jobs");

    let mut failure_count = 0;

    loop {
        if let Err(e) = runner.run_all_pending_jobs() {
            failure_count += 1;
            if failure_count < 5 {
                warn!(?failure_count, err = ?e, "Error running jobs -- retrying");
                runner = build_runner();
            } else {
                panic!("Failed to begin running jobs 5 times. Restarting the process");
            }
        }
        sleep(Duration::from_secs(1));
    }
}
<|endoftext|>


<commit_before>#![feature(async_await, await_macro)]

extern crate glib;
extern crate gio;
use gio::prelude::*;

use std::str;

// Throughout our chained futures, we convert all errors to strings
// via map_err() return them directly.
async fn read_file(file: gio::File) -> Result<(), String> {
    // Try to open the file.
    let strm = await!(file.read_async_future(glib::PRIORITY_DEFAULT))
        .map_err(|err| format!("Failed to open file: {}", err))?;

    // If opening the file succeeds, we asynchronously loop and
    // read the file in up to 64 byte chunks and re-use the same
    // vec for each read.
    let mut buf = vec![0; 64];
    let mut idx = 0;

    loop {
        let (b, len) = await!(strm.read_async_future(buf, glib::PRIORITY_DEFAULT))
            .map_err(|(_buf, err)| format!("Failed to read from stream: {}", err))?;

        // Once 0 is returned, we know that we're done with reading, otherwise
        // loop again and read another chunk.
        if len == 0 {
            break;
        }

        buf = b;

        println!("line {}: {:?}", idx, str::from_utf8(&buf[0..len]).unwrap());

        idx += 1;
    }

    // Asynchronously close the stream in the end.
    let _ = await!(strm.close_async_future(glib::PRIORITY_DEFAULT))
        .map_err(|err| format!("Failed to close stream: {}", err))?;

    Ok(())
}

fn main() {
    let c = glib::MainContext::default();
    let l = glib::MainLoop::new(Some(&c), false);

    c.push_thread_default();

    let file = gio::File::new_for_path("Cargo.toml");

    let l_clone = l.clone();
    let future = async move {
        match await!(read_file(file)) {
            Ok(()) => (),
            Err(err) => eprintln!("Got error: {}", err),
        }
        l_clone.quit();
    };

    c.spawn_local(future);

    l.run();

    c.pop_thread_default();
}
<commit_msg>Move gio_futures_await example from the await! macro to the await keyword<commit_after>#![feature(async_await)]

extern crate gio;
extern crate glib;
use gio::prelude::*;

use futures::prelude::*;

use std::str;

// Throughout our chained futures, we convert all errors to strings
// via map_err() return them directly.
async fn read_file(file: gio::File) -> Result<(), String> {
    // Try to open the file.
    let strm = file
        .read_async_future(glib::PRIORITY_DEFAULT)
        .map_err(|err| format!("Failed to open file: {}", err))
        .await?;

    // If opening the file succeeds, we asynchronously loop and
    // read the file in up to 64 byte chunks and re-use the same
    // vec for each read.
    let mut buf = vec![0; 64];
    let mut idx = 0;

    loop {
        let (b, len) = strm
            .read_async_future(buf, glib::PRIORITY_DEFAULT)
            .map_err(|(_buf, err)| format!("Failed to read from stream: {}", err))
            .await?;

        // Once 0 is returned, we know that we're done with reading, otherwise
        // loop again and read another chunk.
        if len == 0 {
            break;
        }

        buf = b;

        println!("line {}: {:?}", idx, str::from_utf8(&buf[0..len]).unwrap());

        idx += 1;
    }

    // Asynchronously close the stream in the end.
    let _ = strm
        .close_async_future(glib::PRIORITY_DEFAULT)
        .map_err(|err| format!("Failed to close stream: {}", err))
        .await?;

    Ok(())
}

fn main() {
    let c = glib::MainContext::default();
    let l = glib::MainLoop::new(Some(&c), false);

    c.push_thread_default();

    let file = gio::File::new_for_path("Cargo.toml");

    let l_clone = l.clone();
    let future = async move {
        match read_file(file).await {
            Ok(()) => (),
            Err(err) => eprintln!("Got error: {}", err),
        }
        l_clone.quit();
    };

    c.spawn_local(future);

    l.run();

    c.pop_thread_default();
}
<|endoftext|>


<commit_before>extern crate env_logger;

extern crate gears;
use gears::dsl::command::*;
use gears::structure::model::Model;
use gears::structure::domain::Domain;

#[test]
fn test_dsl_tokens_to_tree() {
    let _ = env_logger::init();

    let tokens = vec![
        DslToken::With("domain".to_owned()),
        DslToken::BlockOpen,

        DslToken::With("entity post".to_owned()),
        DslToken::BlockOpen,

        DslToken::With("attibute name".to_owned()),
        DslToken::BlockOpen,

        DslToken::Comment("This adds a default value of 'zork'".to_owned()),
        DslToken::Command("add default 'zork'".to_owned()),

        DslToken::BlockClose,
        DslToken::BlockClose,
        DslToken::BlockClose,
    ];

    match tokens_as_tree(&tokens) {
        Ok(tree) => {
            assert_eq!(tree.len(), 1);
            match tree[0] {
                DslTree::Scope(ref e, ref v) => assert_eq!(e, "domain"),
                _ => assert!(false),
            }
        }
        Err(_) => {
            assert!(false);
        }
    }
}

#[test]
fn test_dsl_domain() {
    let _ = env_logger::init();

    let mut domain = Domain::default();
    let e_count = domain.entities.len();
    domain.interpret_dsl("add entity abc;");
    assert_eq!(domain.entities.len(), e_count + 1);

    let dsl = domain.generate_dsl();
    assert_eq!(dsl.len(), 1);
}

#[test]
fn test_dsl_domain_multiple_commands() {
    let _ = env_logger::init();

    let mut domain = Domain::default();

    let e_count = domain.entities.len();

    domain.interpret_dsl("add entity abc; remove entity abc; add entity post;");
    assert_eq!(domain.entities.len(), e_count + 1);

    let dsl = domain.generate_dsl();
    assert_eq!(dsl.len(), 1);

}

#[test]
fn test_dsl_domain_generate_and_consume() {
    let _ = env_logger::init();

    let mut domain = Domain::default();

    let e_count = domain.entities.len();

    domain
        .interpret_dsl("add entity abc; remove entity abc; add entity post; add entity comment; add entity log;")
        .ok();

    let script = domain.to_text_dsl();

    let mut next_domain = Domain::default();
    next_domain.interpret_dsl(&script).ok();

    assert_eq!(domain, next_domain);
}

#[test]
fn test_dsl_model_interpret() {
    let _ = env_logger::init();

    let mut model = Model::default();
    assert_eq!(model.domain.doc.entities.len(), 0);

    let _ = model.interpret_dsl("with domain { add entity zork; };");
    assert_eq!(model.domain.doc.entities.len(), 1);

    let _ = model.interpret_dsl("with domain { remove entity zork; };");
    assert_eq!(model.domain.doc.entities.len(), 0);

}

#[test]
fn test_dsl_model_interpret_multiline() {
    let _ = env_logger::init();

    let mut model = Model::default();
    assert_eq!(model.domain.doc.entities.len(), 0);

    let _ = model.interpret_dsl(
        &r#"with domain { add entity zork; add entity bork; add entity fnord; };"#,
    );
    assert_eq!(model.domain.doc.entities.len(), 3);

    let _ = model.interpret_dsl(
        &r#"with domain { remove entity zork; remove entity bork; };"#,
    );
    assert_eq!(model.domain.doc.entities.len(), 1);

}
<commit_msg>test: Check dsl results<commit_after>extern crate env_logger;

extern crate gears;
use gears::dsl::command::*;
use gears::structure::model::Model;
use gears::structure::domain::Domain;

#[test]
fn test_dsl_tokens_to_tree() {
    let _ = env_logger::init();

    let tokens = vec![
        DslToken::With("domain".to_owned()),
        DslToken::BlockOpen,

        DslToken::With("entity post".to_owned()),
        DslToken::BlockOpen,

        DslToken::With("attibute name".to_owned()),
        DslToken::BlockOpen,

        DslToken::Comment("This adds a default value of 'zork'".to_owned()),
        DslToken::Command("add default 'zork'".to_owned()),

        DslToken::BlockClose,
        DslToken::BlockClose,
        DslToken::BlockClose,
    ];

    match tokens_as_tree(&tokens) {
        Ok(tree) => {
            assert_eq!(tree.len(), 1);
            match tree[0] {
                DslTree::Scope(ref e, ref v) => assert_eq!(e, "domain"),
                _ => assert!(false),
            }
        }
        Err(_) => {
            assert!(false);
        }
    }
}

#[test]
fn test_dsl_domain() {
    let _ = env_logger::init();

    let mut domain = Domain::default();
    let e_count = domain.entities.len();
    domain.interpret_dsl("add entity abc;").ok();
    assert_eq!(domain.entities.len(), e_count + 1);

    let dsl = domain.generate_dsl();
    assert_eq!(dsl.len(), 1);
}

#[test]
fn test_dsl_domain_multiple_commands() {
    let _ = env_logger::init();

    let mut domain = Domain::default();

    let e_count = domain.entities.len();

    domain
        .interpret_dsl("add entity abc; remove entity abc; add entity post;")
        .ok();
    assert_eq!(domain.entities.len(), e_count + 1);

    let dsl = domain.generate_dsl();
    assert_eq!(dsl.len(), 1);

}

#[test]
fn test_dsl_domain_generate_and_consume() {
    let _ = env_logger::init();

    let mut domain = Domain::default();

    domain
        .interpret_dsl("add entity abc; remove entity abc; add entity post; add entity comment; add entity log;")
        .ok();

    let script = domain.to_text_dsl();

    let mut next_domain = Domain::default();
    next_domain.interpret_dsl(&script).ok();

    assert_eq!(domain, next_domain);
}

#[test]
fn test_dsl_model_interpret() {
    let _ = env_logger::init();

    let mut model = Model::default();
    assert_eq!(model.domain.doc.entities.len(), 0);

    model
        .interpret_dsl("with domain { add entity zork; };")
        .ok();
    assert_eq!(model.domain.doc.entities.len(), 1);

    model
        .interpret_dsl("with domain { remove entity zork; };")
        .ok();
    assert_eq!(model.domain.doc.entities.len(), 0);

}

#[test]
fn test_dsl_model_interpret_translations() {
    let _ = env_logger::init();

    let mut model = Model::default();
    assert_eq!(model.translations.len(), 0);

    /*
    // implement gears-dsl for documentlist first
    model
        .interpret_dsl("with translations { add translation en_GB; };")
        .is_ok();
    assert_eq!(model.translations.len(), 1);
    */
}

#[test]
fn test_dsl_model_interpret_multiline() {
    let _ = env_logger::init();

    let mut model = Model::default();
    assert_eq!(model.domain.doc.entities.len(), 0);

    model
        .interpret_dsl(
            &r#"with domain { add entity zork; add entity bork; add entity fnord; };"#,
        )
        .ok();

    assert_eq!(model.domain.doc.entities.len(), 3);

    model
        .interpret_dsl(
            &r#"with domain { remove entity zork; remove entity bork; };"#,
        )
        .ok();
    assert_eq!(model.domain.doc.entities.len(), 1);

}
<|endoftext|>


<commit_before>// A port of the simplistic benchmark from
//
//    http://github.com/PaulKeeble/ScalaVErlangAgents
//
// I *think* it's the same, more or less.

extern crate native;
extern crate time;
extern crate zmq;
extern crate time;

use std::comm;
use std::os;

fn server(mut pull_socket: zmq::Socket, mut push_socket: zmq::Socket, mut workers: uint) {
    let mut count = 0u;
    let mut msg = zmq::Message::new();

    while workers != 0 {
        match pull_socket.recv(&mut msg, 0) {
            Err(e) => fail!(e.to_str()),
            Ok(()) => {
                msg.with_str(|s| {
                    if s == "" {
                        workers -= 1;
                    } else {
                        count += from_str::<uint>(s).unwrap();
                    }
                })
            }
        }
    }

    match push_socket.send_str(count.to_str(), 0) {
        Ok(()) => { }
        Err(e) => fail!(e.to_str()),
    }
}

fn spawn_server(ctx: &mut zmq::Context, workers: uint) -> comm::Chan<()> {
    let mut pull_socket = ctx.socket(zmq::PULL).unwrap();
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();

    pull_socket.bind("inproc://server-pull").unwrap();
    push_socket.bind("inproc://server-push").unwrap();

    // Spawn the server.
    let (ready_po, ready_ch) = comm::Chan::new();
    let (start_po, start_ch) = comm::Chan::new();

    // Mutable sockets cannot be implicitly captured.
    let pull_socket = pull_socket;
    let push_socket = push_socket;

    native::task::spawn(proc() {
        // Let the main thread know we're ready.
        ready_ch.send(());

        // Wait until we need to start.
        start_po.recv();

        server(pull_socket, push_socket, workers);
    });

    // Wait for the server to start.
    ready_po.recv();

    start_ch
}

fn worker(mut push_socket: zmq::Socket, count: uint) {
    for _ in range(0, count) {
        push_socket.send_str(100u.to_str(), 0).unwrap();
    }

    // Let the server know we're done.
    push_socket.send_str("", 0).unwrap();
}

fn spawn_worker(ctx: &mut zmq::Context, count: uint) -> comm::Port<()> {
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();

    push_socket.connect("inproc://server-pull").unwrap();
    //push_socket.connect("tcp://127.0.0.1:3456").unwrap();

    // Mutable sockets cannot be implicitly captured.
    let push_socket = push_socket;

    // Spawn the worker.
    let (po, ch) = comm::Chan::new();
    native::task::spawn(proc() {
        // Let the main thread we're ready.
        ch.send(());

        worker(push_socket, count);

        ch.send(());
    });

    // Wait for the worker to start.
    po.recv();

    po
}

fn run(ctx: &mut zmq::Context, size: uint, workers: uint) {
    let start_ch = spawn_server(ctx, workers);

    // Create some command/control sockets.
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();
    let mut pull_socket = ctx.socket(zmq::PULL).unwrap();

    push_socket.connect("inproc://server-pull").unwrap();
    pull_socket.connect("inproc://server-push").unwrap();
    //push_socket.connect("tcp://127.0.0.1:3456").unwrap();
    //pull_socket.connect("tcp://127.0.0.1:3457").unwrap();

    // Spawn all the workers.
    let mut worker_results = ~[];
    for _ in range(0, workers) {
        worker_results.push(spawn_worker(ctx, size / workers));
    }

    let start = time::precise_time_s();

    start_ch.send(());

    // Block until all the workers finish.
    for po in worker_results.iter() {
        po.recv();
    }

    // Receive the final count.
    let result = match pull_socket.recv_msg(0) {
        Ok(msg) => msg.with_str(|s| from_str::<uint>(s).unwrap()),
        Err(e) => fail!(e.to_str()),
    };

    let end = time::precise_time_s();
    let elapsed = end - start;

    println!("Count is {}", result);
    println!("Test took {} seconds", elapsed);
    let thruput = ((size / workers * workers) as f64) / elapsed;
    println!("Throughput={:f} per sec", thruput);
}

fn main() {
    let args = os::args();

    let args = if os::getenv("RUST_BENCH").is_some() {
        ~[~"", ~"1000000", ~"10000"]
    } else if args.len() <= 1u {
        ~[~"", ~"10000", ~"4"]
    } else {
        args
    };

    let size = from_str::<uint>(args[1]).unwrap();
    let workers = from_str::<uint>(args[2]).unwrap();

    let mut ctx = zmq::Context::new();

    run(&mut ctx, size, workers);
}
<commit_msg>remove duplicated `extern crate`<commit_after>// A port of the simplistic benchmark from
//
//    http://github.com/PaulKeeble/ScalaVErlangAgents
//
// I *think* it's the same, more or less.

extern crate native;
extern crate time;
extern crate zmq;

use std::comm;
use std::os;

fn server(mut pull_socket: zmq::Socket, mut push_socket: zmq::Socket, mut workers: uint) {
    let mut count = 0u;
    let mut msg = zmq::Message::new();

    while workers != 0 {
        match pull_socket.recv(&mut msg, 0) {
            Err(e) => fail!(e.to_str()),
            Ok(()) => {
                msg.with_str(|s| {
                    if s == "" {
                        workers -= 1;
                    } else {
                        count += from_str::<uint>(s).unwrap();
                    }
                })
            }
        }
    }

    match push_socket.send_str(count.to_str(), 0) {
        Ok(()) => { }
        Err(e) => fail!(e.to_str()),
    }
}

fn spawn_server(ctx: &mut zmq::Context, workers: uint) -> comm::Chan<()> {
    let mut pull_socket = ctx.socket(zmq::PULL).unwrap();
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();

    pull_socket.bind("inproc://server-pull").unwrap();
    push_socket.bind("inproc://server-push").unwrap();

    // Spawn the server.
    let (ready_po, ready_ch) = comm::Chan::new();
    let (start_po, start_ch) = comm::Chan::new();

    // Mutable sockets cannot be implicitly captured.
    let pull_socket = pull_socket;
    let push_socket = push_socket;

    native::task::spawn(proc() {
        // Let the main thread know we're ready.
        ready_ch.send(());

        // Wait until we need to start.
        start_po.recv();

        server(pull_socket, push_socket, workers);
    });

    // Wait for the server to start.
    ready_po.recv();

    start_ch
}

fn worker(mut push_socket: zmq::Socket, count: uint) {
    for _ in range(0, count) {
        push_socket.send_str(100u.to_str(), 0).unwrap();
    }

    // Let the server know we're done.
    push_socket.send_str("", 0).unwrap();
}

fn spawn_worker(ctx: &mut zmq::Context, count: uint) -> comm::Port<()> {
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();

    push_socket.connect("inproc://server-pull").unwrap();
    //push_socket.connect("tcp://127.0.0.1:3456").unwrap();

    // Mutable sockets cannot be implicitly captured.
    let push_socket = push_socket;

    // Spawn the worker.
    let (po, ch) = comm::Chan::new();
    native::task::spawn(proc() {
        // Let the main thread we're ready.
        ch.send(());

        worker(push_socket, count);

        ch.send(());
    });

    // Wait for the worker to start.
    po.recv();

    po
}

fn run(ctx: &mut zmq::Context, size: uint, workers: uint) {
    let start_ch = spawn_server(ctx, workers);

    // Create some command/control sockets.
    let mut push_socket = ctx.socket(zmq::PUSH).unwrap();
    let mut pull_socket = ctx.socket(zmq::PULL).unwrap();

    push_socket.connect("inproc://server-pull").unwrap();
    pull_socket.connect("inproc://server-push").unwrap();
    //push_socket.connect("tcp://127.0.0.1:3456").unwrap();
    //pull_socket.connect("tcp://127.0.0.1:3457").unwrap();

    // Spawn all the workers.
    let mut worker_results = ~[];
    for _ in range(0, workers) {
        worker_results.push(spawn_worker(ctx, size / workers));
    }

    let start = time::precise_time_s();

    start_ch.send(());

    // Block until all the workers finish.
    for po in worker_results.iter() {
        po.recv();
    }

    // Receive the final count.
    let result = match pull_socket.recv_msg(0) {
        Ok(msg) => msg.with_str(|s| from_str::<uint>(s).unwrap()),
        Err(e) => fail!(e.to_str()),
    };

    let end = time::precise_time_s();
    let elapsed = end - start;

    println!("Count is {}", result);
    println!("Test took {} seconds", elapsed);
    let thruput = ((size / workers * workers) as f64) / elapsed;
    println!("Throughput={:f} per sec", thruput);
}

fn main() {
    let args = os::args();

    let args = if os::getenv("RUST_BENCH").is_some() {
        ~[~"", ~"1000000", ~"10000"]
    } else if args.len() <= 1u {
        ~[~"", ~"10000", ~"4"]
    } else {
        args
    };

    let size = from_str::<uint>(args[1]).unwrap();
    let workers = from_str::<uint>(args[2]).unwrap();

    let mut ctx = zmq::Context::new();

    run(&mut ctx, size, workers);
}
<|endoftext|>


<commit_before>extern crate env_logger;

use super::{MaxMindDBError, Reader};

use std::net::IpAddr;
use std::str::FromStr;

#[test]
fn test_decoder() {
    let _ = env_logger::try_init();

    #[allow(non_snake_case)]
    #[derive(Deserialize, Debug, Eq, PartialEq)]
    struct MapXType {
        arrayX: Vec<u32>,
        utf8_stringX: String,
    };

    #[allow(non_snake_case)]
    #[derive(Deserialize, Debug, Eq, PartialEq)]
    struct MapType {
        mapX: MapXType,
    };

    #[derive(Deserialize, Debug)]
    struct TestType {
        array: Vec<u32>,
        boolean: bool,
        bytes: Vec<u8>,
        double: f64,
        float: f32,
        int32: i32,
        map: MapType,
        uint16: u16,
        uint32: u32,
        uint64: u64,
        uint128: Vec<u8>,
        utf8_string: String,
    }

    let r = Reader::open("test-data/test-data/MaxMind-DB-test-decoder.mmdb");
    if let Err(err) = r {
        panic!(format!("error opening mmdb: {:?}", err));
    }
    let r = r.unwrap();
    let ip: IpAddr = FromStr::from_str("1.1.1.0").unwrap();
    let result: TestType = r.lookup(ip).unwrap();

    assert_eq!(result.array, vec![1u32, 2u32, 3u32]);
    assert_eq!(result.boolean, true);
    assert_eq!(result.bytes, vec![0u8, 0u8, 0u8, 42u8]);
    assert_eq!(result.double, 42.123456);
    assert_eq!(result.float, 1.1);
    assert_eq!(result.int32, -268435456);

    assert_eq!(
        result.map,
        MapType {
            mapX: MapXType {
                arrayX: vec![7, 8, 9],
                utf8_stringX: "hello".to_string(),
            },
        }
    );

    assert_eq!(result.uint16, 100);
    assert_eq!(result.uint32, 268435456);
    assert_eq!(result.uint64, 1152921504606846976);
    assert_eq!(
        result.uint128,
        vec![1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
    );

    assert_eq!(result.utf8_string, "unicode! ☯ - ♫".to_string());
}

#[test]
fn test_broken_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("test-data/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb")
        .ok()
        .unwrap();
    let ip: IpAddr = FromStr::from_str("2001:220::").unwrap();

    #[derive(Deserialize, Debug)]
    struct TestType;
    match r.lookup::<TestType>(ip) {
        Err(e) => assert_eq!(
            e,
            MaxMindDBError::InvalidDatabaseError("double of size 2".to_string())
        ),
        Ok(_) => panic!("Error expected"),
    }
}

#[test]
fn test_missing_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("file-does-not-exist.mmdb");
    match r {
        Ok(_) => panic!("Received Reader when opening non-existent file"),
        Err(MaxMindDBError::IoError(_)) => assert!(true),
        Err(_) => assert!(false),
    }
}

#[test]
fn test_non_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("README.md");
    match r {
        Ok(_) => panic!("Received Reader when opening a non-MMDB file"),
        Err(e) => assert_eq!(
            e,
            MaxMindDBError::InvalidDatabaseError(
                "Could not find MaxMind DB metadata \
                 in file."
                    .to_string(),
            )
        ),
    }
}

#[test]
fn test_reader() {
    let _ = env_logger::try_init();

    let sizes = [24usize, 28, 32];
    for record_size in sizes.iter() {
        let versions = [4usize, 6];
        for ip_version in versions.iter() {
            let filename = format!(
                "test-data/test-data/MaxMind-DB-test-ipv{}-{}.mmdb",
                ip_version, record_size
            );
            let reader = Reader::open(filename).ok().unwrap();

            check_metadata(&reader, *ip_version, *record_size);
            check_ip(&reader, *ip_version);
        }
    }
}

/// Create Reader by explicitly reading the entire file into a buffer.
#[test]
#[cfg(feature = "mmap")]
fn test_reader_readfile() {
    let _ = env_logger::try_init();

    let sizes = [24usize, 28, 32];
    for record_size in sizes.iter() {
        let versions = [4usize, 6];
        for ip_version in versions.iter() {
            let filename = format!(
                "test-data/test-data/MaxMind-DB-test-ipv{}-{}.mmdb",
                ip_version, record_size
            );
            let reader = Reader::open_readfile(filename).ok().unwrap();

            check_metadata(&reader, *ip_version, *record_size);
            check_ip(&reader, *ip_version);
        }
    }
}

#[test]
fn test_lookup_city() {
    use super::geoip2::City;
    let _ = env_logger::try_init();

    let filename = "test-data/test-data/GeoIP2-City-Test.mmdb";

    let reader = Reader::open(filename).unwrap();

    let ip: IpAddr = FromStr::from_str("89.160.20.112").unwrap();
    let city: City = reader.lookup(ip).unwrap();

    let iso_code = city.country.and_then(|cy| cy.iso_code);

    assert_eq!(iso_code, Some("SE".to_owned()));
}

fn check_metadata(reader: &Reader, ip_version: usize, record_size: usize) {
    let metadata = &reader.metadata;

    assert_eq!(metadata.binary_format_major_version, 2u16);

    assert_eq!(metadata.binary_format_minor_version, 0u16);
    assert!(metadata.build_epoch >= 1397457605);
    assert_eq!(metadata.database_type, "Test".to_string());

    assert_eq!(
        *metadata.description.get(&"en".to_string()).unwrap(),
        "Test Database".to_string()
    );
    assert_eq!(
        *metadata.description.get(&"zh".to_string()).unwrap(),
        "Test Database Chinese".to_string()
    );

    assert_eq!(metadata.ip_version, ip_version as u16);
    assert_eq!(metadata.languages, vec!["en".to_string(), "zh".to_string()]);

    if ip_version == 4 {
        assert_eq!(metadata.node_count, 164)
    } else {
        assert_eq!(metadata.node_count, 416)
    }

    assert_eq!(metadata.record_size, record_size as u16)
}

fn check_ip(reader: &Reader, ip_version: usize) {
    let subnets = match ip_version {
        6 => [
            "::1:ffff:ffff",
            "::2:0:0",
            "::2:0:0",
            "::2:0:0",
            "::2:0:0",
            "::2:0:40",
            "::2:0:40",
            "::2:0:40",
            "::2:0:50",
            "::2:0:50",
            "::2:0:50",
            "::2:0:58",
            "::2:0:58",
        ],
        _ => [
            "1.1.1.1", "1.1.1.2", "1.1.1.2", "1.1.1.4", "1.1.1.4", "1.1.1.4", "1.1.1.4", "1.1.1.8",
            "1.1.1.8", "1.1.1.8", "1.1.1.16", "1.1.1.16", "1.1.1.16",
        ],
    };

    #[derive(Deserialize, Debug)]
    struct IpType {
        ip: String,
    }

    for subnet in subnets.iter() {
        let ip: IpAddr = FromStr::from_str(&subnet).unwrap();
        let value: IpType = reader.lookup(ip).unwrap();

        assert_eq!(value.ip, subnet.to_string());
    }

    let no_record = ["1.1.1.33", "255.254.253.123", "89fa::"];

    for &address in no_record.iter() {
        let ip: IpAddr = FromStr::from_str(address).unwrap();
        match reader.lookup::<IpType>(ip) {
            Ok(v) => panic!("received an unexpected value: {:?}", v),
            Err(e) => assert_eq!(
                e,
                MaxMindDBError::AddressNotFoundError("Address not found in database".to_string())
            ),
        }
    }
}
<commit_msg>Add an explicit test for open_mmap<commit_after>extern crate env_logger;

use super::{MaxMindDBError, Reader};

use std::net::IpAddr;
use std::str::FromStr;

#[test]
fn test_decoder() {
    let _ = env_logger::try_init();

    #[allow(non_snake_case)]
    #[derive(Deserialize, Debug, Eq, PartialEq)]
    struct MapXType {
        arrayX: Vec<u32>,
        utf8_stringX: String,
    };

    #[allow(non_snake_case)]
    #[derive(Deserialize, Debug, Eq, PartialEq)]
    struct MapType {
        mapX: MapXType,
    };

    #[derive(Deserialize, Debug)]
    struct TestType {
        array: Vec<u32>,
        boolean: bool,
        bytes: Vec<u8>,
        double: f64,
        float: f32,
        int32: i32,
        map: MapType,
        uint16: u16,
        uint32: u32,
        uint64: u64,
        uint128: Vec<u8>,
        utf8_string: String,
    }

    let r = Reader::open("test-data/test-data/MaxMind-DB-test-decoder.mmdb");
    if let Err(err) = r {
        panic!(format!("error opening mmdb: {:?}", err));
    }
    let r = r.unwrap();
    let ip: IpAddr = FromStr::from_str("1.1.1.0").unwrap();
    let result: TestType = r.lookup(ip).unwrap();

    assert_eq!(result.array, vec![1u32, 2u32, 3u32]);
    assert_eq!(result.boolean, true);
    assert_eq!(result.bytes, vec![0u8, 0u8, 0u8, 42u8]);
    assert_eq!(result.double, 42.123456);
    assert_eq!(result.float, 1.1);
    assert_eq!(result.int32, -268435456);

    assert_eq!(
        result.map,
        MapType {
            mapX: MapXType {
                arrayX: vec![7, 8, 9],
                utf8_stringX: "hello".to_string(),
            },
        }
    );

    assert_eq!(result.uint16, 100);
    assert_eq!(result.uint32, 268435456);
    assert_eq!(result.uint64, 1152921504606846976);
    assert_eq!(
        result.uint128,
        vec![1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
    );

    assert_eq!(result.utf8_string, "unicode! ☯ - ♫".to_string());
}

#[test]
fn test_broken_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("test-data/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb")
        .ok()
        .unwrap();
    let ip: IpAddr = FromStr::from_str("2001:220::").unwrap();

    #[derive(Deserialize, Debug)]
    struct TestType;
    match r.lookup::<TestType>(ip) {
        Err(e) => assert_eq!(
            e,
            MaxMindDBError::InvalidDatabaseError("double of size 2".to_string())
        ),
        Ok(_) => panic!("Error expected"),
    }
}

#[test]
fn test_missing_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("file-does-not-exist.mmdb");
    match r {
        Ok(_) => panic!("Received Reader when opening non-existent file"),
        Err(MaxMindDBError::IoError(_)) => assert!(true),
        Err(_) => assert!(false),
    }
}

#[test]
fn test_non_database() {
    let _ = env_logger::try_init();

    let r = Reader::open("README.md");
    match r {
        Ok(_) => panic!("Received Reader when opening a non-MMDB file"),
        Err(e) => assert_eq!(
            e,
            MaxMindDBError::InvalidDatabaseError(
                "Could not find MaxMind DB metadata \
                 in file."
                    .to_string(),
            )
        ),
    }
}

#[test]
fn test_reader() {
    let _ = env_logger::try_init();

    let sizes = [24usize, 28, 32];
    for record_size in sizes.iter() {
        let versions = [4usize, 6];
        for ip_version in versions.iter() {
            let filename = format!(
                "test-data/test-data/MaxMind-DB-test-ipv{}-{}.mmdb",
                ip_version, record_size
            );
            let reader = Reader::open(filename).ok().unwrap();

            check_metadata(&reader, *ip_version, *record_size);
            check_ip(&reader, *ip_version);
        }
    }
}

/// Create Reader by explicitly reading the entire file into a buffer.
#[test]
fn test_reader_readfile() {
    let _ = env_logger::try_init();

    let sizes = [24usize, 28, 32];
    for record_size in sizes.iter() {
        let versions = [4usize, 6];
        for ip_version in versions.iter() {
            let filename = format!(
                "test-data/test-data/MaxMind-DB-test-ipv{}-{}.mmdb",
                ip_version, record_size
            );
            let reader = Reader::open_readfile(filename).ok().unwrap();

            check_metadata(&reader, *ip_version, *record_size);
            check_ip(&reader, *ip_version);
        }
    }
}

#[test]
#[cfg(feature = "mmap")]
fn test_reader_mmap() {
    let _ = env_logger::try_init();

    let sizes = [24usize, 28, 32];
    for record_size in sizes.iter() {
        let versions = [4usize, 6];
        for ip_version in versions.iter() {
            let filename = format!(
                "test-data/test-data/MaxMind-DB-test-ipv{}-{}.mmdb",
                ip_version, record_size
            );
            let reader = Reader::open_mmap(filename).ok().unwrap();

            check_metadata(&reader, *ip_version, *record_size);
            check_ip(&reader, *ip_version);
        }
    }
}

#[test]
fn test_lookup_city() {
    use super::geoip2::City;
    let _ = env_logger::try_init();

    let filename = "test-data/test-data/GeoIP2-City-Test.mmdb";

    let reader = Reader::open(filename).unwrap();

    let ip: IpAddr = FromStr::from_str("89.160.20.112").unwrap();
    let city: City = reader.lookup(ip).unwrap();

    let iso_code = city.country.and_then(|cy| cy.iso_code);

    assert_eq!(iso_code, Some("SE".to_owned()));
}

fn check_metadata(reader: &Reader, ip_version: usize, record_size: usize) {
    let metadata = &reader.metadata;

    assert_eq!(metadata.binary_format_major_version, 2u16);

    assert_eq!(metadata.binary_format_minor_version, 0u16);
    assert!(metadata.build_epoch >= 1397457605);
    assert_eq!(metadata.database_type, "Test".to_string());

    assert_eq!(
        *metadata.description.get(&"en".to_string()).unwrap(),
        "Test Database".to_string()
    );
    assert_eq!(
        *metadata.description.get(&"zh".to_string()).unwrap(),
        "Test Database Chinese".to_string()
    );

    assert_eq!(metadata.ip_version, ip_version as u16);
    assert_eq!(metadata.languages, vec!["en".to_string(), "zh".to_string()]);

    if ip_version == 4 {
        assert_eq!(metadata.node_count, 164)
    } else {
        assert_eq!(metadata.node_count, 416)
    }

    assert_eq!(metadata.record_size, record_size as u16)
}

fn check_ip(reader: &Reader, ip_version: usize) {
    let subnets = match ip_version {
        6 => [
            "::1:ffff:ffff",
            "::2:0:0",
            "::2:0:0",
            "::2:0:0",
            "::2:0:0",
            "::2:0:40",
            "::2:0:40",
            "::2:0:40",
            "::2:0:50",
            "::2:0:50",
            "::2:0:50",
            "::2:0:58",
            "::2:0:58",
        ],
        _ => [
            "1.1.1.1", "1.1.1.2", "1.1.1.2", "1.1.1.4", "1.1.1.4", "1.1.1.4", "1.1.1.4", "1.1.1.8",
            "1.1.1.8", "1.1.1.8", "1.1.1.16", "1.1.1.16", "1.1.1.16",
        ],
    };

    #[derive(Deserialize, Debug)]
    struct IpType {
        ip: String,
    }

    for subnet in subnets.iter() {
        let ip: IpAddr = FromStr::from_str(&subnet).unwrap();
        let value: IpType = reader.lookup(ip).unwrap();

        assert_eq!(value.ip, subnet.to_string());
    }

    let no_record = ["1.1.1.33", "255.254.253.123", "89fa::"];

    for &address in no_record.iter() {
        let ip: IpAddr = FromStr::from_str(address).unwrap();
        match reader.lookup::<IpType>(ip) {
            Ok(v) => panic!("received an unexpected value: {:?}", v),
            Err(e) => assert_eq!(
                e,
                MaxMindDBError::AddressNotFoundError("Address not found in database".to_string())
            ),
        }
    }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! Base classes to work with IDL callbacks.

use dom::bindings::error::{Error, Fallible};
use dom::bindings::global::global_root_from_object;
use dom::bindings::reflector::Reflectable;
use js::jsapi::GetGlobalForObjectCrossCompartment;
use js::jsapi::JSAutoCompartment;
use js::jsapi::{Heap, MutableHandleObject, RootedObject, RootedValue};
use js::jsapi::{IsCallable, JSContext, JSObject, JS_WrapObject};
use js::jsapi::{JSCompartment, JS_EnterCompartment, JS_LeaveCompartment};
use js::jsapi::{JS_BeginRequest, JS_EndRequest};
use js::jsapi::{JS_GetProperty, JS_IsExceptionPending, JS_ReportPendingException};
use js::jsapi::{JS_RestoreFrameChain, JS_SaveFrameChain};
use js::jsval::{JSVal, UndefinedValue};
use std::default::Default;
use std::ffi::CString;
use std::intrinsics::return_address;
use std::ptr;
use std::rc::Rc;

/// The exception handling used for a call.
#[derive(Copy, Clone, PartialEq)]
pub enum ExceptionHandling {
    /// Report any exception and don't throw it to the caller code.
    Report,
    /// Throw any exception to the caller code.
    Rethrow,
}

/// A common base class for representing IDL callback function types.
#[derive(JSTraceable, PartialEq)]
pub struct CallbackFunction {
    object: CallbackObject,
}

impl CallbackFunction {
    /// Create a new `CallbackFunction` for this object.
    pub fn new() -> CallbackFunction {
        CallbackFunction {
            object: CallbackObject {
                callback: Heap::default(),
            },
        }
    }

    /// Initialize the callback function with a value.
    /// Should be called once this object is done moving.
    pub fn init(&mut self, callback: *mut JSObject) {
        self.object.callback.set(callback);
    }
}

/// A common base class for representing IDL callback interface types.
#[derive(JSTraceable, PartialEq)]
pub struct CallbackInterface {
    object: CallbackObject,
}

/// A common base class for representing IDL callback function and
/// callback interface types.
#[derive(JSTraceable)]
struct CallbackObject {
    /// The underlying `JSObject`.
    callback: Heap<*mut JSObject>,
}

impl PartialEq for CallbackObject {
    fn eq(&self, other: &CallbackObject) -> bool {
        self.callback.get() == other.callback.get()
    }
}

/// A trait to be implemented by concrete IDL callback function and
/// callback interface types.
pub trait CallbackContainer {
    /// Create a new CallbackContainer object for the given `JSObject`.
    fn new(callback: *mut JSObject) -> Rc<Self>;
    /// Returns the underlying `JSObject`.
    fn callback(&self) -> *mut JSObject;
}

impl CallbackInterface {
    /// Returns the underlying `JSObject`.
    pub fn callback(&self) -> *mut JSObject {
        self.object.callback.get()
    }
}

impl CallbackFunction {
    /// Returns the underlying `JSObject`.
    pub fn callback(&self) -> *mut JSObject {
        self.object.callback.get()
    }
}

impl CallbackInterface {
    /// Create a new CallbackInterface object for the given `JSObject`.
    pub fn new() -> CallbackInterface {
        CallbackInterface {
            object: CallbackObject {
                callback: Heap::default(),
            },
        }
    }

    /// Initialize the callback function with a value.
    /// Should be called once this object is done moving.
    pub fn init(&mut self, callback: *mut JSObject) {
        self.object.callback.set(callback);
    }

    /// Returns the property with the given `name`, if it is a callable object,
    /// or an error otherwise.
    pub fn get_callable_property(&self, cx: *mut JSContext, name: &str) -> Fallible<JSVal> {
        let mut callable = RootedValue::new(cx, UndefinedValue());
        let obj = RootedObject::new(cx, self.callback());
        unsafe {
            let c_name = CString::new(name).unwrap();
            if !JS_GetProperty(cx, obj.handle(), c_name.as_ptr(), callable.handle_mut()) {
                return Err(Error::JSFailed);
            }

            if !callable.ptr.is_object() || !IsCallable(callable.ptr.to_object()) {
                return Err(Error::Type(format!("The value of the {} property is not callable",
                                               name)));
            }
        }
        Ok(callable.ptr)
    }
}

/// Wraps the reflector for `p` into the compartment of `cx`.
pub fn wrap_call_this_object<T: Reflectable>(cx: *mut JSContext,
                                             p: &T,
                                             rval: MutableHandleObject) {
    rval.set(p.reflector().get_jsobject().get());
    assert!(!rval.get().is_null());

    unsafe {
        if !JS_WrapObject(cx, rval) {
            rval.set(ptr::null_mut());
        }
    }
}

/// A class that performs whatever setup we need to safely make a call while
/// this class is on the stack. After `new` returns, the call is safe to make.
pub struct CallSetup {
    /// The compartment for reporting exceptions.
    /// As a RootedObject, this must be the first field in order to
    /// determine the final address on the stack correctly.
    exception_compartment: RootedObject,
    /// The `JSContext` used for the call.
    cx: *mut JSContext,
    /// The compartment we were in before the call.
    old_compartment: *mut JSCompartment,
    /// The exception handling used for the call.
    handling: ExceptionHandling,
}

impl CallSetup {
    /// Performs the setup needed to make a call.
    #[allow(unrooted_must_root)]
    pub fn new<T: CallbackContainer>(callback: &T, handling: ExceptionHandling) -> CallSetup {
        let global = global_root_from_object(callback.callback());
        let cx = global.r().get_cx();
        unsafe {
            JS_BeginRequest(cx);
        }

        let exception_compartment = unsafe {
            GetGlobalForObjectCrossCompartment(callback.callback())
        };
        CallSetup {
            exception_compartment: RootedObject::new_with_addr(cx,
                                                               exception_compartment,
                                                               unsafe { return_address() }),
            cx: cx,
            old_compartment: unsafe { JS_EnterCompartment(cx, callback.callback()) },
            handling: handling,
        }
    }

    /// Returns the `JSContext` used for the call.
    pub fn get_context(&self) -> *mut JSContext {
        self.cx
    }
}

impl Drop for CallSetup {
    fn drop(&mut self) {
        unsafe {
            JS_LeaveCompartment(self.cx, self.old_compartment);
        }
        let need_to_deal_with_exception = self.handling == ExceptionHandling::Report &&
                                          unsafe { JS_IsExceptionPending(self.cx) };
        if need_to_deal_with_exception {
            unsafe {
                let old_global = RootedObject::new(self.cx, self.exception_compartment.ptr);
                let saved = JS_SaveFrameChain(self.cx);
                {
                    let _ac = JSAutoCompartment::new(self.cx, old_global.ptr);
                    JS_ReportPendingException(self.cx);
                }
                if saved {
                    JS_RestoreFrameChain(self.cx);
                }
            }
        }
        unsafe {
            JS_EndRequest(self.cx);
        }
    }
}
<commit_msg>Auto merge of #11230 - Ms2ger:request, r=nox<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//! Base classes to work with IDL callbacks.

use dom::bindings::error::{Error, Fallible};
use dom::bindings::global::global_root_from_object;
use dom::bindings::reflector::Reflectable;
use js::jsapi::GetGlobalForObjectCrossCompartment;
use js::jsapi::JSAutoCompartment;
use js::jsapi::{Heap, MutableHandleObject, RootedObject, RootedValue};
use js::jsapi::{IsCallable, JSContext, JSObject, JS_WrapObject};
use js::jsapi::{JSCompartment, JS_EnterCompartment, JS_LeaveCompartment};
use js::jsapi::{JS_GetProperty, JS_IsExceptionPending, JS_ReportPendingException};
use js::jsapi::{JS_RestoreFrameChain, JS_SaveFrameChain};
use js::jsval::{JSVal, UndefinedValue};
use std::default::Default;
use std::ffi::CString;
use std::intrinsics::return_address;
use std::ptr;
use std::rc::Rc;

/// The exception handling used for a call.
#[derive(Copy, Clone, PartialEq)]
pub enum ExceptionHandling {
    /// Report any exception and don't throw it to the caller code.
    Report,
    /// Throw any exception to the caller code.
    Rethrow,
}

/// A common base class for representing IDL callback function types.
#[derive(JSTraceable, PartialEq)]
pub struct CallbackFunction {
    object: CallbackObject,
}

impl CallbackFunction {
    /// Create a new `CallbackFunction` for this object.
    pub fn new() -> CallbackFunction {
        CallbackFunction {
            object: CallbackObject {
                callback: Heap::default(),
            },
        }
    }

    /// Initialize the callback function with a value.
    /// Should be called once this object is done moving.
    pub fn init(&mut self, callback: *mut JSObject) {
        self.object.callback.set(callback);
    }
}

/// A common base class for representing IDL callback interface types.
#[derive(JSTraceable, PartialEq)]
pub struct CallbackInterface {
    object: CallbackObject,
}

/// A common base class for representing IDL callback function and
/// callback interface types.
#[derive(JSTraceable)]
struct CallbackObject {
    /// The underlying `JSObject`.
    callback: Heap<*mut JSObject>,
}

impl PartialEq for CallbackObject {
    fn eq(&self, other: &CallbackObject) -> bool {
        self.callback.get() == other.callback.get()
    }
}

/// A trait to be implemented by concrete IDL callback function and
/// callback interface types.
pub trait CallbackContainer {
    /// Create a new CallbackContainer object for the given `JSObject`.
    fn new(callback: *mut JSObject) -> Rc<Self>;
    /// Returns the underlying `JSObject`.
    fn callback(&self) -> *mut JSObject;
}

impl CallbackInterface {
    /// Returns the underlying `JSObject`.
    pub fn callback(&self) -> *mut JSObject {
        self.object.callback.get()
    }
}

impl CallbackFunction {
    /// Returns the underlying `JSObject`.
    pub fn callback(&self) -> *mut JSObject {
        self.object.callback.get()
    }
}

impl CallbackInterface {
    /// Create a new CallbackInterface object for the given `JSObject`.
    pub fn new() -> CallbackInterface {
        CallbackInterface {
            object: CallbackObject {
                callback: Heap::default(),
            },
        }
    }

    /// Initialize the callback function with a value.
    /// Should be called once this object is done moving.
    pub fn init(&mut self, callback: *mut JSObject) {
        self.object.callback.set(callback);
    }

    /// Returns the property with the given `name`, if it is a callable object,
    /// or an error otherwise.
    pub fn get_callable_property(&self, cx: *mut JSContext, name: &str) -> Fallible<JSVal> {
        let mut callable = RootedValue::new(cx, UndefinedValue());
        let obj = RootedObject::new(cx, self.callback());
        unsafe {
            let c_name = CString::new(name).unwrap();
            if !JS_GetProperty(cx, obj.handle(), c_name.as_ptr(), callable.handle_mut()) {
                return Err(Error::JSFailed);
            }

            if !callable.ptr.is_object() || !IsCallable(callable.ptr.to_object()) {
                return Err(Error::Type(format!("The value of the {} property is not callable",
                                               name)));
            }
        }
        Ok(callable.ptr)
    }
}

/// Wraps the reflector for `p` into the compartment of `cx`.
pub fn wrap_call_this_object<T: Reflectable>(cx: *mut JSContext,
                                             p: &T,
                                             rval: MutableHandleObject) {
    rval.set(p.reflector().get_jsobject().get());
    assert!(!rval.get().is_null());

    unsafe {
        if !JS_WrapObject(cx, rval) {
            rval.set(ptr::null_mut());
        }
    }
}

/// A class that performs whatever setup we need to safely make a call while
/// this class is on the stack. After `new` returns, the call is safe to make.
pub struct CallSetup {
    /// The compartment for reporting exceptions.
    /// As a RootedObject, this must be the first field in order to
    /// determine the final address on the stack correctly.
    exception_compartment: RootedObject,
    /// The `JSContext` used for the call.
    cx: *mut JSContext,
    /// The compartment we were in before the call.
    old_compartment: *mut JSCompartment,
    /// The exception handling used for the call.
    handling: ExceptionHandling,
}

impl CallSetup {
    /// Performs the setup needed to make a call.
    #[allow(unrooted_must_root)]
    pub fn new<T: CallbackContainer>(callback: &T, handling: ExceptionHandling) -> CallSetup {
        let global = global_root_from_object(callback.callback());
        let cx = global.r().get_cx();

        let exception_compartment = unsafe {
            GetGlobalForObjectCrossCompartment(callback.callback())
        };
        CallSetup {
            exception_compartment: RootedObject::new_with_addr(cx,
                                                               exception_compartment,
                                                               unsafe { return_address() }),
            cx: cx,
            old_compartment: unsafe { JS_EnterCompartment(cx, callback.callback()) },
            handling: handling,
        }
    }

    /// Returns the `JSContext` used for the call.
    pub fn get_context(&self) -> *mut JSContext {
        self.cx
    }
}

impl Drop for CallSetup {
    fn drop(&mut self) {
        unsafe {
            JS_LeaveCompartment(self.cx, self.old_compartment);
        }
        let need_to_deal_with_exception = self.handling == ExceptionHandling::Report &&
                                          unsafe { JS_IsExceptionPending(self.cx) };
        if need_to_deal_with_exception {
            unsafe {
                let old_global = RootedObject::new(self.cx, self.exception_compartment.ptr);
                let saved = JS_SaveFrameChain(self.cx);
                {
                    let _ac = JSAutoCompartment::new(self.cx, old_global.ptr);
                    JS_ReportPendingException(self.cx);
                }
                if saved {
                    JS_RestoreFrameChain(self.cx);
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>#![feature(test)]

extern crate serde_yaml;
extern crate test;

extern crate liquid;

// Mirrors handlebars' benchmark
static ITERATE: &'static str = "

    <title>{{year}}</title>

CSL {{year}}

    {% for team in teams %}

      {{team.name}}: {{team.score}}

    {% endfor %}

";
static ITERATE_OBJECT: &'static str = "
year: 2015
teams:
  - name: Jiangsu
    score: 43
  - name: Beijing
    score: 27
  - name: Guangzhou
    score: 22
  - name: Shandong
    score: 12
";

#[bench]
fn parse_template(b: &mut test::Bencher) {
    let parser = liquid::ParserBuilder::with_liquid().extra_filters().build();
    b.iter(|| parser.parse(ITERATE));
}

#[bench]
fn render_template(b: &mut test::Bencher) {
    let parser = liquid::ParserBuilder::with_liquid().extra_filters().build();
    let template = parser
        .parse(ITERATE)
        .expect("Benchmark template parsing failed");

    let data: liquid::value::Object =
        serde_yaml::from_str(ITERATE_OBJECT).expect("Benchmark object parsing failed");

    template.render(&data).unwrap();
    b.iter(|| template.render(&data));
}

static LOOP: &'static str = "BEFORE\n{% for this in real%}{{this}}{%endfor%}AFTER";

#[bench]
fn large_loop_helper(b: &mut test::Bencher) {
    let parser = liquid::ParserBuilder::with_liquid().extra_filters().build();
    let template = parser
        .parse(LOOP)
        .expect("Benchmark template parsing failed");

    let data_wrapper = liquid::value::Value::array(
        (1..1000)
            .map(|i| format!("n={}", i))
            .map(liquid::value::Value::scalar),
    );
    let row_wrapper: liquid::value::Object = vec![
        ("real".into(), data_wrapper.clone()),
        ("dummy".into(), data_wrapper.clone()),
    ].into_iter()
    .collect();

    template.render(&row_wrapper).unwrap();
    b.iter(|| template.render(&row_wrapper));
}
<commit_msg>fix: Typo in benchmark<commit_after>#![feature(test)]

extern crate serde_yaml;
extern crate test;

extern crate liquid;

// Mirrors handlebars' benchmark
static ITERATE: &'static str = "

    <title>{{year}}</title>

CSL {{year}}

    {% for team in teams %}

      {{team.name}}: {{team.score}}

    {% endfor %}

";
static ITERATE_OBJECT: &'static str = "
year: 2015
teams:
  - name: Jiangsu
    score: 43
  - name: Beijing
    score: 27
  - name: Guangzhou
    score: 22
  - name: Shandong
    score: 12
";

#[bench]
fn parse_template(b: &mut test::Bencher) {
    let parser = liquid::ParserBuilder::with_liquid().extra_filters().build();
    b.iter(|| parser.parse(ITERATE));
}

#[bench]
fn render_template(b: &mut test::Bencher) {
    let parser = liquid::ParserBuilder::with_liquid().extra_filters().build();
    let template = parser
        .parse(ITERATE)
        .expect("Benchmark template parsing failed");

    let data: liquid::value::Object =
        serde_yaml::from_str(ITERATE_OBJECT).expect("Benchmark object parsing failed");

    template.render(&data).unwrap();
    b.iter(|| template.render(&data));
}

static LOOP: &'static str = "BEFORE\n{% for this in real%}{{this}}{%endfor%}AFTER";

#[bench]
fn large_loop_helper(b: &mut test::Bencher) {
    let parser = liquid::ParserBuilder::with_liquid().extra_filters().build();
    let template = parser
        .parse(LOOP)
        .expect("Benchmark template parsing failed");

    let data_wrapper = liquid::value::Value::array(
        (1..1000)
            .map(|i| format!("n={}", i))
            .map(liquid::value::Value::scalar),
    );
    let row_wrapper: liquid::value::Object = vec![
        ("real".into(), data_wrapper.clone()),
        ("dummy".into(), data_wrapper.clone()),
    ].into_iter()
    .collect();

    template.render(&row_wrapper).unwrap();
    b.iter(|| template.render(&row_wrapper));
}
<|endoftext|>


<commit_before><commit_msg>libintrusive: Add implementation of link::Link<commit_after>use core::ptr;
use core::mem;

#[derive(Debug)]
pub struct Link<T> {
    ptr: *mut T,
}

/// Non owner raw link base on *mut pointers
impl<T> Link<T> {
    pub fn none() -> Self {
        Link {
            ptr: ptr::null_mut(),
        }
    }

    pub fn some(ptr: *mut T) -> Self {
        Link {
            ptr: ptr,
        }
    }

    pub fn take(&mut self) -> Link<T> {
        mem::replace(self, Self::none())
    }

    pub fn as_ref(&self) -> Option<&T> {
        if self.ptr.is_null() {
            None
        } else {
            unsafe { Some(& *self.ptr) }
        }
    }

    pub fn as_mut(&mut self) -> Option<&mut T> {
        if self.ptr.is_null() {
            None
        } else {
            unsafe { Some(&mut *self.ptr) }
        }
    }
}

impl<T> PartialEq for Link<T> {
    fn eq(&self, rhs: &Self) -> bool {
        self.ptr == rhs.ptr
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate rustfbp;

extern crate capnp;

use std::fs;

mod contract_capnp {
    include!("path.rs");
    include!("option_path.rs");
    include!("fbp_graph.rs");
}
use contract_capnp::path;
use contract_capnp::option_path;
use contract_capnp::fbp_graph;

#[derive(Debug)]
struct Graph {
    errors: bool,
    nodes: Vec<(String, String)>,
    edges: Vec<(String, String, String, String, String, String)>,
    iips: Vec<(String, String, String, String)>,
    ext_in: Vec<(String, String, String, String)>,
    ext_out: Vec<(String, String, String, String)>,
}

component! {
    fvm,
    inputs(input: fbp_graph, new_path: option_path, error: any),
    inputs_array(),
    outputs(output: fbp_graph, ask_graph: path, ask_path: path),
    outputs_array(),
    option(),
    acc(),
    fn run(&mut self) -> Result<()>{
        let mut graph = Graph { errors: false,
            nodes: vec![], edges: vec![], iips: vec![],
            ext_in: vec![], ext_out: vec![],
        };

        // retrieve the asked graph
        let mut ip = try!(self.ports.recv("input"));
        let i_graph = try!(ip.get_reader());
        let i_graph: fbp_graph::Reader = try!(i_graph.get_root());

        try!(add_graph(self, &mut graph, i_graph, ""));

        if !graph.errors {
            try!(send_graph(&self, &graph))
        }
        Ok(())
    }
}

fn add_graph(component: &fvm, mut graph: &mut Graph, new_graph: fbp_graph::Reader, name: &str) -> Result<()> {

    if try!(new_graph.get_path()) == "error" { graph.errors = true; }

    for n in try!(new_graph.borrow().get_edges()).iter() {
        graph.edges.push((format!("{}-{}", name, try!(n.get_o_name())),
                          try!(n.get_o_port()).into(), try!(n.get_o_selection()).into(),
                          try!(n.get_i_port()).into(), try!(n.get_i_selection()).into(),
                          format!("{}-{}", name, try!(n.get_i_name()))));
    }
    for n in try!(new_graph.borrow().get_iips()).iter() {
        graph.iips.push((try!(n.get_iip()).into(),
                         try!(n.get_port()).into(), try!(n.get_selection()).into(),
                         format!("{}-{}", name, try!(n.get_comp())) ));
    }
    for n in try!(new_graph.borrow().get_external_inputs()).iter() {
        // TODO : replace existing links
        for edge in &mut graph.edges {
            if edge.5 == name && edge.3 == try!(n.get_name()) {
                edge.5 = format!("{}-{}", name, try!(n.get_comp()));
                edge.3 = try!(n.get_port()).into();
                edge.4 = try!(n.get_selection()).into();
            }
        }

        for iip in &mut graph.iips {
            if iip.3 == name && iip.1 == try!(n.get_name()) {
                iip.3 = format!("{}-{}", name, try!(n.get_comp()));
                iip.1 = try!(n.get_port()).into();
                iip.2 = try!(n.get_selection()).into();
            }
        }
    }
    for n in try!(new_graph.borrow().get_external_outputs()).iter() {
        for edge in &mut graph.edges {
            if edge.0 == name && edge.1 == try!(n.get_name()) {
                edge.0 = format!("{}-{}", name, try!(n.get_comp()));
                edge.1 = try!(n.get_port()).into();
                edge.2 = try!(n.get_selection()).into();
            }
        }
    }

    for n in try!(new_graph.borrow().get_nodes()).iter() {
        let c_sort = try!(n.get_sort());
        let c_name = try!(n.get_name());

        // get new path
        let mut msg = capnp::message::Builder::new_default();
        {
            let mut path = msg.init_root::<path::Builder>();
            path.set_path(c_sort);
        }
        let mut ip = IP::new();
        ip.write_builder(&mut msg);
        try!(component.ports.send("ask_path", ip));

        // retrieve the asked graph
        let mut ip = try!(component.ports.recv("new_path"));
        let i_graph = try!(ip.get_reader());
        let i_graph: option_path::Reader = try!(i_graph.get_root());

        let new_path: Option<String> = match try!(i_graph.which()) {
            option_path::Path(p) => { Some(try!(p).into()) },
            option_path::None(()) => { None }
        };

        let mut is_subnet = true;
        let path = match new_path {
            Some(hash_name) => {
                let path = format!("{}{}{}", "/nix/store/", hash_name, "/lib/libcomponent.so");
                if fs::metadata(&path).is_ok() {
                    is_subnet = false;
                    path
                } else {
                    format!("{}{}{}", "/nix/store/", hash_name, "/lib/lib.subnet")
                }

            },
            None => {
                println!("Error in {} : ", try!(new_graph.get_path()));
                println!("component {}({}) doesn't exist", c_name, c_sort);
                graph.errors = true;
                continue;
            }
        };

        if is_subnet {
            let mut msg = capnp::message::Builder::new_default();
            {
                let mut number = msg.init_root::<path::Builder>();
                number.set_path(&path);
            }
            let mut ip = IP::new();
            ip.write_builder(&mut msg);

            try!(component.ports.send("ask_graph", ip));

            // retrieve the asked graph
            let mut ip = try!(component.ports.recv("input"));
            let i_graph = try!(ip.get_reader());
            let i_graph: fbp_graph::Reader = try!(i_graph.get_root());

            add_graph(component, &mut graph, i_graph, &format!("{}-{}", name, c_name));
        } else {
            graph.nodes.push((format!("{}-{}", name, c_name).into(), path.into()));
        }
    }
    Ok(())
}

fn send_graph(comp: &fvm, graph: &Graph) -> Result<()> {
    let mut new_ip = capnp::message::Builder::new_default();
    {
        let mut ip = new_ip.init_root::<fbp_graph::Builder>();
        ip.set_path("");
        {
            let mut nodes = ip.borrow().init_nodes(graph.nodes.len() as u32);
            let mut i = 0;
            for n in &graph.nodes {
                nodes.borrow().get(i).set_name(&n.0[..]);
                nodes.borrow().get(i).set_sort(&n.1[..]);
                i += 1;
            }
        }
        {
            let mut edges = ip.borrow().init_edges(graph.edges.len() as u32);
            let mut i = 0;
            for e in &graph.edges {
                edges.borrow().get(i).set_o_name(&e.0[..]);
                edges.borrow().get(i).set_o_port(&e.1[..]);
                edges.borrow().get(i).set_o_selection(&e.2[..]);
                edges.borrow().get(i).set_i_port(&e.3[..]);
                edges.borrow().get(i).set_i_selection(&e.4[..]);
                edges.borrow().get(i).set_i_name(&e.5[..]);
                i += 1;
            }
        }
        {
            let mut iips = ip.borrow().init_iips(graph.iips.len() as u32);
            let mut i = 0;
            for iip in &graph.iips {
                iips.borrow().get(i).set_iip(&iip.0[..]);
                iips.borrow().get(i).set_port(&iip.1[..]);
                iips.borrow().get(i).set_selection(&iip.2[..]);
                iips.borrow().get(i).set_comp(&iip.3[..]);
                i += 1;
            }
        }
        {
            let mut ext = ip.borrow().init_external_inputs(graph.ext_in.len() as u32);
            let mut i = 0;
            for e in &graph.ext_in {
                ext.borrow().get(i).set_name(&e.0[..]);
                ext.borrow().get(i).set_port(&e.1[..]);
                ext.borrow().get(i).set_selection(&e.2[..]);
                ext.borrow().get(i).set_comp(&e.3[..]);
                i += 1;
            }
        }
        {
            let mut ext = ip.borrow().init_external_outputs(graph.ext_out.len() as u32);
            let mut i = 0;
            for e in &graph.ext_out {
                ext.borrow().get(i).set_comp(&e.0[..]);
                ext.borrow().get(i).set_port(&e.1[..]);
                ext.borrow().get(i).set_selection(&e.2[..]);
                ext.borrow().get(i).set_name(&e.3[..]);
                i += 1;
            }
        }
    }
    let mut send_ip = IP::new();
    try!(send_ip.write_builder(&new_ip));
    let _ = comp.ports.send("output", send_ip);
    Ok(())
}
<commit_msg>remove ask_graph<commit_after>#[macro_use]
extern crate rustfbp;

extern crate capnp;

use std::fs;

mod contract_capnp {
    include!("path.rs");
    include!("fbp_graph.rs");
}
use contract_capnp::path;
use contract_capnp::fbp_graph;

#[derive(Debug)]
struct Graph {
    errors: bool,
    nodes: Vec<(String, String)>,
    edges: Vec<(String, String, String, String, String, String)>,
    iips: Vec<(String, String, String, String)>,
    ext_in: Vec<(String, String, String, String)>,
    ext_out: Vec<(String, String, String, String)>,
}

component! {
    fvm,
    inputs(input: fbp_graph, error: any),
    inputs_array(),
    outputs(output: fbp_graph, ask_graph: path),
    outputs_array(),
    option(),
    acc(),
    fn run(&mut self) -> Result<()>{
        let mut graph = Graph { errors: false,
            nodes: vec![], edges: vec![], iips: vec![],
            ext_in: vec![], ext_out: vec![],
        };

        // retrieve the asked graph
        let mut ip = try!(self.ports.recv("input"));
        let i_graph = try!(ip.get_reader());
        let i_graph: fbp_graph::Reader = try!(i_graph.get_root());

        try!(add_graph(self, &mut graph, i_graph, ""));

        if !graph.errors {
            try!(send_graph(&self, &graph))
        }
        Ok(())
    }
}

fn add_graph(component: &fvm, mut graph: &mut Graph, new_graph: fbp_graph::Reader, name: &str) -> Result<()> {

    if try!(new_graph.get_path()) == "error" { graph.errors = true; }

    for n in try!(new_graph.borrow().get_edges()).iter() {
        graph.edges.push((format!("{}-{}", name, try!(n.get_o_name())),
                          try!(n.get_o_port()).into(), try!(n.get_o_selection()).into(),
                          try!(n.get_i_port()).into(), try!(n.get_i_selection()).into(),
                          format!("{}-{}", name, try!(n.get_i_name()))));
    }
    for n in try!(new_graph.borrow().get_iips()).iter() {
        graph.iips.push((try!(n.get_iip()).into(),
                         try!(n.get_port()).into(), try!(n.get_selection()).into(),
                         format!("{}-{}", name, try!(n.get_comp())) ));
    }
    for n in try!(new_graph.borrow().get_external_inputs()).iter() {
        // TODO : replace existing links
        for edge in &mut graph.edges {
            if edge.5 == name && edge.3 == try!(n.get_name()) {
                edge.5 = format!("{}-{}", name, try!(n.get_comp()));
                edge.3 = try!(n.get_port()).into();
                edge.4 = try!(n.get_selection()).into();
            }
        }

        for iip in &mut graph.iips {
            if iip.3 == name && iip.1 == try!(n.get_name()) {
                iip.3 = format!("{}-{}", name, try!(n.get_comp()));
                iip.1 = try!(n.get_port()).into();
                iip.2 = try!(n.get_selection()).into();
            }
        }
    }
    for n in try!(new_graph.borrow().get_external_outputs()).iter() {
        for edge in &mut graph.edges {
            if edge.0 == name && edge.1 == try!(n.get_name()) {
                edge.0 = format!("{}-{}", name, try!(n.get_comp()));
                edge.1 = try!(n.get_port()).into();
                edge.2 = try!(n.get_selection()).into();
            }
        }
    }

    for n in try!(new_graph.borrow().get_nodes()).iter() {
        let c_sort = try!(n.get_sort());
        let c_name = try!(n.get_name());

        let mut is_subnet = true;
        let path = format!("{}{}", c_sort, "/lib/libcomponent.so");
        if fs::metadata(&path).is_ok() {
            is_subnet = false;
            path
        } else {
            format!("{}{}", c_sort, "/lib/lib.subnet")
        }

        if is_subnet {
            let mut msg = capnp::message::Builder::new_default();
            {
                let mut number = msg.init_root::<path::Builder>();
                number.set_path(&path);
            }
            let mut ip = IP::new();
            ip.write_builder(&mut msg);

            try!(component.ports.send("ask_graph", ip));

            // retrieve the asked graph
            let mut ip = try!(component.ports.recv("input"));
            let i_graph = try!(ip.get_reader());
            let i_graph: fbp_graph::Reader = try!(i_graph.get_root());

            add_graph(component, &mut graph, i_graph, &format!("{}-{}", name, c_name));
        } else {
            graph.nodes.push((format!("{}-{}", name, c_name).into(), path.into()));
        }
    }
    Ok(())
}

fn send_graph(comp: &fvm, graph: &Graph) -> Result<()> {
    let mut new_ip = capnp::message::Builder::new_default();
    {
        let mut ip = new_ip.init_root::<fbp_graph::Builder>();
        ip.set_path("");
        {
            let mut nodes = ip.borrow().init_nodes(graph.nodes.len() as u32);
            let mut i = 0;
            for n in &graph.nodes {
                nodes.borrow().get(i).set_name(&n.0[..]);
                nodes.borrow().get(i).set_sort(&n.1[..]);
                i += 1;
            }
        }
        {
            let mut edges = ip.borrow().init_edges(graph.edges.len() as u32);
            let mut i = 0;
            for e in &graph.edges {
                edges.borrow().get(i).set_o_name(&e.0[..]);
                edges.borrow().get(i).set_o_port(&e.1[..]);
                edges.borrow().get(i).set_o_selection(&e.2[..]);
                edges.borrow().get(i).set_i_port(&e.3[..]);
                edges.borrow().get(i).set_i_selection(&e.4[..]);
                edges.borrow().get(i).set_i_name(&e.5[..]);
                i += 1;
            }
        }
        {
            let mut iips = ip.borrow().init_iips(graph.iips.len() as u32);
            let mut i = 0;
            for iip in &graph.iips {
                iips.borrow().get(i).set_iip(&iip.0[..]);
                iips.borrow().get(i).set_port(&iip.1[..]);
                iips.borrow().get(i).set_selection(&iip.2[..]);
                iips.borrow().get(i).set_comp(&iip.3[..]);
                i += 1;
            }
        }
        {
            let mut ext = ip.borrow().init_external_inputs(graph.ext_in.len() as u32);
            let mut i = 0;
            for e in &graph.ext_in {
                ext.borrow().get(i).set_name(&e.0[..]);
                ext.borrow().get(i).set_port(&e.1[..]);
                ext.borrow().get(i).set_selection(&e.2[..]);
                ext.borrow().get(i).set_comp(&e.3[..]);
                i += 1;
            }
        }
        {
            let mut ext = ip.borrow().init_external_outputs(graph.ext_out.len() as u32);
            let mut i = 0;
            for e in &graph.ext_out {
                ext.borrow().get(i).set_comp(&e.0[..]);
                ext.borrow().get(i).set_port(&e.1[..]);
                ext.borrow().get(i).set_selection(&e.2[..]);
                ext.borrow().get(i).set_name(&e.3[..]);
                i += 1;
            }
        }
    }
    let mut send_ip = IP::new();
    try!(send_ip.write_builder(&new_ip));
    let _ = comp.ports.send("output", send_ip);
    Ok(())
}
<|endoftext|>