<commit_before>// Bloom
//
// HTTP REST API caching middleware
// Copyright: 2017, Valerian Saliou <valerian@valeriansaliou.name>
// License: Mozilla Public License v2.0 (MPL v2.0)

use std::cmp;
use std::time::Duration;
use r2d2::Pool;
use r2d2::config::Config;
use r2d2_redis::{RedisConnectionManager, Error};
use redis::{self, Value, Connection, Commands, PipelineCommands};
use futures::future::Future;
use futures_cpupool::CpuPool;

use super::route::ROUTE_PREFIX;
use APP_CONF;

static KEY_BODY: &'static str = "b";
static KEY_FINGERPRINT: &'static str = "f";
static KEY_TAGS: &'static str = "t";
static KEY_TAGS_SEPARATOR: &'static str = ",";

lazy_static! {
    pub static ref EXECUTOR_POOL: CpuPool = CpuPool::new(APP_CONF.cache.executor_pool as usize);
}

pub struct CacheStoreBuilder;

pub struct CacheStore {
    pool: Pool<RedisConnectionManager>,
}

#[derive(Debug)]
pub enum CacheStoreError {
    Disconnected,
    Failed,
    Invalid,
    Corrupted,
    Partial,
    TooLarge,
}

#[derive(Debug)]
pub enum CachePurgeVariant {
    Bucket,
    Auth,
}

type CacheReadResultFuture = Box<Future<Item = Option<String>, Error = CacheStoreError>>;
type CacheWriteResult = Result<(String), (CacheStoreError, String)>;
type CacheWriteResultFuture = Box<Future<Item = CacheWriteResult, Error = ()>>;
type CachePurgeResult = Result<(), CacheStoreError>;

impl CacheStoreBuilder {
    pub fn new() -> CacheStore {
        info!(
            "binding to store backend at {}:{}",
            APP_CONF.redis.host,
            APP_CONF.redis.port
        );

        let addr_auth = match APP_CONF.redis.password {
            Some(ref password) => format!(":{}@", password),
            None => "".to_string(),
        };

        let tcp_addr_raw =
            format!(
            "redis://{}{}:{}/{}",
            &addr_auth,
            APP_CONF.redis.host,
            APP_CONF.redis.port,
            APP_CONF.redis.database,
        );

        debug!("will connect to redis at: {}", tcp_addr_raw);

        match RedisConnectionManager::new(tcp_addr_raw.as_ref()) {
            Ok(manager) => {
                let config = Config::<Connection, Error>::builder()
                    .initialization_fail_fast(false)
                    .test_on_check_out(false)
                    .pool_size(APP_CONF.redis.pool_size)
                    .max_lifetime(Some(
                        Duration::from_secs(APP_CONF.redis.max_lifetime_seconds),
                    ))
                    .idle_timeout(Some(
                        Duration::from_secs(APP_CONF.redis.idle_timeout_seconds),
                    ))
                    .connection_timeout(Duration::from_secs(
                        APP_CONF.redis.connection_timeout_seconds,
                    ))
                    .build();

                match Pool::new(config, manager) {
                    Ok(pool) => {
                        info!("bound to store backend");

                        CacheStore { pool: pool }
                    }
                    Err(_) => panic!("could not spawn redis pool"),
                }
            }
            Err(_) => panic!("could not create redis connection manager"),
        }
    }
}

impl CacheStore {
    pub fn get_meta(&self, shard: u8, key: String) -> CacheReadResultFuture {
        let pool = self.pool.to_owned();

        Box::new(EXECUTOR_POOL.spawn_fn(move || {
            get_cache_store_client!(pool, CacheStoreError::Disconnected, client {
                    match (*client).hget::<_, _, (Value, Value)>(key, (KEY_FINGERPRINT, KEY_TAGS)) {
                        Ok(value) => {
                            match value {
                                (Value::Data(fingerprint_bytes), tags_bytes) => {
                                    // Parse tags and bump their last access time
                                    if let Value::Data(tags_bytes_data) = tags_bytes {
                                        if let Ok(tags_data) = String::from_utf8(
                                            tags_bytes_data) {
                                            if tags_data.is_empty() == false {
                                                let tags = tags_data.split(KEY_TAGS_SEPARATOR)
                                                    .map(|tag| {
                                                        format!(
                                                            "{}:{}:{}", ROUTE_PREFIX, shard, tag
                                                        )
                                                    })
                                                    .collect::<Vec<String>>();

                                                // Proceed a soft bump of last access time of \
                                                //   associated tag keys. This prevents a \
                                                //   frequently accessed cache namespace to \
                                                //   become 'orphan' (ie. one or more tag keys \
                                                //   are LRU-expired), and thus cache namespace \
                                                //   not to be properly removed on purge of an \
                                                //   associated tag.
                                                // Also, count bumped keys. It may happen that \
                                                //   some tag keys are incorrectly removed by \
                                                //   Redis LRU system, as it is probabilistic \
                                                //   and thus might sample some keys incorrectly.
                                                // The conditions explained above only happens on \
                                                //   Redis instances with used memory going over \
                                                //   the threshold of the max memory policy.
                                                let tags_count = tags.len();

                                                match redis::cmd("TOUCH").arg(tags)
                                                    .query::<usize>(&*client) {
                                                    Ok(bump_count) => {
                                                        // Partial bump count? Do not serve cache.
                                                        if bump_count < tags_count {
                                                            return Err(CacheStoreError::Partial)
                                                        }
                                                    },
                                                    Err(err) => {
                                                        error!(
                                                            "error bumping access time of tags: {}",
                                                            err
                                                        );
                                                    }
                                                }
                                            }
                                        }
                                    }

                                    // Decode raw bytes to string
                                    if let Ok(fingerprint) = String::from_utf8(fingerprint_bytes) {
                                        Ok(Some(fingerprint))
                                    } else {
                                        Err(CacheStoreError::Corrupted)
                                    }
                                },
                                (Value::Nil, _) | (_, Value::Nil) => Ok(None),
                                _ => Err(CacheStoreError::Invalid),
                            }
                        },
                        _ => Err(CacheStoreError::Failed),
                    }
                })
        }))
    }

    pub fn get_body(&self, key: String) -> CacheReadResultFuture {
        let pool = self.pool.to_owned();

        Box::new(EXECUTOR_POOL.spawn_fn(move || {
            get_cache_store_client!(pool, CacheStoreError::Disconnected, client {
                    match (*client).hget::<_, _, Value>(key, KEY_BODY) {
                        Ok(value) => {
                            match value {
                                Value::Data(body_bytes) => {
                                    // Decode raw bytes to string
                                    if let Ok(body) = String::from_utf8(body_bytes) {
                                        Ok(Some(body))
                                    } else {
                                        Err(CacheStoreError::Corrupted)
                                    }
                                },
                                Value::Nil => Ok(None),
                                _ => Err(CacheStoreError::Invalid),
                            }
                        },
                        _ => Err(CacheStoreError::Failed),
                    }
                })
        }))
    }

    pub fn set(
        &self,
        key: String,
        key_mask: String,
        value: String,
        fingerprint: String,
        ttl: usize,
        key_tags: Vec<(String, String)>,
    ) -> CacheWriteResultFuture {
        let pool = self.pool.to_owned();

        Box::new(EXECUTOR_POOL.spawn_fn(move || {
            Ok(get_cache_store_client!(
                    pool,
                    (CacheStoreError::Disconnected, fingerprint),

                    client {
                        // Cap TTL to 'max_key_expiration'
                        let ttl_cap = cmp::min(ttl, APP_CONF.redis.max_key_expiration);

                        // Ensure value is not larger than 'max_key_size'
                        if value.len() > APP_CONF.redis.max_key_size {
                            Err((CacheStoreError::TooLarge, fingerprint))
                        } else {
                            let mut pipeline = redis::pipe();

                            // Append storage command
                            {
                                let key_tag_masks = key_tags.iter()
                                    .map(|key_tag| key_tag.1.as_ref())
                                    .collect::<Vec<&str>>();

                                pipeline.hset_multiple(
                                    &key, &[
                                        (KEY_FINGERPRINT, &fingerprint),
                                        (KEY_TAGS, &key_tag_masks.join(KEY_TAGS_SEPARATOR)),
                                        (KEY_BODY, &value)
                                    ]
                                ).ignore();
                            }

                            pipeline.expire(&key, ttl_cap).ignore();

                            for key_tag in key_tags {
                                pipeline.sadd(&key_tag.0, &key_mask).ignore();
                                pipeline.expire(&key_tag.0, APP_CONF.redis.max_key_expiration);
                            }

                            // Bucket (MULTI operation for main data + bucket marker)
                            match pipeline.query::<()>(&*client) {
                                Ok(_) => Ok(fingerprint),
                                Err(err) => {
                                    error!("got store error: {}", err);

                                    Err((CacheStoreError::Failed, fingerprint))
                                }
                            }
                        }
                    }
                ))
        }))
    }

    pub fn purge_tag(
        &self,
        variant: &CachePurgeVariant,
        shard: u8,
        key_tag: &str,
    ) -> CachePurgeResult {
        get_cache_store_client!(self.pool, CacheStoreError::Disconnected, client {
            // Invoke keyspace cleanup script for key tag
            let result = redis::Script::new(variant.get_script())
                .arg(ROUTE_PREFIX)
                .arg(shard)
                .arg(key_tag)
                .invoke::<()>(&*client);

            result
                .and(Ok(()))
                .or(Err(CacheStoreError::Failed))
        })
    }
}

impl CachePurgeVariant {
    fn get_script(&self) -> &'static str {
        match *self {
            CachePurgeVariant::Bucket |
            CachePurgeVariant::Auth => {
                r#"
                    local targets = {}

                    for _, tag in pairs(redis.call('SMEMBERS', ARGV[3])) do
                        table.insert(targets, ARGV[1] .. ":" .. ARGV[2] .. ":c:" .. tag)
                    end

                    table.insert(targets, ARGV[3])

                    redis.call('DEL', unpack(targets))
                "#
            }
        }
    }
}
<commit_msg>Fix<commit_after>// Bloom
//
// HTTP REST API caching middleware
// Copyright: 2017, Valerian Saliou <valerian@valeriansaliou.name>
// License: Mozilla Public License v2.0 (MPL v2.0)

use std::cmp;
use std::time::Duration;
use r2d2::Pool;
use r2d2::config::Config;
use r2d2_redis::{RedisConnectionManager, Error};
use redis::{self, Value, Connection, Commands, PipelineCommands};
use futures::future::Future;
use futures_cpupool::CpuPool;

use super::route::ROUTE_PREFIX;
use APP_CONF;

static KEY_BODY: &'static str = "b";
static KEY_FINGERPRINT: &'static str = "f";
static KEY_TAGS: &'static str = "t";
static KEY_TAGS_SEPARATOR: &'static str = ",";

lazy_static! {
    pub static ref EXECUTOR_POOL: CpuPool = CpuPool::new(APP_CONF.cache.executor_pool as usize);
}

pub struct CacheStoreBuilder;

pub struct CacheStore {
    pool: Pool<RedisConnectionManager>,
}

#[derive(Debug)]
pub enum CacheStoreError {
    Disconnected,
    Failed,
    Invalid,
    Corrupted,
    TooLarge,
}

#[derive(Debug)]
pub enum CachePurgeVariant {
    Bucket,
    Auth,
}

type CacheReadResultFuture = Box<Future<Item = Option<String>, Error = CacheStoreError>>;
type CacheWriteResult = Result<(String), (CacheStoreError, String)>;
type CacheWriteResultFuture = Box<Future<Item = CacheWriteResult, Error = ()>>;
type CachePurgeResult = Result<(), CacheStoreError>;

impl CacheStoreBuilder {
    pub fn new() -> CacheStore {
        info!(
            "binding to store backend at {}:{}",
            APP_CONF.redis.host,
            APP_CONF.redis.port
        );

        let addr_auth = match APP_CONF.redis.password {
            Some(ref password) => format!(":{}@", password),
            None => "".to_string(),
        };

        let tcp_addr_raw =
            format!(
            "redis://{}{}:{}/{}",
            &addr_auth,
            APP_CONF.redis.host,
            APP_CONF.redis.port,
            APP_CONF.redis.database,
        );

        debug!("will connect to redis at: {}", tcp_addr_raw);

        match RedisConnectionManager::new(tcp_addr_raw.as_ref()) {
            Ok(manager) => {
                let config = Config::<Connection, Error>::builder()
                    .initialization_fail_fast(false)
                    .test_on_check_out(false)
                    .pool_size(APP_CONF.redis.pool_size)
                    .max_lifetime(Some(
                        Duration::from_secs(APP_CONF.redis.max_lifetime_seconds),
                    ))
                    .idle_timeout(Some(
                        Duration::from_secs(APP_CONF.redis.idle_timeout_seconds),
                    ))
                    .connection_timeout(Duration::from_secs(
                        APP_CONF.redis.connection_timeout_seconds,
                    ))
                    .build();

                match Pool::new(config, manager) {
                    Ok(pool) => {
                        info!("bound to store backend");

                        CacheStore { pool: pool }
                    }
                    Err(_) => panic!("could not spawn redis pool"),
                }
            }
            Err(_) => panic!("could not create redis connection manager"),
        }
    }
}

impl CacheStore {
    pub fn get_meta(&self, shard: u8, key: String) -> CacheReadResultFuture {
        let pool = self.pool.to_owned();

        Box::new(EXECUTOR_POOL.spawn_fn(move || {
            get_cache_store_client!(pool, CacheStoreError::Disconnected, client {
                    match (*client).hget::<_, _, (Value, Value)>(key, (KEY_FINGERPRINT, KEY_TAGS)) {
                        Ok(value) => {
                            match value {
                                (Value::Data(fingerprint_bytes), tags_bytes) => {
                                    // Parse tags and bump their last access time
                                    if let Value::Data(tags_bytes_data) = tags_bytes {
                                        if let Ok(tags_data) = String::from_utf8(
                                            tags_bytes_data) {
                                            if tags_data.is_empty() == false {
                                                let tags = tags_data.split(KEY_TAGS_SEPARATOR)
                                                    .map(|tag| {
                                                        format!(
                                                            "{}:{}:{}", ROUTE_PREFIX, shard, tag
                                                        )
                                                    })
                                                    .collect::<Vec<String>>();

                                                // Proceed a soft bump of last access time of \
                                                //   associated tag keys. This prevents a \
                                                //   frequently accessed cache namespace to \
                                                //   become 'orphan' (ie. one or more tag keys \
                                                //   are LRU-expired), and thus cache namespace \
                                                //   not to be properly removed on purge of an \
                                                //   associated tag.
                                                // Also, count bumped keys. It may happen that \
                                                //   some tag keys are incorrectly removed by \
                                                //   Redis LRU system, as it is probabilistic \
                                                //   and thus might sample some keys incorrectly.
                                                // The conditions explained above only happens on \
                                                //   Redis instances with used memory going over \
                                                //   the threshold of the max memory policy.
                                                let tags_count = tags.len();

                                                match redis::cmd("TOUCH").arg(tags)
                                                    .query::<usize>(&*client) {
                                                    Ok(bump_count) => {
                                                        // Partial bump count? Consider cache as \
                                                        //   non-existing
                                                        if bump_count < tags_count {
                                                            return Ok(None);
                                                        }
                                                    },
                                                    Err(err) => {
                                                        error!(
                                                            "error bumping access time of tags: {}",
                                                            err
                                                        );
                                                    }
                                                }
                                            }
                                        }
                                    }

                                    // Decode raw bytes to string
                                    if let Ok(fingerprint) = String::from_utf8(fingerprint_bytes) {
                                        Ok(Some(fingerprint))
                                    } else {
                                        Err(CacheStoreError::Corrupted)
                                    }
                                },
                                (Value::Nil, _) | (_, Value::Nil) => Ok(None),
                                _ => Err(CacheStoreError::Invalid),
                            }
                        },
                        _ => Err(CacheStoreError::Failed),
                    }
                })
        }))
    }

    pub fn get_body(&self, key: String) -> CacheReadResultFuture {
        let pool = self.pool.to_owned();

        Box::new(EXECUTOR_POOL.spawn_fn(move || {
            get_cache_store_client!(pool, CacheStoreError::Disconnected, client {
                    match (*client).hget::<_, _, Value>(key, KEY_BODY) {
                        Ok(value) => {
                            match value {
                                Value::Data(body_bytes) => {
                                    // Decode raw bytes to string
                                    if let Ok(body) = String::from_utf8(body_bytes) {
                                        Ok(Some(body))
                                    } else {
                                        Err(CacheStoreError::Corrupted)
                                    }
                                },
                                Value::Nil => Ok(None),
                                _ => Err(CacheStoreError::Invalid),
                            }
                        },
                        _ => Err(CacheStoreError::Failed),
                    }
                })
        }))
    }

    pub fn set(
        &self,
        key: String,
        key_mask: String,
        value: String,
        fingerprint: String,
        ttl: usize,
        key_tags: Vec<(String, String)>,
    ) -> CacheWriteResultFuture {
        let pool = self.pool.to_owned();

        Box::new(EXECUTOR_POOL.spawn_fn(move || {
            Ok(get_cache_store_client!(
                    pool,
                    (CacheStoreError::Disconnected, fingerprint),

                    client {
                        // Cap TTL to 'max_key_expiration'
                        let ttl_cap = cmp::min(ttl, APP_CONF.redis.max_key_expiration);

                        // Ensure value is not larger than 'max_key_size'
                        if value.len() > APP_CONF.redis.max_key_size {
                            Err((CacheStoreError::TooLarge, fingerprint))
                        } else {
                            let mut pipeline = redis::pipe();

                            // Append storage command
                            {
                                let key_tag_masks = key_tags.iter()
                                    .map(|key_tag| key_tag.1.as_ref())
                                    .collect::<Vec<&str>>();

                                pipeline.hset_multiple(
                                    &key, &[
                                        (KEY_FINGERPRINT, &fingerprint),
                                        (KEY_TAGS, &key_tag_masks.join(KEY_TAGS_SEPARATOR)),
                                        (KEY_BODY, &value)
                                    ]
                                ).ignore();
                            }

                            pipeline.expire(&key, ttl_cap).ignore();

                            for key_tag in key_tags {
                                pipeline.sadd(&key_tag.0, &key_mask).ignore();
                                pipeline.expire(&key_tag.0, APP_CONF.redis.max_key_expiration);
                            }

                            // Bucket (MULTI operation for main data + bucket marker)
                            match pipeline.query::<()>(&*client) {
                                Ok(_) => Ok(fingerprint),
                                Err(err) => {
                                    error!("got store error: {}", err);

                                    Err((CacheStoreError::Failed, fingerprint))
                                }
                            }
                        }
                    }
                ))
        }))
    }

    pub fn purge_tag(
        &self,
        variant: &CachePurgeVariant,
        shard: u8,
        key_tag: &str,
    ) -> CachePurgeResult {
        get_cache_store_client!(self.pool, CacheStoreError::Disconnected, client {
            // Invoke keyspace cleanup script for key tag
            let result = redis::Script::new(variant.get_script())
                .arg(ROUTE_PREFIX)
                .arg(shard)
                .arg(key_tag)
                .invoke::<()>(&*client);

            result
                .and(Ok(()))
                .or(Err(CacheStoreError::Failed))
        })
    }
}

impl CachePurgeVariant {
    fn get_script(&self) -> &'static str {
        match *self {
            CachePurgeVariant::Bucket |
            CachePurgeVariant::Auth => {
                r#"
                    local targets = {}

                    for _, tag in pairs(redis.call('SMEMBERS', ARGV[3])) do
                        table.insert(targets, ARGV[1] .. ":" .. ARGV[2] .. ":c:" .. tag)
                    end

                    table.insert(targets, ARGV[3])

                    redis.call('DEL', unpack(targets))
                "#
            }
        }
    }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Code to handle a single block device.

use std::collections::HashMap;
use std::io::ErrorKind;
use std::fs::{OpenOptions, read_dir};
use std::os::linux::fs::MetadataExt;
use std::path::PathBuf;

use nix::Errno;
use nix::sys::stat::{S_IFBLK, S_IFMT};
use serde_json;

use engine::{EngineResult, EngineError, ErrorEnum, PoolUuid};
use super::metadata::{BDA, StaticHeader};
use super::engine::DevOwnership;
use super::serde_structs::PoolSave;

/// Find all Stratis devices.
///
/// Returns a map of pool uuids to a vector of devices for each pool.
pub fn find_all() -> EngineResult<HashMap<PoolUuid, Vec<PathBuf>>> {

    let mut pool_map = HashMap::new();
    for dir_e in try!(read_dir("/dev")) {
        let dir_e = try!(dir_e);
        let mode = try!(dir_e.metadata()).st_mode();

        // Device node can't belong to Stratis if it is not a block device
        if mode & S_IFMT.bits() != S_IFBLK.bits() {
            continue;
        }

        let devnode = dir_e.path();

        let f = OpenOptions::new().read(true).open(&devnode);

        // There are some reasons for OpenOptions::open() to return an error
        // which are not reasons for this method to return an error.
        // Try to distinguish. Non-error conditions are:
        // 1. The device does not exist anymore. This means that the device
        // was volatile for some reason; in that case it can not belong to
        // Stratis so it is safe to ignore it.
        //
        // Note that it is better to be conservative and return with an
        // error in any case where failure to read the device could result
        // in bad data for Stratis. Additional exceptions may be added,
        // but only with a complete justification.
        if f.is_err() {
            let err = f.unwrap_err();
            match err.kind() {
                ErrorKind::NotFound => {
                    continue;
                }
                _ => {
                    if let Some(errno) = err.raw_os_error() {
                        if Errno::from_i32(errno) == Errno::ENXIO {
                            continue;
                        } else {
                            return Err(EngineError::Io(err));
                        }
                    } else {
                        return Err(EngineError::Io(err));
                    }
                }
            }
        }

        let mut f = f.expect("unreachable if f is err");
        if let DevOwnership::Ours(uuid) = try!(StaticHeader::determine_ownership(&mut f)) {
            pool_map
                .entry(uuid)
                .or_insert_with(Vec::new)
                .push(devnode)
        };
    }

    Ok(pool_map)
}

/// Get the most recent metadata from a set of Devices for a given pool UUID.
/// Returns None if no metadata found for this pool.
pub fn get_metadata(pool_uuid: &PoolUuid, devnodes: &[PathBuf]) -> EngineResult<Option<PoolSave>> {

    // No device nodes means no metadata
    if devnodes.is_empty() {
        return Ok(None);
    }

    // Get pairs of device nodes and matching BDAs
    // If no BDA, or BDA UUID does not match pool UUID, skip.
    // If there is an error reading the BDA, error. There could have been
    // vital information on that BDA, for example, it may have contained
    // the newest metadata.
    let mut bdas = Vec::new();
    for devnode in devnodes {
        let bda = try!(BDA::load(&mut try!(OpenOptions::new().read(true).open(devnode))));
        if bda.is_none() {
            continue;
        }
        let bda = bda.expect("unreachable if bda is None");

        if bda.pool_uuid() != pool_uuid {
            continue;
        }
        bdas.push((devnode, bda));
    }

    // We may have had no devices with BDAs for this pool, so return if no BDAs.
    if bdas.is_empty() {
        return Ok(None);
    }

    // Get a most recent BDA
    let &(_, ref most_recent_bda) = bdas.iter()
        .max_by_key(|p| p.1.last_update_time())
        .expect("bdas is not empty, must have a max");

    // Most recent time should never be None if this was a properly
    // created pool; this allows for the method to be called in other
    // circumstances.
    let most_recent_time = most_recent_bda.last_update_time();
    if most_recent_time.is_none() {
        return Ok(None);
    }

    // Try to read from all available devnodes that could contain most
    // recent metadata. In the event of errors, continue to try until all are
    // exhausted.
    for &(devnode, ref bda) in
        bdas.iter()
            .filter(|p| p.1.last_update_time() == most_recent_time) {

        let f = OpenOptions::new().read(true).open(devnode);
        if f.is_err() {
            continue;
        }
        let mut f = f.expect("f is not err");

        if let Ok(Some(data)) = bda.load_state(&mut f) {
            let json: serde_json::Result<PoolSave> = serde_json::from_slice(&data);
            if let Ok(pool) = json {
                return Ok(Some(pool));
            } else {
                continue;
            }
        } else {
            continue;
        }
    }

    // If no data has yet returned, we have an error. That is, we should have
    // some metadata, because we have a most recent time, but we failed to
    // get any.
    let err_str = "timestamp indicates data was written, but no data succesfully read";
    Err(EngineError::Engine(ErrorEnum::NotFound, err_str.into()))
}

/// Get the most recent metadata for each pool.
/// Since the metadata is written immediately after a pool is created, it
/// is considered an error for a pool to be w/out metadata.
pub fn get_pool_metadata(pool_table: &HashMap<PoolUuid, Vec<PathBuf>>)
                         -> EngineResult<HashMap<PoolUuid, PoolSave>> {
    let mut metadata = HashMap::new();
    for (pool_uuid, devices) in pool_table.iter() {
        if let Ok(Some(pool)) = get_metadata(pool_uuid, &devices) {
            metadata.insert(pool_uuid.clone(), pool);
        } else {
            return Err(EngineError::Engine(ErrorEnum::NotFound,
                                           format!("no metadata for pool {}", pool_uuid)));
        }
    }
    Ok(metadata)
}
<commit_msg>Ignore Errno::ENOMEDIUM for blockdev::find_all()<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Code to handle a single block device.

use std::collections::HashMap;
use std::io::ErrorKind;
use std::fs::{OpenOptions, read_dir};
use std::os::linux::fs::MetadataExt;
use std::path::PathBuf;

use nix::Errno;
use nix::sys::stat::{S_IFBLK, S_IFMT};
use serde_json;

use engine::{EngineResult, EngineError, ErrorEnum, PoolUuid};
use super::metadata::{BDA, StaticHeader};
use super::engine::DevOwnership;
use super::serde_structs::PoolSave;

/// Find all Stratis devices.
///
/// Returns a map of pool uuids to a vector of devices for each pool.
pub fn find_all() -> EngineResult<HashMap<PoolUuid, Vec<PathBuf>>> {

    let mut pool_map = HashMap::new();
    for dir_e in try!(read_dir("/dev")) {
        let dir_e = try!(dir_e);
        let mode = try!(dir_e.metadata()).st_mode();

        // Device node can't belong to Stratis if it is not a block device
        if mode & S_IFMT.bits() != S_IFBLK.bits() {
            continue;
        }

        let devnode = dir_e.path();

        let f = OpenOptions::new().read(true).open(&devnode);

        // There are some reasons for OpenOptions::open() to return an error
        // which are not reasons for this method to return an error.
        // Try to distinguish. Non-error conditions are:
        //
        // 1. ENXIO: The device does not exist anymore. This means that the device
        // was volatile for some reason; in that case it can not belong to
        // Stratis so it is safe to ignore it.
        //
        // 2. ENOMEDIUM: The device has no medium. An example of this case is an
        // empty optical drive.
        //
        // Note that it is better to be conservative and return with an
        // error in any case where failure to read the device could result
        // in bad data for Stratis. Additional exceptions may be added,
        // but only with a complete justification.
        if f.is_err() {
            let err = f.unwrap_err();
            match err.kind() {
                ErrorKind::NotFound => {
                    continue;
                }
                _ => {
                    if let Some(errno) = err.raw_os_error() {
                        match Errno::from_i32(errno) {
                            Errno::ENXIO | Errno::ENOMEDIUM => continue,
                            _ => return Err(EngineError::Io(err)),
                        };
                    } else {
                        return Err(EngineError::Io(err));
                    }
                }
            }
        }

        let mut f = f.expect("unreachable if f is err");
        if let DevOwnership::Ours(uuid) = try!(StaticHeader::determine_ownership(&mut f)) {
            pool_map
                .entry(uuid)
                .or_insert_with(Vec::new)
                .push(devnode)
        };
    }

    Ok(pool_map)
}

/// Get the most recent metadata from a set of Devices for a given pool UUID.
/// Returns None if no metadata found for this pool.
pub fn get_metadata(pool_uuid: &PoolUuid, devnodes: &[PathBuf]) -> EngineResult<Option<PoolSave>> {

    // No device nodes means no metadata
    if devnodes.is_empty() {
        return Ok(None);
    }

    // Get pairs of device nodes and matching BDAs
    // If no BDA, or BDA UUID does not match pool UUID, skip.
    // If there is an error reading the BDA, error. There could have been
    // vital information on that BDA, for example, it may have contained
    // the newest metadata.
    let mut bdas = Vec::new();
    for devnode in devnodes {
        let bda = try!(BDA::load(&mut try!(OpenOptions::new().read(true).open(devnode))));
        if bda.is_none() {
            continue;
        }
        let bda = bda.expect("unreachable if bda is None");

        if bda.pool_uuid() != pool_uuid {
            continue;
        }
        bdas.push((devnode, bda));
    }

    // We may have had no devices with BDAs for this pool, so return if no BDAs.
    if bdas.is_empty() {
        return Ok(None);
    }

    // Get a most recent BDA
    let &(_, ref most_recent_bda) = bdas.iter()
        .max_by_key(|p| p.1.last_update_time())
        .expect("bdas is not empty, must have a max");

    // Most recent time should never be None if this was a properly
    // created pool; this allows for the method to be called in other
    // circumstances.
    let most_recent_time = most_recent_bda.last_update_time();
    if most_recent_time.is_none() {
        return Ok(None);
    }

    // Try to read from all available devnodes that could contain most
    // recent metadata. In the event of errors, continue to try until all are
    // exhausted.
    for &(devnode, ref bda) in
        bdas.iter()
            .filter(|p| p.1.last_update_time() == most_recent_time) {

        let f = OpenOptions::new().read(true).open(devnode);
        if f.is_err() {
            continue;
        }
        let mut f = f.expect("f is not err");

        if let Ok(Some(data)) = bda.load_state(&mut f) {
            let json: serde_json::Result<PoolSave> = serde_json::from_slice(&data);
            if let Ok(pool) = json {
                return Ok(Some(pool));
            } else {
                continue;
            }
        } else {
            continue;
        }
    }

    // If no data has yet returned, we have an error. That is, we should have
    // some metadata, because we have a most recent time, but we failed to
    // get any.
    let err_str = "timestamp indicates data was written, but no data succesfully read";
    Err(EngineError::Engine(ErrorEnum::NotFound, err_str.into()))
}

/// Get the most recent metadata for each pool.
/// Since the metadata is written immediately after a pool is created, it
/// is considered an error for a pool to be w/out metadata.
pub fn get_pool_metadata(pool_table: &HashMap<PoolUuid, Vec<PathBuf>>)
                         -> EngineResult<HashMap<PoolUuid, PoolSave>> {
    let mut metadata = HashMap::new();
    for (pool_uuid, devices) in pool_table.iter() {
        if let Ok(Some(pool)) = get_metadata(pool_uuid, &devices) {
            metadata.insert(pool_uuid.clone(), pool);
        } else {
            return Err(EngineError::Engine(ErrorEnum::NotFound,
                                           format!("no metadata for pool {}", pool_uuid)));
        }
    }
    Ok(metadata)
}
<|endoftext|>


<commit_before>// Higher level OpenCL wrappers.

use CL::*;
use CL::ll::*;

struct Platform {
    id: cl_platform_id
}

pub fn get_platforms() -> ~[Platform] {
    let mut num_platforms = 0;

    // TODO: Check result status
    clGetPlatformIDs(0, ptr::null(), ptr::addr_of(&num_platforms));

    let ids = vec::to_mut(vec::from_elem(num_platforms as uint,
                                         0 as cl_platform_id));

    do vec::as_imm_buf(ids) |ids, len| {
        clGetPlatformIDs(len as cl_uint,
                         ids, ptr::addr_of(&num_platforms))
    };

    do ids.map |id| { Platform { id: *id } }
}

struct Device {
    id: cl_device_id
}

pub fn get_devices(platform: Platform) -> ~[Device] {
    let mut num_devices = 0;

    clGetDeviceIDs(platform.id, CL_DEVICE_TYPE_GPU, 0, ptr::null(),
                   ptr::addr_of(&num_devices));

    let ids = vec::to_mut(vec::from_elem(num_devices as uint,
                                         0 as cl_device_id));
    do vec::as_imm_buf(ids) |ids, len| {
        clGetDeviceIDs(platform.id, CL_DEVICE_TYPE_GPU, len as cl_uint,
                       ids, ptr::addr_of(&num_devices));
    };

    do ids.map |id| { Device {id: *id }}
}

struct Context {
    ctx: cl_context,

    drop {
        clReleaseContext(self.ctx);
    }
}

pub fn create_context(device: Device) -> Context {
    // TODO: Support for multiple devices
    let mut errcode = 0;

    // TODO: Proper error messages
    let ctx = clCreateContext(ptr::null(), 1, ptr::addr_of(&device.id),
                              ptr::null(), ptr::null(), ptr::addr_of(&errcode));

    if errcode != CL_SUCCESS {
        fail ~"Failed to create opencl context!"
    }

    Context { ctx: ctx }
}

struct CommandQueue {
    cqueue: cl_command_queue,

    drop {
        clReleaseCommandQueue(self.cqueue);
    }
}

pub fn create_commandqueue(ctx: &Context, device: Device) -> CommandQueue {
    let mut errcode = 0;

    let cqueue = clCreateCommandQueue(ctx.ctx, device.id, 0, ptr::addr_of(&errcode));

    if errcode != CL_SUCCESS {
        fail ~"Failed to create command queue!"
    }

    CommandQueue { cqueue: cqueue }
}

struct Buffer {
    buffer: cl_mem,
    size: int,

    drop {
        clReleaseMemObject(self.buffer);
    }
}

// TODO: How to make this function cleaner and nice
pub fn create_buffer(ctx: &Context, size: int, flags: cl_mem_flags) -> Buffer {
    let mut errcode = 0;

    let buffer = clCreateBuffer(ctx.ctx, flags, size as libc::size_t, ptr::null(),
                                ptr::addr_of(&errcode));

    if errcode != CL_SUCCESS {
        fail ~"Failed to create buffer!"
    }

    Buffer { buffer: buffer, size: size }
}

pub fn enqueue_write_buffer(cqueue: CommandQueue, buf: Buffer, host_vector: ~[int]) unsafe {
    let ret = clEnqueueWriteBuffer(cqueue.cqueue, buf.buffer, CL_TRUE, 0,
                                   buf.size as libc::size_t,
                                   vec::raw::to_ptr(host_vector) as *libc::c_void,
                                   0, ptr::null(), ptr::null());
    if ret != CL_SUCCESS {
        fail ~"Failed to enqueue write buffer!"
    }
}

struct Program {
    prg: cl_program,

    drop {
        clReleaseProgram(self.prg);
    }
}

// TODO: Support multiple devices
pub fn create_program_with_binary(ctx: Context, device: Device, binary_path: & Path){
    let mut errcode = 0;
    let binary = io::read_whole_file_str(binary_path).get();
    let program = do str::as_c_str(binary) |kernel_binary| {
        clCreateProgramWithBinary(ctx.ctx, 1, ptr::addr_of(&device.id),
                                  ptr::addr_of(&(binary.len() + 1)) as *libc::size_t,
                                  ptr::addr_of(&kernel_binary) as **libc::c_uchar,
                                  ptr::null(),
                                  ptr::addr_of(&errcode))
    };

    if errcode != CL_SUCCESS {
        fail ~"Failed to create open cl program with binary!"
    }

    Program { prg: program }
}

<commit_msg>fixed create_program_with_binary return type issue.<commit_after>// Higher level OpenCL wrappers.

use CL::*;
use CL::ll::*;

struct Platform {
    id: cl_platform_id
}

pub fn get_platforms() -> ~[Platform] {
    let mut num_platforms = 0;

    // TODO: Check result status
    clGetPlatformIDs(0, ptr::null(), ptr::addr_of(&num_platforms));

    let ids = vec::to_mut(vec::from_elem(num_platforms as uint,
                                         0 as cl_platform_id));

    do vec::as_imm_buf(ids) |ids, len| {
        clGetPlatformIDs(len as cl_uint,
                         ids, ptr::addr_of(&num_platforms))
    };

    do ids.map |id| { Platform { id: *id } }
}

struct Device {
    id: cl_device_id
}

pub fn get_devices(platform: Platform) -> ~[Device] {
    let mut num_devices = 0;

    clGetDeviceIDs(platform.id, CL_DEVICE_TYPE_GPU, 0, ptr::null(),
                   ptr::addr_of(&num_devices));

    let ids = vec::to_mut(vec::from_elem(num_devices as uint,
                                         0 as cl_device_id));
    do vec::as_imm_buf(ids) |ids, len| {
        clGetDeviceIDs(platform.id, CL_DEVICE_TYPE_GPU, len as cl_uint,
                       ids, ptr::addr_of(&num_devices));
    };

    do ids.map |id| { Device {id: *id }}
}

struct Context {
    ctx: cl_context,

    drop {
        clReleaseContext(self.ctx);
    }
}

pub fn create_context(device: Device) -> Context {
    // TODO: Support for multiple devices
    let mut errcode = 0;

    // TODO: Proper error messages
    let ctx = clCreateContext(ptr::null(), 1, ptr::addr_of(&device.id),
                              ptr::null(), ptr::null(), ptr::addr_of(&errcode));

    if errcode != CL_SUCCESS {
        fail ~"Failed to create opencl context!"
    }

    Context { ctx: ctx }
}

struct CommandQueue {
    cqueue: cl_command_queue,

    drop {
        clReleaseCommandQueue(self.cqueue);
    }
}

pub fn create_commandqueue(ctx: &Context, device: Device) -> CommandQueue {
    let mut errcode = 0;

    let cqueue = clCreateCommandQueue(ctx.ctx, device.id, 0, ptr::addr_of(&errcode));

    if errcode != CL_SUCCESS {
        fail ~"Failed to create command queue!"
    }

    CommandQueue { cqueue: cqueue }
}

struct Buffer {
    buffer: cl_mem,
    size: int,

    drop {
        clReleaseMemObject(self.buffer);
    }
}

// TODO: How to make this function cleaner and nice
pub fn create_buffer(ctx: &Context, size: int, flags: cl_mem_flags) -> Buffer {
    let mut errcode = 0;

    let buffer = clCreateBuffer(ctx.ctx, flags, size as libc::size_t, ptr::null(),
                                ptr::addr_of(&errcode));

    if errcode != CL_SUCCESS {
        fail ~"Failed to create buffer!"
    }

    Buffer { buffer: buffer, size: size }
}

pub fn enqueue_write_buffer(cqueue: CommandQueue, buf: Buffer, host_vector: ~[int]) unsafe {
    let ret = clEnqueueWriteBuffer(cqueue.cqueue, buf.buffer, CL_TRUE, 0,
                                   buf.size as libc::size_t,
                                   vec::raw::to_ptr(host_vector) as *libc::c_void,
                                   0, ptr::null(), ptr::null());
    if ret != CL_SUCCESS {
        fail ~"Failed to enqueue write buffer!"
    }
}

struct Program {
    prg: cl_program,

    drop {
        clReleaseProgram(self.prg);
    }
}

// TODO: Support multiple devices
pub fn create_program_with_binary(ctx: Context, device: Device, binary_path: & Path) -> Program{
    let mut errcode = 0;
    let binary = io::read_whole_file_str(binary_path).get();
    let program = do str::as_c_str(binary) |kernel_binary| {
        clCreateProgramWithBinary(ctx.ctx, 1, ptr::addr_of(&device.id),
                                  ptr::addr_of(&(binary.len() + 1)) as *libc::size_t,
                                  ptr::addr_of(&kernel_binary) as **libc::c_uchar,
                                  ptr::null(),
                                  ptr::addr_of(&errcode))
    };

    if errcode != CL_SUCCESS {
        fail ~"Failed to create open cl program with binary!"
    }

    Program { prg: program }
}

<|endoftext|>


<commit_before><commit_msg>s/interpret/eval, s/statement/stmt<commit_after><|endoftext|>


<commit_before><commit_msg>Remove comment about refcounted pointers; there are no refcounted pointers<commit_after><|endoftext|>


<commit_before>extern mod std;
extern mod cql_client (name="cql_client", vers="0.0.1");

use core::rand;
use std::time;

static str_seq:&'static str = "qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM";
static str_seq_len:uint = 26 * 2;

fn rand_str(size: uint) -> ~str {
    let mut bytes:~[u8] = ~[];
    for uint::range(0, size) |_| {
        bytes.push(str_seq[rand::random::<uint>() % str_seq_len]);
    }
    str::from_bytes(bytes)
}

fn time_diff(start: time::Timespec, end: time::Timespec) -> float {
    return (end.sec - start.sec) as float + (end.nsec - start.nsec) as float/1e9f;
}

fn main() {
    let res = cql_client::connect(~"127.0.0.1", 9042, None);
    if res.is_err() {
        io::println(fmt!("%?", res.get_err()));
        fail!(~"Failed to connect");
    }

    let client = res.get();

    let mut res;
    res = client.query(~"create keyspace rust with replication = \
        {'class': 'SimpleStrategy', 'replication_factor':1}", cql_client::ConsistencyOne);
    io::println(fmt!("%?", res));

    res = client.query(~"create table rust.test (id text primary key, value float)",
         cql_client::ConsistencyOne);
    io::println(fmt!("%?", res));

    res = client.query(~"insert into rust.test (id, value) values ('asdf', 1.2345)",
         cql_client::ConsistencyOne);
    io::println(fmt!("%?", res));

    res = client.query(~"select * from rust.test",
         cql_client::ConsistencyOne);
    io::println(fmt!("%?", res));
}
<commit_msg>New set of tests using the new library<commit_after>extern crate std;
extern crate cql;

fn main() {
    println!("Connecting ...!");
    let mut client = match cql::connect("127.0.0.1", 9042, None) {
        Ok(c) => c,
        Err(e) => fail!(format!("Failed to connect: {}", e.desc))
    };
    println!("Connected with CQL binary version v{}", client.version);

    let mut q = "create keyspace rust with replication = {'class': 'SimpleStrategy', 'replication_factor':1}";
    println!("Query: {}", q);
    let mut res = client.exec_query(q, cql::Consistency::One);
    println!("Result: {} \n", res);

    q = "create table rust.test (id text primary key, f32 float, f64 double, i32 int, i64 bigint, b boolean, ip inet)";
    println!("Query: {}", q);
    res = client.exec_query(q, cql::Consistency::One);
    println!("Result: {} \n", res);

    q = "insert into rust.test (id, f32, f64, i32, i64, b, ip) values ('asdf', 1.2345, 3.14159, 47, 59, true, '127.0.0.1')";
    println!("Query: {}", q);
    res = client.exec_query(q, cql::Consistency::One);
    println!("Result: {} \n", res);

    q = "select * from rust.test";
    println!("Query: {}", q);
    res = client.exec_query(q, cql::Consistency::One);
    println!("Result: {} \n", res);

    q = "insert into rust.test (id, f32) values (?, ?)";
    println!("Create prepared: {}", q);
    let res_id = client.prepared_statement(q);
    println!("Result prepared: {} \n", res);

    if res_id.is_err() {
        fail!("Error in creating prepared statement")
    }

    let ps_id = res_id.unwrap();
    println!("Execute prepared: {}", ps_id);
    let params: Vec<cql::CqlValue> = vec![cql::CqlVarchar(Some("ttrwe".into_maybe_owned())), cql::CqlFloat(Some(15.1617))];
    res = client.exec_prepared(ps_id, params, cql::Consistency::One);
    println!("Result: {} \n", res);

    q = "select * from rust.test";
    println!("Query: {}", q);
    res = client.exec_query(q, cql::Consistency::One);
    println!("Result: {} \n", res);

    q = "create table rust.test2 (id text primary key, l list<int>, m map<int, text>, s set<float>)";
    println!("Query: {}", q);
    res = client.exec_query(q, cql::Consistency::One);
    println!("Result: {} \n", res);

    q = "insert into rust.test2 (id, l, m, s) values ('asdf', [1,2,3,4,5], {0: 'aa', 1: 'bbb', 3: 'cccc'}, {1.234, 2.345, 3.456, 4.567})";
    println!("Query: {}", q);
    res = client.exec_query(q, cql::Consistency::One);
    println!("Result: {} \n", res);

    q = "select * from rust.test2";
    println!("Query: {}", q);
    res = client.exec_query(q, cql::Consistency::One);
    println!("Result: {} \n", res);
}
<|endoftext|>


<commit_before>/**
Katy Williamson kcw4sn
Ben Terner bt3ze
@virginia.edu

Final Project for CS4414 - Operating Systems
A parallel MST implementation on images in Rust

*/

extern mod extra;
use std::{os, num,str} ;
use extra::{arc,priority_queue};

struct Pixel{
    r: int,
    b: int,
    g: int,
    color: int,
    x: int,
    y: int
}

impl Pixel {
    fn new(red: int, blue: int, green: int) -> Pixel {
        Pixel{
            r: red,
            b: blue,
            g: green,
            color: 0,
            x: 0,
            y: 0
        }
    }
}

struct Edge {
    source: Pixel,
    dest: Pixel,
    cost: float
}

impl Edge {
    fn new(s: Pixel, d: Pixel) -> Edge {
        Edge { // these need to be references to pixels, not copies.
            // for standard prim, it's alright for now
            source: s,
            dest: d,
            cost: edgeCost(s,d)
        }
    }
}

impl Ord for Edge {
    fn lt(&self, other: &Edge) -> bool {
        self.cost < other.cost
    }
}

fn edgeCost(a: Pixel, b: Pixel) -> float {
    num::sqrt( (
            a.r*a.r-b.r*b.r + a.g*a.g-b.g*b.g + a.b*a.b-b.b*b.b ) as float)
}

fn readImage( filename: &~str) -> ~ [ ~[Pixel]] {
    ~[~[]]
}

fn main(){
    let argv =  os::args();
    println(argv.to_str());
    let length = match from_str::<int>(argv[2]) {
        Some(x) => x,
        None() => 0
    };
    let width = match from_str::<int>(argv[3]) {
        Some(x) => x,
        None() => 0
    };

    let pixels: ~[ ~[Pixel] ] = readImage(&argv[1]);
    let mut find_arcs: ~[ ~[arc::RWArc<Pixel>] ] =  ~[ ~[]];
    let mut len = 0;
    let mut wid = 0;
    loop {
        loop {
            find_arcs[len][wid] = arc::RWArc::new(pixels[len][wid]);
                wid+=1;
            if(wid == width) {
                break;
            }
        }
        len+=1;
        if(len == length) {
            break;
        }
    }

    let arcs = find_arcs; // now we have an immutable array that can be shared

    let mut x= 0;
    let mut y= 0;
    let mut queue: priority_queue::PriorityQueue<Edge> =  priority_queue::PriorityQueue::new();

    loop { // iterate through items in the queue, which contains all of the edges/vertices

        // add (x,y+1), (x,y-1), (x-1,y), (x+1,y) to queue if not already visited or colored
        let neighbors = [ (x,y-1),(x+1,y), (x,y+1), (x,y-1)];
        for &coord in neighbors.iter() {
            let (w,z) = coord;
            if w >=0 && w < width && z >=0 && z <= length { // bounds checking
                // if neighbor at coord has not been visited or colored
                if pixels[w][z].color != 0 { // then we have a new vertex
                    // should really update this to make sure the vertex has not already been visited
                    queue.push( Edge::new(pixels[x][y],pixels[w][z]) );
                }

            }
        }

        let edge = queue.maybe_pop();
        match edge {
            Some(e) => {
                let col = e.dest.color;  // check e.destination's color
                if col == 0 {
                    // not in any tree
                    x = e.dest.x;
                    y = e.dest.y;
                    // use an arc to write the following
                    // e.dest.color = e.source.color;
                }
            }
            None => { break; }
        }
    }

}
<commit_msg>more on prim algoore prim algorithm. initial edge additions still not covered, should be.<commit_after>/**
Katy Williamson kcw4sn
Ben Terner bt3ze
@virginia.edu

Final Project for CS4414 - Operating Systems
A parallel MST implementation on images in Rust

*/

extern mod extra;
use std::{os, num,hashmap,str} ;
use extra::{arc,priority_queue};

struct Pixel{
    r: int,
    b: int,
    g: int,
    color: int,
    x: int,
    y: int
}

impl Pixel {
    fn new(red: int, blue: int, green: int) -> Pixel {
        Pixel{
            r: red,
            b: blue,
            g: green,
            color: 0,
            x: 0,
            y: 0
        }
    }
}

struct Edge {
    source: Pixel,
    dest: Pixel,
    cost: float
}

impl Edge {
    fn new(s: Pixel, d: Pixel) -> Edge {
        Edge { // these need to be references to pixels, not copies.
            // for standard prim, it's alright for now
            source: s,
            dest: d,
            cost: edgeCost(s,d)
        }
    }
}

impl Ord for Edge {
    fn lt(&self, other: &Edge) -> bool {
        self.cost < other.cost
    }
}

fn edgeCost(a: Pixel, b: Pixel) -> float {
    num::sqrt( (
            a.r*a.r-b.r*b.r + a.g*a.g-b.g*b.g + a.b*a.b-b.b*b.b ) as float)
}

fn readImage( filename: &~str) -> ~ [ ~[Pixel]] {
    ~[~[]]
}

fn main(){
    let argv =  os::args();
    println(argv.to_str());
    let length = match from_str::<int>(argv[2]) {
        Some(x) => x,
        None() => 0
    };
    let width = match from_str::<int>(argv[3]) {
        Some(x) => x,
        None() => 0
    };

    let pixels: ~[ ~[Pixel] ] = readImage(&argv[1]);
    let mut find_arcs: ~[ ~[arc::RWArc<Pixel>] ] =  ~[ ~[]];
    let mut len = 0;
    let mut wid = 0;
    loop {
        loop {
            find_arcs[len][wid] = arc::RWArc::new(pixels[len][wid]);
                wid+=1;
            if(wid == width) {
                wid = 0;
                break;
            }
        }
        len+=1;
        if(len == length) {
            break;
        }
    }

    let arcs = find_arcs; // now we have an immutable array that can be shared

    let mut x= 0;
    let mut y= 0;
    let mut queue: priority_queue::PriorityQueue<Edge> =  priority_queue::PriorityQueue::new();
    let mut visited: hashmap::HashMap<(int,int),bool> = hashmap::HashMap::new();
    loop { // iterate through items in the queue, which contains all of the edges/vertices

        let edge = queue.maybe_pop();
        match edge {
            Some(e) => {
                if e.dest.color == 0 {
                    // not in any tree
                    let coord = (e.dest.x,e.dest.y);
                    if *visited.get(&coord) == false { // use visited hashmap to figure out if we're at a new vertex
                        visited.insert(coord,true);
                        x = e.dest.x;
                        y = e.dest.y;
                        let neighbors = [ (x,y-1),(x+1,y), (x,y+1), (x,y-1)];
                        for &coord in neighbors.iter() {
                            let (w,z) = coord;
                            if w >=0 && w < width && z >=0 && z <= length { // bounds checking
                                // if neighbor at coord has not been or colored
                                if pixels[w][z].color != 0 { // then we have a new vertex
                                    queue.push( Edge::new(pixels[x][y],pixels[w][z]) );
                                }
                            }
                        }
                    }

                    // use an arc to write the following
                    // e.dest.color = e.source.color;
                }
            }
            None => { break; } // this should really never execute
        }
    }

}
<|endoftext|>


<commit_before>pub struct<commit_msg>Cleaning up<commit_after><|endoftext|>


<commit_before><commit_msg>use self.deref(...) (see #653)<commit_after><|endoftext|>


<commit_before>//! Convenience functions for making requests over HTTP
//!
//! This module provides a few functions that make it easier to make
//! requests over HTTP in a consistent manner e.g. with the 'User-Agent` header
//! set and respecting cache control headers in responses. In addition to reducing
//! the number of network requests for the client, several APIs ask clients
//! to implement caching to reduce load on their servers.

use std::{env, fs::create_dir_all, fs::File, io, io::Write, path::Path};

use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache};
use reqwest::header::{self, HeaderMap, HeaderName, HeaderValue};
use reqwest::Response;
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};

use common::{
    dirs,
    eyre::Result,
    once_cell::sync::Lazy,
    serde::{de::DeserializeOwned, Serialize},
    tempfile::{self, NamedTempFile},
    tokio,
};

// Re-exports for consumers of this crate
pub use http;
pub use reqwest;
pub use reqwest_middleware;
pub use url;
pub use urlencoding;

pub static USER_AGENT: &str = concat!("stencila/", env!("CARGO_PKG_VERSION"),);

/// Get the directory of the HTTP cache
pub fn cache_dir() -> String {
    let user_cache_dir = dirs::cache_dir().unwrap_or_else(|| env::current_dir().unwrap());
    match env::consts::OS {
        "macos" | "windows" => user_cache_dir.join("Stencila").join("HTTP-Cache"),
        _ => user_cache_dir.join("stencila").join("http-cache"),
    }
    .to_string_lossy()
    .to_string()
}

pub static CLIENT: Lazy<ClientWithMiddleware> = Lazy::new(|| {
    let client = reqwest::Client::builder()
        .user_agent(USER_AGENT)
        .build()
        .expect("Should be able to build HTTP client");
    let caching_middleware = Cache(HttpCache {
        mode: CacheMode::Default,
        manager: CACacheManager { path: cache_dir() },
        options: None,
    });
    ClientBuilder::new(client).with(caching_middleware).build()
});

/// Convert an array of tuples to a `reqwest::HeaderMap`
fn header_map(headers: &[(HeaderName, String)]) -> Result<HeaderMap> {
    let mut header_map = HeaderMap::new();
    header_map.insert(header::ACCEPT, HeaderValue::from_str("application/json")?);
    for (key, value) in headers {
        header_map.insert(key, value.parse()?);
    }
    Ok(header_map)
}

/// Send a GET request
pub async fn get<T: DeserializeOwned>(url: &str) -> Result<T> {
    get_with(url, &[]).await
}

/// Send a GET request with additional request headers
pub async fn get_with<T: DeserializeOwned>(
    url: &str,
    headers: &[(HeaderName, String)],
) -> Result<T> {
    let response = get_response(url, headers).await?;
    let json = response.error_for_status()?.json().await?;
    Ok(json)
}

/// Send a GET request with additional request headers and return response
///
/// Use this when you want to handle response errors or content yourself.
pub async fn get_response(url: &str, headers: &[(HeaderName, String)]) -> Result<Response> {
    let response = CLIENT.get(url).headers(header_map(headers)?).send().await?;
    Ok(response)
}

/// Send a POST request
pub async fn post<B: Serialize, T: DeserializeOwned>(url: &str, body: B) -> Result<T> {
    post_with(url, body, &[]).await
}

/// Send a POST request with additional request headers
pub async fn post_with<B: Serialize, T: DeserializeOwned>(
    url: &str,
    body: B,
    headers: &[(HeaderName, String)],
) -> Result<T> {
    let response = post_response(url, body, headers).await?;
    let json = response.error_for_status()?.json().await?;
    Ok(json)
}

/// Send a POST request with additional request headers and return response
pub async fn post_response<B: Serialize>(
    url: &str,
    body: B,
    headers: &[(HeaderName, String)],
) -> Result<Response> {
    let response = CLIENT
        .post(url)
        .json(&body)
        .headers(header_map(headers)?)
        .send()
        .await?;
    Ok(response)
}

/// Send a DELETE request
pub async fn delete(url: &str) -> Result<()> {
    delete_with(url, &[]).await
}

/// Send a DELETE request with additional request headers
pub async fn delete_with(url: &str, headers: &[(HeaderName, String)]) -> Result<()> {
    CLIENT
        .delete(url)
        .headers(header_map(headers)?)
        .send()
        .await?
        .error_for_status()?;
    Ok(())
}

/// Download a file from to a path
pub async fn download(url: &str, path: &Path) -> Result<()> {
    download_with(url, path, &[]).await
}

/// Download a file from to a path with additional request headers
pub async fn download_with(url: &str, path: &Path, headers: &[(HeaderName, String)]) -> Result<()> {
    let response = CLIENT
        .get(url)
        .headers(header_map(headers)?)
        .send()
        .await?
        .error_for_status()?;

    if let Some(parent) = path.parent() {
        create_dir_all(parent)?
    }

    let bytes = response.bytes().await?;
    let mut file = File::create(&path)?;
    io::copy(&mut bytes.as_ref(), &mut file)?;
    file.flush()?;

    Ok(())
}

/// Download a file from to a path synchronously
pub fn download_sync(url: &str, path: &Path) -> Result<()> {
    download_with_sync(url, path, &[])
}

/// Download a file from to a path synchronously with additional request headers
pub fn download_with_sync(url: &str, path: &Path, headers: &[(HeaderName, String)]) -> Result<()> {
    let url = url.to_owned();
    let path = path.to_owned();
    let headers = headers.to_vec();
    let (sender, receiver) = std::sync::mpsc::channel();
    tokio::spawn(async move {
        let result = download_with(&url, &path, &headers).await;
        sender.send(result)
    });
    receiver.recv()?
}

/// Download a file from to a temporary file
pub async fn download_temp(url: &str, ext: Option<&str>) -> Result<NamedTempFile> {
    download_temp_with(url, ext, &[]).await
}

/// Download a file from to a temporary file with additional request headers
///
/// Returns a `NamedTempFile` which will remove the temporary file when it
/// is dropped. Be aware of that and the security implications of long-lived temp files:
/// https://docs.rs/tempfile/latest/tempfile/struct.NamedTempFile.html
pub async fn download_temp_with(
    url: &str,
    ext: Option<&str>,
    headers: &[(HeaderName, String)],
) -> Result<NamedTempFile> {
    let suffix = ext.map_or_else(String::new, |ext| [".", ext].concat());
    let temp = tempfile::Builder::new().suffix(&suffix).tempfile()?;
    download_with(url, temp.path(), headers).await?;
    Ok(temp)
}

/// Create a `Response`
///
/// A convenience function for creating a HTTP response with a status code and
/// a text body.
pub fn response(status: http::StatusCode, content: &str) -> http::Response<String> {
    http::Response::builder()
        .status(status)
        .body(content.into())
        .expect("Unable to create HTTP response")
}
<commit_msg>refactor(HTTP utilities): Change cache directory name to `http`<commit_after>//! Convenience functions for making requests over HTTP
//!
//! This module provides a few functions that make it easier to make
//! requests over HTTP in a consistent manner e.g. with the 'User-Agent` header
//! set and respecting cache control headers in responses. In addition to reducing
//! the number of network requests for the client, several APIs ask clients
//! to implement caching to reduce load on their servers.

use std::{env, fs::create_dir_all, fs::File, io, io::Write, path::Path};

use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache};
use reqwest::header::{self, HeaderMap, HeaderName, HeaderValue};
use reqwest::Response;
use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};

use common::{
    dirs,
    eyre::Result,
    once_cell::sync::Lazy,
    serde::{de::DeserializeOwned, Serialize},
    tempfile::{self, NamedTempFile},
    tokio,
};

// Re-exports for consumers of this crate
pub use http;
pub use reqwest;
pub use reqwest_middleware;
pub use url;
pub use urlencoding;

pub static USER_AGENT: &str = concat!("stencila/", env!("CARGO_PKG_VERSION"),);

/// Get the directory of the HTTP cache
pub fn cache_dir() -> String {
    let user_cache_dir = dirs::cache_dir().unwrap_or_else(|| env::current_dir().unwrap());
    match env::consts::OS {
        "macos" | "windows" => user_cache_dir.join("Stencila").join("HTTP-Cache"),
        _ => user_cache_dir.join("stencila").join("http"),
    }
    .to_string_lossy()
    .to_string()
}

pub static CLIENT: Lazy<ClientWithMiddleware> = Lazy::new(|| {
    let client = reqwest::Client::builder()
        .user_agent(USER_AGENT)
        .build()
        .expect("Should be able to build HTTP client");
    let caching_middleware = Cache(HttpCache {
        mode: CacheMode::Default,
        manager: CACacheManager { path: cache_dir() },
        options: None,
    });
    ClientBuilder::new(client).with(caching_middleware).build()
});

/// Convert an array of tuples to a `reqwest::HeaderMap`
fn header_map(headers: &[(HeaderName, String)]) -> Result<HeaderMap> {
    let mut header_map = HeaderMap::new();
    header_map.insert(header::ACCEPT, HeaderValue::from_str("application/json")?);
    for (key, value) in headers {
        header_map.insert(key, value.parse()?);
    }
    Ok(header_map)
}

/// Send a GET request
pub async fn get<T: DeserializeOwned>(url: &str) -> Result<T> {
    get_with(url, &[]).await
}

/// Send a GET request with additional request headers
pub async fn get_with<T: DeserializeOwned>(
    url: &str,
    headers: &[(HeaderName, String)],
) -> Result<T> {
    let response = get_response(url, headers).await?;
    let json = response.error_for_status()?.json().await?;
    Ok(json)
}

/// Send a GET request with additional request headers and return response
///
/// Use this when you want to handle response errors or content yourself.
pub async fn get_response(url: &str, headers: &[(HeaderName, String)]) -> Result<Response> {
    let response = CLIENT.get(url).headers(header_map(headers)?).send().await?;
    Ok(response)
}

/// Send a POST request
pub async fn post<B: Serialize, T: DeserializeOwned>(url: &str, body: B) -> Result<T> {
    post_with(url, body, &[]).await
}

/// Send a POST request with additional request headers
pub async fn post_with<B: Serialize, T: DeserializeOwned>(
    url: &str,
    body: B,
    headers: &[(HeaderName, String)],
) -> Result<T> {
    let response = post_response(url, body, headers).await?;
    let json = response.error_for_status()?.json().await?;
    Ok(json)
}

/// Send a POST request with additional request headers and return response
pub async fn post_response<B: Serialize>(
    url: &str,
    body: B,
    headers: &[(HeaderName, String)],
) -> Result<Response> {
    let response = CLIENT
        .post(url)
        .json(&body)
        .headers(header_map(headers)?)
        .send()
        .await?;
    Ok(response)
}

/// Send a DELETE request
pub async fn delete(url: &str) -> Result<()> {
    delete_with(url, &[]).await
}

/// Send a DELETE request with additional request headers
pub async fn delete_with(url: &str, headers: &[(HeaderName, String)]) -> Result<()> {
    CLIENT
        .delete(url)
        .headers(header_map(headers)?)
        .send()
        .await?
        .error_for_status()?;
    Ok(())
}

/// Download a file from to a path
pub async fn download(url: &str, path: &Path) -> Result<()> {
    download_with(url, path, &[]).await
}

/// Download a file from to a path with additional request headers
pub async fn download_with(url: &str, path: &Path, headers: &[(HeaderName, String)]) -> Result<()> {
    let response = CLIENT
        .get(url)
        .headers(header_map(headers)?)
        .send()
        .await?
        .error_for_status()?;

    if let Some(parent) = path.parent() {
        create_dir_all(parent)?
    }

    let bytes = response.bytes().await?;
    let mut file = File::create(&path)?;
    io::copy(&mut bytes.as_ref(), &mut file)?;
    file.flush()?;

    Ok(())
}

/// Download a file from to a path synchronously
pub fn download_sync(url: &str, path: &Path) -> Result<()> {
    download_with_sync(url, path, &[])
}

/// Download a file from to a path synchronously with additional request headers
pub fn download_with_sync(url: &str, path: &Path, headers: &[(HeaderName, String)]) -> Result<()> {
    let url = url.to_owned();
    let path = path.to_owned();
    let headers = headers.to_vec();
    let (sender, receiver) = std::sync::mpsc::channel();
    tokio::spawn(async move {
        let result = download_with(&url, &path, &headers).await;
        sender.send(result)
    });
    receiver.recv()?
}

/// Download a file from to a temporary file
pub async fn download_temp(url: &str, ext: Option<&str>) -> Result<NamedTempFile> {
    download_temp_with(url, ext, &[]).await
}

/// Download a file from to a temporary file with additional request headers
///
/// Returns a `NamedTempFile` which will remove the temporary file when it
/// is dropped. Be aware of that and the security implications of long-lived temp files:
/// https://docs.rs/tempfile/latest/tempfile/struct.NamedTempFile.html
pub async fn download_temp_with(
    url: &str,
    ext: Option<&str>,
    headers: &[(HeaderName, String)],
) -> Result<NamedTempFile> {
    let suffix = ext.map_or_else(String::new, |ext| [".", ext].concat());
    let temp = tempfile::Builder::new().suffix(&suffix).tempfile()?;
    download_with(url, temp.path(), headers).await?;
    Ok(temp)
}

/// Create a `Response`
///
/// A convenience function for creating a HTTP response with a status code and
/// a text body.
pub fn response(status: http::StatusCode, content: &str) -> http::Response<String> {
    http::Response::builder()
        .status(status)
        .body(content.into())
        .expect("Unable to create HTTP response")
}
<|endoftext|>


<commit_before>/// For all the linkers we support, and information they might
/// need out of the shared crate context before we get rid of it.

use rustc::session::{Session, config};
use rustc::session::search_paths::PathKind;
use rustc::middle::dependency_format::Linkage;
use rustc::middle::cstore::LibSource;
use rustc_target::spec::LinkerFlavor;
use rustc::hir::def_id::CrateNum;

use super::command::Command;
use crate::CrateInfo;

use cc::windows_registry;
use std::fs;
use std::path::{Path, PathBuf};
use std::env;

pub fn remove(sess: &Session, path: &Path) {
    if let Err(e) = fs::remove_file(path) {
        sess.err(&format!("failed to remove {}: {}",
                          path.display(),
                          e));
    }
}

// The third parameter is for env vars, used on windows to set up the
// path for MSVC to find its DLLs, and gcc to find its bundled
// toolchain
pub fn get_linker(sess: &Session, linker: &Path, flavor: LinkerFlavor) -> (PathBuf, Command) {
    let msvc_tool = windows_registry::find_tool(&sess.opts.target_triple.triple(), "link.exe");

    // If our linker looks like a batch script on Windows then to execute this
    // we'll need to spawn `cmd` explicitly. This is primarily done to handle
    // emscripten where the linker is `emcc.bat` and needs to be spawned as
    // `cmd /c emcc.bat ...`.
    //
    // This worked historically but is needed manually since #42436 (regression
    // was tagged as #42791) and some more info can be found on #44443 for
    // emscripten itself.
    let mut cmd = match linker.to_str() {
        Some(linker) if cfg!(windows) && linker.ends_with(".bat") => Command::bat_script(linker),
        _ => match flavor {
            LinkerFlavor::Lld(f) => Command::lld(linker, f),
            LinkerFlavor::Msvc
                if sess.opts.cg.linker.is_none() && sess.target.target.options.linker.is_none() =>
            {
                Command::new(msvc_tool.as_ref().map(|t| t.path()).unwrap_or(linker))
            },
            _ => Command::new(linker),
        }
    };

    // The compiler's sysroot often has some bundled tools, so add it to the
    // PATH for the child.
    let mut new_path = sess.host_filesearch(PathKind::All)
                           .get_tools_search_paths();
    let mut msvc_changed_path = false;
    if sess.target.target.options.is_like_msvc {
        if let Some(ref tool) = msvc_tool {
            cmd.args(tool.args());
            for &(ref k, ref v) in tool.env() {
                if k == "PATH" {
                    new_path.extend(env::split_paths(v));
                    msvc_changed_path = true;
                } else {
                    cmd.env(k, v);
                }
            }
        }
    }

    if !msvc_changed_path {
        if let Some(path) = env::var_os("PATH") {
            new_path.extend(env::split_paths(&path));
        }
    }
    cmd.env("PATH", env::join_paths(new_path).unwrap());

    (linker.to_path_buf(), cmd)
}

pub fn each_linked_rlib(sess: &Session,
                               info: &CrateInfo,
                               f: &mut dyn FnMut(CrateNum, &Path)) -> Result<(), String> {
    let crates = info.used_crates_static.iter();
    let fmts = sess.dependency_formats.borrow();
    let fmts = fmts.get(&config::CrateType::Executable)
                   .or_else(|| fmts.get(&config::CrateType::Staticlib))
                   .or_else(|| fmts.get(&config::CrateType::Cdylib))
                   .or_else(|| fmts.get(&config::CrateType::ProcMacro));
    let fmts = match fmts {
        Some(f) => f,
        None => return Err("could not find formats for rlibs".to_string())
    };
    for &(cnum, ref path) in crates {
        match fmts.get(cnum.as_usize() - 1) {
            Some(&Linkage::NotLinked) |
            Some(&Linkage::IncludedFromDylib) => continue,
            Some(_) => {}
            None => return Err("could not find formats for rlibs".to_string())
        }
        let name = &info.crate_name[&cnum];
        let path = match *path {
            LibSource::Some(ref p) => p,
            LibSource::MetadataOnly => {
                return Err(format!("could not find rlib for: `{}`, found rmeta (metadata) file",
                                   name))
            }
            LibSource::None => {
                return Err(format!("could not find rlib for: `{}`", name))
            }
        };
        f(cnum, &path);
    }
    Ok(())
}

/// Returns a boolean indicating whether the specified crate should be ignored
/// during LTO.
///
/// Crates ignored during LTO are not lumped together in the "massive object
/// file" that we create and are linked in their normal rlib states. See
/// comments below for what crates do not participate in LTO.
///
/// It's unusual for a crate to not participate in LTO. Typically only
/// compiler-specific and unstable crates have a reason to not participate in
/// LTO.
pub fn ignored_for_lto(sess: &Session, info: &CrateInfo, cnum: CrateNum) -> bool {
    // If our target enables builtin function lowering in LLVM then the
    // crates providing these functions don't participate in LTO (e.g.
    // no_builtins or compiler builtins crates).
    !sess.target.target.options.no_builtins &&
        (info.compiler_builtins == Some(cnum) || info.is_no_builtins.contains(&cnum))
}

pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
    fn infer_from(
        sess: &Session,
        linker: Option<PathBuf>,
        flavor: Option<LinkerFlavor>,
    ) -> Option<(PathBuf, LinkerFlavor)> {
        match (linker, flavor) {
            (Some(linker), Some(flavor)) => Some((linker, flavor)),
            // only the linker flavor is known; use the default linker for the selected flavor
            (None, Some(flavor)) => Some((PathBuf::from(match flavor {
                LinkerFlavor::Em  => if cfg!(windows) { "emcc.bat" } else { "emcc" },
                LinkerFlavor::Gcc => "cc",
                LinkerFlavor::Ld => "ld",
                LinkerFlavor::Msvc => "link.exe",
                LinkerFlavor::Lld(_) => "lld",
                LinkerFlavor::PtxLinker => "rust-ptx-linker",
            }), flavor)),
            (Some(linker), None) => {
                let stem = if linker.extension().and_then(|ext| ext.to_str()) == Some("exe") {
                    linker.file_stem().and_then(|stem| stem.to_str())
                } else {
                    linker.to_str()
                }.unwrap_or_else(|| {
                    sess.fatal("couldn't extract file stem from specified linker");
                }).to_owned();

                let flavor = if stem == "emcc" {
                    LinkerFlavor::Em
                } else if stem == "gcc" || stem.ends_with("-gcc") {
                    LinkerFlavor::Gcc
                } else if stem == "ld" || stem == "ld.lld" || stem.ends_with("-ld") {
                    LinkerFlavor::Ld
                } else if stem == "link" || stem == "lld-link" {
                    LinkerFlavor::Msvc
                } else if stem == "lld" || stem == "rust-lld" {
                    LinkerFlavor::Lld(sess.target.target.options.lld_flavor)
                } else {
                    // fall back to the value in the target spec
                    sess.target.target.linker_flavor
                };

                Some((linker, flavor))
            },
            (None, None) => None,
        }
    }

    // linker and linker flavor specified via command line have precedence over what the target
    // specification specifies
    if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) {
        return ret;
    }

    if let Some(ret) = infer_from(
        sess,
        sess.target.target.options.linker.clone().map(PathBuf::from),
        Some(sess.target.target.linker_flavor),
    ) {
        return ret;
    }

    bug!("Not enough information provided to determine how to invoke the linker");
}
<commit_msg>Rollup merge of #59318 - alexcrichton:check-for-clang, r=michaelwoerister<commit_after>/// For all the linkers we support, and information they might
/// need out of the shared crate context before we get rid of it.

use rustc::session::{Session, config};
use rustc::session::search_paths::PathKind;
use rustc::middle::dependency_format::Linkage;
use rustc::middle::cstore::LibSource;
use rustc_target::spec::LinkerFlavor;
use rustc::hir::def_id::CrateNum;

use super::command::Command;
use crate::CrateInfo;

use cc::windows_registry;
use std::fs;
use std::path::{Path, PathBuf};
use std::env;

pub fn remove(sess: &Session, path: &Path) {
    if let Err(e) = fs::remove_file(path) {
        sess.err(&format!("failed to remove {}: {}",
                          path.display(),
                          e));
    }
}

// The third parameter is for env vars, used on windows to set up the
// path for MSVC to find its DLLs, and gcc to find its bundled
// toolchain
pub fn get_linker(sess: &Session, linker: &Path, flavor: LinkerFlavor) -> (PathBuf, Command) {
    let msvc_tool = windows_registry::find_tool(&sess.opts.target_triple.triple(), "link.exe");

    // If our linker looks like a batch script on Windows then to execute this
    // we'll need to spawn `cmd` explicitly. This is primarily done to handle
    // emscripten where the linker is `emcc.bat` and needs to be spawned as
    // `cmd /c emcc.bat ...`.
    //
    // This worked historically but is needed manually since #42436 (regression
    // was tagged as #42791) and some more info can be found on #44443 for
    // emscripten itself.
    let mut cmd = match linker.to_str() {
        Some(linker) if cfg!(windows) && linker.ends_with(".bat") => Command::bat_script(linker),
        _ => match flavor {
            LinkerFlavor::Lld(f) => Command::lld(linker, f),
            LinkerFlavor::Msvc
                if sess.opts.cg.linker.is_none() && sess.target.target.options.linker.is_none() =>
            {
                Command::new(msvc_tool.as_ref().map(|t| t.path()).unwrap_or(linker))
            },
            _ => Command::new(linker),
        }
    };

    // The compiler's sysroot often has some bundled tools, so add it to the
    // PATH for the child.
    let mut new_path = sess.host_filesearch(PathKind::All)
                           .get_tools_search_paths();
    let mut msvc_changed_path = false;
    if sess.target.target.options.is_like_msvc {
        if let Some(ref tool) = msvc_tool {
            cmd.args(tool.args());
            for &(ref k, ref v) in tool.env() {
                if k == "PATH" {
                    new_path.extend(env::split_paths(v));
                    msvc_changed_path = true;
                } else {
                    cmd.env(k, v);
                }
            }
        }
    }

    if !msvc_changed_path {
        if let Some(path) = env::var_os("PATH") {
            new_path.extend(env::split_paths(&path));
        }
    }
    cmd.env("PATH", env::join_paths(new_path).unwrap());

    (linker.to_path_buf(), cmd)
}

pub fn each_linked_rlib(sess: &Session,
                               info: &CrateInfo,
                               f: &mut dyn FnMut(CrateNum, &Path)) -> Result<(), String> {
    let crates = info.used_crates_static.iter();
    let fmts = sess.dependency_formats.borrow();
    let fmts = fmts.get(&config::CrateType::Executable)
                   .or_else(|| fmts.get(&config::CrateType::Staticlib))
                   .or_else(|| fmts.get(&config::CrateType::Cdylib))
                   .or_else(|| fmts.get(&config::CrateType::ProcMacro));
    let fmts = match fmts {
        Some(f) => f,
        None => return Err("could not find formats for rlibs".to_string())
    };
    for &(cnum, ref path) in crates {
        match fmts.get(cnum.as_usize() - 1) {
            Some(&Linkage::NotLinked) |
            Some(&Linkage::IncludedFromDylib) => continue,
            Some(_) => {}
            None => return Err("could not find formats for rlibs".to_string())
        }
        let name = &info.crate_name[&cnum];
        let path = match *path {
            LibSource::Some(ref p) => p,
            LibSource::MetadataOnly => {
                return Err(format!("could not find rlib for: `{}`, found rmeta (metadata) file",
                                   name))
            }
            LibSource::None => {
                return Err(format!("could not find rlib for: `{}`", name))
            }
        };
        f(cnum, &path);
    }
    Ok(())
}

/// Returns a boolean indicating whether the specified crate should be ignored
/// during LTO.
///
/// Crates ignored during LTO are not lumped together in the "massive object
/// file" that we create and are linked in their normal rlib states. See
/// comments below for what crates do not participate in LTO.
///
/// It's unusual for a crate to not participate in LTO. Typically only
/// compiler-specific and unstable crates have a reason to not participate in
/// LTO.
pub fn ignored_for_lto(sess: &Session, info: &CrateInfo, cnum: CrateNum) -> bool {
    // If our target enables builtin function lowering in LLVM then the
    // crates providing these functions don't participate in LTO (e.g.
    // no_builtins or compiler builtins crates).
    !sess.target.target.options.no_builtins &&
        (info.compiler_builtins == Some(cnum) || info.is_no_builtins.contains(&cnum))
}

pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
    fn infer_from(
        sess: &Session,
        linker: Option<PathBuf>,
        flavor: Option<LinkerFlavor>,
    ) -> Option<(PathBuf, LinkerFlavor)> {
        match (linker, flavor) {
            (Some(linker), Some(flavor)) => Some((linker, flavor)),
            // only the linker flavor is known; use the default linker for the selected flavor
            (None, Some(flavor)) => Some((PathBuf::from(match flavor {
                LinkerFlavor::Em  => if cfg!(windows) { "emcc.bat" } else { "emcc" },
                LinkerFlavor::Gcc => "cc",
                LinkerFlavor::Ld => "ld",
                LinkerFlavor::Msvc => "link.exe",
                LinkerFlavor::Lld(_) => "lld",
                LinkerFlavor::PtxLinker => "rust-ptx-linker",
            }), flavor)),
            (Some(linker), None) => {
                let stem = linker
                    .file_stem()
                    .and_then(|stem| stem.to_str())
                    .unwrap_or_else(|| {
                        sess.fatal("couldn't extract file stem from specified linker")
                    });

                let flavor = if stem == "emcc" {
                    LinkerFlavor::Em
                } else if stem == "gcc"
                    || stem.ends_with("-gcc")
                    || stem == "clang"
                    || stem.ends_with("-clang")
                {
                    LinkerFlavor::Gcc
                } else if stem == "ld" || stem == "ld.lld" || stem.ends_with("-ld") {
                    LinkerFlavor::Ld
                } else if stem == "link" || stem == "lld-link" {
                    LinkerFlavor::Msvc
                } else if stem == "lld" || stem == "rust-lld" {
                    LinkerFlavor::Lld(sess.target.target.options.lld_flavor)
                } else {
                    // fall back to the value in the target spec
                    sess.target.target.linker_flavor
                };

                Some((linker, flavor))
            },
            (None, None) => None,
        }
    }

    // linker and linker flavor specified via command line have precedence over what the target
    // specification specifies
    if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) {
        return ret;
    }

    if let Some(ret) = infer_from(
        sess,
        sess.target.target.options.linker.clone().map(PathBuf::from),
        Some(sess.target.target.linker_flavor),
    ) {
        return ret;
    }

    bug!("Not enough information provided to determine how to invoke the linker");
}
<|endoftext|>


<commit_before>use rustc::hir::def_id::LOCAL_CRATE;
use rustc::mir::mono::MonoItem;
use rustc::session::config::OptLevel;
use rustc::ty::{self, TyCtxt, Instance};
use rustc::ty::subst::InternalSubsts;
use rustc::ty::print::obsolete::DefPathBasedNames;
use syntax::attr::InlineAttr;
use std::fmt;
use rustc::mir::mono::Linkage;
use syntax_pos::symbol::InternedString;
use syntax::source_map::Span;

/// Describes how a monomorphization will be instantiated in object files.
#[derive(PartialEq, Eq, Clone, Copy, Debug, Hash)]
pub enum InstantiationMode {
    /// There will be exactly one instance of the given MonoItem. It will have
    /// external linkage so that it can be linked to from other codegen units.
    GloballyShared {
        /// In some compilation scenarios we may decide to take functions that
        /// are typically `LocalCopy` and instead move them to `GloballyShared`
        /// to avoid codegenning them a bunch of times. In this situation,
        /// however, our local copy may conflict with other crates also
        /// inlining the same function.
        ///
        /// This flag indicates that this situation is occurring, and informs
        /// symbol name calculation that some extra mangling is needed to
        /// avoid conflicts. Note that this may eventually go away entirely if
        /// ThinLTO enables us to *always* have a globally shared instance of a
        /// function within one crate's compilation.
        may_conflict: bool,
    },

    /// Each codegen unit containing a reference to the given MonoItem will
    /// have its own private copy of the function (with internal linkage).
    LocalCopy,
}

pub trait MonoItemExt<'tcx>: fmt::Debug {
    fn as_mono_item(&self) -> &MonoItem<'tcx>;

    fn is_generic_fn(&self) -> bool {
        match *self.as_mono_item() {
            MonoItem::Fn(ref instance) => {
                instance.substs.non_erasable_generics().next().is_some()
            }
            MonoItem::Static(..) |
            MonoItem::GlobalAsm(..) => false,
        }
    }

    fn symbol_name(&self, tcx: TyCtxt<'tcx>) -> ty::SymbolName {
        match *self.as_mono_item() {
            MonoItem::Fn(instance) => tcx.symbol_name(instance),
            MonoItem::Static(def_id) => {
                tcx.symbol_name(Instance::mono(tcx, def_id))
            }
            MonoItem::GlobalAsm(hir_id) => {
                let def_id = tcx.hir().local_def_id(hir_id);
                ty::SymbolName {
                    name: InternedString::intern(&format!("global_asm_{:?}", def_id))
                }
            }
        }
    }
    fn instantiation_mode(&self, tcx: TyCtxt<'tcx>) -> InstantiationMode {
        let inline_in_all_cgus =
            tcx.sess.opts.debugging_opts.inline_in_all_cgus.unwrap_or_else(|| {
                tcx.sess.opts.optimize != OptLevel::No
            }) && !tcx.sess.opts.cg.link_dead_code;

        match *self.as_mono_item() {
            MonoItem::Fn(ref instance) => {
                let entry_def_id = tcx.entry_fn(LOCAL_CRATE).map(|(id, _)| id);
                // If this function isn't inlined or otherwise has explicit
                // linkage, then we'll be creating a globally shared version.
                if self.explicit_linkage(tcx).is_some() ||
                    !instance.def.requires_local(tcx) ||
                    Some(instance.def_id()) == entry_def_id
                {
                    return InstantiationMode::GloballyShared  { may_conflict: false }
                }

                // At this point we don't have explicit linkage and we're an
                // inlined function. If we're inlining into all CGUs then we'll
                // be creating a local copy per CGU
                if inline_in_all_cgus {
                    return InstantiationMode::LocalCopy
                }

                // Finally, if this is `#[inline(always)]` we're sure to respect
                // that with an inline copy per CGU, but otherwise we'll be
                // creating one copy of this `#[inline]` function which may
                // conflict with upstream crates as it could be an exported
                // symbol.
                match tcx.codegen_fn_attrs(instance.def_id()).inline {
                    InlineAttr::Always => InstantiationMode::LocalCopy,
                    _ => {
                        InstantiationMode::GloballyShared  { may_conflict: true }
                    }
                }
            }
            MonoItem::Static(..) |
            MonoItem::GlobalAsm(..) => {
                InstantiationMode::GloballyShared { may_conflict: false }
            }
        }
    }

    fn explicit_linkage(&self, tcx: TyCtxt<'tcx>) -> Option<Linkage> {
        let def_id = match *self.as_mono_item() {
            MonoItem::Fn(ref instance) => instance.def_id(),
            MonoItem::Static(def_id) => def_id,
            MonoItem::GlobalAsm(..) => return None,
        };

        let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
        codegen_fn_attrs.linkage
    }

    /// Returns `true` if this instance is instantiable - whether it has no unsatisfied
    /// predicates.
    ///
    /// In order to codegen an item, all of its predicates must hold, because
    /// otherwise the item does not make sense. Type-checking ensures that
    /// the predicates of every item that is *used by* a valid item *do*
    /// hold, so we can rely on that.
    ///
    /// However, we codegen collector roots (reachable items) and functions
    /// in vtables when they are seen, even if they are not used, and so they
    /// might not be instantiable. For example, a programmer can define this
    /// public function:
    ///
    ///     pub fn foo<'a>(s: &'a mut ()) where &'a mut (): Clone {
    ///         <&mut () as Clone>::clone(&s);
    ///     }
    ///
    /// That function can't be codegened, because the method `<&mut () as Clone>::clone`
    /// does not exist. Luckily for us, that function can't ever be used,
    /// because that would require for `&'a mut (): Clone` to hold, so we
    /// can just not emit any code, or even a linker reference for it.
    ///
    /// Similarly, if a vtable method has such a signature, and therefore can't
    /// be used, we can just not emit it and have a placeholder (a null pointer,
    /// which will never be accessed) in its place.
    fn is_instantiable(&self, tcx: TyCtxt<'tcx>) -> bool {
        debug!("is_instantiable({:?})", self);
        let (def_id, substs) = match *self.as_mono_item() {
            MonoItem::Fn(ref instance) => (instance.def_id(), instance.substs),
            MonoItem::Static(def_id) => (def_id, InternalSubsts::empty()),
            // global asm never has predicates
            MonoItem::GlobalAsm(..) => return true
        };

        tcx.substitute_normalize_and_test_predicates((def_id, &substs))
    }

    fn to_string(&self, tcx: TyCtxt<'tcx>, debug: bool) -> String {
        return match *self.as_mono_item() {
            MonoItem::Fn(instance) => {
                to_string_internal(tcx, "fn ", instance, debug)
            },
            MonoItem::Static(def_id) => {
                let instance = Instance::new(def_id, tcx.intern_substs(&[]));
                to_string_internal(tcx, "static ", instance, debug)
            },
            MonoItem::GlobalAsm(..) => {
                "global_asm".to_string()
            }
        };

        fn to_string_internal<'a, 'tcx>(
            tcx: TyCtxt<'tcx>,
            prefix: &str,
            instance: Instance<'tcx>,
            debug: bool,
        ) -> String {
            let mut result = String::with_capacity(32);
            result.push_str(prefix);
            let printer = DefPathBasedNames::new(tcx, false, false);
            printer.push_instance_as_string(instance, &mut result, debug);
            result
        }
    }

    fn local_span(&self, tcx: TyCtxt<'tcx>) -> Option<Span> {
        match *self.as_mono_item() {
            MonoItem::Fn(Instance { def, .. }) => {
                tcx.hir().as_local_hir_id(def.def_id())
            }
            MonoItem::Static(def_id) => {
                tcx.hir().as_local_hir_id(def_id)
            }
            MonoItem::GlobalAsm(hir_id) => {
                Some(hir_id)
            }
        }.map(|hir_id| tcx.hir().span(hir_id))
    }
}

impl MonoItemExt<'tcx> for MonoItem<'tcx> {
    fn as_mono_item(&self) -> &MonoItem<'tcx> {
        self
    }
}
<commit_msg>Remove dead module<commit_after><|endoftext|>