<commit_before>use super::super::sm::client::SMClient;
use super::super::trees::*;
pub use super::btree::level::{LEVEL_1 as MIGRATE_SIZE, LEVEL_M as BLOCK_SIZE};
use super::btree::storage;
use super::tree::*;
use crate::client::AsyncClient;
use crate::ram::types::Id;
use crate::ram::types::RandValue;
use bifrost::conshash::ConsistentHashing;
use bifrost_plugins::hash_ident;
use futures::future::BoxFuture;
use futures::prelude::*;
use lightning::map::HashMap;
use lightning::map::Map;
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use std::mem;
use std::time::Duration;

pub type IdBlock = [Id; BLOCK_SIZE];
pub static DEFAULT_SERVICE_ID: u64 = hash_ident!(LSM_TREE_RPC_SERVICE) as u64;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Boundary {
    upper: EntryKey,
    lower: EntryKey,
}

#[derive(Clone, Serialize, Deserialize)]
pub enum OpResult<T> {
    Successful(T),
    NotFound,
    OutOfBound,
    EpochMissMatch(u64, u64),
    Migrating,
}

#[derive(Clone, Serialize, Deserialize)]
pub struct ServBlock {
    pub buffer: Vec<Id>,
    pub next: Option<EntryKey>,
}

pub struct DistLSMTree {
    id: Id,
    tree: LSMTree,
    prop: RwLock<DistProp>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DistProp {
    boundary: Boundary,
    migration: Option<Migration>,
    epoch: u64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Migration {
    pivot: EntryKey,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct BTreeStat {
    pub size: usize,
    pub count: usize,
    pub mid_key: Option<EntryKey>,
    pub head: Id,
    pub ideal_cap: usize,
    pub oversized: bool,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct LSMTreeStat {
    pub id: Id,
    pub prop: DistProp,
    pub trees: Vec<BTreeStat>,
}

service! {
    rpc crate_tree(id: Id, boundary: Boundary, epoch: u64);
    rpc load_tree(id: Id, boundary: Boundary, epoch: u64);
    rpc insert(id: Id, entry: EntryKey, epoch: u64) -> OpResult<bool>;
    rpc delete(id: Id, entry: EntryKey, epoch: u64) -> OpResult<bool>;
    rpc seek(id: Id, entry: EntryKey, ordering: Ordering, buffer_size: u16, epoch: u64)
        -> OpResult<ServBlock>;
    rpc stat(id: Id) -> OpResult<LSMTreeStat>;
}

pub struct LSMTreeService {
    client: Arc<AsyncClient>,
    trees: Arc<HashMap<Id, Arc<DistLSMTree>>>,
}

impl Service for LSMTreeService {
    fn crate_tree(&self, id: Id, boundary: Boundary, epoch: u64) -> BoxFuture<()> {
        async move {
            if self.trees.contains_key(&id) {
                return;
            }
            let tree = LSMTree::create(&self.client, &id).await;
            self.trees.insert(
                &id,
                Arc::new(DistLSMTree::new(id, tree, boundary, None, epoch)),
            );
        }
        .boxed()
    }

    fn load_tree(&self, id: Id, boundary: Boundary, epoch: u64) -> BoxFuture<()> {
        async move {
            if self.trees.contains_key(&id) {
                debug!("Tree loaded, skip {:?}", id);
                return;
            }
            info!("Called to load tree {:?}, boundary {:?}", id, boundary);
            let tree = LSMTree::recover(&self.client, &id).await;
            debug!(
                "LSM tree loaded with {} keys, capacity {}.",
                tree.count(),
                tree.ideal_capacity()
            );
            self.trees.insert(
                &id,
                Arc::new(DistLSMTree::new(id, tree, boundary, None, epoch)),
            );
        }
        .boxed()
    }

    fn insert(&self, id: Id, entry: EntryKey, epoch: u64) -> BoxFuture<OpResult<bool>> {
        self.apply_in_ranged_tree(id, entry, epoch, |entry, tree| {
            if tree.insert(&entry) {
                OpResult::Successful(true)
            } else {
                OpResult::Successful(false)
            }
        })
    }

    fn delete(&self, id: Id, entry: EntryKey, epoch: u64) -> BoxFuture<OpResult<bool>> {
        self.apply_in_ranged_tree(id, entry, epoch, |entry, tree| {
            if tree.delete(&entry) {
                OpResult::Successful(true)
            } else {
                OpResult::Successful(false)
            }
        })
    }

    fn seek(
        &self,
        id: Id,
        entry: EntryKey,
        ordering: Ordering,
        buffer_size: u16,
        epoch: u64,
    ) -> BoxFuture<OpResult<ServBlock>> {
        self.apply_in_ranged_tree(id, entry, epoch, |entry, tree| {
            let buffer_size = buffer_size as usize;
            let mut tree_cursor = tree.seek(&entry, ordering);
            let mut buffer = Vec::with_capacity(buffer_size);
            for _ in 0..buffer_size {
                if let Some(key) = tree_cursor.next() {
                    buffer.push(key.id());
                } else {
                    break;
                }
            }
            let next = mem::take(&mut tree_cursor.current.map(|(_, k)| k));
            let lsm_cursor = ServBlock { buffer, next };
            OpResult::Successful(lsm_cursor)
        })
    }

    fn stat(&self, id: Id) -> BoxFuture<OpResult<LSMTreeStat>> {
        future::ready(if let Some(tree) = self.trees.get(&id) {
            OpResult::Successful(LSMTreeStat {
                id,
                prop: tree.prop.read().clone(),
                trees: tree
                    .tree
                    .disk_trees
                    .iter()
                    .map(|t| BTreeStat {
                        size: t.size(),
                        count: t.count(),
                        mid_key: t.mid_key(),
                        head: t.head_id(),
                        ideal_cap: t.ideal_capacity(),
                        oversized: t.oversized(),
                    })
                    .collect(),
            })
        } else {
            OpResult::NotFound
        })
        .boxed()
    }
}

impl LSMTreeService {
    pub fn new(client: &Arc<AsyncClient>, sm_client: &Arc<SMClient>) -> Self {
        info!("Initializing LSM tree service");
        let trees_map = Arc::new(HashMap::with_capacity(32));
        super::btree::storage::start_external_nodes_write_back(client);
        Self::start_tree_balancer(&trees_map, client, sm_client);
        Self {
            client: client.clone(),
            trees: trees_map,
        }
    }

    pub fn start_tree_balancer(
        trees_map: &Arc<HashMap<Id, Arc<DistLSMTree>>>,
        client: &Arc<AsyncClient>,
        sm_client: &Arc<SMClient>,
    ) {
        debug!("Starting range indexer tree balancer");
        let trees_map = trees_map.clone();
        let client = client.clone();
        let sm_client = sm_client.clone();
        tokio::spawn(async move {
            loop {
                let mut fast_mode = false;
                for (_, dist_tree) in trees_map.entries() {
                    let tree = &dist_tree.tree;
                    fast_mode = tree.merge_levels().await | fast_mode;
                    if tree.oversized() {
                        info!("LSM Tree oversized {:?}, start migration", dist_tree.id);
                        // Tree oversized, need to migrate
                        let mid_key = tree.mid_key().unwrap();
                        let migration_target_id = Id::rand();
                        debug!(
                            "Creating migration target tree {:?} split at {:?}",
                            migration_target_id, mid_key
                        );
                        let migration_tree = LSMTree::create(&client, &migration_target_id).await;
                        {
                            let mut dist_tree_prop = dist_tree.prop.write();
                            dist_tree_prop.migration = Some(Migration {
                                pivot: mid_key.clone(),
                            });
                        }
                        debug!("Marking migration for tree {:?}", dist_tree.id);
                        tree.mark_migration(&dist_tree.id, Some(migration_target_id), &client)
                            .await;
                        let buffer_size = MIGRATE_SIZE << 4;
                        let mut cursor = tree.seek(&mid_key, Ordering::Forward);
                        let mut entry_buffer = Vec::with_capacity(buffer_size);
                        debug!(
                            "Start moving keys from {:?} to {:?}",
                            dist_tree.id, migration_target_id
                        );
                        while cursor.current().is_some() {
                            if let Some(entry) = cursor.next() {
                                entry_buffer.push(entry);
                                if entry_buffer.len() >= buffer_size {
                                    debug!("Merging entry buffer, size {}", entry_buffer.len());
                                    migration_tree.merge_keys(entry_buffer);
                                    entry_buffer = Vec::with_capacity(buffer_size);
                                }
                            }
                        }
                        debug!("Merging last batch of keys, size {}", entry_buffer.len());
                        migration_tree.merge_keys(entry_buffer);
                        debug!("Waiting for new tree {:?} persisted", migration_target_id);
                        storage::wait_until_updated().await;
                        debug!("Calling placement for split to {:?}", migration_target_id);
                        sm_client
                            .split(&dist_tree.id, &migration_target_id, &mid_key)
                            .await
                            .unwrap();
                        // Reset state on current tree
                        {
                            let mut dist_prop = dist_tree.prop.write();
                            dist_prop.boundary.upper = mid_key.clone();
                            dist_prop.migration = None;
                            dist_prop.epoch += 1;
                        }
                        debug!("Unmark migration {:?}", dist_tree.id);
                        tree.mark_migration(&dist_tree.id, None, &client).await;
                        tree.retain(&mid_key);
                    }
                }
                if !fast_mode {
                    // Sleep for a while to check for trees to be merge in levels
                    tokio::time::sleep(Duration::from_millis(500)).await;
                }
            }
        });
    }

    fn apply_in_ranged_tree<F, R>(
        &self,
        id: Id,
        entry: EntryKey,
        epoch: u64,
        func: F,
    ) -> BoxFuture<OpResult<R>>
    where
        F: Fn(&EntryKey, &LSMTree) -> OpResult<R>,
        R: Send + 'static,
    {
        future::ready(if let Some(tree) = self.trees.get(&id) {
            let tree_prop = tree.prop.read();
            if epoch < tree_prop.epoch {
                OpResult::EpochMissMatch(tree_prop.epoch, epoch)
            } else if tree_prop.boundary.in_boundary(&entry) {
                if let &Some(ref migration) = &tree_prop.migration {
                    if entry < migration.pivot {
                        // Entries lower than pivot should be safe to work on
                        func(&entry, &tree.tree)
                    } else {
                        OpResult::Migrating
                    }
                } else {
                    func(&entry, &tree.tree)
                }
            } else {
                OpResult::OutOfBound
            }
        } else {
            OpResult::NotFound
        })
        .boxed()
    }
}

impl DistLSMTree {
    fn new(
        id: Id,
        tree: LSMTree,
        boundary: Boundary,
        migration: Option<Migration>,
        epoch: u64,
    ) -> Self {
        let prop = RwLock::new(DistProp {
            boundary,
            migration,
            epoch,
        });
        Self { id, tree, prop }
    }
}

impl Boundary {
    pub fn new(lower: EntryKey, upper: EntryKey) -> Self {
        Boundary { lower, upper }
    }
    fn in_boundary(&self, entry: &EntryKey) -> bool {
        // Allow max/min query as special cases
        (entry >= &self.lower && entry < &self.upper)
            || entry == &*MIN_ENTRY_KEY
            || entry == &*MAX_ENTRY_KEY
    }
}

dispatch_rpc_service_functions!(LSMTreeService);

unsafe impl Send for DistLSMTree {}
unsafe impl Sync for DistLSMTree {}

pub fn client_by_rpc_client(rpc: &Arc<RPCClient>) -> Arc<AsyncServiceClient> {
    AsyncServiceClient::new(DEFAULT_SERVICE_ID, rpc)
}

pub async fn locate_tree_server_from_conshash(
    id: &Id,
    conshash: &Arc<ConsistentHashing>,
) -> Result<Arc<AsyncServiceClient>, RPCError> {
    let server_id = conshash.get_server_id_by(id).unwrap();
    DEFAULT_CLIENT_POOL
        .get_by_id(server_id, move |sid| conshash.to_server_name(sid))
        .await
        .map_err(|e| RPCError::IOError(e))
        .map(|c| client_by_rpc_client(&c))
}
<commit_msg>better logging<commit_after>use super::super::sm::client::SMClient;
use super::super::trees::*;
pub use super::btree::level::{LEVEL_1 as MIGRATE_SIZE, LEVEL_M as BLOCK_SIZE};
use super::btree::storage;
use super::tree::*;
use crate::client::AsyncClient;
use crate::ram::types::Id;
use crate::ram::types::RandValue;
use bifrost::conshash::ConsistentHashing;
use bifrost_plugins::hash_ident;
use futures::future::BoxFuture;
use futures::prelude::*;
use lightning::map::HashMap;
use lightning::map::Map;
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use std::mem;
use std::time::Duration;

pub type IdBlock = [Id; BLOCK_SIZE];
pub static DEFAULT_SERVICE_ID: u64 = hash_ident!(LSM_TREE_RPC_SERVICE) as u64;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Boundary {
    upper: EntryKey,
    lower: EntryKey,
}

#[derive(Clone, Serialize, Deserialize)]
pub enum OpResult<T> {
    Successful(T),
    NotFound,
    OutOfBound,
    EpochMissMatch(u64, u64),
    Migrating,
}

#[derive(Clone, Serialize, Deserialize)]
pub struct ServBlock {
    pub buffer: Vec<Id>,
    pub next: Option<EntryKey>,
}

pub struct DistLSMTree {
    id: Id,
    tree: LSMTree,
    prop: RwLock<DistProp>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DistProp {
    boundary: Boundary,
    migration: Option<Migration>,
    epoch: u64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Migration {
    pivot: EntryKey,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct BTreeStat {
    pub size: usize,
    pub count: usize,
    pub mid_key: Option<EntryKey>,
    pub head: Id,
    pub ideal_cap: usize,
    pub oversized: bool,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct LSMTreeStat {
    pub id: Id,
    pub prop: DistProp,
    pub trees: Vec<BTreeStat>,
}

service! {
    rpc crate_tree(id: Id, boundary: Boundary, epoch: u64);
    rpc load_tree(id: Id, boundary: Boundary, epoch: u64);
    rpc insert(id: Id, entry: EntryKey, epoch: u64) -> OpResult<bool>;
    rpc delete(id: Id, entry: EntryKey, epoch: u64) -> OpResult<bool>;
    rpc seek(id: Id, entry: EntryKey, ordering: Ordering, buffer_size: u16, epoch: u64)
        -> OpResult<ServBlock>;
    rpc stat(id: Id) -> OpResult<LSMTreeStat>;
}

pub struct LSMTreeService {
    client: Arc<AsyncClient>,
    trees: Arc<HashMap<Id, Arc<DistLSMTree>>>,
}

impl Service for LSMTreeService {
    fn crate_tree(&self, id: Id, boundary: Boundary, epoch: u64) -> BoxFuture<()> {
        async move {
            if self.trees.contains_key(&id) {
                return;
            }
            let tree = LSMTree::create(&self.client, &id).await;
            self.trees.insert(
                &id,
                Arc::new(DistLSMTree::new(id, tree, boundary, None, epoch)),
            );
        }
        .boxed()
    }

    fn load_tree(&self, id: Id, boundary: Boundary, epoch: u64) -> BoxFuture<()> {
        async move {
            if self.trees.contains_key(&id) {
                debug!("Tree loaded, skip {:?}", id);
                return;
            }
            info!("Called to load tree {:?}, boundary {:?}", id, boundary);
            let tree = LSMTree::recover(&self.client, &id).await;
            debug!(
                "LSM tree loaded with {} keys, capacity {}.",
                tree.count(),
                tree.ideal_capacity()
            );
            self.trees.insert(
                &id,
                Arc::new(DistLSMTree::new(id, tree, boundary, None, epoch)),
            );
        }
        .boxed()
    }

    fn insert(&self, id: Id, entry: EntryKey, epoch: u64) -> BoxFuture<OpResult<bool>> {
        self.apply_in_ranged_tree(id, entry, epoch, |entry, tree| {
            if tree.insert(&entry) {
                OpResult::Successful(true)
            } else {
                OpResult::Successful(false)
            }
        })
    }

    fn delete(&self, id: Id, entry: EntryKey, epoch: u64) -> BoxFuture<OpResult<bool>> {
        self.apply_in_ranged_tree(id, entry, epoch, |entry, tree| {
            if tree.delete(&entry) {
                OpResult::Successful(true)
            } else {
                OpResult::Successful(false)
            }
        })
    }

    fn seek(
        &self,
        id: Id,
        entry: EntryKey,
        ordering: Ordering,
        buffer_size: u16,
        epoch: u64,
    ) -> BoxFuture<OpResult<ServBlock>> {
        self.apply_in_ranged_tree(id, entry, epoch, |entry, tree| {
            let buffer_size = buffer_size as usize;
            let mut tree_cursor = tree.seek(&entry, ordering);
            let mut buffer = Vec::with_capacity(buffer_size);
            for _ in 0..buffer_size {
                if let Some(key) = tree_cursor.next() {
                    buffer.push(key.id());
                } else {
                    break;
                }
            }
            let next = mem::take(&mut tree_cursor.current.map(|(_, k)| k));
            let lsm_cursor = ServBlock { buffer, next };
            OpResult::Successful(lsm_cursor)
        })
    }

    fn stat(&self, id: Id) -> BoxFuture<OpResult<LSMTreeStat>> {
        future::ready(if let Some(tree) = self.trees.get(&id) {
            OpResult::Successful(LSMTreeStat {
                id,
                prop: tree.prop.read().clone(),
                trees: tree
                    .tree
                    .disk_trees
                    .iter()
                    .map(|t| BTreeStat {
                        size: t.size(),
                        count: t.count(),
                        mid_key: t.mid_key(),
                        head: t.head_id(),
                        ideal_cap: t.ideal_capacity(),
                        oversized: t.oversized(),
                    })
                    .collect(),
            })
        } else {
            OpResult::NotFound
        })
        .boxed()
    }
}

impl LSMTreeService {
    pub fn new(client: &Arc<AsyncClient>, sm_client: &Arc<SMClient>) -> Self {
        info!("Initializing LSM tree service");
        let trees_map = Arc::new(HashMap::with_capacity(32));
        super::btree::storage::start_external_nodes_write_back(client);
        Self::start_tree_balancer(&trees_map, client, sm_client);
        Self {
            client: client.clone(),
            trees: trees_map,
        }
    }

    pub fn start_tree_balancer(
        trees_map: &Arc<HashMap<Id, Arc<DistLSMTree>>>,
        client: &Arc<AsyncClient>,
        sm_client: &Arc<SMClient>,
    ) {
        debug!("Starting range indexer tree balancer");
        let trees_map = trees_map.clone();
        let client = client.clone();
        let sm_client = sm_client.clone();
        tokio::spawn(async move {
            loop {
                let mut fast_mode = false;
                for (_, dist_tree) in trees_map.entries() {
                    let tree = &dist_tree.tree;
                    fast_mode = tree.merge_levels().await | fast_mode;
                    if tree.oversized() {
                        info!("LSM Tree oversized {:?}, start migration", dist_tree.id);
                        // Tree oversized, need to migrate
                        let mid_key = tree.mid_key().unwrap();
                        let migration_target_id = Id::rand();
                        debug!(
                            "Creating migration target tree {:?} split at {:?}",
                            migration_target_id, mid_key
                        );
                        let migration_tree = LSMTree::create(&client, &migration_target_id).await;
                        {
                            let mut dist_tree_prop = dist_tree.prop.write();
                            dist_tree_prop.migration = Some(Migration {
                                pivot: mid_key.clone(),
                            });
                        }
                        debug!("Marking migration for tree {:?}", dist_tree.id);
                        tree.mark_migration(&dist_tree.id, Some(migration_target_id), &client)
                            .await;
                        let buffer_size = MIGRATE_SIZE << 4;
                        let mut cursor = tree.seek(&mid_key, Ordering::Forward);
                        let mut entry_buffer = Vec::with_capacity(buffer_size);
                        debug!(
                            "Start moving keys from {:?} to {:?}",
                            dist_tree.id, migration_target_id
                        );
                        while cursor.current().is_some() {
                            if let Some(entry) = cursor.next() {
                                entry_buffer.push(entry);
                                if entry_buffer.len() >= buffer_size {
                                    debug!("Merging entry buffer, size {}", entry_buffer.len());
                                    migration_tree.merge_keys(entry_buffer);
                                    entry_buffer = Vec::with_capacity(buffer_size);
                                }
                            }
                        }
                        debug!("Merging last batch of keys, size {}", entry_buffer.len());
                        migration_tree.merge_keys(entry_buffer);
                        debug!("Waiting for new tree {:?} persisted", migration_target_id);
                        storage::wait_until_updated().await;
                        debug!("Calling placement for split to {:?}", migration_target_id);
                        sm_client
                            .split(&dist_tree.id, &migration_target_id, &mid_key)
                            .await
                            .unwrap();
                        // Reset state on current tree
                        {
                            let mut dist_prop = dist_tree.prop.write();
                            dist_prop.boundary.upper = mid_key.clone();
                            dist_prop.migration = None;
                            dist_prop.epoch += 1;
                        }
                        debug!("Unmark migration {:?}", dist_tree.id);
                        tree.mark_migration(&dist_tree.id, None, &client).await;
                        tree.retain(&mid_key);
                        debug!("LSM tree migration from {:?} to {:?} succeed", dist_tree.id, migration_target_id);
                    }
                }
                if !fast_mode {
                    // Sleep for a while to check for trees to be merge in levels
                    tokio::time::sleep(Duration::from_millis(500)).await;
                }
            }
        });
    }

    fn apply_in_ranged_tree<F, R>(
        &self,
        id: Id,
        entry: EntryKey,
        epoch: u64,
        func: F,
    ) -> BoxFuture<OpResult<R>>
    where
        F: Fn(&EntryKey, &LSMTree) -> OpResult<R>,
        R: Send + 'static,
    {
        future::ready(if let Some(tree) = self.trees.get(&id) {
            let tree_prop = tree.prop.read();
            if epoch < tree_prop.epoch {
                OpResult::EpochMissMatch(tree_prop.epoch, epoch)
            } else if tree_prop.boundary.in_boundary(&entry) {
                if let &Some(ref migration) = &tree_prop.migration {
                    if entry < migration.pivot {
                        // Entries lower than pivot should be safe to work on
                        func(&entry, &tree.tree)
                    } else {
                        OpResult::Migrating
                    }
                } else {
                    func(&entry, &tree.tree)
                }
            } else {
                OpResult::OutOfBound
            }
        } else {
            OpResult::NotFound
        })
        .boxed()
    }
}

impl DistLSMTree {
    fn new(
        id: Id,
        tree: LSMTree,
        boundary: Boundary,
        migration: Option<Migration>,
        epoch: u64,
    ) -> Self {
        let prop = RwLock::new(DistProp {
            boundary,
            migration,
            epoch,
        });
        Self { id, tree, prop }
    }
}

impl Boundary {
    pub fn new(lower: EntryKey, upper: EntryKey) -> Self {
        Boundary { lower, upper }
    }
    fn in_boundary(&self, entry: &EntryKey) -> bool {
        // Allow max/min query as special cases
        (entry >= &self.lower && entry < &self.upper)
            || entry == &*MIN_ENTRY_KEY
            || entry == &*MAX_ENTRY_KEY
    }
}

dispatch_rpc_service_functions!(LSMTreeService);

unsafe impl Send for DistLSMTree {}
unsafe impl Sync for DistLSMTree {}

pub fn client_by_rpc_client(rpc: &Arc<RPCClient>) -> Arc<AsyncServiceClient> {
    AsyncServiceClient::new(DEFAULT_SERVICE_ID, rpc)
}

pub async fn locate_tree_server_from_conshash(
    id: &Id,
    conshash: &Arc<ConsistentHashing>,
) -> Result<Arc<AsyncServiceClient>, RPCError> {
    let server_id = conshash.get_server_id_by(id).unwrap();
    DEFAULT_CLIENT_POOL
        .get_by_id(server_id, move |sid| conshash.to_server_name(sid))
        .await
        .map_err(|e| RPCError::IOError(e))
        .map(|c| client_by_rpc_client(&c))
}
<|endoftext|>


<commit_before>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// The crate store - a central repo for information collected about external
// crates and libraries

use schema;

use rustc::hir::def_id::{CRATE_DEF_INDEX, CrateNum, DefIndex};
use rustc::hir::map::definitions::DefPathTable;
use rustc::hir::svh::Svh;
use rustc::middle::cstore::{DepKind, ExternCrate, MetadataLoader};
use rustc::session::{Session, CrateDisambiguator};
use rustc_back::PanicStrategy;
use rustc_data_structures::indexed_vec::IndexVec;
use rustc::util::nodemap::{FxHashMap, FxHashSet, NodeMap};

use std::cell::{RefCell, Cell};
use std::rc::Rc;
use rustc_data_structures::owning_ref::ErasedBoxRef;
use syntax::{ast, attr};
use syntax::ext::base::SyntaxExtension;
use syntax::symbol::Symbol;
use syntax_pos;

pub use rustc::middle::cstore::{NativeLibrary, NativeLibraryKind, LinkagePreference};
pub use rustc::middle::cstore::NativeLibraryKind::*;
pub use rustc::middle::cstore::{CrateSource, LibSource};

pub use cstore_impl::{provide, provide_extern};

// A map from external crate numbers (as decoded from some crate file) to
// local crate numbers (as generated during this session). Each external
// crate may refer to types in other external crates, and each has their
// own crate numbers.
pub type CrateNumMap = IndexVec<CrateNum, CrateNum>;

pub struct MetadataBlob(pub ErasedBoxRef<[u8]>);

/// Holds information about a syntax_pos::FileMap imported from another crate.
/// See `imported_filemaps()` for more information.
pub struct ImportedFileMap {
    /// This FileMap's byte-offset within the codemap of its original crate
    pub original_start_pos: syntax_pos::BytePos,
    /// The end of this FileMap within the codemap of its original crate
    pub original_end_pos: syntax_pos::BytePos,
    /// The imported FileMap's representation within the local codemap
    pub translated_filemap: Rc<syntax_pos::FileMap>,
}

pub struct CrateMetadata {
    pub name: Symbol,

    /// Information about the extern crate that caused this crate to
    /// be loaded. If this is `None`, then the crate was injected
    /// (e.g., by the allocator)
    pub extern_crate: Cell<Option<ExternCrate>>,

    pub blob: MetadataBlob,
    pub cnum_map: RefCell<CrateNumMap>,
    pub cnum: CrateNum,
    pub codemap_import_info: RefCell<Vec<ImportedFileMap>>,
    pub attribute_cache: RefCell<[Vec<Option<Rc<[ast::Attribute]>>>; 2]>,

    pub root: schema::CrateRoot,

    /// For each public item in this crate, we encode a key.  When the
    /// crate is loaded, we read all the keys and put them in this
    /// hashmap, which gives the reverse mapping.  This allows us to
    /// quickly retrace a `DefPath`, which is needed for incremental
    /// compilation support.
    pub def_path_table: Rc<DefPathTable>,

    pub exported_symbols: FxHashSet<DefIndex>,

    pub trait_impls: FxHashMap<(u32, DefIndex), schema::LazySeq<DefIndex>>,

    pub dep_kind: Cell<DepKind>,
    pub source: CrateSource,

    pub proc_macros: Option<Vec<(ast::Name, Rc<SyntaxExtension>)>>,
    // Foreign items imported from a dylib (Windows only)
    pub dllimport_foreign_items: FxHashSet<DefIndex>,
}

pub struct CStore {
    metas: RefCell<FxHashMap<CrateNum, Rc<CrateMetadata>>>,
    /// Map from NodeId's of local extern crate statements to crate numbers
    extern_mod_crate_map: RefCell<NodeMap<CrateNum>>,
    pub metadata_loader: Box<MetadataLoader>,
}

impl CStore {
    pub fn new(metadata_loader: Box<MetadataLoader>) -> CStore {
        CStore {
            metas: RefCell::new(FxHashMap()),
            extern_mod_crate_map: RefCell::new(FxHashMap()),
            metadata_loader,
        }
    }

    pub fn next_crate_num(&self) -> CrateNum {
        CrateNum::new(self.metas.borrow().len() + 1)
    }

    pub fn get_crate_data(&self, cnum: CrateNum) -> Rc<CrateMetadata> {
        self.metas.borrow().get(&cnum).unwrap().clone()
    }

    pub fn set_crate_data(&self, cnum: CrateNum, data: Rc<CrateMetadata>) {
        self.metas.borrow_mut().insert(cnum, data);
    }

    pub fn iter_crate_data<I>(&self, mut i: I)
        where I: FnMut(CrateNum, &Rc<CrateMetadata>)
    {
        for (&k, v) in self.metas.borrow().iter() {
            i(k, v);
        }
    }

    pub fn crate_dependencies_in_rpo(&self, krate: CrateNum) -> Vec<CrateNum> {
        let mut ordering = Vec::new();
        self.push_dependencies_in_postorder(&mut ordering, krate);
        ordering.reverse();
        ordering
    }

    pub fn push_dependencies_in_postorder(&self, ordering: &mut Vec<CrateNum>, krate: CrateNum) {
        if ordering.contains(&krate) {
            return;
        }

        let data = self.get_crate_data(krate);
        for &dep in data.cnum_map.borrow().iter() {
            if dep != krate {
                self.push_dependencies_in_postorder(ordering, dep);
            }
        }

        ordering.push(krate);
    }

    pub fn do_postorder_cnums_untracked(&self) -> Vec<CrateNum> {
        let mut ordering = Vec::new();
        for (&num, _) in self.metas.borrow().iter() {
            self.push_dependencies_in_postorder(&mut ordering, num);
        }
        return ordering
    }

    pub fn add_extern_mod_stmt_cnum(&self, emod_id: ast::NodeId, cnum: CrateNum) {
        self.extern_mod_crate_map.borrow_mut().insert(emod_id, cnum);
    }

    pub fn do_extern_mod_stmt_cnum(&self, emod_id: ast::NodeId) -> Option<CrateNum> {
        self.extern_mod_crate_map.borrow().get(&emod_id).cloned()
    }
}

impl CrateMetadata {
    pub fn name(&self) -> Symbol {
        self.root.name
    }
    pub fn hash(&self) -> Svh {
        self.root.hash
    }
    pub fn disambiguator(&self) -> CrateDisambiguator {
        self.root.disambiguator
    }

    pub fn needs_allocator(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "needs_allocator")
    }

    pub fn has_global_allocator(&self) -> bool {
        self.root.has_global_allocator.clone()
    }

    pub fn has_default_lib_allocator(&self) -> bool {
        self.root.has_default_lib_allocator.clone()
    }

    pub fn is_panic_runtime(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "panic_runtime")
    }

    pub fn needs_panic_runtime(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "needs_panic_runtime")
    }

    pub fn is_compiler_builtins(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "compiler_builtins")
    }

    pub fn is_sanitizer_runtime(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "sanitizer_runtime")
    }

    pub fn is_profiler_runtime(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "profiler_runtime")
    }

    pub fn is_no_builtins(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "no_builtins")
    }

     pub fn has_copy_closures(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_feature_attr(&attrs, "copy_closures")
    }

    pub fn has_clone_closures(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_feature_attr(&attrs, "clone_closures")
    }

    pub fn panic_strategy(&self) -> PanicStrategy {
        self.root.panic_strategy.clone()
    }
}
<commit_msg>Auto merge of #46913 - Eh2406:master, r=eddyb<commit_after>// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// The crate store - a central repo for information collected about external
// crates and libraries

use schema;

use rustc::hir::def_id::{CRATE_DEF_INDEX, CrateNum, DefIndex};
use rustc::hir::map::definitions::DefPathTable;
use rustc::hir::svh::Svh;
use rustc::middle::cstore::{DepKind, ExternCrate, MetadataLoader};
use rustc::session::{Session, CrateDisambiguator};
use rustc_back::PanicStrategy;
use rustc_data_structures::indexed_vec::IndexVec;
use rustc::util::nodemap::{FxHashMap, FxHashSet, NodeMap};

use std::cell::{RefCell, Cell};
use std::rc::Rc;
use rustc_data_structures::owning_ref::ErasedBoxRef;
use syntax::{ast, attr};
use syntax::ext::base::SyntaxExtension;
use syntax::symbol::Symbol;
use syntax_pos;

pub use rustc::middle::cstore::{NativeLibrary, NativeLibraryKind, LinkagePreference};
pub use rustc::middle::cstore::NativeLibraryKind::*;
pub use rustc::middle::cstore::{CrateSource, LibSource};

pub use cstore_impl::{provide, provide_extern};

// A map from external crate numbers (as decoded from some crate file) to
// local crate numbers (as generated during this session). Each external
// crate may refer to types in other external crates, and each has their
// own crate numbers.
pub type CrateNumMap = IndexVec<CrateNum, CrateNum>;

pub struct MetadataBlob(pub ErasedBoxRef<[u8]>);

/// Holds information about a syntax_pos::FileMap imported from another crate.
/// See `imported_filemaps()` for more information.
pub struct ImportedFileMap {
    /// This FileMap's byte-offset within the codemap of its original crate
    pub original_start_pos: syntax_pos::BytePos,
    /// The end of this FileMap within the codemap of its original crate
    pub original_end_pos: syntax_pos::BytePos,
    /// The imported FileMap's representation within the local codemap
    pub translated_filemap: Rc<syntax_pos::FileMap>,
}

pub struct CrateMetadata {
    pub name: Symbol,

    /// Information about the extern crate that caused this crate to
    /// be loaded. If this is `None`, then the crate was injected
    /// (e.g., by the allocator)
    pub extern_crate: Cell<Option<ExternCrate>>,

    pub blob: MetadataBlob,
    pub cnum_map: RefCell<CrateNumMap>,
    pub cnum: CrateNum,
    pub codemap_import_info: RefCell<Vec<ImportedFileMap>>,
    pub attribute_cache: RefCell<[Vec<Option<Rc<[ast::Attribute]>>>; 2]>,

    pub root: schema::CrateRoot,

    /// For each public item in this crate, we encode a key.  When the
    /// crate is loaded, we read all the keys and put them in this
    /// hashmap, which gives the reverse mapping.  This allows us to
    /// quickly retrace a `DefPath`, which is needed for incremental
    /// compilation support.
    pub def_path_table: Rc<DefPathTable>,

    pub exported_symbols: FxHashSet<DefIndex>,

    pub trait_impls: FxHashMap<(u32, DefIndex), schema::LazySeq<DefIndex>>,

    pub dep_kind: Cell<DepKind>,
    pub source: CrateSource,

    pub proc_macros: Option<Vec<(ast::Name, Rc<SyntaxExtension>)>>,
    // Foreign items imported from a dylib (Windows only)
    pub dllimport_foreign_items: FxHashSet<DefIndex>,
}

pub struct CStore {
    metas: RefCell<IndexVec<CrateNum, Option<Rc<CrateMetadata>>>>,
    /// Map from NodeId's of local extern crate statements to crate numbers
    extern_mod_crate_map: RefCell<NodeMap<CrateNum>>,
    pub metadata_loader: Box<MetadataLoader>,
}

impl CStore {
    pub fn new(metadata_loader: Box<MetadataLoader>) -> CStore {
        CStore {
            metas: RefCell::new(IndexVec::new()),
            extern_mod_crate_map: RefCell::new(FxHashMap()),
            metadata_loader,
        }
    }

    pub fn next_crate_num(&self) -> CrateNum {
        CrateNum::new(self.metas.borrow().len() + 1)
    }

    pub fn get_crate_data(&self, cnum: CrateNum) -> Rc<CrateMetadata> {
        self.metas.borrow()[cnum].clone().unwrap()
    }

    pub fn set_crate_data(&self, cnum: CrateNum, data: Rc<CrateMetadata>) {
        use rustc_data_structures::indexed_vec::Idx;
        let mut met = self.metas.borrow_mut();
        while met.len() <= cnum.index() {
            met.push(None);
        }
        met[cnum] = Some(data);
    }

    pub fn iter_crate_data<I>(&self, mut i: I)
        where I: FnMut(CrateNum, &Rc<CrateMetadata>)
    {
        for (k, v) in self.metas.borrow().iter_enumerated() {
            if let &Some(ref v) = v {
                i(k, v);
            }
        }
    }

    pub fn crate_dependencies_in_rpo(&self, krate: CrateNum) -> Vec<CrateNum> {
        let mut ordering = Vec::new();
        self.push_dependencies_in_postorder(&mut ordering, krate);
        ordering.reverse();
        ordering
    }

    pub fn push_dependencies_in_postorder(&self, ordering: &mut Vec<CrateNum>, krate: CrateNum) {
        if ordering.contains(&krate) {
            return;
        }

        let data = self.get_crate_data(krate);
        for &dep in data.cnum_map.borrow().iter() {
            if dep != krate {
                self.push_dependencies_in_postorder(ordering, dep);
            }
        }

        ordering.push(krate);
    }

    pub fn do_postorder_cnums_untracked(&self) -> Vec<CrateNum> {
        let mut ordering = Vec::new();
        for (num, v) in self.metas.borrow().iter_enumerated() {
            if let &Some(_) = v {
                self.push_dependencies_in_postorder(&mut ordering, num);
            }
        }
        return ordering
    }

    pub fn add_extern_mod_stmt_cnum(&self, emod_id: ast::NodeId, cnum: CrateNum) {
        self.extern_mod_crate_map.borrow_mut().insert(emod_id, cnum);
    }

    pub fn do_extern_mod_stmt_cnum(&self, emod_id: ast::NodeId) -> Option<CrateNum> {
        self.extern_mod_crate_map.borrow().get(&emod_id).cloned()
    }
}

impl CrateMetadata {
    pub fn name(&self) -> Symbol {
        self.root.name
    }
    pub fn hash(&self) -> Svh {
        self.root.hash
    }
    pub fn disambiguator(&self) -> CrateDisambiguator {
        self.root.disambiguator
    }

    pub fn needs_allocator(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "needs_allocator")
    }

    pub fn has_global_allocator(&self) -> bool {
        self.root.has_global_allocator.clone()
    }

    pub fn has_default_lib_allocator(&self) -> bool {
        self.root.has_default_lib_allocator.clone()
    }

    pub fn is_panic_runtime(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "panic_runtime")
    }

    pub fn needs_panic_runtime(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "needs_panic_runtime")
    }

    pub fn is_compiler_builtins(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "compiler_builtins")
    }

    pub fn is_sanitizer_runtime(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "sanitizer_runtime")
    }

    pub fn is_profiler_runtime(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "profiler_runtime")
    }

    pub fn is_no_builtins(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_name(&attrs, "no_builtins")
    }

     pub fn has_copy_closures(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_feature_attr(&attrs, "copy_closures")
    }

    pub fn has_clone_closures(&self, sess: &Session) -> bool {
        let attrs = self.get_item_attrs(CRATE_DEF_INDEX, sess);
        attr::contains_feature_attr(&attrs, "clone_closures")
    }

    pub fn panic_strategy(&self) -> PanicStrategy {
        self.root.panic_strategy.clone()
    }
}
<|endoftext|>


<commit_before>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use syntax_pos::symbol::Symbol;
use back::write::create_target_machine;
use llvm;
use rustc::session::Session;
use rustc::session::config::PrintRequest;
use libc::{c_int, c_char};
use std::ffi::CString;

use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Once;

pub fn init(sess: &Session) {
    unsafe {
        // Before we touch LLVM, make sure that multithreading is enabled.
        static POISONED: AtomicBool = AtomicBool::new(false);
        static INIT: Once = Once::new();
        INIT.call_once(|| {
            if llvm::LLVMStartMultithreaded() != 1 {
                // use an extra bool to make sure that all future usage of LLVM
                // cannot proceed despite the Once not running more than once.
                POISONED.store(true, Ordering::SeqCst);
            }

            configure_llvm(sess);
        });

        if POISONED.load(Ordering::SeqCst) {
            bug!("couldn't enable multi-threaded LLVM");
        }
    }
}

unsafe fn configure_llvm(sess: &Session) {
    let mut llvm_c_strs = Vec::new();
    let mut llvm_args = Vec::new();

    {
        let mut add = |arg: &str| {
            let s = CString::new(arg).unwrap();
            llvm_args.push(s.as_ptr());
            llvm_c_strs.push(s);
        };
        add("rustc"); // fake program name
        if sess.time_llvm_passes() { add("-time-passes"); }
        if sess.print_llvm_passes() { add("-debug-pass=Structure"); }

        for arg in &sess.opts.cg.llvm_args {
            add(&(*arg));
        }
    }

    llvm::LLVMInitializePasses();

    llvm::initialize_available_targets();

    llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int,
                                 llvm_args.as_ptr());
}

// WARNING: the features must be known to LLVM or the feature
// detection code will walk past the end of the feature array,
// leading to crashes.

const ARM_WHITELIST: &'static [&'static str] = &["neon\0", "vfp2\0", "vfp3\0", "vfp4\0"];

const AARCH64_WHITELIST: &'static [&'static str] = &["neon\0"];

const X86_WHITELIST: &'static [&'static str] = &["avx\0", "avx2\0", "bmi\0", "bmi2\0", "sse\0",
                                                 "sse2\0", "sse3\0", "sse4.1\0", "sse4.2\0",
                                                 "ssse3\0", "tbm\0", "lzcnt\0", "popcnt\0",
                                                 "sse4a\0", "rdrnd\0", "rdseed\0", "fma\0",
                                                 "xsave\0", "xsaveopt\0", "xsavec\0",
                                                 "xsaves\0",
                                                 "avx512bw\0", "avx512cd\0",
                                                 "avx512dq\0", "avx512er\0",
                                                 "avx512f\0", "avx512ifma\0",
                                                 "avx512pf\0", "avx512vbmi\0",
                                                 "avx512vl\0", "avx512vpopcntdq\0"];

const HEXAGON_WHITELIST: &'static [&'static str] = &["hvx\0", "hvx-double\0"];

const POWERPC_WHITELIST: &'static [&'static str] = &["altivec\0",
                                                     "power8-altivec\0", "power9-altivec\0",
                                                     "power8-vector\0", "power9-vector\0",
                                                     "vsx\0"];

pub fn target_features(sess: &Session) -> Vec<Symbol> {
    let target_machine = create_target_machine(sess);

    let whitelist = match &*sess.target.target.arch {
        "arm" => ARM_WHITELIST,
        "aarch64" => AARCH64_WHITELIST,
        "x86" | "x86_64" => X86_WHITELIST,
        "hexagon" => HEXAGON_WHITELIST,
        "powerpc" | "powerpc64" => POWERPC_WHITELIST,
        _ => &[],
    };

    let mut features = Vec::new();
    for feat in whitelist {
        assert_eq!(feat.chars().last(), Some('\0'));
        if unsafe { llvm::LLVMRustHasFeature(target_machine, feat.as_ptr() as *const c_char) } {
            features.push(Symbol::intern(&feat[..feat.len() - 1]));
        }
    }
    features
}

pub fn print_version() {
    unsafe {
        println!("LLVM version: {}.{}",
                 llvm::LLVMRustVersionMajor(), llvm::LLVMRustVersionMinor());
    }
}

pub fn print_passes() {
    unsafe { llvm::LLVMRustPrintPasses(); }
}

pub fn print(req: PrintRequest, sess: &Session) {
    let tm = create_target_machine(sess);
    unsafe {
        match req {
            PrintRequest::TargetCPUs => llvm::LLVMRustPrintTargetCPUs(tm),
            PrintRequest::TargetFeatures => llvm::LLVMRustPrintTargetFeatures(tm),
            _ => bug!("rustc_trans can't handle print request: {:?}", req),
        }
    }
}

pub fn enable_llvm_debug() {
    unsafe { llvm::LLVMRustSetDebug(1); }
}
<commit_msg>white list MMX and MSA target features<commit_after>// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use syntax_pos::symbol::Symbol;
use back::write::create_target_machine;
use llvm;
use rustc::session::Session;
use rustc::session::config::PrintRequest;
use libc::{c_int, c_char};
use std::ffi::CString;

use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Once;

pub fn init(sess: &Session) {
    unsafe {
        // Before we touch LLVM, make sure that multithreading is enabled.
        static POISONED: AtomicBool = AtomicBool::new(false);
        static INIT: Once = Once::new();
        INIT.call_once(|| {
            if llvm::LLVMStartMultithreaded() != 1 {
                // use an extra bool to make sure that all future usage of LLVM
                // cannot proceed despite the Once not running more than once.
                POISONED.store(true, Ordering::SeqCst);
            }

            configure_llvm(sess);
        });

        if POISONED.load(Ordering::SeqCst) {
            bug!("couldn't enable multi-threaded LLVM");
        }
    }
}

unsafe fn configure_llvm(sess: &Session) {
    let mut llvm_c_strs = Vec::new();
    let mut llvm_args = Vec::new();

    {
        let mut add = |arg: &str| {
            let s = CString::new(arg).unwrap();
            llvm_args.push(s.as_ptr());
            llvm_c_strs.push(s);
        };
        add("rustc"); // fake program name
        if sess.time_llvm_passes() { add("-time-passes"); }
        if sess.print_llvm_passes() { add("-debug-pass=Structure"); }

        for arg in &sess.opts.cg.llvm_args {
            add(&(*arg));
        }
    }

    llvm::LLVMInitializePasses();

    llvm::initialize_available_targets();

    llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int,
                                 llvm_args.as_ptr());
}

// WARNING: the features must be known to LLVM or the feature
// detection code will walk past the end of the feature array,
// leading to crashes.

const ARM_WHITELIST: &'static [&'static str] = &["neon\0", "vfp2\0", "vfp3\0", "vfp4\0"];

const AARCH64_WHITELIST: &'static [&'static str] = &["neon\0"];

const X86_WHITELIST: &'static [&'static str] = &["avx\0", "avx2\0", "bmi\0", "bmi2\0", "sse\0",
                                                 "sse2\0", "sse3\0", "sse4.1\0", "sse4.2\0",
                                                 "ssse3\0", "tbm\0", "lzcnt\0", "popcnt\0",
                                                 "sse4a\0", "rdrnd\0", "rdseed\0", "fma\0",
                                                 "xsave\0", "xsaveopt\0", "xsavec\0",
                                                 "xsaves\0",
                                                 "avx512bw\0", "avx512cd\0",
                                                 "avx512dq\0", "avx512er\0",
                                                 "avx512f\0", "avx512ifma\0",
                                                 "avx512pf\0", "avx512vbmi\0",
                                                 "avx512vl\0", "avx512vpopcntdq\0", "mmx\0"];

const HEXAGON_WHITELIST: &'static [&'static str] = &["hvx\0", "hvx-double\0"];

const POWERPC_WHITELIST: &'static [&'static str] = &["altivec\0",
                                                     "power8-altivec\0", "power9-altivec\0",
                                                     "power8-vector\0", "power9-vector\0",
                                                     "vsx\0"];

const MIPS_WHITELIST: &'static [&'static str] = &["msa\0"];

pub fn target_features(sess: &Session) -> Vec<Symbol> {
    let target_machine = create_target_machine(sess);

    let whitelist = match &*sess.target.target.arch {
        "arm" => ARM_WHITELIST,
        "aarch64" => AARCH64_WHITELIST,
        "x86" | "x86_64" => X86_WHITELIST,
        "hexagon" => HEXAGON_WHITELIST,
        "mips" | "mips64" => MIPS_WHITELIST,
        "powerpc" | "powerpc64" => POWERPC_WHITELIST,
        _ => &[],
    };

    let mut features = Vec::new();
    for feat in whitelist {
        assert_eq!(feat.chars().last(), Some('\0'));
        if unsafe { llvm::LLVMRustHasFeature(target_machine, feat.as_ptr() as *const c_char) } {
            features.push(Symbol::intern(&feat[..feat.len() - 1]));
        }
    }
    features
}

pub fn print_version() {
    unsafe {
        println!("LLVM version: {}.{}",
                 llvm::LLVMRustVersionMajor(), llvm::LLVMRustVersionMinor());
    }
}

pub fn print_passes() {
    unsafe { llvm::LLVMRustPrintPasses(); }
}

pub fn print(req: PrintRequest, sess: &Session) {
    let tm = create_target_machine(sess);
    unsafe {
        match req {
            PrintRequest::TargetCPUs => llvm::LLVMRustPrintTargetCPUs(tm),
            PrintRequest::TargetFeatures => llvm::LLVMRustPrintTargetFeatures(tm),
            _ => bug!("rustc_trans can't handle print request: {:?}", req),
        }
    }
}

pub fn enable_llvm_debug() {
    unsafe { llvm::LLVMRustSetDebug(1); }
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, ilammy
//
// Licensed under MIT license (see LICENSE file in the root directory).
// This file may be copied, distributed, and modified only in accordance
// with the terms specified by this license.

//! Unicode normalization algorithms.
//!
//! This module implements _Unicode Normalization Forms_ as defined by [Unicode Standard
//! Annex #15][UAX-15]. Unicode normalization is used to ensure that visually equivalent
//! strings have equivalent binary representations.
//!
//! [UAX-15]: http://www.unicode.org/reports/tr15/

use std::iter::{FromIterator, IntoIterator};
use tables::{decomposition_mappings, composition_mappings};

//
// Definitions of Normalization Forms
//

/// Normalize a string according to **Normalization Form D** (_D118_).
pub fn nfd(s: &str) -> String {
    let v = canonical_decomposition(s.chars());
    return String::from_iter(v.iter().map(|&(c, _)| c));
}

/// Normalize a string according to **Normalization Form KD** (_D119_).
pub fn nfkd(s: &str) -> String {
    let v = compatibility_decomposition(s.chars());
    return String::from_iter(v.iter().map(|&(c, _)| c));
}

/// Normalize a string according to **Normalization Form C** (_D120_).
pub fn nfc(s: &str) -> String {
    let mut v = canonical_decomposition(s.chars());
    compose_canonically(&mut v);
    return String::from_iter(v.iter().map(|&(c, _)| c));
}

/// Normalize a string according to **Normalization Form KC** (_D121_).
pub fn nfkc(s: &str) -> String {
    let mut v = compatibility_decomposition(s.chars());
    compose_canonically(&mut v);
    return String::from_iter(v.iter().map(|&(c, _)| c));
}

//
// Decomposition
//

/// Produce a Compatibility decomposition (D65) of a character sequence.
fn compatibility_decomposition<I>(chars: I) -> Vec<(char, u8)>
    where I: IntoIterator<Item=char>
{
    let mut buffer = Vec::new();

    for c in chars {
        push_compatibility_decomposition(c, &mut buffer);
    }

    reorder_canonically(&mut buffer[..]);

    return buffer;
}

/// Produce a Canonical decomposition (D68) of a character sequence.
fn canonical_decomposition<I>(chars: I) -> Vec<(char, u8)>
    where I: IntoIterator<Item=char>
{
    let mut buffer = Vec::new();

    for c in chars {
        push_canonical_decomposition(c, &mut buffer);
    }

    reorder_canonically(&mut buffer[..]);

    return buffer;
}

/// Push a Compatibility decomposition (D65) of a single character into the given buffer.
fn push_compatibility_decomposition(c: char, vec: &mut Vec<(char, u8)>) {
    use tables::properties::canonical_combining_class as ccc;

    if push_hangul_decomposition(c, vec) {
        return;
    }

    match decomposition_mappings::compatibility_mapping(c) {
        Some(decomposition) => {
            vec.extend_from_slice(decomposition);
        }
        None => {
            vec.push((c, ccc(c)));
        }
    }
}

/// Push a Canonical decomposition (D68) of a single character into the given buffer.
fn push_canonical_decomposition(c: char, vec: &mut Vec<(char, u8)>) {
    use tables::properties::canonical_combining_class as ccc;

    if push_hangul_decomposition(c, vec) {
        return;
    }

    match decomposition_mappings::canonical_mapping(c) {
        Some(decomposition) => {
            vec.extend_from_slice(decomposition);
        }
        None => {
            vec.push((c, ccc(c)));
        }
    }
}

//
// Conjoining Jamo Behavior
//

const S_BASE: u32 = 0xAC00;
const L_BASE: u32 = 0x1100;
const V_BASE: u32 = 0x1161;
const T_BASE: u32 = 0x11A7;
const L_COUNT: u32 = 19;
const V_COUNT: u32 = 21;
const T_COUNT: u32 = 28;
const N_COUNT: u32 = V_COUNT * T_COUNT;
const S_COUNT: u32 = L_COUNT * N_COUNT;

/// If a character is a Precomposed Hangul syllable (D132) then push its full decomposition into
/// the given buffer and return true. Otherwise do not modify the buffer and return false.
fn push_hangul_decomposition(c: char, vec: &mut Vec<(char, u8)>) -> bool {
    use std::char;
    use tables::properties::canonical_combining_class as ccc;

    if ((c as u32) < S_BASE) || ((S_BASE + S_COUNT) <= (c as u32)) {
        return false;
    }

    let s_index = (c as u32) - S_BASE;

    let l_index = s_index / N_COUNT;
    let v_index = (s_index % N_COUNT) / T_COUNT;
    let t_index = s_index % T_COUNT;

    // Computed Hangul syllables are guaranteed to be valid Unicode codepoints (cf. ranges).
    // They also have been assigned canonical combining class zero and canonical combining
    // class is guaranteed to not change in Unicode, so we can save a table lookup on that.

    if t_index > 0 {
        let l = char::from_u32(L_BASE + l_index).unwrap();
        let v = char::from_u32(V_BASE + v_index).unwrap();
        let t = char::from_u32(T_BASE + t_index).unwrap();

        assert!((ccc(l) == 0) && (ccc(v) == 0) && (ccc(t) == 0));

        vec.push((l, 0));
        vec.push((v, 0));
        vec.push((t, 0));
    } else {
        let l = char::from_u32(L_BASE + l_index).unwrap();
        let v = char::from_u32(V_BASE + v_index).unwrap();

        assert!((ccc(l) == 0) && (ccc(v) == 0));

        vec.push((l, 0));
        vec.push((v, 0));
    }

    return true;
}

/// If a character pair forms a Precomposed Hangul syllable (D132) when it is canonically composed
/// then return this Some composition. Otherwise return None.
fn compose_hangul(c1: char, c2: char) -> Option<(char, u8)> {
    use std::char;
    use tables::properties::canonical_combining_class as ccc;

    // In the same way as with decomposition, arithmetics and character ranges guarantee codepoint
    // validity here, and precomposed Hangul syllables also have canonical combining class zero.

    // <L, V> pair
    if ((L_BASE <= (c1 as u32)) && ((c1 as u32) < L_BASE + L_COUNT)) &&
       ((V_BASE <= (c2 as u32)) && ((c2 as u32) < V_BASE + V_COUNT))
    {
        let l_index = (c1 as u32) - L_BASE;
        let v_index = (c2 as u32) - V_BASE;

        let lv_index = l_index * N_COUNT + v_index * T_COUNT;

        let lv = char::from_u32(S_BASE + lv_index).unwrap();

        assert!(ccc(lv) == 0);

        return Some((lv, 0));
    }

    // <LV, T> pair
    if ((S_BASE <= (c1 as u32)) && ((c1 as u32) < S_BASE + S_COUNT)) &&
       (((c1 as u32) - S_BASE) % T_COUNT == 0) &&
       (((T_BASE + 1) <= (c2 as u32)) && ((c2 as u32) < T_BASE + T_COUNT))
    {
        let t_index = (c2 as u32) - T_BASE;

        let lvt = char::from_u32((c1 as u32) + t_index).unwrap();

        assert!(ccc(lvt) == 0);

        return Some((lvt, 0));
    }

    // Anything else
    return None;
}

//
// Canonical Ordering Algorithm
//

/// Check whether adjacent characters form a Reorderable pair (D108).
fn reorderable_pair(a: (char, u8), b: (char, u8)) -> bool {
    (a.1 > b.1) && (b.1 > 0)
}

/// Apply the Canonical Ordering Algorithm (D109) to a character slice.
fn reorder_canonically(slice: &mut [(char, u8)]) {
    // Bubble sort, yay!
    for lim in (1..slice.len()).rev() {
        for i in 0..lim {
            if reorderable_pair(slice[i], slice[i + 1]) {
                slice.swap(i, i + 1);
            }
        }
    }
}

//
// Canonical Composition Algorithm
//

/// Apply the Canonical Composition Algorithm (D117) to a character buffer.
fn compose_canonically(buffer: &mut Vec<(char, u8)>) {
    let mut ci = 1;
    while ci < buffer.len() {
        if let Some(li) = find_starter(&buffer[..], ci) {
            if !blocked(&buffer[..], li, ci) {
                if let Some(p) = primary_composite(buffer[li].0, buffer[ci].0) {
                    buffer[li] = p;
                    buffer.remove(ci);
                    continue;
                }
            }
        }
        ci += 1;
    }
}

/// Find the last Starter (D107) preceding C in a character slice.
/// This is step R1 of the Canonical Composition Algorithm (D117).
fn find_starter(slice: &[(char, u8)], ci: usize) -> Option<usize> {
    for li in (0..ci).rev() {
        if slice[li].1 == 0 {
            return Some(li);
        }
    }
    return None;
}

/// Verify that A is not blocked (D115) from C in a character slice.
/// This is the first part of step R2 of the Canonical Composition Algorithm (D117).
fn blocked(slice: &[(char, u8)], ai: usize, ci: usize) -> bool {
    assert!(ai < ci);

    let ccc_a = slice[ai].1;
    let ccc_c = slice[ci].1;

    if ccc_a == 0 {
        for bi in (ai + 1)..ci {
            let ccc_b = slice[bi].1;

            if (ccc_b == 0) || (ccc_b >= ccc_c) {
                return true;
            }
        }
    }

    return false;
}

/// Check for a Primary Composite (D114) equivalent to the given pair of characters.
/// This is the second part of step R2 of the Canonical Composition Algorithm (D117).
fn primary_composite(c1: char, c2: char) -> Option<(char, u8)> {
    compose_hangul(c1, c2).or_else(|| composition_mappings::primary(c1, c2))
}
<commit_msg>Optimize canonical reordering algorithm<commit_after>// Copyright (c) 2016, ilammy
//
// Licensed under MIT license (see LICENSE file in the root directory).
// This file may be copied, distributed, and modified only in accordance
// with the terms specified by this license.

//! Unicode normalization algorithms.
//!
//! This module implements _Unicode Normalization Forms_ as defined by [Unicode Standard
//! Annex #15][UAX-15]. Unicode normalization is used to ensure that visually equivalent
//! strings have equivalent binary representations.
//!
//! [UAX-15]: http://www.unicode.org/reports/tr15/

use std::iter::{FromIterator, IntoIterator};
use tables::{decomposition_mappings, composition_mappings};

//
// Definitions of Normalization Forms
//

/// Normalize a string according to **Normalization Form D** (_D118_).
pub fn nfd(s: &str) -> String {
    let v = canonical_decomposition(s.chars());
    return String::from_iter(v.iter().map(|&(c, _)| c));
}

/// Normalize a string according to **Normalization Form KD** (_D119_).
pub fn nfkd(s: &str) -> String {
    let v = compatibility_decomposition(s.chars());
    return String::from_iter(v.iter().map(|&(c, _)| c));
}

/// Normalize a string according to **Normalization Form C** (_D120_).
pub fn nfc(s: &str) -> String {
    let mut v = canonical_decomposition(s.chars());
    compose_canonically(&mut v);
    return String::from_iter(v.iter().map(|&(c, _)| c));
}

/// Normalize a string according to **Normalization Form KC** (_D121_).
pub fn nfkc(s: &str) -> String {
    let mut v = compatibility_decomposition(s.chars());
    compose_canonically(&mut v);
    return String::from_iter(v.iter().map(|&(c, _)| c));
}

//
// Decomposition
//

/// Produce a Compatibility decomposition (D65) of a character sequence.
fn compatibility_decomposition<I>(chars: I) -> Vec<(char, u8)>
    where I: IntoIterator<Item=char>
{
    let mut buffer = Vec::new();

    for c in chars {
        push_compatibility_decomposition(c, &mut buffer);
    }

    reorder_canonically(&mut buffer[..]);

    return buffer;
}

/// Produce a Canonical decomposition (D68) of a character sequence.
fn canonical_decomposition<I>(chars: I) -> Vec<(char, u8)>
    where I: IntoIterator<Item=char>
{
    let mut buffer = Vec::new();

    for c in chars {
        push_canonical_decomposition(c, &mut buffer);
    }

    reorder_canonically(&mut buffer[..]);

    return buffer;
}

/// Push a Compatibility decomposition (D65) of a single character into the given buffer.
fn push_compatibility_decomposition(c: char, vec: &mut Vec<(char, u8)>) {
    use tables::properties::canonical_combining_class as ccc;

    if push_hangul_decomposition(c, vec) {
        return;
    }

    match decomposition_mappings::compatibility_mapping(c) {
        Some(decomposition) => {
            vec.extend_from_slice(decomposition);
        }
        None => {
            vec.push((c, ccc(c)));
        }
    }
}

/// Push a Canonical decomposition (D68) of a single character into the given buffer.
fn push_canonical_decomposition(c: char, vec: &mut Vec<(char, u8)>) {
    use tables::properties::canonical_combining_class as ccc;

    if push_hangul_decomposition(c, vec) {
        return;
    }

    match decomposition_mappings::canonical_mapping(c) {
        Some(decomposition) => {
            vec.extend_from_slice(decomposition);
        }
        None => {
            vec.push((c, ccc(c)));
        }
    }
}

//
// Conjoining Jamo Behavior
//

const S_BASE: u32 = 0xAC00;
const L_BASE: u32 = 0x1100;
const V_BASE: u32 = 0x1161;
const T_BASE: u32 = 0x11A7;
const L_COUNT: u32 = 19;
const V_COUNT: u32 = 21;
const T_COUNT: u32 = 28;
const N_COUNT: u32 = V_COUNT * T_COUNT;
const S_COUNT: u32 = L_COUNT * N_COUNT;

/// If a character is a Precomposed Hangul syllable (D132) then push its full decomposition into
/// the given buffer and return true. Otherwise do not modify the buffer and return false.
fn push_hangul_decomposition(c: char, vec: &mut Vec<(char, u8)>) -> bool {
    use std::char;
    use tables::properties::canonical_combining_class as ccc;

    if ((c as u32) < S_BASE) || ((S_BASE + S_COUNT) <= (c as u32)) {
        return false;
    }

    let s_index = (c as u32) - S_BASE;

    let l_index = s_index / N_COUNT;
    let v_index = (s_index % N_COUNT) / T_COUNT;
    let t_index = s_index % T_COUNT;

    // Computed Hangul syllables are guaranteed to be valid Unicode codepoints (cf. ranges).
    // They also have been assigned canonical combining class zero and canonical combining
    // class is guaranteed to not change in Unicode, so we can save a table lookup on that.

    if t_index > 0 {
        let l = char::from_u32(L_BASE + l_index).unwrap();
        let v = char::from_u32(V_BASE + v_index).unwrap();
        let t = char::from_u32(T_BASE + t_index).unwrap();

        assert!((ccc(l) == 0) && (ccc(v) == 0) && (ccc(t) == 0));

        vec.push((l, 0));
        vec.push((v, 0));
        vec.push((t, 0));
    } else {
        let l = char::from_u32(L_BASE + l_index).unwrap();
        let v = char::from_u32(V_BASE + v_index).unwrap();

        assert!((ccc(l) == 0) && (ccc(v) == 0));

        vec.push((l, 0));
        vec.push((v, 0));
    }

    return true;
}

/// If a character pair forms a Precomposed Hangul syllable (D132) when it is canonically composed
/// then return this Some composition. Otherwise return None.
fn compose_hangul(c1: char, c2: char) -> Option<(char, u8)> {
    use std::char;
    use tables::properties::canonical_combining_class as ccc;

    // In the same way as with decomposition, arithmetics and character ranges guarantee codepoint
    // validity here, and precomposed Hangul syllables also have canonical combining class zero.

    // <L, V> pair
    if ((L_BASE <= (c1 as u32)) && ((c1 as u32) < L_BASE + L_COUNT)) &&
       ((V_BASE <= (c2 as u32)) && ((c2 as u32) < V_BASE + V_COUNT))
    {
        let l_index = (c1 as u32) - L_BASE;
        let v_index = (c2 as u32) - V_BASE;

        let lv_index = l_index * N_COUNT + v_index * T_COUNT;

        let lv = char::from_u32(S_BASE + lv_index).unwrap();

        assert!(ccc(lv) == 0);

        return Some((lv, 0));
    }

    // <LV, T> pair
    if ((S_BASE <= (c1 as u32)) && ((c1 as u32) < S_BASE + S_COUNT)) &&
       (((c1 as u32) - S_BASE) % T_COUNT == 0) &&
       (((T_BASE + 1) <= (c2 as u32)) && ((c2 as u32) < T_BASE + T_COUNT))
    {
        let t_index = (c2 as u32) - T_BASE;

        let lvt = char::from_u32((c1 as u32) + t_index).unwrap();

        assert!(ccc(lvt) == 0);

        return Some((lvt, 0));
    }

    // Anything else
    return None;
}

//
// Canonical Ordering Algorithm
//

/// Apply the Canonical Ordering Algorithm (D109) to a character slice.
fn reorder_canonically(slice: &mut [(char, u8)]) {
    // Actually, this is a bubble sort, but with one important detail: starter characters are never
    // reordered. That is, we must sort only the grapheme clusters. Therefore we can replace O(n^2)
    // bubble sort of the entire string with an O(n) pass over the clusters. Each grapheme cluster
    // still takes O(n^2) time to sort. However, in this case 'n' is the length of the cluster,
    // which is a small number in real-world texts (less than 10 in sane texts, usually 2..5).
    // And usually the combining marks are almost sorted, so the bubble sort works just fine.

    let len = slice.len();

    let mut cur = 0;
    while cur < len {
        // Skip over sequences of starters. We are looking for non-starters.
        if slice[cur].1 == 0 {
            cur += 1;
            continue;
        }

        // Now find the next starter so we know where to stop.
        let mut next = cur + 1;
        while next < len {
            if slice[next].1 == 0 {
                break;
            }
            next += 1;
        }

        // Apply bubble sort to the cluster.
        for limit in (cur..next).rev() {
            for i in cur..limit {
                if slice[i].1 > slice[i + 1].1 {
                    slice.swap(i, i + 1);
                }
            }
        }

        // We're done with this cluster, move on to the next one.
        cur = next;
    }
}

//
// Canonical Composition Algorithm
//

/// Apply the Canonical Composition Algorithm (D117) to a character buffer.
fn compose_canonically(buffer: &mut Vec<(char, u8)>) {
    let mut ci = 1;
    while ci < buffer.len() {
        if let Some(li) = find_starter(&buffer[..], ci) {
            if !blocked(&buffer[..], li, ci) {
                if let Some(p) = primary_composite(buffer[li].0, buffer[ci].0) {
                    buffer[li] = p;
                    buffer.remove(ci);
                    continue;
                }
            }
        }
        ci += 1;
    }
}

/// Find the last Starter (D107) preceding C in a character slice.
/// This is step R1 of the Canonical Composition Algorithm (D117).
fn find_starter(slice: &[(char, u8)], ci: usize) -> Option<usize> {
    for li in (0..ci).rev() {
        if slice[li].1 == 0 {
            return Some(li);
        }
    }
    return None;
}

/// Verify that A is not blocked (D115) from C in a character slice.
/// This is the first part of step R2 of the Canonical Composition Algorithm (D117).
fn blocked(slice: &[(char, u8)], ai: usize, ci: usize) -> bool {
    assert!(ai < ci);

    let ccc_a = slice[ai].1;
    let ccc_c = slice[ci].1;

    if ccc_a == 0 {
        for bi in (ai + 1)..ci {
            let ccc_b = slice[bi].1;

            if (ccc_b == 0) || (ccc_b >= ccc_c) {
                return true;
            }
        }
    }

    return false;
}

/// Check for a Primary Composite (D114) equivalent to the given pair of characters.
/// This is the second part of step R2 of the Canonical Composition Algorithm (D117).
fn primary_composite(c1: char, c2: char) -> Option<(char, u8)> {
    compose_hangul(c1, c2).or_else(|| composition_mappings::primary(c1, c2))
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#[feature(macro_rules)];

pub mod tokenizer;

struct TokenPrinter;

impl tokenizer::TokenSink for TokenPrinter {
    fn process_token(&mut self, token: tokenizer::Token) {
        println!("{:?}", token);
    }
}

fn main() {
    let mut sink = TokenPrinter;
    let mut tok = tokenizer::Tokenizer::new(&mut sink);
    tok.feed("<div novalue unquoted=foo singlequoted='bar' doublequoted=\"baz\">Hello, world!</div>");
}
<commit_msg>Enhance tokenizer test program<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#[feature(macro_rules)];

use std::io;
use std::char;

use tokenizer::{TokenSink, Token, Tokenizer};
use tokenizer::{CharacterToken, TagToken, StartTag, EndTag};

pub mod tokenizer;

struct TokenPrinter {
    in_char_run: bool,
}

impl TokenPrinter {
    fn is_char(&mut self, is_char: bool) {
        match (self.in_char_run, is_char) {
            (false, true ) => print!("CHAR : "),
            (true,  false) => println!(""),
            _ => (),
        }
        self.in_char_run = is_char;
    }
}

impl TokenSink for TokenPrinter {
    fn process_token(&mut self, token: Token) {
        match token {
            CharacterToken(c) => {
                self.is_char(true);
                char::escape_default(c, |d| print!("{:c}", d));
            }
            TagToken(tag) => {
                self.is_char(false);
                // This is not proper HTML serialization, of course.
                match tag.kind {
                    StartTag => print!("TAG  : <\x1b[32m{:s}\x1b[0m", tag.name),
                    EndTag   => print!("TAG  : <\x1b[31m/{:s}\x1b[0m", tag.name),
                }
                for attr in tag.attrs.iter() {
                    print!(" \x1b[36m{:s}\x1b[0m='\x1b[34m{:s}\x1b[0m'", attr.name, attr.value);
                }
                if tag.self_closing {
                    print!(" \x1b[31m/\x1b[0m");
                }
                println!(">");
            }
            _ => {
                self.is_char(false);
                println!("OTHER: {:?}", token);
            }
        }
    }
}

fn main() {
    let mut sink = TokenPrinter {
        in_char_run: false,
    };
    {
        let mut tok = Tokenizer::new(&mut sink);
        tok.feed(io::stdin().read_to_str());
    }
    sink.is_char(false);
}
<|endoftext|>


<commit_before>extern mod extra;
extern mod http_client;

use extra::json;
use extra::json::{Object, List, String, Number};
use extra::net::url::Url;
use extra::net::url;
//use extra::serialize::{Decodable, Encodable};

use std::hashmap::HashMap;
use std::str;
use std::result::{Ok, Err};

use http_client::uv_http_request;
use http_client::StatusCode;

pub struct RepoResponse {
    rawJson: ~[~str]
}

impl RepoResponse {
    fn add(&mut self, data: ~str) {
        self.rawJson.push(data)
    }
}

/* https://github.com/huonw/isrustfastyet/blob/fce8777e289eb874a94242dbd07803248d320d12/mem/process.rs
#[deriving(Encodable)]
struct Repo {
    id: uint,
    name: ~str,
    full_name: ~str
}
*/
/*
fn handleRepo(repo: & json::Json) {
    //println(fmt!("decoded: %?\n", json::Decoder(copy *repo).read_str(~"name")));
    match repo {
        & Object(o) => {
            if (o.contains_key(& ~"name")) {
                let foo = match o.find(& ~"name") {
                    Some( & String(value)) =>  value,
                    Some(_) => fail!("foo was wrong type"),
                    None => ~"Unknown Name"
                };*/
                /*
                match o.find(&~"name") {
                    Some(& String(name)) => {
                        println(name);
                    },
                    _ => {

                    }
                }*/
                /*
            }
        },
        _ => fail!("Why you no Repo?")
    }
}
*/
fn readJson(json: json::Json) {
    match json {
        json::List(l) => {
            println(fmt!("Got %? items", l.len()));
            //for l.consume_iter().advance |repo| {
            for l.iter().advance |r| {
                //println(fmt!("repo=%?\n\n", repo));
                //handleRepo(repo)

                let repo: & json::Json = r;
                println(fmt!("repo=\n%?\n\n", repo));

                match repo {
                    &Object(o2) => {}
                    &Number(n) => {}
                    _ => {}
                }

                /*
                match repo {
                    & Object(o) => {
                        let yolo = copy o;
                        if (o.contains_key(& ~"name")) {
                            let foo = match o.find(& ~"name") {
                                Some( & String(value)) =>  value,
                                Some(_) => fail!("foo was wrong type"),
                                None => ~"Unknown Name"
                            };
                        }
                    },
                    _ => fail!("Why you no Repo?")
                }
*/

                        //x.find_copy(~"name");
                        //let d: Repo = Decodable::decode(&mut json::Decoder(json));
                        //println(fmt!("%?\n", repo.contains_key(~"git_commits_url")));

                    /*json::Json.Object(orepo) => {
                        println(fmt!("%?\n", orepo.contains_key(~"git_commits_url")));
                    }*/

                /*
                match repo {
                    json::Object(repo) => {

                    },
                    _ => {
                        fail!("We expected a list of objects");
                    }
                }*/
            }

            println("A list of Objects, perhaps")
        }
        _ => {
            println("ERROR: Unrecognized JSON format")
        }
    }
}

fn main() {

    // http://ozten.com/psto/
    // https://api.github.com/repositories
    let u: Url = url::from_str("http://ozten.com/random/sample.json").get();
    let mut request = uv_http_request(u);

    let res = @mut RepoResponse{rawJson: ~[]};

    do request.begin |event| {
        match event {
            http_client::Error(e) => {
                println(fmt!("Ouch... error %?", e));
            },
            http_client::Status(s) =>
                match s {
                    // TODO wait... how did I break how match works here
                    // I should need the pattern guard.
                    StatusOK if s == StatusOK => {
                        println(fmt!("Status %?", s));
                        match json::from_str(res.rawJson.concat()) {
                            Ok(json) => {
                                readJson(json);
                            }
                            Err(e) => {
                                println(fmt!("Error parsing JSON %?", e));
                                fail!("Can't read JSON");
                            }
                        }
                    }
                    StatusFound if s == StatusFound => {
                        println(fmt!("Redirected? %?", s));
                    }
                    StatusUnknown => {
                        println(fmt!("hmmm status is unknown %?", s));
                        fail!("No JSON of Repositiories");
                    }
                },

            http_client::Payload(p) => {
                let data = p.take();
                res.rawJson.push(str::from_bytes(data));
            }
        }
    }

    // Problem #2 How to output parts of JSON after parsing
    // I think this is working...
    match json::from_str("{\"ham\": \"bone\"}") {
        Ok(obj) => {
            match obj {
                json::Object(o) => {
                    //println(fmt!("Payload! %?", o.get(& ~"ham")))
                }
                _ => { println("Skipping") }
            }

        }
        Err(e) => {
            println("Error")
        }
    }

    // HashMap play - One type of Json is Object which is a
    // type over HashMap<~str, Json>
    let mut h: HashMap<~str, ~str> = HashMap::new();
    h.insert(~"foo", ~"bar");
    h.get(& ~"foo");

    let mut a:~[~str] = ~[];
    a.push(~"Hello");
    a.push(~" World");
    //println(a.concat());
}<commit_msg>Get further with JSON list of objects<commit_after>extern mod extra;
extern mod http_client;

use extra::json;
use extra::json::{Object, List, String, Number};
use extra::net::url::Url;
use extra::net::url;
//use extra::serialize::{Decodable, Encodable};

use std::hashmap::HashMap;
use std::str;
use std::result::{Ok, Err};

use http_client::uv_http_request;
use http_client::StatusCode;

pub struct RepoResponse {
    rawJson: ~[~str]
}

impl RepoResponse {
    fn add(&mut self, data: ~str) {
        self.rawJson.push(data)
    }
}

/* https://github.com/huonw/isrustfastyet/blob/fce8777e289eb874a94242dbd07803248d320d12/mem/process.rs
#[deriving(Encodable)]
struct Repo {
    id: uint,
    name: ~str,
    full_name: ~str
}
*/

fn handleRepo(repo: json::Json) {
    //println(fmt!("decoded: %?\n", json::Decoder(copy *repo).read_str(~"name")));
    match repo {
        Object(o) => {
            if (o.contains_key(& ~"name")) {
                let name = match copy *o.get(&~"name") {
                    String(value) =>  value,
                    _ => fail!("foo was wrong type")
                };
                println(fmt!("REPO NAME: %?", name));
            }
        },
        _ => fail!("Why you no Repo?")
    }
}

fn readJson(json: json::Json) {
    match json {
        json::List(l) => {
            println(fmt!("Got %? items", l.len()));
            //for l.consume_iter().advance |repo| {
            for l.iter().advance |repo| {
                //println(fmt!("repo=%?\n\n", repo));
                handleRepo(copy *repo);

                        //x.find_copy(~"name");
                        //let d: Repo = Decodable::decode(&mut json::Decoder(json));
                        //println(fmt!("%?\n", repo.contains_key(~"git_commits_url")));

                    /*json::Json.Object(orepo) => {
                        println(fmt!("%?\n", orepo.contains_key(~"git_commits_url")));
                    }*/

                /*
                match repo {
                    json::Object(repo) => {

                    },
                    _ => {
                        fail!("We expected a list of objects");
                    }
                }*/
            }

            println("A list of Objects, perhaps")
        }
        _ => {
            println("ERROR: Unrecognized JSON format")
        }
    }
}

fn main() {

    // http://ozten.com/psto/
    // https://api.github.com/repositories
    let u: Url = url::from_str("http://ozten.com/random/sample.json").get();
    let mut request = uv_http_request(u);

    let res = @mut RepoResponse{rawJson: ~[]};

    do request.begin |event| {
        match event {
            http_client::Error(e) => {
                println(fmt!("Ouch... error %?", e));
            },
            http_client::Status(s) =>
                match s {
                    // TODO wait... how did I break how match works here
                    // I should need the pattern guard.
                    StatusOK if s == StatusOK => {
                        println(fmt!("Status %?", s));
                        match json::from_str(res.rawJson.concat()) {
                            Ok(json) => {
                                readJson(json);
                            }
                            Err(e) => {
                                println(fmt!("Error parsing JSON %?", e));
                                fail!("Can't read JSON");
                            }
                        }
                    }
                    StatusFound if s == StatusFound => {
                        println(fmt!("Redirected? %?", s));
                    }
                    StatusUnknown => {
                        println(fmt!("hmmm status is unknown %?", s));
                        fail!("No JSON of Repositiories");
                    }
                },

            http_client::Payload(p) => {
                let data = p.take();
                res.rawJson.push(str::from_bytes(data));
            }
        }
    }

    // Problem #2 How to output parts of JSON after parsing
    // I think this is working...
    match json::from_str("{\"ham\": \"bone\"}") {
        Ok(obj) => {
            match obj {
                json::Object(o) => {
                    //println(fmt!("Payload! %?", o.get(& ~"ham")))
                }
                _ => { println("Skipping") }
            }

        }
        Err(e) => {
            println("Error")
        }
    }

    // HashMap play - One type of Json is Object which is a
    // type over HashMap<~str, Json>
    let mut h: HashMap<~str, ~str> = HashMap::new();
    h.insert(~"foo", ~"bar");
    h.get(& ~"foo");

    let mut a:~[~str] = ~[];
    a.push(~"Hello");
    a.push(~" World");
    //println(a.concat());
}<|endoftext|>


<commit_before>#![feature(test)]
#![feature(step_by)]
extern crate raw_nbits_vec;
extern crate test;
extern crate num;

mod tests {
    macro_rules! bench_test {
        ($(($m: ident, $nbits: ident, $storage: ident)),*) => {
            mod get {
                $(
                    mod $m {
                        use raw_nbits_vec::*;
                        use test::{self, Bencher};
                        #[bench]
                        fn bench(b: &mut Bencher) {
                            let n = test::black_box(100);
                            let mut vec: NbitsVec<$nbits, $storage> = NbitsVec::with_capacity(n);
                            unsafe { vec.set_len(n) }
                            b.iter(|| {
                                (0..n).fold(0, |_a, i| vec.get(i));
                            })
                        }
                    }
                )*
            }
            mod resize {
                $(
                    mod $m {
                        use raw_nbits_vec::*;
                        use test::{self, Bencher};
                        use num::Zero;
                        #[bench]
                        fn bench(b: &mut Bencher) {
                            let n = test::black_box(100);
                            let mut vec: NbitsVec<$nbits, $storage> = NbitsVec::new();
                            b.iter(|| {
                                for i in (0..n).step_by(10) {
                                    vec.resize(i, $storage::zero());
                                }
                            })
                        }
                    }
                )*
            }
            mod set {
                $(
                    mod $m {
                        use raw_nbits_vec::*;
                        use test::{self, Bencher};
                        #[bench]
                        fn bench(b: &mut Bencher) {
                            b.iter(|| {
                                let n = test::black_box(100);
                                let mut vec: NbitsVec<$nbits, $storage> = NbitsVec::with_capacity(n);
                                unsafe { vec.set_len(n) };
                                for i in 0..n {
                                    vec.set(i, ::num::cast(i).unwrap());
                                }
                            });
                        }
                    }
                )*
            }
            mod new {
                $(
                    mod $m {
                        use raw_nbits_vec::*;
                        use test::{self, Bencher};
                        #[bench]
                        fn bench(b: &mut Bencher) {
                            b.iter(|| {
                                let n = test::black_box(100);
                                let vec: NbitsVec<$nbits, $storage> = NbitsVec::with_capacity(n);
                                vec
                            });
                        }
                    }
                 )*
            }
            mod push {
                $(
                    mod $m {
                        use raw_nbits_vec::*;
                        use test::{self, Bencher};
                        #[bench]
                        fn push(b: &mut Bencher) {
                            let n = test::black_box(100);
                            let mut vec: NbitsVec<$nbits, $storage> = NbitsVec::with_capacity(n);
                            b.iter(|| {
                                for i in 0..n {
                                    vec.push(::num::cast(i).unwrap())
                                }
                            });
                        }
                    }
                 )*
            }
        }
    }
    bench_test! {
        (test_4bits_usize, N4, usize),
        (test_4bits_u64, N4, u64),
        (test_4bits_u32, N4, u32),
        (test_4bits_u16, N4, u16),
        (test_4bits_u8, N4, u8),
        (test_3bits_usize, N3, usize),
        (test_3bits_u64, N3, u64),
        (test_3bits_u32, N3, u32),
        (test_3bits_u16, N3, u16),
        (test_3bits_u8, N3, u8),
        (test_2bits_usize, N2, usize),
        (test_2bits_u64, N2, u64),
        (test_2bits_u32, N2, u32),
        (test_2bits_u16, N2, u16),
        (test_2bits_u8, N2, u8),
        (test_1bits_usize, N1, usize),
        (test_1bits_u64, N1, u64),
        (test_1bits_u32, N1, u32),
        (test_1bits_u16, N1, u16),
        (test_1bits_u8, N1, u8)
    }
}
<commit_msg>Fix bench tests error in large size.<commit_after>#![feature(test)]
#![feature(step_by)]
extern crate raw_nbits_vec;
extern crate test;
extern crate num;

mod tests {
    macro_rules! bench_test {
        ($(($m: ident, $nbits: ident, $storage: ident)),*) => {
            mod get {
                $(
                    mod $m {
                        use raw_nbits_vec::*;
                        use test::{self, Bencher};
                        #[bench]
                        fn bench(b: &mut Bencher) {
                            let n = test::black_box(1000);
                            let mut vec: NbitsVec<$nbits, $storage> = NbitsVec::with_capacity(n);
                            unsafe { vec.set_len(n) }
                            b.iter(|| {
                                (0..n).fold(0, |_a, i| vec.get(i));
                            })
                        }
                    }
                )*
            }
            mod resize {
                $(
                    mod $m {
                        use raw_nbits_vec::*;
                        use test::{self, Bencher};
                        use num::Zero;
                        #[bench]
                        fn bench(b: &mut Bencher) {
                            let n = test::black_box(1000);
                            let mut vec: NbitsVec<$nbits, $storage> = NbitsVec::new();
                            b.iter(|| {
                                for i in (0..n).step_by(10) {
                                    vec.resize(i, $storage::zero());
                                }
                            })
                        }
                    }
                )*
            }
            mod set {
                $(
                    mod $m {
                        use raw_nbits_vec::*;
                        use test::{self, Bencher};
                        #[bench]
                        fn bench(b: &mut Bencher) {
                            b.iter(|| {
                                let n = test::black_box(1000);
                                let mut vec: NbitsVec<$nbits, $storage> = NbitsVec::with_capacity(n);
                                unsafe { vec.set_len(n) };
                                for i in 0..n {
                                    vec.set(i, 0b1010);
                                }
                            });
                        }
                    }
                )*
            }
            mod new {
                $(
                    mod $m {
                        use raw_nbits_vec::*;
                        use test::{self, Bencher};
                        #[bench]
                        fn bench(b: &mut Bencher) {
                            b.iter(|| {
                                let n = test::black_box(1000);
                                let vec: NbitsVec<$nbits, $storage> = NbitsVec::with_capacity(n);
                                vec
                            });
                        }
                    }
                 )*
            }
            mod push {
                $(
                    mod $m {
                        use raw_nbits_vec::*;
                        use test::{self, Bencher};
                        #[bench]
                        fn push(b: &mut Bencher) {
                            let n = test::black_box(1000);
                            let mut vec: NbitsVec<$nbits, $storage> = NbitsVec::with_capacity(n);
                            b.iter(|| {
                                for i in 0..n {
                                    vec.push(1);
                                }
                            });
                        }
                    }
                 )*
            }
        }
    }
    bench_test! {
        (test_4bits_usize, N4, usize),
        (test_4bits_u64, N4, u64),
        (test_4bits_u32, N4, u32),
        (test_4bits_u16, N4, u16),
        (test_4bits_u8, N4, u8),
        (test_3bits_usize, N3, usize),
        (test_3bits_u64, N3, u64),
        (test_3bits_u32, N3, u32),
        (test_3bits_u16, N3, u16),
        (test_3bits_u8, N3, u8),
        (test_2bits_usize, N2, usize),
        (test_2bits_u64, N2, u64),
        (test_2bits_u32, N2, u32),
        (test_2bits_u16, N2, u16),
        (test_2bits_u8, N2, u8),
        (test_1bits_usize, N1, usize),
        (test_1bits_u64, N1, u64),
        (test_1bits_u32, N1, u32),
        (test_1bits_u16, N1, u16),
        (test_1bits_u8, N1, u8)
    }
}
<|endoftext|>


<commit_before>//! Endpoints for room membership.

pub mod ban_user;
pub mod forget_room;
pub mod get_member_events;
pub mod invite_user;
pub mod join_room_by_id;
pub mod join_room_by_id_or_alias;
pub mod joined_members;
pub mod joined_rooms;
pub mod kick_user;
pub mod leave_room;
pub mod unban_user;

use std::collections::BTreeMap;

use ruma_common::thirdparty::Medium;
use ruma_identifiers::{ServerNameBox, ServerSigningKeyId};
use ruma_serde::Outgoing;
use serde::Serialize;

/// A signature of an `m.third_party_invite` token to prove that this user owns a third party
/// identity which has been invited to the room.
#[derive(Clone, Debug, Outgoing, Serialize)]
pub struct ThirdPartySigned<'a> {
    /// The Matrix ID of the user who issued the invite.
    pub sender: &'a str,

    /// The Matrix ID of the invitee.
    pub mxid: &'a str,

    /// The state key of the m.third_party_invite event.
    pub token: &'a str,

    /// A signatures object containing a signature of the entire signed object.
    pub signatures: BTreeMap<ServerNameBox, BTreeMap<ServerSigningKeyId, String>>,
}

/// Represents third party IDs to invite to the room.
#[derive(Clone, Debug, PartialEq, Outgoing, Serialize)]
#[incoming_derive(PartialEq)]
pub struct Invite3pid<'a> {
    /// Hostname and port of identity server to be used for account lookups.
    pub id_server: &'a str,

    /// An access token registered with the identity server.
    pub id_access_token: &'a str,

    /// Type of third party ID.
    pub medium: Medium,

    /// Third party identifier.
    pub address: &'a str,
}
<commit_msg>client-api: Fix missing backticks in doc comment<commit_after>//! Endpoints for room membership.

pub mod ban_user;
pub mod forget_room;
pub mod get_member_events;
pub mod invite_user;
pub mod join_room_by_id;
pub mod join_room_by_id_or_alias;
pub mod joined_members;
pub mod joined_rooms;
pub mod kick_user;
pub mod leave_room;
pub mod unban_user;

use std::collections::BTreeMap;

use ruma_common::thirdparty::Medium;
use ruma_identifiers::{ServerNameBox, ServerSigningKeyId};
use ruma_serde::Outgoing;
use serde::Serialize;

/// A signature of an `m.third_party_invite` token to prove that this user owns a third party
/// identity which has been invited to the room.
#[derive(Clone, Debug, Outgoing, Serialize)]
pub struct ThirdPartySigned<'a> {
    /// The Matrix ID of the user who issued the invite.
    pub sender: &'a str,

    /// The Matrix ID of the invitee.
    pub mxid: &'a str,

    /// The state key of the `m.third_party_invite` event.
    pub token: &'a str,

    /// A signatures object containing a signature of the entire signed object.
    pub signatures: BTreeMap<ServerNameBox, BTreeMap<ServerSigningKeyId, String>>,
}

/// Represents third party IDs to invite to the room.
#[derive(Clone, Debug, PartialEq, Outgoing, Serialize)]
#[incoming_derive(PartialEq)]
pub struct Invite3pid<'a> {
    /// Hostname and port of identity server to be used for account lookups.
    pub id_server: &'a str,

    /// An access token registered with the identity server.
    pub id_access_token: &'a str,

    /// Type of third party ID.
    pub medium: Medium,

    /// Third party identifier.
    pub address: &'a str,
}
<|endoftext|>