<commit_before>// Copyright 2017 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! This crate defines the oracle for Exonum.

/*
#![deny(missing_debug_implementations)]
#![deny(missing_docs)]
*/

extern crate serde;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate exonum;
extern crate router;
extern crate bodyparser;
extern crate iron;

use iron::prelude::*;
use iron::Handler;
use router::Router;

use std::time::SystemTime;

use exonum::blockchain::{Blockchain, Service, ServiceContext, Schema, Transaction, ApiContext};
use exonum::messages::{RawTransaction, FromRaw, Message};
use exonum::encoding::serialize::json::reexport::Value;
use exonum::storage::{Fork, Snapshot, MapIndex, Entry};
use exonum::crypto::{PublicKey, Hash};
use exonum::encoding;
use exonum::helpers::fabric::{ServiceFactory, Context};
use exonum::api::Api;

const SERVICE_ID: u16 = 3;
const TX_TIME_ID: u16 = 1;

// SCHEMA

encoding_struct! {
    struct Time {
        const SIZE = 12;

        field time:     SystemTime  [00 => 12]
    }
}

pub struct TimeSchema<T> {
    view: T,
}

impl<T: AsRef<Snapshot>> TimeSchema<T> {
    pub fn new(view: T) -> Self {
        TimeSchema { view }
    }

    pub fn validators_time(&self) -> MapIndex<&Snapshot, PublicKey, Time> {
        MapIndex::new("exonum_time.validators_time", self.view.as_ref())
    }

    pub fn time(&self) -> Entry<&Snapshot, Time> {
        Entry::new("exonum_time.time", self.view.as_ref())
    }
}

impl<'a> TimeSchema<&'a mut Fork> {
    pub fn validators_time_mut(&mut self) -> MapIndex<&mut Fork, PublicKey, Time> {
        MapIndex::new("exonum_time.validators_time", self.view)
    }

    pub fn time_mut(&mut self) -> Entry<&mut Fork, Time> {
        Entry::new("exonum_time.time", self.view)
    }
}

// TRANSACTION

message! {
    struct TxTime {
        const TYPE = SERVICE_ID;
        const ID = TX_TIME_ID;
        const SIZE = 44;

        field time:     SystemTime  [00 => 12]
        field pub_key:  &PublicKey  [12 => 44]
    }
}

impl Transaction for TxTime {
    fn verify(&self) -> bool {
        self.verify_signature(self.pub_key())
    }

    fn execute(&self, view: &mut Fork) {
        let validator_keys = Schema::new(&view).actual_configuration().validator_keys;

        if !validator_keys.iter().any(|&validator| {
            validator.service_key == *self.pub_key()
        })
        {
            return;
        }

        let mut schema = TimeSchema::new(view);
        match schema.validators_time().get(self.pub_key()) {
            Some(ref storage_time) if storage_time.time() >= self.time() => {
                return;
            }
            _ => {
                schema.validators_time_mut().put(
                    self.pub_key(),
                    Time::new(self.time()),
                )
            }
        }

        let mut validators_time: Vec<SystemTime>;

        {
            let idx = schema.validators_time();
            validators_time = idx.iter()
                .filter_map(|pair| if validator_keys.iter().any(|validator| {
                    validator.service_key == pair.0
                })
                {
                    Some(pair.1.time())
                } else {
                    None
                })
                .collect();
        }

        let max_byzantine_nodes = validator_keys.len() / 3;
        if validators_time.len() <= max_byzantine_nodes {
            return;
        }

        validators_time.sort_by(|a, b| b.cmp(a));

        match schema.time().get() {
            Some(ref current_time)
                if current_time.time() >= validators_time[max_byzantine_nodes] => {
                return;
            }
            _ => {
                schema.time_mut().set(Time::new(
                    validators_time[max_byzantine_nodes],
                ));
            }
        }
    }
}

// API

#[derive(Serialize, Deserialize)]
pub struct TxResponse {
    pub tx_hash: Hash,
}

#[derive(Clone)]
struct TimeApi {
    blockchain: Blockchain,
}

impl TimeApi {
    fn get_current_time(&self, _: &mut Request) -> IronResult<Response> {
        let view = self.blockchain.snapshot();
        let schema = TimeSchema::new(&view);
        let current_time = schema.time().get();
        self.ok_response(&serde_json::to_value(current_time).unwrap())
    }

    fn get_validators_time(&self, _: &mut Request) -> IronResult<Response> {
        let view = self.blockchain.snapshot();
        let schema = TimeSchema::new(&view);
        let idx = schema.validators_time();
        let validators_time: Vec<Time> = idx.values().collect();
        if validators_time.is_empty() {
            self.not_found_response(&serde_json::to_value("Validators time database if empty")
                .unwrap())
        } else {
            self.ok_response(&serde_json::to_value(validators_time).unwrap())
        }
    }

    fn wire_private(&self, router: &mut Router) {
        let self_ = self.clone();
        let get_validators_time = move |req: &mut Request| self_.get_validators_time(req);
        router.get(
            "/validators_time",
            get_validators_time,
            "get_validators_time",
        );
    }
}

impl Api for TimeApi {
    fn wire(&self, router: &mut Router) {
        let self_ = self.clone();
        let get_current_time = move |req: &mut Request| self_.get_current_time(req);
        router.get("/current_time", get_current_time, "get_current_time");
    }
}

// SERVICE DECLARATION

pub struct TimeService;

impl TimeService {
    pub fn new() -> TimeService {
        TimeService {}
    }
}

impl Service for TimeService {
    fn service_name(&self) -> &'static str {
        "exonum_time"
    }

    fn service_id(&self) -> u16 {
        SERVICE_ID
    }

    fn tx_from_raw(&self, raw: RawTransaction) -> Result<Box<Transaction>, encoding::Error> {
        match raw.message_type() {
            TX_TIME_ID => Ok(Box::new(TxTime::from_raw(raw)?)),
            _ => {
                let error =
                    encoding::Error::IncorrectMessageType { message_type: raw.message_type() };
                Err(error)
            }
        }
    }

    fn initialize(&self, _fork: &mut Fork) -> Value {
        Value::Null
    }

    fn handle_commit(&self, context: &mut ServiceContext) {
        let (pub_key, sec_key) = (*context.public_key(), context.secret_key().clone());
        context.add_transaction(Box::new(TxTime::new(SystemTime::now(), &pub_key, &sec_key)));
    }

    fn private_api_handler(&self, ctx: &ApiContext) -> Option<Box<Handler>> {
        let mut router = Router::new();
        let api = TimeApi { blockchain: ctx.blockchain().clone() };
        api.wire_private(&mut router);
        Some(Box::new(router))
    }

    fn public_api_handler(&self, ctx: &ApiContext) -> Option<Box<Handler>> {
        let mut router = Router::new();
        let api = TimeApi { blockchain: ctx.blockchain().clone() };
        api.wire(&mut router);
        Some(Box::new(router))
    }
}

pub struct TimeServiceFactory;

impl ServiceFactory for TimeServiceFactory {
    fn make_service(&mut self, _: &Context) -> Box<Service> {
        Box::new(TimeService::new())
    }
}
<commit_msg>Change crate type. Rewrite 'execute' for transaction<commit_after>// Copyright 2017 The Exonum Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
#![deny(missing_debug_implementations)]
#![deny(missing_docs)]
*/

extern crate serde;
extern crate serde_json;
#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate exonum;
extern crate router;
extern crate bodyparser;
extern crate iron;

use iron::prelude::*;
use iron::Handler;
use router::Router;

use std::time::SystemTime;

use exonum::blockchain::{Blockchain, Service, ServiceContext, Schema, Transaction, ApiContext};
use exonum::messages::{RawTransaction, FromRaw, Message};
use exonum::encoding::serialize::json::reexport::Value;
use exonum::storage::{Fork, Snapshot, MapIndex, Entry};
use exonum::crypto::{PublicKey, Hash};
use exonum::encoding;
use exonum::helpers::fabric::{ServiceFactory, Context};
use exonum::api::Api;

const SERVICE_ID: u16 = 3;
const TX_TIME_ID: u16 = 1;

// SCHEMA

encoding_struct! {
    struct Time {
        const SIZE = 12;

        field time:     SystemTime  [00 => 12]
    }
}

pub struct TimeSchema<T> {
    view: T,
}

impl<T: AsRef<Snapshot>> TimeSchema<T> {
    pub fn new(view: T) -> Self {
        TimeSchema { view }
    }

    pub fn validators_time(&self) -> MapIndex<&Snapshot, PublicKey, Time> {
        MapIndex::new("exonum_time.validators_time", self.view.as_ref())
    }

    pub fn time(&self) -> Entry<&Snapshot, Time> {
        Entry::new("exonum_time.time", self.view.as_ref())
    }
}

impl<'a> TimeSchema<&'a mut Fork> {
    pub fn validators_time_mut(&mut self) -> MapIndex<&mut Fork, PublicKey, Time> {
        MapIndex::new("exonum_time.validators_time", self.view)
    }

    pub fn time_mut(&mut self) -> Entry<&mut Fork, Time> {
        Entry::new("exonum_time.time", self.view)
    }
}

// TRANSACTION

message! {
    struct TxTime {
        const TYPE = SERVICE_ID;
        const ID = TX_TIME_ID;
        const SIZE = 44;

        field time:     SystemTime  [00 => 12]
        field pub_key:  &PublicKey  [12 => 44]
    }
}

impl Transaction for TxTime {
    fn verify(&self) -> bool {
        self.verify_signature(self.pub_key())
    }

    fn execute(&self, view: &mut Fork) {
        let validator_keys = Schema::new(&view).actual_configuration().validator_keys;

        if !validator_keys.iter().any(|&validator| {
            validator.service_key == *self.pub_key()
        })
        {
            return;
        }

        let mut schema = TimeSchema::new(view);
        match schema.validators_time().get(self.pub_key()) {
            Some(ref storage_time) if storage_time.time() >= self.time() => {
                return;
            }
            _ => {
                schema.validators_time_mut().put(
                    self.pub_key(),
                    Time::new(self.time()),
                )
            }
        }

        let mut validators_time: Vec<SystemTime>;

        {
            let idx = schema.validators_time();
            validators_time = idx.iter()
                .filter_map(|pair| if validator_keys.iter().any(|validator| {
                    validator.service_key == pair.0
                })
                {
                    Some(pair.1.time())
                } else {
                    None
                })
                .collect();
        }

        let max_byzantine_nodes = validator_keys.len() / 3;
        if validators_time.len() <= max_byzantine_nodes {
            return;
        }

        validators_time.sort_by(|a, b| b.cmp(a));

        match schema.time().get() {
            Some(ref current_time)
                if current_time.time() >= validators_time[max_byzantine_nodes] => {
                return;
            }
            _ => {
                schema.time_mut().set(Time::new(
                    validators_time[max_byzantine_nodes],
                ));
            }
        }
    }
}

// API

#[derive(Serialize, Deserialize)]
pub struct TxResponse {
    pub tx_hash: Hash,
}

#[derive(Clone)]
struct TimeApi {
    blockchain: Blockchain,
}

impl TimeApi {
    fn get_current_time(&self, _: &mut Request) -> IronResult<Response> {
        let view = self.blockchain.snapshot();
        let schema = TimeSchema::new(&view);
        let current_time = schema.time().get();
        self.ok_response(&serde_json::to_value(current_time).unwrap())
    }

    fn get_validators_time(&self, _: &mut Request) -> IronResult<Response> {
        let view = self.blockchain.snapshot();
        let schema = TimeSchema::new(&view);
        let idx = schema.validators_time();
        let validators_time: Vec<Time> = idx.values().collect();
        if validators_time.is_empty() {
            self.not_found_response(&serde_json::to_value("Validators time database if empty")
                .unwrap())
        } else {
            self.ok_response(&serde_json::to_value(validators_time).unwrap())
        }
    }

    fn wire_private(&self, router: &mut Router) {
        let self_ = self.clone();
        let get_validators_time = move |req: &mut Request| self_.get_validators_time(req);
        router.get(
            "/validators_time",
            get_validators_time,
            "get_validators_time",
        );
    }
}

impl Api for TimeApi {
    fn wire(&self, router: &mut Router) {
        let self_ = self.clone();
        let get_current_time = move |req: &mut Request| self_.get_current_time(req);
        router.get("/current_time", get_current_time, "get_current_time");
    }
}

// SERVICE DECLARATION

pub struct TimeService;

impl TimeService {
    pub fn new() -> TimeService {
        TimeService {}
    }
}

impl Service for TimeService {
    fn service_name(&self) -> &'static str {
        "exonum_time"
    }

    fn service_id(&self) -> u16 {
        SERVICE_ID
    }

    fn tx_from_raw(&self, raw: RawTransaction) -> Result<Box<Transaction>, encoding::Error> {
        match raw.message_type() {
            TX_TIME_ID => Ok(Box::new(TxTime::from_raw(raw)?)),
            _ => {
                let error =
                    encoding::Error::IncorrectMessageType { message_type: raw.message_type() };
                Err(error)
            }
        }
    }

    fn initialize(&self, _fork: &mut Fork) -> Value {
        Value::Null
    }

    fn handle_commit(&self, context: &mut ServiceContext) {
        let (pub_key, sec_key) = (*context.public_key(), context.secret_key().clone());
        context.add_transaction(Box::new(TxTime::new(SystemTime::now(), &pub_key, &sec_key)));
    }

    fn private_api_handler(&self, ctx: &ApiContext) -> Option<Box<Handler>> {
        let mut router = Router::new();
        let api = TimeApi { blockchain: ctx.blockchain().clone() };
        api.wire_private(&mut router);
        Some(Box::new(router))
    }

    fn public_api_handler(&self, ctx: &ApiContext) -> Option<Box<Handler>> {
        let mut router = Router::new();
        let api = TimeApi { blockchain: ctx.blockchain().clone() };
        api.wire(&mut router);
        Some(Box::new(router))
    }
}

pub struct TimeServiceFactory;

impl ServiceFactory for TimeServiceFactory {
    fn make_service(&mut self, _: &Context) -> Box<Service> {
        Box::new(TimeService::new())
    }
}
<|endoftext|>


<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! A (mostly) lock-free concurrent work-stealing deque
//!
//! This module contains an implementation of the Chase-Lev work stealing deque
//! described in "Dynamic Circular Work-Stealing Deque". The implementation is
//! heavily based on the pseudocode found in the paper.
//!
//! This implementation does not want to have the restriction of a garbage
//! collector for reclamation of buffers, and instead it uses a shared pool of
//! buffers. This shared pool is required for correctness in this
//! implementation.
//!
//! The only lock-synchronized portions of this deque are the buffer allocation
//! and deallocation portions. Otherwise all operations are lock-free.
//!
//! # Example
//!
//!     use deque::BufferPool;
//!
//!     let mut pool = BufferPool::new();
//!     let (mut worker, mut stealer) = pool.deque();
//!
//!     // Only the worker may push/pop
//!     worker.push(1);
//!     worker.pop();
//!
//!     // Stealers take data from the other end of the deque
//!     worker.push(1);
//!     stealer.steal();
//!
//!     // Stealers can be cloned to have many stealers stealing in parallel
//!     worker.push(1);
//!     let mut stealer2 = stealer.clone();
//!     stealer2.steal();

// NB: the "buffer pool" strategy is not done for speed, but rather for
//     correctness. For more info, see the comment on `swap_buffer`

// FIXME: all atomic operations in this module use a SeqCst ordering. That is
//      probably overkill

pub use self::Stolen::*;

use std::sync::{Arc, Mutex};
use std::boxed::Box;
use std::vec::Vec;
use std::mem::{forget, size_of, transmute};
use std::ptr;
use std::slice;

use std::sync::atomic::{AtomicIsize, AtomicPtr};
use std::sync::atomic::Ordering::SeqCst;

// Once the queue is less than 1/K full, then it will be downsized. Note that
// the deque requires that this number be less than 2.
static K: isize = 4;

// Minimum number of bits that a buffer size should be. No buffer will resize to
// under this value, and all deques will initially contain a buffer of this
// size.
//
// The size in question is 1 << MIN_BITS
static MIN_BITS: usize = 7;

struct Deque<T: Send> {
    bottom: AtomicIsize,
    top: AtomicIsize,
    array: AtomicPtr<Buffer<T>>,
    pool: BufferPool<T>,
}

/// Worker half of the work-stealing deque. This worker has exclusive access to
/// one side of the deque, and uses `push` and `pop` method to manipulate it.
///
/// There may only be one worker per deque.
pub struct Worker<T: Send> {
    deque: Arc<Deque<T>>,
}

unsafe impl<T: Send> Send for Worker<T> { }

/// The stealing half of the work-stealing deque. Stealers have access to the
/// opposite end of the deque from the worker, and they only have access to the
/// `steal` method.
pub struct Stealer<T: Send> {
    deque: Arc<Deque<T>>,
}

unsafe impl<T: Send> Send for Stealer<T> { }

/// When stealing some data, this is an enumeration of the possible outcomes.
#[derive(PartialEq, Debug)]
pub enum Stolen<T> {
    /// The deque was empty at the time of stealing
    Empty,
    /// The stealer lost the race for stealing data, and a retry may return more
    /// data.
    Abort,
    /// The stealer has successfully stolen some data.
    Data(T),
}

/// The allocation pool for buffers used by work-stealing deques. Right now this
/// structure is used for reclamation of memory after it is no longer in use by
/// deques.
///
/// This data structure is protected by a mutex, but it is rarely used. Deques
/// will only use this structure when allocating a new buffer or deallocating a
/// previous one.
pub struct BufferPool<T: Send> {
    pool: Arc<Mutex<Vec<Box<Buffer<T>>>>>,
}

/// An internal buffer used by the chase-lev deque. This structure is actually
/// implemented as a circular buffer, and is used as the intermediate storage of
/// the data in the deque.
///
/// This type is implemented with *T instead of Vec<T> for two reasons:
///
///   1. There is nothing safe about using this buffer. This easily allows the
///      same value to be read twice in to rust, and there is nothing to
///      prevent this. The usage by the deque must ensure that one of the
///      values is forgotten. Furthermore, we only ever want to manually run
///      destructors for values in this buffer (on drop) because the bounds
///      are defined by the deque it's owned by.
///
///   2. We can certainly avoid bounds checks using *T instead of Vec<T>, although
///      LLVM is probably pretty good at doing this already.
struct Buffer<T: Send> {
    storage: *const T,
    log_size: usize,
}

unsafe impl<T: Send> Send for Buffer<T> { }

impl<T: Send> BufferPool<T> {
    /// Allocates a new buffer pool which in turn can be used to allocate new
    /// deques.
    pub fn new() -> BufferPool<T> {
        BufferPool { pool: Arc::new(Mutex::new(Vec::new())) }
    }

    /// Allocates a new work-stealing deque which will send/receiving memory to
    /// and from this buffer pool.
    pub fn deque(&self) -> (Worker<T>, Stealer<T>) {
        let a = Arc::new(Deque::new(self.clone()));
        let b = a.clone();
        (Worker { deque: a }, Stealer { deque: b })
    }

    fn alloc(&mut self, bits: usize) -> Box<Buffer<T>> {
        unsafe {
            let mut pool = self.pool.lock().unwrap();
            match pool.iter().position(|x| x.size() >= (1 << bits)) {
                Some(i) => pool.remove(i),
                None => Box::new(Buffer::new(bits))
            }
        }
    }

    fn free(&self, buf: Box<Buffer<T>>) {
        let mut pool = self.pool.lock().unwrap();
        match pool.iter().position(|v| v.size() > buf.size()) {
            Some(i) => pool.insert(i, buf),
            None => pool.push(buf),
        }
    }
}

impl<T: Send> Clone for BufferPool<T> {
    fn clone(&self) -> BufferPool<T> { BufferPool { pool: self.pool.clone() } }
}

impl<T: Send> Worker<T> {
    /// Pushes data onto the front of this work queue.
    pub fn push(&self, t: T) {
        unsafe { self.deque.push(t) }
    }
    /// Pops data off the front of the work queue, returning `None` on an empty
    /// queue.
    pub fn pop(&self) -> Option<T> {
        unsafe { self.deque.pop() }
    }

    /// Gets access to the buffer pool that this worker is attached to. This can
    /// be used to create more deques which share the same buffer pool as this
    /// deque.
    pub fn pool<'a>(&'a self) -> &'a BufferPool<T> {
        &self.deque.pool
    }
}

impl<T: Send> Stealer<T> {
    /// Steals work off the end of the queue (opposite of the worker's end)
    pub fn steal(&self) -> Stolen<T> {
        unsafe { self.deque.steal() }
    }

    /// Gets access to the buffer pool that this stealer is attached to. This
    /// can be used to create more deques which share the same buffer pool as
    /// this deque.
    pub fn pool<'a>(&'a self) -> &'a BufferPool<T> {
        &self.deque.pool
    }
}

impl<T: Send> Clone for Stealer<T> {
    fn clone(&self) -> Stealer<T> {
        Stealer { deque: self.deque.clone() }
    }
}

// Almost all of this code can be found directly in the paper so I'm not
// personally going to heavily comment what's going on here.

impl<T: Send> Deque<T> {
    fn new(mut pool: BufferPool<T>) -> Deque<T> {
        let buf = pool.alloc(MIN_BITS);
        Deque {
            bottom: AtomicIsize::new(0),
            top: AtomicIsize::new(0),
            array: AtomicPtr::new(unsafe { transmute(buf) }),
            pool: pool,
        }
    }

    unsafe fn push(&self, data: T) {
        let mut b = self.bottom.load(SeqCst);
        let t = self.top.load(SeqCst);
        let mut a = self.array.load(SeqCst);
        let size = b - t;
        if size >= (*a).size() - 1 {
            // You won't find this code in the chase-lev deque paper. This is
            // alluded to in a small footnote, however. We always free a buffer
            // when growing in order to prevent leaks.
            a = self.swap_buffer(b, a, (*a).resize(b, t, 1));
            b = self.bottom.load(SeqCst);
        }
        (*a).put(b, data);
        self.bottom.store(b + 1, SeqCst);
    }

    unsafe fn pop(&self) -> Option<T> {
        let b = self.bottom.load(SeqCst);
        let a = self.array.load(SeqCst);
        let b = b - 1;
        self.bottom.store(b, SeqCst);
        let t = self.top.load(SeqCst);
        let size = b - t;
        if size < 0 {
            self.bottom.store(t, SeqCst);
            return None;
        }
        let data = (*a).get(b);
        if size > 0 {
            self.maybe_shrink(b, t);
            return Some(data);
        }
        if self.top.compare_and_swap(t, t + 1, SeqCst) == t {
            self.bottom.store(t + 1, SeqCst);
            return Some(data);
        } else {
            self.bottom.store(t + 1, SeqCst);
            forget(data); // someone else stole this value
            return None;
        }
    }

    unsafe fn steal(&self) -> Stolen<T> {
        let t = self.top.load(SeqCst);
        let old = self.array.load(SeqCst);
        let b = self.bottom.load(SeqCst);
        let a = self.array.load(SeqCst);
        let size = b - t;
        if size <= 0 { return Empty }
        if size % (*a).size() == 0 {
            if a == old && t == self.top.load(SeqCst) {
                return Empty
            }
            return Abort
        }
        let data = (*a).get(t);
        if self.top.compare_and_swap(t, t + 1, SeqCst) == t {
            Data(data)
        } else {
            forget(data); // someone else stole this value
            Abort
        }
    }

    unsafe fn maybe_shrink(&self, b: isize, t: isize) {
        let a = self.array.load(SeqCst);
        if b - t < (*a).size() / K && b - t > (1 << MIN_BITS) {
            self.swap_buffer(b, a, (*a).resize(b, t, -1));
        }
    }

    // Helper routine not mentioned in the paper which is used in growing and
    // shrinking buffers to swap in a new buffer into place. As a bit of a
    // recap, the whole point that we need a buffer pool rather than just
    // calling malloc/free directly is that stealers can continue using buffers
    // after this method has called 'free' on it. The continued usage is simply
    // a read followed by a forget, but we must make sure that the memory can
    // continue to be read after we flag this buffer for reclamation.
    unsafe fn swap_buffer(&self, b: isize, old: *mut Buffer<T>,
                          buf: Buffer<T>) -> *mut Buffer<T> {
        let newbuf: *mut Buffer<T> = transmute(Box::new(buf));
        self.array.store(newbuf, SeqCst);
        let ss = (*newbuf).size();
        self.bottom.store(b + ss, SeqCst);
        let t = self.top.load(SeqCst);
        if self.top.compare_and_swap(t, t + ss, SeqCst) != t {
            self.bottom.store(b, SeqCst);
        }
        self.pool.free(transmute(old));
        return newbuf;
    }
}

impl<T: Send> Drop for Deque<T> {
    fn drop(&mut self) {
        let t = self.top.load(SeqCst);
        let b = self.bottom.load(SeqCst);
        let a = self.array.load(SeqCst);
        // Free whatever is leftover in the dequeue, and then move the buffer
        // back into the pool.
        for i in t..b {
            let _: T = unsafe { (*a).get(i) };
        }
        self.pool.free(unsafe { transmute(a) });
    }
}

#[inline]
fn buffer_alloc_size<T>(log_size: usize) -> usize {
    (1 << log_size) * size_of::<T>()
}

impl<T: Send> Buffer<T> {
    unsafe fn new(log_size: usize) -> Buffer<T> {
        let size = buffer_alloc_size::<T>(log_size);

        let mut buffer_vec = Vec::with_capacity(size);
        buffer_vec.set_len(size);
        let buffer: *const T = buffer_vec.get_unchecked(0);
        forget(buffer_vec);

        Buffer {
            storage: buffer as *const T,
            log_size: log_size,
        }
    }

    fn size(&self) -> isize { 1 << self.log_size }

    // Apparently LLVM cannot optimize (foo % (1 << bar)) into this implicitly
    fn mask(&self) -> isize { (1 << self.log_size) - 1 }

    unsafe fn elem(&self, i: isize) -> *const T {
        self.storage.offset(i & self.mask())
    }

    // This does not protect against loading duplicate values of the same cell,
    // nor does this clear out the contents contained within. Hence, this is a
    // very unsafe method which the caller needs to treat specially in case a
    // race is lost.
    unsafe fn get(&self, i: isize) -> T {
        ptr::read(self.elem(i))
    }

    // Unsafe because this unsafely overwrites possibly uninitialized or
    // initialized data.
    unsafe fn put(&self, i: isize, t: T) {
        ptr::write(self.elem(i) as *mut T, t);
    }

    // Again, unsafe because this has incredibly dubious ownership violations.
    // It is assumed that this buffer is immediately dropped.
    unsafe fn resize(&self, b: isize, t: isize, delta: isize) -> Buffer<T> {
        // NB: not entirely obvious, but thanks to 2's complement,
        // casting delta to usize and then adding gives the desired
        // effect.
        let buf = Buffer::new(self.log_size.wrapping_add(delta as usize));
        for i in t..b {
            buf.put(i, self.get(i));
        }
        return buf;
    }
}

impl<T: Send> Drop for Buffer<T> {
    fn drop(&mut self) {
        // It is assumed that all buffers are empty on drop.
        let size = buffer_alloc_size::<T>(self.log_size);
        unsafe {
            //deallocate(self.storage as *mut u8, size, min_align_of::<T>());
            let buffer = slice::from_raw_parts(self.storage, size);
            drop(buffer);
        }
    }
}
<commit_msg>Removed commented deallocate code<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

//! A (mostly) lock-free concurrent work-stealing deque
//!
//! This module contains an implementation of the Chase-Lev work stealing deque
//! described in "Dynamic Circular Work-Stealing Deque". The implementation is
//! heavily based on the pseudocode found in the paper.
//!
//! This implementation does not want to have the restriction of a garbage
//! collector for reclamation of buffers, and instead it uses a shared pool of
//! buffers. This shared pool is required for correctness in this
//! implementation.
//!
//! The only lock-synchronized portions of this deque are the buffer allocation
//! and deallocation portions. Otherwise all operations are lock-free.
//!
//! # Example
//!
//!     use deque::BufferPool;
//!
//!     let mut pool = BufferPool::new();
//!     let (mut worker, mut stealer) = pool.deque();
//!
//!     // Only the worker may push/pop
//!     worker.push(1);
//!     worker.pop();
//!
//!     // Stealers take data from the other end of the deque
//!     worker.push(1);
//!     stealer.steal();
//!
//!     // Stealers can be cloned to have many stealers stealing in parallel
//!     worker.push(1);
//!     let mut stealer2 = stealer.clone();
//!     stealer2.steal();

// NB: the "buffer pool" strategy is not done for speed, but rather for
//     correctness. For more info, see the comment on `swap_buffer`

// FIXME: all atomic operations in this module use a SeqCst ordering. That is
//      probably overkill

pub use self::Stolen::*;

use std::sync::{Arc, Mutex};
use std::boxed::Box;
use std::vec::Vec;
use std::mem::{forget, size_of, transmute};
use std::ptr;
use std::slice;

use std::sync::atomic::{AtomicIsize, AtomicPtr};
use std::sync::atomic::Ordering::SeqCst;

// Once the queue is less than 1/K full, then it will be downsized. Note that
// the deque requires that this number be less than 2.
static K: isize = 4;

// Minimum number of bits that a buffer size should be. No buffer will resize to
// under this value, and all deques will initially contain a buffer of this
// size.
//
// The size in question is 1 << MIN_BITS
static MIN_BITS: usize = 7;

struct Deque<T: Send> {
    bottom: AtomicIsize,
    top: AtomicIsize,
    array: AtomicPtr<Buffer<T>>,
    pool: BufferPool<T>,
}

/// Worker half of the work-stealing deque. This worker has exclusive access to
/// one side of the deque, and uses `push` and `pop` method to manipulate it.
///
/// There may only be one worker per deque.
pub struct Worker<T: Send> {
    deque: Arc<Deque<T>>,
}

unsafe impl<T: Send> Send for Worker<T> { }

/// The stealing half of the work-stealing deque. Stealers have access to the
/// opposite end of the deque from the worker, and they only have access to the
/// `steal` method.
pub struct Stealer<T: Send> {
    deque: Arc<Deque<T>>,
}

unsafe impl<T: Send> Send for Stealer<T> { }

/// When stealing some data, this is an enumeration of the possible outcomes.
#[derive(PartialEq, Debug)]
pub enum Stolen<T> {
    /// The deque was empty at the time of stealing
    Empty,
    /// The stealer lost the race for stealing data, and a retry may return more
    /// data.
    Abort,
    /// The stealer has successfully stolen some data.
    Data(T),
}

/// The allocation pool for buffers used by work-stealing deques. Right now this
/// structure is used for reclamation of memory after it is no longer in use by
/// deques.
///
/// This data structure is protected by a mutex, but it is rarely used. Deques
/// will only use this structure when allocating a new buffer or deallocating a
/// previous one.
pub struct BufferPool<T: Send> {
    pool: Arc<Mutex<Vec<Box<Buffer<T>>>>>,
}

/// An internal buffer used by the chase-lev deque. This structure is actually
/// implemented as a circular buffer, and is used as the intermediate storage of
/// the data in the deque.
///
/// This type is implemented with *T instead of Vec<T> for two reasons:
///
///   1. There is nothing safe about using this buffer. This easily allows the
///      same value to be read twice in to rust, and there is nothing to
///      prevent this. The usage by the deque must ensure that one of the
///      values is forgotten. Furthermore, we only ever want to manually run
///      destructors for values in this buffer (on drop) because the bounds
///      are defined by the deque it's owned by.
///
///   2. We can certainly avoid bounds checks using *T instead of Vec<T>, although
///      LLVM is probably pretty good at doing this already.
struct Buffer<T: Send> {
    storage: *const T,
    log_size: usize,
}

unsafe impl<T: Send> Send for Buffer<T> { }

impl<T: Send> BufferPool<T> {
    /// Allocates a new buffer pool which in turn can be used to allocate new
    /// deques.
    pub fn new() -> BufferPool<T> {
        BufferPool { pool: Arc::new(Mutex::new(Vec::new())) }
    }

    /// Allocates a new work-stealing deque which will send/receiving memory to
    /// and from this buffer pool.
    pub fn deque(&self) -> (Worker<T>, Stealer<T>) {
        let a = Arc::new(Deque::new(self.clone()));
        let b = a.clone();
        (Worker { deque: a }, Stealer { deque: b })
    }

    fn alloc(&mut self, bits: usize) -> Box<Buffer<T>> {
        unsafe {
            let mut pool = self.pool.lock().unwrap();
            match pool.iter().position(|x| x.size() >= (1 << bits)) {
                Some(i) => pool.remove(i),
                None => Box::new(Buffer::new(bits))
            }
        }
    }

    fn free(&self, buf: Box<Buffer<T>>) {
        let mut pool = self.pool.lock().unwrap();
        match pool.iter().position(|v| v.size() > buf.size()) {
            Some(i) => pool.insert(i, buf),
            None => pool.push(buf),
        }
    }
}

impl<T: Send> Clone for BufferPool<T> {
    fn clone(&self) -> BufferPool<T> { BufferPool { pool: self.pool.clone() } }
}

impl<T: Send> Worker<T> {
    /// Pushes data onto the front of this work queue.
    pub fn push(&self, t: T) {
        unsafe { self.deque.push(t) }
    }
    /// Pops data off the front of the work queue, returning `None` on an empty
    /// queue.
    pub fn pop(&self) -> Option<T> {
        unsafe { self.deque.pop() }
    }

    /// Gets access to the buffer pool that this worker is attached to. This can
    /// be used to create more deques which share the same buffer pool as this
    /// deque.
    pub fn pool<'a>(&'a self) -> &'a BufferPool<T> {
        &self.deque.pool
    }
}

impl<T: Send> Stealer<T> {
    /// Steals work off the end of the queue (opposite of the worker's end)
    pub fn steal(&self) -> Stolen<T> {
        unsafe { self.deque.steal() }
    }

    /// Gets access to the buffer pool that this stealer is attached to. This
    /// can be used to create more deques which share the same buffer pool as
    /// this deque.
    pub fn pool<'a>(&'a self) -> &'a BufferPool<T> {
        &self.deque.pool
    }
}

impl<T: Send> Clone for Stealer<T> {
    fn clone(&self) -> Stealer<T> {
        Stealer { deque: self.deque.clone() }
    }
}

// Almost all of this code can be found directly in the paper so I'm not
// personally going to heavily comment what's going on here.

impl<T: Send> Deque<T> {
    fn new(mut pool: BufferPool<T>) -> Deque<T> {
        let buf = pool.alloc(MIN_BITS);
        Deque {
            bottom: AtomicIsize::new(0),
            top: AtomicIsize::new(0),
            array: AtomicPtr::new(unsafe { transmute(buf) }),
            pool: pool,
        }
    }

    unsafe fn push(&self, data: T) {
        let mut b = self.bottom.load(SeqCst);
        let t = self.top.load(SeqCst);
        let mut a = self.array.load(SeqCst);
        let size = b - t;
        if size >= (*a).size() - 1 {
            // You won't find this code in the chase-lev deque paper. This is
            // alluded to in a small footnote, however. We always free a buffer
            // when growing in order to prevent leaks.
            a = self.swap_buffer(b, a, (*a).resize(b, t, 1));
            b = self.bottom.load(SeqCst);
        }
        (*a).put(b, data);
        self.bottom.store(b + 1, SeqCst);
    }

    unsafe fn pop(&self) -> Option<T> {
        let b = self.bottom.load(SeqCst);
        let a = self.array.load(SeqCst);
        let b = b - 1;
        self.bottom.store(b, SeqCst);
        let t = self.top.load(SeqCst);
        let size = b - t;
        if size < 0 {
            self.bottom.store(t, SeqCst);
            return None;
        }
        let data = (*a).get(b);
        if size > 0 {
            self.maybe_shrink(b, t);
            return Some(data);
        }
        if self.top.compare_and_swap(t, t + 1, SeqCst) == t {
            self.bottom.store(t + 1, SeqCst);
            return Some(data);
        } else {
            self.bottom.store(t + 1, SeqCst);
            forget(data); // someone else stole this value
            return None;
        }
    }

    unsafe fn steal(&self) -> Stolen<T> {
        let t = self.top.load(SeqCst);
        let old = self.array.load(SeqCst);
        let b = self.bottom.load(SeqCst);
        let a = self.array.load(SeqCst);
        let size = b - t;
        if size <= 0 { return Empty }
        if size % (*a).size() == 0 {
            if a == old && t == self.top.load(SeqCst) {
                return Empty
            }
            return Abort
        }
        let data = (*a).get(t);
        if self.top.compare_and_swap(t, t + 1, SeqCst) == t {
            Data(data)
        } else {
            forget(data); // someone else stole this value
            Abort
        }
    }

    unsafe fn maybe_shrink(&self, b: isize, t: isize) {
        let a = self.array.load(SeqCst);
        if b - t < (*a).size() / K && b - t > (1 << MIN_BITS) {
            self.swap_buffer(b, a, (*a).resize(b, t, -1));
        }
    }

    // Helper routine not mentioned in the paper which is used in growing and
    // shrinking buffers to swap in a new buffer into place. As a bit of a
    // recap, the whole point that we need a buffer pool rather than just
    // calling malloc/free directly is that stealers can continue using buffers
    // after this method has called 'free' on it. The continued usage is simply
    // a read followed by a forget, but we must make sure that the memory can
    // continue to be read after we flag this buffer for reclamation.
    unsafe fn swap_buffer(&self, b: isize, old: *mut Buffer<T>,
                          buf: Buffer<T>) -> *mut Buffer<T> {
        let newbuf: *mut Buffer<T> = transmute(Box::new(buf));
        self.array.store(newbuf, SeqCst);
        let ss = (*newbuf).size();
        self.bottom.store(b + ss, SeqCst);
        let t = self.top.load(SeqCst);
        if self.top.compare_and_swap(t, t + ss, SeqCst) != t {
            self.bottom.store(b, SeqCst);
        }
        self.pool.free(transmute(old));
        return newbuf;
    }
}

impl<T: Send> Drop for Deque<T> {
    fn drop(&mut self) {
        let t = self.top.load(SeqCst);
        let b = self.bottom.load(SeqCst);
        let a = self.array.load(SeqCst);
        // Free whatever is leftover in the dequeue, and then move the buffer
        // back into the pool.
        for i in t..b {
            let _: T = unsafe { (*a).get(i) };
        }
        self.pool.free(unsafe { transmute(a) });
    }
}

#[inline]
fn buffer_alloc_size<T>(log_size: usize) -> usize {
    (1 << log_size) * size_of::<T>()
}

impl<T: Send> Buffer<T> {
    unsafe fn new(log_size: usize) -> Buffer<T> {
        let size = buffer_alloc_size::<T>(log_size);

        let mut buffer_vec = Vec::with_capacity(size);
        buffer_vec.set_len(size);
        let buffer: *const T = buffer_vec.get_unchecked(0);
        forget(buffer_vec);

        Buffer {
            storage: buffer as *const T,
            log_size: log_size,
        }
    }

    fn size(&self) -> isize { 1 << self.log_size }

    // Apparently LLVM cannot optimize (foo % (1 << bar)) into this implicitly
    fn mask(&self) -> isize { (1 << self.log_size) - 1 }

    unsafe fn elem(&self, i: isize) -> *const T {
        self.storage.offset(i & self.mask())
    }

    // This does not protect against loading duplicate values of the same cell,
    // nor does this clear out the contents contained within. Hence, this is a
    // very unsafe method which the caller needs to treat specially in case a
    // race is lost.
    unsafe fn get(&self, i: isize) -> T {
        ptr::read(self.elem(i))
    }

    // Unsafe because this unsafely overwrites possibly uninitialized or
    // initialized data.
    unsafe fn put(&self, i: isize, t: T) {
        ptr::write(self.elem(i) as *mut T, t);
    }

    // Again, unsafe because this has incredibly dubious ownership violations.
    // It is assumed that this buffer is immediately dropped.
    unsafe fn resize(&self, b: isize, t: isize, delta: isize) -> Buffer<T> {
        // NB: not entirely obvious, but thanks to 2's complement,
        // casting delta to usize and then adding gives the desired
        // effect.
        let buf = Buffer::new(self.log_size.wrapping_add(delta as usize));
        for i in t..b {
            buf.put(i, self.get(i));
        }
        return buf;
    }
}

impl<T: Send> Drop for Buffer<T> {
    fn drop(&mut self) {
        // It is assumed that all buffers are empty on drop.
        let size = buffer_alloc_size::<T>(self.log_size);
        unsafe {
            let buffer = slice::from_raw_parts(self.storage, size);
            drop(buffer);
        }
    }
}
<|endoftext|>


<commit_before>//! cpython-json converts native Python objects (via cpython `PyObject`s) to serde_json `Value`s
//! and back again.
//!
//! It was developed for [crowbar](https://crates.io/crates/crowbar), a shim for writing native
//! Rust code in [AWS Lambda](https://aws.amazon.com/lambda/) using the Python execution
//! environment. Because Lambda is a JSON-in, JSON-out API, all objects passing through crowbar are
//! JSON serializable.
//!
//! Values are not actually converted to JSON as part of this process; serializing and
//! deserializing JSON is slow. Instead, `PyObject`s are natively casted to a reasonably matching
//! type of `Value`, and `PyObject`s are created directly from pattern-matching `Value`s.
//!
//! Data types that the Python `json` module can convert to JSON can be converted with this. (If
//! you find something that works in the Python `json` module that doesn't work in `cpython-json`,
//! please [file an issue](https://github.com/ilianaw/rust-cpython-json/issues) with your test
//! case.)
//!
//! ## Usage
//!
//! Add `cpython-json` to your `Cargo.toml` alongside `cpython`:
//!
//! ```toml
//! [dependencies]
//! cpython = "*"
//! cpython-json = "0.1"
//! ```
//!
//! Similar to `cpython`, Python 3 is used by default. To use Python 2:
//!
//! ```toml
//! [dependencies.cpython-json]
//! version = "0.1"
//! default-features = false
//! features = ["python27-sys"]
//! ```
//!
//! Example code which reads `sys.hexversion` and bitbangs something resembling the version string
//! (release level and serial not included for brevity):
//!
//! ```rust
//! extern crate cpython;
//! extern crate cpython_json;
//! extern crate serde_json;
//!
//! use cpython::*;
//! use cpython_json::to_json;
//! use serde_json::Value;
//!
//! fn main() {
//!     let gil = Python::acquire_gil();
//!     println!("{}", version(gil.python()).expect("failed to get Python version"));
//! }
//!
//! fn version(py: Python) -> PyResult<String> {
//!     let sys = py.import("sys")?;
//!     let py_hexversion = sys.get(py, "hexversion")?;
//!     let hexversion = match to_json(py, &py_hexversion).map_err(|e| e.to_pyerr(py))? {
//!         Value::U64(x) => x,
//!         Value::I64(x) => x as u64,
//!         _ => panic!("hexversion is not an int"),
//!     };
//!
//!     Ok(format!("{}.{}.{}", hexversion >> 24, hexversion >> 16 & 0xff, hexversion >> 8 & 0xff))
//! }
//! ```

extern crate cpython;
#[macro_use]
extern crate quick_error;
extern crate serde_json;

use cpython::*;
use serde_json::value::Value;
use std::collections::BTreeMap;
use std::convert::From;

quick_error! {
    /// The `Error` enum returned by this crate.
    ///
    /// Most of the time you will just want a `PyErr`, and `to_pyerr` will convert to one for you.
    #[derive(Debug)]
    pub enum JsonError {
        /// A Python exception occurred.
        PythonError(err: PyErr) {
            from()
        }
        /// The PyObject passed could not be converted to a `serde_json::Value` object.
        ///
        /// The error tuple is the type name, then the `repr` of the object.
        ///
        /// This usually means that Python's `json` module wouldn't be able to serialize the object
        /// either. If the Python `json` module works but `cpython-json` doesn't, please [file an
        /// issue] (https://github.com/ilianaw/rust-cpython-json/issues) with your test case.
        TypeError(type_name: String, repr: PyResult<String>) {}
        /// A `dict` key was not a string object, and so it couldn't be converted to an object. JSON
        /// object keys must always be strings.
        DictKeyNotString(obj: PyObject) {}
    }
}

impl JsonError {
    /// Convenience method for converting a `JsonError` to a `PyErr`.
    pub fn to_pyerr(&self, py: Python) -> PyErr {
        match *self {
            JsonError::PythonError(ref err) => err.clone_ref(py),
            JsonError::TypeError(_, ref repr) => {
                match *repr {
                    Ok(ref repr) => {
                        PyErr {
                            ptype: cpython::exc::TypeError::type_object(py).into_object(),
                            pvalue: Some(PyUnicode::new(py,
                                                        &format!("{} is not JSON serializable",
                                                                 repr))
                                .into_object()),
                            ptraceback: None,
                        }
                    }
                    Err(ref err) => err.clone_ref(py),
                }
            }
            JsonError::DictKeyNotString(_) => {
                PyErr {
                    ptype: cpython::exc::TypeError::type_object(py).into_object(),
                    pvalue: Some(PyString::new(py, "keys must be a string").into_object()),
                    ptraceback: None,
                }
            }
        }
    }
}

/// Convert from a `cpython::PyObject` to a `serde_json::Value`.
pub fn to_json(py: Python, obj: &PyObject) -> Result<Value, JsonError> {
    macro_rules! cast {
        ($t:ty, $f:expr) => {
            if let Ok(val) = obj.cast_as::<$t>(py) {
                return $f(val);
            }
        }
    }

    macro_rules! extract {
        ($t:ty) => {
            if let Ok(val) = obj.extract::<$t>(py) {
                return Ok(serde_json::value::to_value(val));
            }
        }
    }

    cast!(PyDict, |x: &PyDict| {
        let mut map = BTreeMap::new();
        for (key_obj, value) in x.items(py) {
            let key = if key_obj == py.None() {
                Ok("null".to_string())
            } else if let Ok(val) = key_obj.extract::<bool>(py) {
                Ok(if val {
                    "true".to_string()
                } else {
                    "false".to_string()
                })
            } else if let Ok(val) = key_obj.str(py) {
                Ok(val.to_string(py)?.into_owned())
            } else {
                Err(JsonError::DictKeyNotString(key_obj))
            };
            map.insert(key?, to_json(py, &value)?);
        }
        Ok(Value::Object(map))
    });

    cast!(PyList,
          |x: &PyList| Ok(Value::Array(try!(x.iter(py).map(|x| to_json(py, &x)).collect()))));
    cast!(PyTuple,
          |x: &PyTuple| Ok(Value::Array(try!(x.iter(py).map(|x| to_json(py, x)).collect()))));

    extract!(String);
    extract!(bool);

    cast!(PyFloat, |x: &PyFloat| Ok(Value::F64(x.value(py))));

    extract!(u64);
    extract!(i64);

    if obj == &py.None() {
        return Ok(Value::Null);
    }

    // At this point we can't cast it, set up the error object
    let repr = obj.repr(py).and_then(|x| x.to_string(py).and_then(|y| Ok(y.into_owned())));
    Err(JsonError::TypeError(obj.get_type(py).name(py).into_owned(), repr))
}

/// Convert from a `serde_json::Value` to a `cpython::PyObject`.
pub fn from_json(py: Python, json: Value) -> Result<PyObject, JsonError> {
    macro_rules! obj {
        ($x:ident) => {
            Ok($x.into_py_object(py).into_object())
        }
    }

    match json {
        Value::I64(x) => obj!(x),
        Value::U64(x) => obj!(x),
        Value::F64(x) => obj!(x),
        Value::String(x) => Ok(PyUnicode::new(py, &x).into_object()),
        Value::Bool(x) => obj!(x),
        Value::Array(vec) => {
            let mut elements = Vec::new();
            for item in vec {
                elements.push(from_json(py, item)?);
            }
            Ok(PyList::new(py, &elements[..]).into_object())
        }
        Value::Object(map) => {
            let dict = PyDict::new(py);
            for (key, value) in map {
                dict.set_item(py, key, from_json(py, value)?)?;
            }
            Ok(dict.into_object())
        }
        Value::Null => Ok(py.None()),
    }
}

#[cfg(test)]
mod tests {
    use cpython::*;
    use cpython::exc::TypeError;
    use serde_json;
    use std::fs::File;
    use std::io::{BufRead, BufReader};
    use super::*;

    #[test]
    fn test_json() {
        let gil = Python::acquire_gil();
        let py = gil.python();

        // use operator.__eq__ to determine equality of PyObjects
        let operator = py.import("operator").unwrap();

        for line in BufReader::new(&File::open("testdata/to_json.txt").unwrap()).lines() {
            let line = line.unwrap();
            if line == "" || line.starts_with("#") {
                continue;
            }
            let mut line: Vec<_> = line.split("\t").collect();

            if line.len() == 2 {
                line.push("");
            }
            assert_eq!(line.len(), 3);

            // test to_json
            let json = serde_json::from_str(line[1]).unwrap();
            if !line[2].contains("skip_to") {
                let obj = py.eval(line[0], None, None).unwrap();
                assert_eq!(to_json(py, &obj).unwrap(),
                           json,
                           "to_json: {} != {}",
                           line[0],
                           line[1]);
            }

            // test from_json
            if !line[2].contains("skip_from") {
                let obj = py.eval(line[0], None, None).unwrap();
                let eq = operator.call(py,
                          "__eq__",
                          PyTuple::new(py, &[from_json(py, json).unwrap(), obj]),
                          None)
                    .unwrap();
                assert!(eq.extract::<bool>(py).unwrap(),
                        "from_json: {} != {}",
                        line[0],
                        line[1]);
            }
        }
    }

    #[test]
    fn test_unserializable() {
        let gil = Python::acquire_gil();
        let py = gil.python();

        // datetime.datetime objects are not JSON serializable
        let datetime = py.import("datetime").unwrap();
        let min = datetime.get(py, "datetime").unwrap().getattr(py, "min").unwrap();
        let err = to_json(py, &min).unwrap_err().to_pyerr(py);
        assert_eq!(err.ptype, TypeError::type_object(py).into_object());
        assert_eq!(err.pvalue.unwrap().to_string(),
                   "datetime.datetime(1, 1, 1, 0, 0) is not JSON serializable");
        assert_eq!(err.ptraceback, None);
    }

    #[test]
    /// The compiler already makes sure that JsonError can derive Debug, but kcov doesn't know
    /// that. This makes JsonError's #[derive(Debug)] show as covered code.
    fn test_jsonerror_debug() {
        let gil = Python::acquire_gil();
        let py = gil.python();
        println!("{:?}", JsonError::DictKeyNotString(py.None()));
    }

    #[test]
    fn test_to_pyerr_python() {
        let gil = Python::acquire_gil();
        let py = gil.python();

        let err = JsonError::PythonError(JsonError::DictKeyNotString(py.None()).to_pyerr(py))
            .to_pyerr(py);
        assert_eq!(err.ptype, TypeError::type_object(py).into_object());
        assert_eq!(err.pvalue.unwrap().to_string(), "keys must be a string");
        assert_eq!(err.ptraceback, None);
    }

    #[test]
    fn test_to_pyerr_type_failed_repr() {
        let gil = Python::acquire_gil();
        let py = gil.python();

        let err = JsonError::TypeError("datetime.datetime".to_string(),
                                       Err(JsonError::DictKeyNotString(py.None()).to_pyerr(py)))
            .to_pyerr(py);
        assert_eq!(err.ptype, TypeError::type_object(py).into_object());
        assert_eq!(err.pvalue.unwrap().to_string(), "keys must be a string");
        assert_eq!(err.ptraceback, None);
    }

    #[test]
    fn test_to_pyerr_dkns() {
        let gil = Python::acquire_gil();
        let py = gil.python();

        let err = JsonError::DictKeyNotString(py.None()).to_pyerr(py);
        assert_eq!(err.ptype, TypeError::type_object(py).into_object());
        assert_eq!(err.pvalue.unwrap().to_string(), "keys must be a string");
        assert_eq!(err.ptraceback, None);
    }
}
<commit_msg>rustfmt<commit_after>//! cpython-json converts native Python objects (via cpython `PyObject`s) to serde_json `Value`s
//! and back again.
//!
//! It was developed for [crowbar](https://crates.io/crates/crowbar), a shim for writing native
//! Rust code in [AWS Lambda](https://aws.amazon.com/lambda/) using the Python execution
//! environment. Because Lambda is a JSON-in, JSON-out API, all objects passing through crowbar are
//! JSON serializable.
//!
//! Values are not actually converted to JSON as part of this process; serializing and
//! deserializing JSON is slow. Instead, `PyObject`s are natively casted to a reasonably matching
//! type of `Value`, and `PyObject`s are created directly from pattern-matching `Value`s.
//!
//! Data types that the Python `json` module can convert to JSON can be converted with this. (If
//! you find something that works in the Python `json` module that doesn't work in `cpython-json`,
//! please [file an issue](https://github.com/ilianaw/rust-cpython-json/issues) with your test
//! case.)
//!
//! ## Usage
//!
//! Add `cpython-json` to your `Cargo.toml` alongside `cpython`:
//!
//! ```toml
//! [dependencies]
//! cpython = "*"
//! cpython-json = "0.1"
//! ```
//!
//! Similar to `cpython`, Python 3 is used by default. To use Python 2:
//!
//! ```toml
//! [dependencies.cpython-json]
//! version = "0.1"
//! default-features = false
//! features = ["python27-sys"]
//! ```
//!
//! Example code which reads `sys.hexversion` and bitbangs something resembling the version string
//! (release level and serial not included for brevity):
//!
//! ```rust
//! extern crate cpython;
//! extern crate cpython_json;
//! extern crate serde_json;
//!
//! use cpython::*;
//! use cpython_json::to_json;
//! use serde_json::Value;
//!
//! fn main() {
//!     let gil = Python::acquire_gil();
//!     println!("{}", version(gil.python()).expect("failed to get Python version"));
//! }
//!
//! fn version(py: Python) -> PyResult<String> {
//!     let sys = py.import("sys")?;
//!     let py_hexversion = sys.get(py, "hexversion")?;
//!     let hexversion = match to_json(py, &py_hexversion).map_err(|e| e.to_pyerr(py))? {
//!         Value::U64(x) => x,
//!         Value::I64(x) => x as u64,
//!         _ => panic!("hexversion is not an int"),
//!     };
//!
//!     Ok(format!("{}.{}.{}", hexversion >> 24, hexversion >> 16 & 0xff, hexversion >> 8 & 0xff))
//! }
//! ```

extern crate cpython;
#[macro_use]
extern crate quick_error;
extern crate serde_json;

use cpython::*;
use serde_json::value::Value;
use std::collections::BTreeMap;
use std::convert::From;

quick_error! {
    /// The `Error` enum returned by this crate.
    ///
    /// Most of the time you will just want a `PyErr`, and `to_pyerr` will convert to one for you.
    #[derive(Debug)]
    pub enum JsonError {
        /// A Python exception occurred.
        PythonError(err: PyErr) {
            from()
        }
        /// The PyObject passed could not be converted to a `serde_json::Value` object.
        ///
        /// The error tuple is the type name, then the `repr` of the object.
        ///
        /// This usually means that Python's `json` module wouldn't be able to serialize the object
        /// either. If the Python `json` module works but `cpython-json` doesn't, please [file an
        /// issue] (https://github.com/ilianaw/rust-cpython-json/issues) with your test case.
        TypeError(type_name: String, repr: PyResult<String>) {}
        /// A `dict` key was not a string object, and so it couldn't be converted to an object. JSON
        /// object keys must always be strings.
        DictKeyNotString(obj: PyObject) {}
    }
}

impl JsonError {
    /// Convenience method for converting a `JsonError` to a `PyErr`.
    pub fn to_pyerr(&self, py: Python) -> PyErr {
        match *self {
            JsonError::PythonError(ref err) => err.clone_ref(py),
            JsonError::TypeError(_, ref repr) => {
                match *repr {
                    Ok(ref repr) => {
                        PyErr {
                            ptype: cpython::exc::TypeError::type_object(py).into_object(),
                            pvalue: Some(PyUnicode::new(py,
                                                        &format!("{} is not JSON serializable",
                                                                repr))
                                                 .into_object()),
                            ptraceback: None,
                        }
                    }
                    Err(ref err) => err.clone_ref(py),
                }
            }
            JsonError::DictKeyNotString(_) => {
                PyErr {
                    ptype: cpython::exc::TypeError::type_object(py).into_object(),
                    pvalue: Some(PyString::new(py, "keys must be a string").into_object()),
                    ptraceback: None,
                }
            }
        }
    }
}

/// Convert from a `cpython::PyObject` to a `serde_json::Value`.
pub fn to_json(py: Python, obj: &PyObject) -> Result<Value, JsonError> {
    macro_rules! cast {
        ($t:ty, $f:expr) => {
            if let Ok(val) = obj.cast_as::<$t>(py) {
                return $f(val);
            }
        }
    }

    macro_rules! extract {
        ($t:ty) => {
            if let Ok(val) = obj.extract::<$t>(py) {
                return Ok(serde_json::value::to_value(val));
            }
        }
    }

    cast!(PyDict, |x: &PyDict| {
        let mut map = BTreeMap::new();
        for (key_obj, value) in x.items(py) {
            let key = if key_obj == py.None() {
                Ok("null".to_string())
            } else if let Ok(val) = key_obj.extract::<bool>(py) {
                Ok(if val {
                       "true".to_string()
                   } else {
                       "false".to_string()
                   })
            } else if let Ok(val) = key_obj.str(py) {
                Ok(val.to_string(py)?.into_owned())
            } else {
                Err(JsonError::DictKeyNotString(key_obj))
            };
            map.insert(key?, to_json(py, &value)?);
        }
        Ok(Value::Object(map))
    });

    cast!(PyList,
          |x: &PyList| Ok(Value::Array(try!(x.iter(py).map(|x| to_json(py, &x)).collect()))));
    cast!(PyTuple,
          |x: &PyTuple| Ok(Value::Array(try!(x.iter(py).map(|x| to_json(py, x)).collect()))));

    extract!(String);
    extract!(bool);

    cast!(PyFloat, |x: &PyFloat| Ok(Value::F64(x.value(py))));

    extract!(u64);
    extract!(i64);

    if obj == &py.None() {
        return Ok(Value::Null);
    }

    // At this point we can't cast it, set up the error object
    let repr = obj.repr(py)
        .and_then(|x| x.to_string(py).and_then(|y| Ok(y.into_owned())));
    Err(JsonError::TypeError(obj.get_type(py).name(py).into_owned(), repr))
}

/// Convert from a `serde_json::Value` to a `cpython::PyObject`.
pub fn from_json(py: Python, json: Value) -> Result<PyObject, JsonError> {
    macro_rules! obj {
        ($x:ident) => {
            Ok($x.into_py_object(py).into_object())
        }
    }

    match json {
        Value::I64(x) => obj!(x),
        Value::U64(x) => obj!(x),
        Value::F64(x) => obj!(x),
        Value::String(x) => Ok(PyUnicode::new(py, &x).into_object()),
        Value::Bool(x) => obj!(x),
        Value::Array(vec) => {
            let mut elements = Vec::new();
            for item in vec {
                elements.push(from_json(py, item)?);
            }
            Ok(PyList::new(py, &elements[..]).into_object())
        }
        Value::Object(map) => {
            let dict = PyDict::new(py);
            for (key, value) in map {
                dict.set_item(py, key, from_json(py, value)?)?;
            }
            Ok(dict.into_object())
        }
        Value::Null => Ok(py.None()),
    }
}

#[cfg(test)]
mod tests {
    use cpython::*;
    use cpython::exc::TypeError;
    use serde_json;
    use std::fs::File;
    use std::io::{BufRead, BufReader};
    use super::*;

    #[test]
    fn test_json() {
        let gil = Python::acquire_gil();
        let py = gil.python();

        // use operator.__eq__ to determine equality of PyObjects
        let operator = py.import("operator").unwrap();

        for line in BufReader::new(&File::open("testdata/to_json.txt").unwrap()).lines() {
            let line = line.unwrap();
            if line == "" || line.starts_with("#") {
                continue;
            }
            let mut line: Vec<_> = line.split("\t").collect();

            if line.len() == 2 {
                line.push("");
            }
            assert_eq!(line.len(), 3);

            // test to_json
            let json = serde_json::from_str(line[1]).unwrap();
            if !line[2].contains("skip_to") {
                let obj = py.eval(line[0], None, None).unwrap();
                assert_eq!(to_json(py, &obj).unwrap(),
                           json,
                           "to_json: {} != {}",
                           line[0],
                           line[1]);
            }

            // test from_json
            if !line[2].contains("skip_from") {
                let obj = py.eval(line[0], None, None).unwrap();
                let eq = operator
                    .call(py,
                          "__eq__",
                          PyTuple::new(py, &[from_json(py, json).unwrap(), obj]),
                          None)
                    .unwrap();
                assert!(eq.extract::<bool>(py).unwrap(),
                        "from_json: {} != {}",
                        line[0],
                        line[1]);
            }
        }
    }

    #[test]
    fn test_unserializable() {
        let gil = Python::acquire_gil();
        let py = gil.python();

        // datetime.datetime objects are not JSON serializable
        let datetime = py.import("datetime").unwrap();
        let min = datetime
            .get(py, "datetime")
            .unwrap()
            .getattr(py, "min")
            .unwrap();
        let err = to_json(py, &min).unwrap_err().to_pyerr(py);
        assert_eq!(err.ptype, TypeError::type_object(py).into_object());
        assert_eq!(err.pvalue.unwrap().to_string(),
                   "datetime.datetime(1, 1, 1, 0, 0) is not JSON serializable");
        assert_eq!(err.ptraceback, None);
    }

    #[test]
    /// The compiler already makes sure that JsonError can derive Debug, but kcov doesn't know
    /// that. This makes JsonError's #[derive(Debug)] show as covered code.
    fn test_jsonerror_debug() {
        let gil = Python::acquire_gil();
        let py = gil.python();
        println!("{:?}", JsonError::DictKeyNotString(py.None()));
    }

    #[test]
    fn test_to_pyerr_python() {
        let gil = Python::acquire_gil();
        let py = gil.python();

        let err = JsonError::PythonError(JsonError::DictKeyNotString(py.None()).to_pyerr(py))
            .to_pyerr(py);
        assert_eq!(err.ptype, TypeError::type_object(py).into_object());
        assert_eq!(err.pvalue.unwrap().to_string(), "keys must be a string");
        assert_eq!(err.ptraceback, None);
    }

    #[test]
    fn test_to_pyerr_type_failed_repr() {
        let gil = Python::acquire_gil();
        let py = gil.python();

        let err = JsonError::TypeError("datetime.datetime".to_string(),
                                       Err(JsonError::DictKeyNotString(py.None()).to_pyerr(py)))
                .to_pyerr(py);
        assert_eq!(err.ptype, TypeError::type_object(py).into_object());
        assert_eq!(err.pvalue.unwrap().to_string(), "keys must be a string");
        assert_eq!(err.ptraceback, None);
    }

    #[test]
    fn test_to_pyerr_dkns() {
        let gil = Python::acquire_gil();
        let py = gil.python();

        let err = JsonError::DictKeyNotString(py.None()).to_pyerr(py);
        assert_eq!(err.ptype, TypeError::type_object(py).into_object());
        assert_eq!(err.pvalue.unwrap().to_string(), "keys must be a string");
        assert_eq!(err.ptraceback, None);
    }
}
<|endoftext|>


<commit_before>#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts)]
#![warn(rust_2018_idioms)]
#![warn(unused_lifetimes)]
#![warn(unreachable_pub)]

extern crate snap;
#[macro_use]
extern crate rustc_middle;
extern crate rustc_ast;
extern crate rustc_codegen_ssa;
extern crate rustc_data_structures;
extern crate rustc_errors;
extern crate rustc_fs_util;
extern crate rustc_hir;
extern crate rustc_incremental;
extern crate rustc_index;
extern crate rustc_session;
extern crate rustc_span;
extern crate rustc_target;

// This prevents duplicating functions and statics that are already part of the host rustc process.
#[allow(unused_extern_crates)]
extern crate rustc_driver;

use std::any::Any;

use rustc_codegen_ssa::traits::CodegenBackend;
use rustc_codegen_ssa::CodegenResults;
use rustc_errors::ErrorReported;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
use rustc_middle::ty::query::Providers;
use rustc_session::config::OutputFilenames;
use rustc_session::Session;

use cranelift_codegen::isa::TargetIsa;
use cranelift_codegen::settings::{self, Configurable};

pub use crate::config::*;
use crate::prelude::*;

mod abi;
mod allocator;
mod analyze;
mod archive;
mod backend;
mod base;
mod cast;
mod codegen_i128;
mod common;
mod compiler_builtins;
mod config;
mod constant;
mod debuginfo;
mod discriminant;
mod driver;
mod inline_asm;
mod intrinsics;
mod linkage;
mod main_shim;
mod metadata;
mod num;
mod optimize;
mod pointer;
mod pretty_clif;
mod toolchain;
mod trap;
mod unsize;
mod value_and_place;
mod vtable;

mod prelude {
    pub(crate) use std::convert::{TryFrom, TryInto};

    pub(crate) use rustc_span::Span;

    pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE};
    pub(crate) use rustc_middle::bug;
    pub(crate) use rustc_middle::mir::{self, *};
    pub(crate) use rustc_middle::ty::layout::{self, TyAndLayout};
    pub(crate) use rustc_middle::ty::{
        self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut,
        TypeFoldable, UintTy,
    };
    pub(crate) use rustc_target::abi::{Abi, LayoutOf, Scalar, Size, VariantIdx};

    pub(crate) use rustc_data_structures::fx::FxHashMap;

    pub(crate) use rustc_index::vec::Idx;

    pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
    pub(crate) use cranelift_codegen::ir::function::Function;
    pub(crate) use cranelift_codegen::ir::types;
    pub(crate) use cranelift_codegen::ir::{
        AbiParam, Block, ExternalName, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc,
        StackSlot, StackSlotData, StackSlotKind, TrapCode, Type, Value,
    };
    pub(crate) use cranelift_codegen::isa::{self, CallConv};
    pub(crate) use cranelift_codegen::Context;
    pub(crate) use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
    pub(crate) use cranelift_module::{self, DataContext, DataId, FuncId, Linkage, Module};

    pub(crate) use crate::abi::*;
    pub(crate) use crate::base::{codegen_operand, codegen_place};
    pub(crate) use crate::cast::*;
    pub(crate) use crate::common::*;
    pub(crate) use crate::debuginfo::{DebugContext, UnwindContext};
    pub(crate) use crate::pointer::Pointer;
    pub(crate) use crate::trap::*;
    pub(crate) use crate::value_and_place::{CPlace, CPlaceInner, CValue};
}

struct PrintOnPanic<F: Fn() -> String>(F);
impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
    fn drop(&mut self) {
        if ::std::thread::panicking() {
            println!("{}", (self.0)());
        }
    }
}

/// The codegen context holds any information shared between the codegen of individual functions
/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module).
struct CodegenCx<'tcx> {
    tcx: TyCtxt<'tcx>,
    global_asm: String,
    cached_context: Context,
    debug_context: Option<DebugContext<'tcx>>,
    unwind_context: UnwindContext,
}

impl<'tcx> CodegenCx<'tcx> {
    fn new(
        tcx: TyCtxt<'tcx>,
        backend_config: BackendConfig,
        isa: &dyn TargetIsa,
        debug_info: bool,
    ) -> Self {
        assert_eq!(pointer_ty(tcx), isa.pointer_type());

        let unwind_context =
            UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot));
        let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None };
        CodegenCx {
            tcx,
            global_asm: String::new(),
            cached_context: Context::new(),
            debug_context,
            unwind_context,
        }
    }
}

pub struct CraneliftCodegenBackend {
    pub config: Option<BackendConfig>,
}

impl CodegenBackend for CraneliftCodegenBackend {
    fn init(&self, sess: &Session) {
        use rustc_session::config::Lto;
        match sess.lto() {
            Lto::No | Lto::ThinLocal => {}
            Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
        }
    }

    fn metadata_loader(&self) -> Box<dyn MetadataLoader + Sync> {
        Box::new(crate::metadata::CraneliftMetadataLoader)
    }

    fn provide(&self, _providers: &mut Providers) {}
    fn provide_extern(&self, _providers: &mut Providers) {}

    fn target_features(&self, _sess: &Session) -> Vec<rustc_span::Symbol> {
        vec![]
    }

    fn codegen_crate(
        &self,
        tcx: TyCtxt<'_>,
        metadata: EncodedMetadata,
        need_metadata_module: bool,
    ) -> Box<dyn Any> {
        tcx.sess.abort_if_errors();
        let config = if let Some(config) = self.config.clone() {
            config
        } else {
            BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
                .unwrap_or_else(|err| tcx.sess.fatal(&err))
        };
        match config.codegen_mode {
            CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
            CodegenMode::Jit | CodegenMode::JitLazy => {
                #[cfg(feature = "jit")]
                let _: ! = driver::jit::run_jit(tcx, config);

                #[cfg(not(feature = "jit"))]
                tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
            }
        }
    }

    fn join_codegen(
        &self,
        ongoing_codegen: Box<dyn Any>,
        _sess: &Session,
    ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorReported> {
        Ok(*ongoing_codegen
            .downcast::<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)>()
            .unwrap())
    }

    fn link(
        &self,
        sess: &Session,
        codegen_results: CodegenResults,
        outputs: &OutputFilenames,
    ) -> Result<(), ErrorReported> {
        use rustc_codegen_ssa::back::link::link_binary;

        let target_cpu = crate::target_triple(sess).to_string();
        link_binary::<crate::archive::ArArchiveBuilder<'_>>(
            sess,
            &codegen_results,
            outputs,
            &codegen_results.crate_name.as_str(),
            &target_cpu,
        );

        Ok(())
    }
}

fn target_triple(sess: &Session) -> target_lexicon::Triple {
    sess.target.llvm_target.parse().unwrap()
}

fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::TargetIsa + 'static> {
    use target_lexicon::BinaryFormat;

    let target_triple = crate::target_triple(sess);

    let mut flags_builder = settings::builder();
    flags_builder.enable("is_pic").unwrap();
    flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
    let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" };
    flags_builder.set("enable_verifier", enable_verifier).unwrap();

    let tls_model = match target_triple.binary_format {
        BinaryFormat::Elf => "elf_gd",
        BinaryFormat::Macho => "macho",
        BinaryFormat::Coff => "coff",
        _ => "none",
    };
    flags_builder.set("tls_model", tls_model).unwrap();

    flags_builder.set("enable_simd", "true").unwrap();

    flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();

    use rustc_session::config::OptLevel;
    match sess.opts.optimize {
        OptLevel::No => {
            flags_builder.set("opt_level", "none").unwrap();
        }
        OptLevel::Less | OptLevel::Default => {}
        OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => {
            flags_builder.set("opt_level", "speed_and_size").unwrap();
        }
    }

    let flags = settings::Flags::new(flags_builder);

    let variant = cranelift_codegen::isa::BackendVariant::MachInst;
    let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
    // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt`
    // is interpreted as `bsr`.
    isa_builder.enable("nehalem").unwrap();
    isa_builder.finish(flags)
}

/// This is the entrypoint for a hot plugged rustc_codegen_cranelift
#[no_mangle]
pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
    Box::new(CraneliftCodegenBackend { config: None })
}
<commit_msg>Support -Ctarget-cpu<commit_after>#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts)]
#![warn(rust_2018_idioms)]
#![warn(unused_lifetimes)]
#![warn(unreachable_pub)]

extern crate snap;
#[macro_use]
extern crate rustc_middle;
extern crate rustc_ast;
extern crate rustc_codegen_ssa;
extern crate rustc_data_structures;
extern crate rustc_errors;
extern crate rustc_fs_util;
extern crate rustc_hir;
extern crate rustc_incremental;
extern crate rustc_index;
extern crate rustc_session;
extern crate rustc_span;
extern crate rustc_target;

// This prevents duplicating functions and statics that are already part of the host rustc process.
#[allow(unused_extern_crates)]
extern crate rustc_driver;

use std::any::Any;

use rustc_codegen_ssa::traits::CodegenBackend;
use rustc_codegen_ssa::CodegenResults;
use rustc_errors::ErrorReported;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
use rustc_middle::ty::query::Providers;
use rustc_session::config::OutputFilenames;
use rustc_session::Session;

use cranelift_codegen::isa::TargetIsa;
use cranelift_codegen::settings::{self, Configurable};

pub use crate::config::*;
use crate::prelude::*;

mod abi;
mod allocator;
mod analyze;
mod archive;
mod backend;
mod base;
mod cast;
mod codegen_i128;
mod common;
mod compiler_builtins;
mod config;
mod constant;
mod debuginfo;
mod discriminant;
mod driver;
mod inline_asm;
mod intrinsics;
mod linkage;
mod main_shim;
mod metadata;
mod num;
mod optimize;
mod pointer;
mod pretty_clif;
mod toolchain;
mod trap;
mod unsize;
mod value_and_place;
mod vtable;

mod prelude {
    pub(crate) use std::convert::{TryFrom, TryInto};

    pub(crate) use rustc_span::Span;

    pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE};
    pub(crate) use rustc_middle::bug;
    pub(crate) use rustc_middle::mir::{self, *};
    pub(crate) use rustc_middle::ty::layout::{self, TyAndLayout};
    pub(crate) use rustc_middle::ty::{
        self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut,
        TypeFoldable, UintTy,
    };
    pub(crate) use rustc_target::abi::{Abi, LayoutOf, Scalar, Size, VariantIdx};

    pub(crate) use rustc_data_structures::fx::FxHashMap;

    pub(crate) use rustc_index::vec::Idx;

    pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
    pub(crate) use cranelift_codegen::ir::function::Function;
    pub(crate) use cranelift_codegen::ir::types;
    pub(crate) use cranelift_codegen::ir::{
        AbiParam, Block, ExternalName, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc,
        StackSlot, StackSlotData, StackSlotKind, TrapCode, Type, Value,
    };
    pub(crate) use cranelift_codegen::isa::{self, CallConv};
    pub(crate) use cranelift_codegen::Context;
    pub(crate) use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
    pub(crate) use cranelift_module::{self, DataContext, DataId, FuncId, Linkage, Module};

    pub(crate) use crate::abi::*;
    pub(crate) use crate::base::{codegen_operand, codegen_place};
    pub(crate) use crate::cast::*;
    pub(crate) use crate::common::*;
    pub(crate) use crate::debuginfo::{DebugContext, UnwindContext};
    pub(crate) use crate::pointer::Pointer;
    pub(crate) use crate::trap::*;
    pub(crate) use crate::value_and_place::{CPlace, CPlaceInner, CValue};
}

struct PrintOnPanic<F: Fn() -> String>(F);
impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
    fn drop(&mut self) {
        if ::std::thread::panicking() {
            println!("{}", (self.0)());
        }
    }
}

/// The codegen context holds any information shared between the codegen of individual functions
/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module).
struct CodegenCx<'tcx> {
    tcx: TyCtxt<'tcx>,
    global_asm: String,
    cached_context: Context,
    debug_context: Option<DebugContext<'tcx>>,
    unwind_context: UnwindContext,
}

impl<'tcx> CodegenCx<'tcx> {
    fn new(
        tcx: TyCtxt<'tcx>,
        backend_config: BackendConfig,
        isa: &dyn TargetIsa,
        debug_info: bool,
    ) -> Self {
        assert_eq!(pointer_ty(tcx), isa.pointer_type());

        let unwind_context =
            UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot));
        let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None };
        CodegenCx {
            tcx,
            global_asm: String::new(),
            cached_context: Context::new(),
            debug_context,
            unwind_context,
        }
    }
}

pub struct CraneliftCodegenBackend {
    pub config: Option<BackendConfig>,
}

impl CodegenBackend for CraneliftCodegenBackend {
    fn init(&self, sess: &Session) {
        use rustc_session::config::Lto;
        match sess.lto() {
            Lto::No | Lto::ThinLocal => {}
            Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
        }
    }

    fn metadata_loader(&self) -> Box<dyn MetadataLoader + Sync> {
        Box::new(crate::metadata::CraneliftMetadataLoader)
    }

    fn provide(&self, _providers: &mut Providers) {}
    fn provide_extern(&self, _providers: &mut Providers) {}

    fn target_features(&self, _sess: &Session) -> Vec<rustc_span::Symbol> {
        vec![]
    }

    fn codegen_crate(
        &self,
        tcx: TyCtxt<'_>,
        metadata: EncodedMetadata,
        need_metadata_module: bool,
    ) -> Box<dyn Any> {
        tcx.sess.abort_if_errors();
        let config = if let Some(config) = self.config.clone() {
            config
        } else {
            BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
                .unwrap_or_else(|err| tcx.sess.fatal(&err))
        };
        match config.codegen_mode {
            CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
            CodegenMode::Jit | CodegenMode::JitLazy => {
                #[cfg(feature = "jit")]
                let _: ! = driver::jit::run_jit(tcx, config);

                #[cfg(not(feature = "jit"))]
                tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
            }
        }
    }

    fn join_codegen(
        &self,
        ongoing_codegen: Box<dyn Any>,
        _sess: &Session,
    ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorReported> {
        Ok(*ongoing_codegen
            .downcast::<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)>()
            .unwrap())
    }

    fn link(
        &self,
        sess: &Session,
        codegen_results: CodegenResults,
        outputs: &OutputFilenames,
    ) -> Result<(), ErrorReported> {
        use rustc_codegen_ssa::back::link::link_binary;

        let target_cpu = crate::target_triple(sess).to_string();
        link_binary::<crate::archive::ArArchiveBuilder<'_>>(
            sess,
            &codegen_results,
            outputs,
            &codegen_results.crate_name.as_str(),
            &target_cpu,
        );

        Ok(())
    }
}

fn target_triple(sess: &Session) -> target_lexicon::Triple {
    sess.target.llvm_target.parse().unwrap()
}

fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::TargetIsa + 'static> {
    use target_lexicon::BinaryFormat;

    let target_triple = crate::target_triple(sess);

    let mut flags_builder = settings::builder();
    flags_builder.enable("is_pic").unwrap();
    flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
    let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" };
    flags_builder.set("enable_verifier", enable_verifier).unwrap();

    let tls_model = match target_triple.binary_format {
        BinaryFormat::Elf => "elf_gd",
        BinaryFormat::Macho => "macho",
        BinaryFormat::Coff => "coff",
        _ => "none",
    };
    flags_builder.set("tls_model", tls_model).unwrap();

    flags_builder.set("enable_simd", "true").unwrap();

    flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();

    use rustc_session::config::OptLevel;
    match sess.opts.optimize {
        OptLevel::No => {
            flags_builder.set("opt_level", "none").unwrap();
        }
        OptLevel::Less | OptLevel::Default => {}
        OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => {
            flags_builder.set("opt_level", "speed_and_size").unwrap();
        }
    }

    let flags = settings::Flags::new(flags_builder);

    let variant = cranelift_codegen::isa::BackendVariant::MachInst;
    let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();

    if let Some(target_cpu) = sess.opts.cg.target_cpu.as_ref() {
        isa_builder.enable(target_cpu).unwrap();
    } else {
        // Don't use "haswell" as the default, as it implies `has_lzcnt`.
        // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`.
        isa_builder.enable("nehalem").unwrap();
    }
    isa_builder.finish(flags)
}

/// This is the entrypoint for a hot plugged rustc_codegen_cranelift
#[no_mangle]
pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
    Box::new(CraneliftCodegenBackend { config: None })
}
<|endoftext|>


<commit_before>extern crate uuid;

use uuid::Uuid;
use std::intrinsics::TypeId;
use std::collections::hashmap::HashMap;
use std::collections::hashmap::{Vacant, Occupied};
use std::any::{Any, AnyRefExt, AnyMutRefExt};

pub trait Component {
    fn get_id(&self) -> Uuid;
}

type ComponentVec = Vec<Box<Any>>;
type EntityMap = HashMap<Uuid, ComponentVec>;

pub struct EntityManager {
    component_map: HashMap<TypeId, EntityMap>,
}

impl EntityManager {
    pub fn new() -> EntityManager {
        EntityManager {
            component_map: HashMap::new()
        }
    }

    pub fn create_entity(&self) -> Uuid {
        Uuid::new_v4()
    }

    pub fn insert<T:Component+'static>(&mut self, entity:Uuid, component:T) {
        let entity_map = match self.component_map.entry(TypeId::of::<T>()) {
            Vacant(entry) => entry.set(HashMap::new()),
            Occupied(entry) => entry.into_mut(),
        };

        let component_vec = match entity_map.entry(entity) {
            Vacant(entry) => entry.set(Vec::new()),
            Occupied(entry) => entry.into_mut(),
        };

        component_vec.push(box component as Box<Any>);
    }

    pub fn find<T:Component+'static>(&self) -> Vec<Box<&T>> {
        let mut result:Vec<Box<&T>> = Vec::new();

        match self.component_map.find(&TypeId::of::<T>()) {
            None => {},
            Some(entity_map) => {
                for (_, component_vec) in entity_map.iter() {
                    for component in component_vec.iter() {
                        result.push(box component.downcast_ref::<T>().unwrap());
                    }
                }
            }
        }

        result
    }

    pub fn find_mut<T:Component+'static>(&mut self) -> Vec<Box<&mut T>> {
        let mut result:Vec<Box<&mut T>> = Vec::new();

        match self.component_map.find_mut(&TypeId::of::<T>()) {
            None => {},
            Some(entity_map) => {
                for (_, component_vec) in entity_map.iter_mut() {
                    for component in component_vec.iter_mut() {
                        result.push(box component.downcast_mut::<T>().unwrap());
                    }
                }
            }
        }

        result
    }

    pub fn find_for<T:Component+'static>(&self, entity:Uuid) -> Vec<Box<&T>> {
        let mut result:Vec<Box<&T>> = Vec::new();

        match self.component_map.find(&TypeId::of::<T>()) {
            None => {},
            Some(entity_map) => {
                match entity_map.find(&entity) {
                    None => {},
                    Some(component_vec) => {
                        for component in component_vec.iter() {
                            result.push(box component.downcast_ref::<T>().unwrap())
                        }
                    }
                }
            }

        }

        result
    }

    pub fn find_for_mut<T:Component+'static>(&mut self, entity:Uuid) -> Vec<Box<&mut T>> {
        let mut result:Vec<Box<&mut T>> = Vec::new();

        match self.component_map.find_mut(&TypeId::of::<T>()) {
            None => {},
            Some(entity_map) => {
                match entity_map.find_mut(&entity) {
                    None => {},
                    Some(component_vec) => {
                        for component in component_vec.iter_mut() {
                            result.push(box component.downcast_mut::<T>().unwrap())
                        }
                    }
                }
            }

        }

        result
    }
}
<commit_msg>more cleanup. Named entities. Benchmarks<commit_after>extern crate uuid;

use uuid::Uuid;
use std::intrinsics::TypeId;
use std::collections::hashmap::HashMap;
use std::collections::hashmap::{Vacant, Occupied};
use std::any::{Any, AnyRefExt, AnyMutRefExt};

pub trait Component {
    fn get_id(&self) -> Uuid;
}

type ComponentVec = Vec<Box<Any>>;
type EntityMap = HashMap<Uuid, ComponentVec>;

pub struct EntityManager {
    named_entities: HashMap<&'static str, Uuid>,
    components: HashMap<TypeId, EntityMap>,
}

impl EntityManager {
    pub fn new() -> EntityManager {
        EntityManager {
            named_entities: HashMap::new(),
            components: HashMap::new(),
        }
    }

    pub fn create_entity(&self) -> Uuid {
        Uuid::new_v4()
    }

    pub fn create_named_entity(&mut self, name: &'static str) -> Uuid {
        let entity = self.create_entity();
        self.named_entities.insert(name, entity);
        entity
    }

    pub fn get_named_entity(&self, name: &'static str) -> Result<Uuid, String> {
        match self.named_entities.find(&name) {
            None => Err(format!("could not find entity for name: {}", name)),
            Some(entity) => Ok(entity.clone()),
        }
    }

    pub fn insert<T:Component+'static>(&mut self, entity:Uuid, component:T) {
        let entity_map = match self.components.entry(TypeId::of::<T>()) {
            Vacant(entry) => entry.set(HashMap::new()),
            Occupied(entry) => entry.into_mut(),
        };

        let component_vec = match entity_map.entry(entity) {
            Vacant(entry) => entry.set(Vec::new()),
            Occupied(entry) => entry.into_mut(),
        };

        component_vec.push(box component as Box<Any>);
    }

    pub fn find<T:Component+Clone+'static>(&self) -> Vec<T> {
        let mut result:Vec<T> = Vec::new();

        match self.components.find(&TypeId::of::<T>()) {
            None => {},
            Some(entity_map) => {
                result.reserve_additional(entity_map.len());
                for (_, component_vec) in entity_map.iter() {
                    if component_vec.len() > 1 {
                        result.reserve_additional(component_vec.len()-1);
                    }
                    for component in component_vec.iter() {
                        result.push(component.downcast_ref::<T>().unwrap().clone());
                    }
                }
            }
        }

        result
    }

    pub fn find_mut<T:Component+'static>(&mut self) -> Vec<&mut T> {
        let mut result:Vec<&mut T> = Vec::new();

        match self.components.find_mut(&TypeId::of::<T>()) {
            None => {},
            Some(entity_map) => {
                result.reserve_additional(entity_map.len());
                for (_, component_vec) in entity_map.iter_mut() {
                    if component_vec.len() > 1 {
                        result.reserve_additional(component_vec.len()-1);
                    }
                    for component in component_vec.iter_mut() {
                        result.push(component.downcast_mut::<T>().unwrap());
                    }
                }
            }
        }

        result
    }

    pub fn find_for<T:Component+Clone+'static>(&self, entity:Uuid) -> Vec<T> {
        let mut result:Vec<T> = Vec::new();

        match self.components.find(&TypeId::of::<T>()) {
            None => {},
            Some(entity_map) => {
                match entity_map.find(&entity) {
                    None => {},
                    Some(component_vec) => {
                        result.reserve_additional(component_vec.len());
                        for component in component_vec.iter() {
                            result.push(component.downcast_ref::<T>().unwrap().clone());
                        }
                    }
                }
            }

        }

        result
    }

    pub fn find_for_mut<T:Component+'static>(&mut self, entity:Uuid) -> Vec<&mut T> {
        let mut result:Vec<&mut T> = Vec::new();

        match self.components.find_mut(&TypeId::of::<T>()) {
            None => {},
            Some(entity_map) => {
                match entity_map.find_mut(&entity) {
                    None => {},
                    Some(component_vec) => {
                        for component in component_vec.iter_mut() {
                            result.push(component.downcast_mut::<T>().unwrap())
                        }
                    }
                }
            }

        }

        result
    }
}

#[cfg(test)]
mod test {
    extern crate test;
    extern crate uuid;
    extern crate entity_system;

    use std::default::Default;
    use uuid::Uuid;

    #[deriving(Default,Clone)]
    struct TestComponent {
        id: Uuid,
        pub name: &'static str,
    }

    impl TestComponent {
        pub fn new() -> TestComponent {
            TestComponent {
                id: Uuid::new_v4(),
                ..Default::default()
            }
        }
    }

    impl entity_system::Component for TestComponent {
        fn get_id(&self) -> Uuid {
            self.id
        }
    }

    #[deriving(Default,Clone)]
    struct OtherComponent {
        id: Uuid,
        pub name: &'static str,
    }

    impl OtherComponent {
        pub fn new() -> OtherComponent {
            OtherComponent {
                id: Uuid::new_v4(),
                ..Default::default()
            }
        }
    }

    impl entity_system::Component for OtherComponent {
        fn get_id(&self) -> Uuid {
            self.id
        }
    }
    #[bench]
    fn bench_insert(b: &mut test::Bencher) {

        let mut em = entity_system::EntityManager::new();
        let entity = em.create_entity();

        b.iter(|| {
            let tc = TestComponent::new();
            em.insert(entity, tc);
        });
    }

    #[bench]
    fn bench_find(b: &mut test::Bencher) {
        let mut em = entity_system::EntityManager::new();
        for _ in range(0u, 1000) {
            let entity = em.create_entity();
            let tc = TestComponent::new();
            em.insert(entity, tc);
            for _ in range(0u, 10) {
                let oc = OtherComponent::new();
                em.insert(entity, oc);
            }
        }

        b.iter(|| {
            em.find::<TestComponent>();
        });
    }

    #[bench]
    fn bench_find_multiple_per_entity(b: &mut test::Bencher) {
        let mut em = entity_system::EntityManager::new();
        for _ in range(0u, 100) {
            let entity = em.create_entity();
            for _ in range(0u, 100) {
                let tc = TestComponent::new();
                em.insert(entity, tc);
            }
        }

        b.iter(|| {
            em.find::<TestComponent>();
        });
    }

    #[bench]
    fn bench_find_for(b: &mut test::Bencher) {

        let mut em = entity_system::EntityManager::new();
        let known_entity = em.create_entity();
        let tc = TestComponent::new();
        em.insert(known_entity, tc);
        for _ in range(0u, 10) {
            let oc = OtherComponent::new();
            em.insert(known_entity, oc);
        }

        for _ in range(0u, 1000) {
            let entity = em.create_entity();
            let tc = TestComponent::new();
            em.insert(entity, tc);
            for _ in range(0u, 10) {
                let oc = OtherComponent::new();
                em.insert(entity, oc);
            }
        }

        b.iter(|| {
            em.find_for::<TestComponent>(known_entity);
        });
    }

    #[bench]
    fn bench_find_mut(b: &mut test::Bencher) {

        let mut em = entity_system::EntityManager::new();

        for _ in range(0u, 1000) {
            let entity = em.create_entity();
            let tc = TestComponent::new();
            em.insert(entity, tc);
            for _ in range(0u, 10) {
                let oc = OtherComponent::new();
                em.insert(entity, oc);
            }
        }

        b.iter(|| {
            em.find_mut::<TestComponent>();
        });
    }
}
<|endoftext|>