<commit_before>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::shared::{get_shuffle_dir, send};
use anyhow::{anyhow, Context, Result};
use diem_config::config::NodeConfig;
use diem_crypto::{
    ed25519::{Ed25519PrivateKey, Ed25519PublicKey},
    PrivateKey,
};
use diem_sdk::{
    client::BlockingClient,
    transaction_builder::{Currency, TransactionFactory},
    types::LocalAccount,
};
use diem_types::{
    account_config, chain_id::ChainId, transaction::authenticator::AuthenticationKey,
};
use generate_key::load_key;
use shuffle_transaction_builder::framework::encode_create_parent_vasp_account_script_function;
use std::{fs, fs::File, io::Write, path::Path};

const NEW_KEY_FILE_CONTENT: &[u8] = include_bytes!("../new_account.key");

// Creates new account from randomly generated private/public key pair.
pub fn handle() -> Result<()> {
    let shuffle_dir = &get_shuffle_dir();
    if !Path::new(shuffle_dir.as_path()).is_dir() {
        return Err(anyhow!(
            "A node hasn't been created yet! Run shuffle node first"
        ));
    }
    println!("{:?}", shuffle_dir);
    let config_path = shuffle_dir.join("nodeconfig/0").join("node.yaml");
    let config = NodeConfig::load(&config_path)
        .with_context(|| format!("Failed to load NodeConfig from file: {:?}", config_path))?;

    let root_key_path = shuffle_dir.join("nodeconfig").join("mint.key");
    let root_account_key = load_key(root_key_path);

    let json_rpc_url = format!("http://0.0.0.0:{}", config.json_rpc.address.port());
    println!("Connecting to {}...", json_rpc_url);
    let client = BlockingClient::new(json_rpc_url);

    let root_seq_num = client
        .get_account(account_config::treasury_compliance_account_address())?
        .into_inner()
        .unwrap()
        .sequence_number;
    let mut root_account = LocalAccount::new(
        account_config::treasury_compliance_account_address(),
        root_account_key,
        root_seq_num,
    );

    generate_shuffle_accounts_dir(shuffle_dir)?;
    let new_account_key = generate_key_file(shuffle_dir).unwrap();
    let public_key = new_account_key.public_key();
    generate_address_file(shuffle_dir, &public_key)?;

    let new_account = LocalAccount::new(
        AuthenticationKey::ed25519(&public_key).derived_address(),
        new_account_key,
        0,
    );

    if client
        .get_account(new_account.address())
        .unwrap()
        .into_inner()
        .is_some()
    {
        println!("Account already exists: {}", new_account.address());
        println!(
            "Private key: {}",
            ::hex::encode(new_account.private_key().to_bytes())
        );
        println!("Public key: {}", new_account.public_key());
        return Ok(());
    }

    // Create a new account.
    println!("Create a new account...",);
    let create_new_account_txn = root_account.sign_with_transaction_builder(
        TransactionFactory::new(ChainId::test()).payload(
            encode_create_parent_vasp_account_script_function(
                Currency::XUS.type_tag(),
                0,
                new_account.address(),
                new_account.authentication_key().prefix().to_vec(),
                vec![],
                false,
            ),
        ),
    );
    send(&client, create_new_account_txn)?;
    println!("Successfully created account {}", new_account.address());
    println!(
        "Private key: {}",
        ::hex::encode(new_account.private_key().to_bytes())
    );
    println!("Public key: {}", new_account.public_key());
    Ok(())
}

pub fn generate_shuffle_accounts_dir(shuffle_dir: &Path) -> Result<()> {
    let account_dir = &shuffle_dir.join("accounts");
    if !account_dir.as_path().is_dir() {
        fs::create_dir(account_dir)?;
    }
    let latest_dir = &account_dir.join("latest");
    if !latest_dir.as_path().is_dir() {
        fs::create_dir(latest_dir)?;
    }

    Ok(())
}

pub fn generate_key_file(shuffle_dir: &Path) -> Result<Ed25519PrivateKey> {
    let latest_dir = &shuffle_dir.join("accounts").join("latest");
    let dev_key_filepath = &latest_dir.join("dev.key");
    fs::write(dev_key_filepath, NEW_KEY_FILE_CONTENT)?;
    let private_key = generate_key::load_key(&dev_key_filepath);
    Ok(private_key)
}

pub fn generate_address_file(shuffle_dir: &Path, public_key: &Ed25519PublicKey) -> Result<()> {
    let latest_dir = &shuffle_dir.join("accounts").join("latest");
    let address = AuthenticationKey::ed25519(public_key).derived_address();
    let account_filepath = &latest_dir.join("address");
    let mut file = File::create(account_filepath)?;
    file.write_all(address.to_string().as_ref())?;
    Ok(())
}

#[cfg(test)]
mod test {
    use crate::account::{generate_address_file, generate_key_file, generate_shuffle_accounts_dir};
    use diem_crypto::PrivateKey;
    use generate_key::generate_key;
    use std::fs;
    use tempfile::tempdir;

    #[test]
    fn test_generate_user_shuffle_dirs() {
        let dir = tempdir().unwrap();
        generate_shuffle_accounts_dir(&dir.path().to_path_buf())
            .expect("Directories weren't created");
        assert_eq!(dir.path().join("accounts").as_path().is_dir(), true);
        assert_eq!(
            dir.path()
                .join("accounts")
                .join("latest")
                .as_path()
                .is_dir(),
            true
        );
    }

    #[test]
    fn test_generate_key_path() {
        let dir = tempdir().unwrap();
        fs::create_dir_all(dir.path().join("accounts").join("latest"))
            .expect("Directories weren't created");
        generate_key_file(&dir.path().to_path_buf()).expect("Key file wasn't generated");
        assert_eq!(
            dir.path()
                .join("accounts")
                .join("latest")
                .join("dev.key")
                .as_path()
                .exists(),
            true
        );
    }

    #[test]
    fn test_generate_address_path() {
        let dir = tempdir().unwrap();
        fs::create_dir_all(dir.path().join("accounts").join("latest"))
            .expect("Directories weren't created");
        let private_key = generate_key();
        let public_key = private_key.public_key();
        generate_address_file(dir.path(), &public_key).expect("Address file wasn't generated");
        assert_eq!(
            dir.path()
                .join("accounts")
                .join("latest")
                .join("address")
                .as_path()
                .exists(),
            true
        );
    }
}
<commit_msg>nitpick: changed usage of get_shuffle_dir()<commit_after>// Copyright (c) The Diem Core Contributors
// SPDX-License-Identifier: Apache-2.0

use crate::shared::{get_shuffle_dir, send};
use anyhow::{anyhow, Context, Result};
use diem_config::config::NodeConfig;
use diem_crypto::{
    ed25519::{Ed25519PrivateKey, Ed25519PublicKey},
    PrivateKey,
};
use diem_sdk::{
    client::BlockingClient,
    transaction_builder::{Currency, TransactionFactory},
    types::LocalAccount,
};
use diem_types::{
    account_config, chain_id::ChainId, transaction::authenticator::AuthenticationKey,
};
use generate_key::load_key;
use shuffle_transaction_builder::framework::encode_create_parent_vasp_account_script_function;
use std::{fs, fs::File, io::Write, path::Path};

const NEW_KEY_FILE_CONTENT: &[u8] = include_bytes!("../new_account.key");

// Creates new account from randomly generated private/public key pair.
pub fn handle() -> Result<()> {
    let shuffle_dir = get_shuffle_dir();
    if !Path::new(shuffle_dir.as_path()).is_dir() {
        return Err(anyhow!(
            "A node hasn't been created yet! Run shuffle node first"
        ));
    }
    println!("{:?}", &shuffle_dir);
    let config_path = &shuffle_dir.join("nodeconfig/0").join("node.yaml");
    let config = NodeConfig::load(&config_path)
        .with_context(|| format!("Failed to load NodeConfig from file: {:?}", config_path))?;

    let root_key_path = &shuffle_dir.join("nodeconfig").join("mint.key");
    let root_account_key = load_key(root_key_path);

    let json_rpc_url = format!("http://0.0.0.0:{}", config.json_rpc.address.port());
    println!("Connecting to {}...", json_rpc_url);
    let client = BlockingClient::new(json_rpc_url);

    let root_seq_num = client
        .get_account(account_config::treasury_compliance_account_address())?
        .into_inner()
        .unwrap()
        .sequence_number;
    let mut root_account = LocalAccount::new(
        account_config::treasury_compliance_account_address(),
        root_account_key,
        root_seq_num,
    );

    generate_shuffle_accounts_dir(&shuffle_dir)?;
    let new_account_key = generate_key_file(&shuffle_dir).unwrap();
    let public_key = new_account_key.public_key();
    generate_address_file(&shuffle_dir, &public_key)?;

    let new_account = LocalAccount::new(
        AuthenticationKey::ed25519(&public_key).derived_address(),
        new_account_key,
        0,
    );

    if client
        .get_account(new_account.address())
        .unwrap()
        .into_inner()
        .is_some()
    {
        println!("Account already exists: {}", new_account.address());
        println!(
            "Private key: {}",
            ::hex::encode(new_account.private_key().to_bytes())
        );
        println!("Public key: {}", new_account.public_key());
        return Ok(());
    }

    // Create a new account.
    println!("Create a new account...",);
    let create_new_account_txn = root_account.sign_with_transaction_builder(
        TransactionFactory::new(ChainId::test()).payload(
            encode_create_parent_vasp_account_script_function(
                Currency::XUS.type_tag(),
                0,
                new_account.address(),
                new_account.authentication_key().prefix().to_vec(),
                vec![],
                false,
            ),
        ),
    );
    send(&client, create_new_account_txn)?;
    println!("Successfully created account {}", new_account.address());
    println!(
        "Private key: {}",
        ::hex::encode(new_account.private_key().to_bytes())
    );
    println!("Public key: {}", new_account.public_key());
    Ok(())
}

pub fn generate_shuffle_accounts_dir(shuffle_dir: &Path) -> Result<()> {
    let account_dir = &shuffle_dir.join("accounts");
    if !account_dir.as_path().is_dir() {
        fs::create_dir(account_dir)?;
    }
    let latest_dir = &account_dir.join("latest");
    if !latest_dir.as_path().is_dir() {
        fs::create_dir(latest_dir)?;
    }

    Ok(())
}

pub fn generate_key_file(shuffle_dir: &Path) -> Result<Ed25519PrivateKey> {
    let latest_dir = &shuffle_dir.join("accounts").join("latest");
    let dev_key_filepath = &latest_dir.join("dev.key");
    fs::write(dev_key_filepath, NEW_KEY_FILE_CONTENT)?;
    let private_key = generate_key::load_key(&dev_key_filepath);
    Ok(private_key)
}

pub fn generate_address_file(shuffle_dir: &Path, public_key: &Ed25519PublicKey) -> Result<()> {
    let latest_dir = &shuffle_dir.join("accounts").join("latest");
    let address = AuthenticationKey::ed25519(public_key).derived_address();
    let account_filepath = &latest_dir.join("address");
    let mut file = File::create(account_filepath)?;
    file.write_all(address.to_string().as_ref())?;
    Ok(())
}

#[cfg(test)]
mod test {
    use crate::account::{generate_address_file, generate_key_file, generate_shuffle_accounts_dir};
    use diem_crypto::PrivateKey;
    use generate_key::generate_key;
    use std::fs;
    use tempfile::tempdir;

    #[test]
    fn test_generate_user_shuffle_dirs() {
        let dir = tempdir().unwrap();
        generate_shuffle_accounts_dir(&dir.path().to_path_buf())
            .expect("Directories weren't created");
        assert_eq!(dir.path().join("accounts").as_path().is_dir(), true);
        assert_eq!(
            dir.path()
                .join("accounts")
                .join("latest")
                .as_path()
                .is_dir(),
            true
        );
    }

    #[test]
    fn test_generate_key_path() {
        let dir = tempdir().unwrap();
        fs::create_dir_all(dir.path().join("accounts").join("latest"))
            .expect("Directories weren't created");
        generate_key_file(&dir.path().to_path_buf()).expect("Key file wasn't generated");
        assert_eq!(
            dir.path()
                .join("accounts")
                .join("latest")
                .join("dev.key")
                .as_path()
                .exists(),
            true
        );
    }

    #[test]
    fn test_generate_address_path() {
        let dir = tempdir().unwrap();
        fs::create_dir_all(dir.path().join("accounts").join("latest"))
            .expect("Directories weren't created");
        let private_key = generate_key();
        let public_key = private_key.public_key();
        generate_address_file(dir.path(), &public_key).expect("Address file wasn't generated");
        assert_eq!(
            dir.path()
                .join("accounts")
                .join("latest")
                .join("address")
                .as_path()
                .exists(),
            true
        );
    }
}
<|endoftext|>


<commit_before>use crate::{
    buffer::Buffer, connection::Connection, connection_status::ConnectionState, Error,
    PromiseResolver, Result,
};
use amq_protocol::frame::{gen_frame, parse_frame, AMQPFrame, GenError};
use log::{error, trace};
use mio::{event::Source, Events, Interest, Poll, Token, Waker};
use std::{
    collections::VecDeque,
    io::{Read, Write},
    sync::{
        atomic::{AtomicBool, Ordering},
        Arc,
    },
    thread::{self, Builder as ThreadBuilder, JoinHandle},
    time::{Duration, Instant},
};

pub(crate) const SOCKET: Token = Token(1);
const WAKER: Token = Token(2);

const FRAMES_STORAGE: usize = 32;

#[derive(Debug, PartialEq)]
enum Status {
    Initial,
    Setup,
    Stop,
}

pub struct IoLoop<T> {
    connection: Connection,
    socket: T,
    status: Status,
    poll: Poll,
    waker: Arc<Waker>,
    hb_handle: Option<JoinHandle<()>>,
    frame_size: usize,
    receive_buffer: Buffer,
    send_buffer: Buffer,
    can_write: bool,
    can_read: bool,
    send_heartbeat: Arc<AtomicBool>,
    poll_timeout: Option<Duration>,
    serialized_frames: VecDeque<(u64, Option<PromiseResolver<()>>)>,
}

impl<T: Source + Read + Write + Send + 'static> IoLoop<T> {
    pub(crate) fn new(
        connection: Connection,
        socket: T,
        poll: Option<(Poll, Token)>,
    ) -> Result<Self> {
        let (poll, registered) = poll
            .map(|t| Ok((t.0, true)))
            .unwrap_or_else(|| Poll::new().map(|poll| (poll, false)))?;
        let frame_size = std::cmp::max(8192, connection.configuration().frame_max() as usize);
        let waker = Arc::new(Waker::new(poll.registry(), WAKER)?);
        let mut inner = Self {
            connection,
            socket,
            status: Status::Initial,
            poll,
            waker,
            hb_handle: None,
            frame_size,
            receive_buffer: Buffer::with_capacity(FRAMES_STORAGE * frame_size),
            send_buffer: Buffer::with_capacity(FRAMES_STORAGE * frame_size),
            can_write: false,
            can_read: false,
            send_heartbeat: Arc::new(AtomicBool::new(false)),
            poll_timeout: None,
            serialized_frames: VecDeque::default(),
        };
        if registered {
            inner.poll.registry().reregister(
                &mut inner.socket,
                SOCKET,
                Interest::READABLE | Interest::WRITABLE,
            )?;
        } else {
            inner.poll.registry().register(
                &mut inner.socket,
                SOCKET,
                Interest::READABLE | Interest::WRITABLE,
            )?;
        }
        Ok(inner)
    }

    fn start_heartbeat(&mut self, interval: Duration) -> Result<()> {
        let connection = self.connection.clone();
        let send_hartbeat = self.send_heartbeat.clone();
        let hb_handle = ThreadBuilder::new()
            .name("heartbeat".to_owned())
            .spawn(move || {
                while connection.status().connected() {
                    let start = Instant::now();
                    let mut remaining = interval;

                    loop {
                        thread::park_timeout(remaining);
                        let elapsed = start.elapsed();
                        if elapsed >= remaining {
                            break;
                        }
                        remaining -= interval - elapsed;
                    }

                    send_hartbeat.store(true, Ordering::Relaxed);
                }
            })?;
        self.hb_handle = Some(hb_handle);
        Ok(())
    }

    fn heartbeat(&mut self) -> Result<()> {
        if self.send_heartbeat.load(Ordering::Relaxed) {
            trace!("send heartbeat");
            self.connection.send_heartbeat()?;
            self.send_heartbeat.store(false, Ordering::Relaxed);
        }
        Ok(())
    }

    fn ensure_setup(&mut self) -> Result<()> {
        if self.status != Status::Setup && self.connection.status().connected() {
            let frame_max = self.connection.configuration().frame_max() as usize;
            self.frame_size = std::cmp::max(self.frame_size, frame_max);
            self.receive_buffer.grow(FRAMES_STORAGE * self.frame_size);
            self.send_buffer.grow(FRAMES_STORAGE * self.frame_size);
            let heartbeat = self.connection.configuration().heartbeat();
            if heartbeat != 0 {
                trace!("io_loop: start heartbeat");
                let heartbeat = Duration::from_secs(u64::from(heartbeat));
                self.start_heartbeat(heartbeat)?;
                self.poll_timeout = Some(heartbeat);
                trace!("io_loop: heartbeat started");
            }
            self.status = Status::Setup;
        }
        Ok(())
    }

    fn has_data(&self) -> bool {
        self.connection.has_pending_frames()
            || self.send_buffer.available_data() > 0
            || !self.serialized_frames.is_empty()
    }

    fn can_write(&self) -> bool {
        self.can_write && self.has_data() && !self.connection.status().blocked()
    }

    fn can_read(&self) -> bool {
        self.can_read
    }

    fn can_parse(&self) -> bool {
        self.receive_buffer.available_data() > 0
    }

    fn should_continue(&self) -> bool {
        let connection_status = self.connection.status();
        (self.status == Status::Initial
            || connection_status.connected()
            || connection_status.closing())
            && self.status != Status::Stop
            && !connection_status.errored()
    }

    pub fn start(mut self) -> Result<()> {
        let waker = self.waker.clone();
        self.connection.clone().set_io_loop(
            ThreadBuilder::new()
                .name("io_loop".to_owned())
                .spawn(move || {
                    let mut events = Events::with_capacity(1024);
                    while self.should_continue() {
                        self.run(&mut events)?;
                    }
                    if let Some(hb_handle) = self.hb_handle.take() {
                        hb_handle.thread().unpark();
                        hb_handle.join().expect("heartbeat loop failed");
                    }
                    Ok(())
                })?,
            waker,
        )
    }

    fn poll(&mut self, events: &mut Events) -> Result<()> {
        trace!("io_loop poll");
        self.poll.poll(events, self.poll_timeout)?;
        trace!("io_loop poll done");
        for event in events.iter() {
            if event.token() == SOCKET {
                if event.is_readable() {
                    self.can_read = true;
                }
                if event.is_writable() {
                    self.can_write = true;
                }
            }
        }
        Ok(())
    }

    fn run(&mut self, events: &mut Events) -> Result<()> {
        trace!("io_loop run");
        self.ensure_setup()?;
        self.poll(events)?;
        self.do_run()
    }

    fn do_run(&mut self) -> Result<()> {
        trace!(
            "io_loop do_run; can_read={}, can_write={}, has_data={}",
            self.can_read,
            self.can_write,
            self.has_data()
        );
        loop {
            self.heartbeat()?;
            self.write()?;
            if self.connection.status().closed() {
                self.status = Status::Stop;
            }
            if self.should_continue() {
                self.read()?;
            }
            self.parse()?;
            self.connection.poll_internal_promises()?;
            if self.stop_looping() {
                self.maybe_continue()?;
                break;
            }
        }
        trace!(
            "io_loop do_run done; can_read={}, can_write={}, has_data={}, status={:?}",
            self.can_read,
            self.can_write,
            self.has_data(),
            self.status
        );
        Ok(())
    }

    fn stop_looping(&self) -> bool {
        !self.can_read()
            || !self.can_write()
            || self.status == Status::Stop
            || self.connection.status().errored()
    }

    fn has_pending_operations(&self) -> bool {
        self.status != Status::Stop && (self.can_read() || self.can_parse() || self.can_write())
    }

    fn maybe_continue(&mut self) -> Result<()> {
        if self.has_pending_operations() {
            trace!(
                "io_loop send continue; can_read={}, can_write={}, has_data={}",
                self.can_read,
                self.can_write,
                self.has_data()
            );
            self.send_continue()?;
        }
        Ok(())
    }

    fn critical_error(&mut self, error: Error) -> Result<()> {
        if let ConnectionState::SentProtocolHeader(resolver, ..) = self.connection.status().state()
        {
            resolver.swear(Err(error.clone()));
            self.status = Status::Stop;
        }
        self.connection.set_error(error.clone())?;
        Err(error)
    }

    fn write(&mut self) -> Result<()> {
        if self.can_write() {
            if let Err(e) = self.write_to_stream() {
                if e.wouldblock() {
                    self.can_write = false
                } else {
                    error!("error writing: {:?}", e);
                    self.critical_error(e)?;
                }
            }
        }
        Ok(())
    }

    fn read(&mut self) -> Result<()> {
        if self.can_read() {
            if let Err(e) = self.read_from_stream() {
                if e.wouldblock() {
                    self.can_read = false
                } else {
                    error!("error reading: {:?}", e);
                    self.critical_error(e)?;
                }
            }
            self.receive_buffer.shift_unless_available(self.frame_size);
        }
        Ok(())
    }

    fn send_continue(&mut self) -> Result<()> {
        self.waker.wake()?;
        Ok(())
    }

    fn write_to_stream(&mut self) -> Result<()> {
        self.serialize()?;

        let sz = self.send_buffer.write_to(&mut self.socket)?;

        trace!("wrote {} bytes", sz);
        self.send_buffer.consume(sz, true);

        let mut written = sz as u64;
        while written > 0 {
            if let Some((to_write, resolver)) = self.serialized_frames.pop_front() {
                if written < to_write {
                    self.serialized_frames
                        .push_front((to_write - written, resolver));
                    written = 0;
                } else {
                    if let Some(resolver) = resolver {
                        resolver.swear(Ok(()));
                    }
                    written -= to_write;
                }
            } else {
                break;
            }
        }

        if sz > 0 && self.send_buffer.available_data() > 0 {
            // We didn't write all the data yet
            self.send_continue()?;
        }
        Ok(())
    }

    fn read_from_stream(&mut self) -> Result<()> {
        match self.connection.status().state() {
            ConnectionState::Closed => Ok(()),
            ConnectionState::Error => Err(Error::InvalidConnectionState(ConnectionState::Error)),
            _ => {
                self.receive_buffer
                    .read_from(&mut self.socket, false)
                    .map(|sz| {
                        trace!("read {} bytes", sz);
                        self.receive_buffer.fill(sz, false);
                    })?;
                Ok(())
            }
        }
    }

    fn serialize(&mut self) -> Result<()> {
        if let Some((next_msg, resolver)) = self.connection.next_frame() {
            trace!("will write to buffer: {:?}", next_msg);
            let checkpoint = self.send_buffer.checkpoint();
            let res = gen_frame(&next_msg)((&mut self.send_buffer).into());
            match res.map(|w| w.into_inner().1) {
                Ok(sz) => {
                    self.serialized_frames.push_back((sz, resolver));
                    Ok(())
                }
                Err(e) => {
                    self.send_buffer.rollback(checkpoint);
                    match e {
                        GenError::BufferTooSmall(_) => {
                            // Requeue msg
                            self.connection.requeue_frame((next_msg, resolver))?;
                            Ok(())
                        }
                        e => {
                            error!("error generating frame: {:?}", e);
                            let error = Error::SerialisationError(Arc::new(e));
                            self.connection.set_error(error.clone())?;
                            Err(error)
                        }
                    }
                }
            }
        } else {
            Ok(())
        }
    }

    fn parse(&mut self) -> Result<()> {
        if self.can_parse() {
            if let Some(frame) = self.do_parse()? {
                self.connection.handle_frame(frame)?;
            }
        }
        Ok(())
    }

    fn do_parse(&mut self) -> Result<Option<AMQPFrame>> {
        match parse_frame(self.receive_buffer.data()) {
            Ok((i, f)) => {
                let consumed = self.receive_buffer.offset(i);
                self.receive_buffer.consume(consumed, false);
                Ok(Some(f))
            }
            Err(e) => {
                if e.is_incomplete() {
                    self.receive_buffer.shift();
                    Ok(None)
                } else {
                    error!("parse error: {:?}", e);
                    let error = Error::ParsingError(e);
                    self.connection.set_error(error.clone())?;
                    Err(error)
                }
            }
        }
    }
}
<commit_msg>IoLoop: heartbeat should be sent when 50% of timeout elapsed<commit_after>use crate::{
    buffer::Buffer, connection::Connection, connection_status::ConnectionState, Error,
    PromiseResolver, Result,
};
use amq_protocol::frame::{gen_frame, parse_frame, AMQPFrame, GenError};
use log::{debug, error, trace};
use mio::{event::Source, Events, Interest, Poll, Token, Waker};
use std::{
    collections::VecDeque,
    io::{Read, Write},
    sync::Arc,
    thread::{self, Builder as ThreadBuilder, JoinHandle},
    time::{Duration, Instant},
};

pub(crate) const SOCKET: Token = Token(1);
const WAKER: Token = Token(2);

const FRAMES_STORAGE: usize = 32;

#[derive(Debug, PartialEq)]
enum Status {
    Initial,
    Setup,
    Stop,
}

pub struct IoLoop<T> {
    connection: Connection,
    socket: T,
    status: Status,
    poll: Poll,
    waker: Arc<Waker>,
    hb_handle: Option<JoinHandle<()>>,
    frame_size: usize,
    receive_buffer: Buffer,
    send_buffer: Buffer,
    can_write: bool,
    can_read: bool,
    poll_timeout: Option<Duration>,
    serialized_frames: VecDeque<(u64, Option<PromiseResolver<()>>)>,
}

impl<T: Source + Read + Write + Send + 'static> IoLoop<T> {
    pub(crate) fn new(
        connection: Connection,
        socket: T,
        poll: Option<(Poll, Token)>,
    ) -> Result<Self> {
        let (poll, registered) = poll
            .map(|t| Ok((t.0, true)))
            .unwrap_or_else(|| Poll::new().map(|poll| (poll, false)))?;
        let frame_size = std::cmp::max(8192, connection.configuration().frame_max() as usize);
        let waker = Arc::new(Waker::new(poll.registry(), WAKER)?);
        let mut inner = Self {
            connection,
            socket,
            status: Status::Initial,
            poll,
            waker,
            hb_handle: None,
            frame_size,
            receive_buffer: Buffer::with_capacity(FRAMES_STORAGE * frame_size),
            send_buffer: Buffer::with_capacity(FRAMES_STORAGE * frame_size),
            can_write: false,
            can_read: false,
            poll_timeout: None,
            serialized_frames: VecDeque::default(),
        };
        if registered {
            inner.poll.registry().reregister(
                &mut inner.socket,
                SOCKET,
                Interest::READABLE | Interest::WRITABLE,
            )?;
        } else {
            inner.poll.registry().register(
                &mut inner.socket,
                SOCKET,
                Interest::READABLE | Interest::WRITABLE,
            )?;
        }
        Ok(inner)
    }

    fn start_heartbeat(&mut self, interval: Duration) -> Result<()> {
        let connection = self.connection.clone();
        let hb_handle = ThreadBuilder::new()
            .name("heartbeat".to_owned())
            .spawn(move || {
                while connection.status().connected() {
                    let start = Instant::now();
                    let mut remaining = interval;

                    loop {
                        thread::park_timeout(remaining);
                        let elapsed = start.elapsed();
                        if elapsed >= remaining {
                            break;
                        }
                        remaining -= interval - elapsed;
                    }

                    debug!("send heartbeat");
                    if let Err(err) = connection.send_heartbeat() {
                        error!("Error while sending heartbeat: {}", err);
                    }
                }
            })?;
        self.hb_handle = Some(hb_handle);
        Ok(())
    }

    fn ensure_setup(&mut self) -> Result<()> {
        if self.status != Status::Setup && self.connection.status().connected() {
            let frame_max = self.connection.configuration().frame_max() as usize;
            self.frame_size = std::cmp::max(self.frame_size, frame_max);
            self.receive_buffer.grow(FRAMES_STORAGE * self.frame_size);
            self.send_buffer.grow(FRAMES_STORAGE * self.frame_size);
            let heartbeat = self.connection.configuration().heartbeat();
            if heartbeat != 0 {
                trace!("io_loop: start heartbeat");
                let heartbeat = Duration::from_secs(u64::from(heartbeat / 2));
                self.start_heartbeat(heartbeat)?;
                self.poll_timeout = Some(heartbeat);
                trace!("io_loop: heartbeat started");
            }
            self.status = Status::Setup;
        }
        Ok(())
    }

    fn has_data(&self) -> bool {
        self.connection.has_pending_frames()
            || self.send_buffer.available_data() > 0
            || !self.serialized_frames.is_empty()
    }

    fn can_write(&self) -> bool {
        self.can_write && self.has_data() && !self.connection.status().blocked()
    }

    fn can_read(&self) -> bool {
        self.can_read
    }

    fn can_parse(&self) -> bool {
        self.receive_buffer.available_data() > 0
    }

    fn should_continue(&self) -> bool {
        let connection_status = self.connection.status();
        (self.status == Status::Initial
            || connection_status.connected()
            || connection_status.closing())
            && self.status != Status::Stop
            && !connection_status.errored()
    }

    pub fn start(mut self) -> Result<()> {
        let waker = self.waker.clone();
        self.connection.clone().set_io_loop(
            ThreadBuilder::new()
                .name("io_loop".to_owned())
                .spawn(move || {
                    let mut events = Events::with_capacity(1024);
                    while self.should_continue() {
                        self.run(&mut events)?;
                    }
                    if let Some(hb_handle) = self.hb_handle.take() {
                        hb_handle.thread().unpark();
                        hb_handle.join().expect("heartbeat loop failed");
                    }
                    Ok(())
                })?,
            waker,
        )
    }

    fn poll(&mut self, events: &mut Events) -> Result<()> {
        trace!("io_loop poll");
        self.poll.poll(events, self.poll_timeout)?;
        trace!("io_loop poll done");
        for event in events.iter() {
            if event.token() == SOCKET {
                if event.is_readable() {
                    self.can_read = true;
                }
                if event.is_writable() {
                    self.can_write = true;
                }
            }
        }
        Ok(())
    }

    fn run(&mut self, events: &mut Events) -> Result<()> {
        trace!("io_loop run");
        self.ensure_setup()?;
        self.poll(events)?;
        self.do_run()
    }

    fn do_run(&mut self) -> Result<()> {
        trace!(
            "io_loop do_run; can_read={}, can_write={}, has_data={}",
            self.can_read,
            self.can_write,
            self.has_data()
        );
        loop {
            self.write()?;
            if self.connection.status().closed() {
                self.status = Status::Stop;
            }
            if self.should_continue() {
                self.read()?;
            }
            self.parse()?;
            self.connection.poll_internal_promises()?;
            if self.stop_looping() {
                self.maybe_continue()?;
                break;
            }
        }
        trace!(
            "io_loop do_run done; can_read={}, can_write={}, has_data={}, status={:?}",
            self.can_read,
            self.can_write,
            self.has_data(),
            self.status
        );
        Ok(())
    }

    fn stop_looping(&self) -> bool {
        !self.can_read()
            || !self.can_write()
            || self.status == Status::Stop
            || self.connection.status().errored()
    }

    fn has_pending_operations(&self) -> bool {
        self.status != Status::Stop && (self.can_read() || self.can_parse() || self.can_write())
    }

    fn maybe_continue(&mut self) -> Result<()> {
        if self.has_pending_operations() {
            trace!(
                "io_loop send continue; can_read={}, can_write={}, has_data={}",
                self.can_read,
                self.can_write,
                self.has_data()
            );
            self.send_continue()?;
        }
        Ok(())
    }

    fn critical_error(&mut self, error: Error) -> Result<()> {
        if let ConnectionState::SentProtocolHeader(resolver, ..) = self.connection.status().state()
        {
            resolver.swear(Err(error.clone()));
            self.status = Status::Stop;
        }
        self.connection.set_error(error.clone())?;
        Err(error)
    }

    fn write(&mut self) -> Result<()> {
        if self.can_write() {
            if let Err(e) = self.write_to_stream() {
                if e.wouldblock() {
                    self.can_write = false
                } else {
                    error!("error writing: {:?}", e);
                    self.critical_error(e)?;
                }
            }
        }
        Ok(())
    }

    fn read(&mut self) -> Result<()> {
        if self.can_read() {
            if let Err(e) = self.read_from_stream() {
                if e.wouldblock() {
                    self.can_read = false
                } else {
                    error!("error reading: {:?}", e);
                    self.critical_error(e)?;
                }
            }
            self.receive_buffer.shift_unless_available(self.frame_size);
        }
        Ok(())
    }

    fn send_continue(&mut self) -> Result<()> {
        self.waker.wake()?;
        Ok(())
    }

    fn write_to_stream(&mut self) -> Result<()> {
        self.serialize()?;

        let sz = self.send_buffer.write_to(&mut self.socket)?;

        trace!("wrote {} bytes", sz);
        self.send_buffer.consume(sz, true);

        let mut written = sz as u64;
        while written > 0 {
            if let Some((to_write, resolver)) = self.serialized_frames.pop_front() {
                if written < to_write {
                    self.serialized_frames
                        .push_front((to_write - written, resolver));
                    written = 0;
                } else {
                    if let Some(resolver) = resolver {
                        resolver.swear(Ok(()));
                    }
                    written -= to_write;
                }
            } else {
                break;
            }
        }

        if sz > 0 && self.send_buffer.available_data() > 0 {
            // We didn't write all the data yet
            self.send_continue()?;
        }
        Ok(())
    }

    fn read_from_stream(&mut self) -> Result<()> {
        match self.connection.status().state() {
            ConnectionState::Closed => Ok(()),
            ConnectionState::Error => Err(Error::InvalidConnectionState(ConnectionState::Error)),
            _ => {
                self.receive_buffer
                    .read_from(&mut self.socket, false)
                    .map(|sz| {
                        trace!("read {} bytes", sz);
                        self.receive_buffer.fill(sz, false);
                    })?;
                Ok(())
            }
        }
    }

    fn serialize(&mut self) -> Result<()> {
        if let Some((next_msg, resolver)) = self.connection.next_frame() {
            trace!("will write to buffer: {:?}", next_msg);
            let checkpoint = self.send_buffer.checkpoint();
            let res = gen_frame(&next_msg)((&mut self.send_buffer).into());
            match res.map(|w| w.into_inner().1) {
                Ok(sz) => {
                    self.serialized_frames.push_back((sz, resolver));
                    Ok(())
                }
                Err(e) => {
                    self.send_buffer.rollback(checkpoint);
                    match e {
                        GenError::BufferTooSmall(_) => {
                            // Requeue msg
                            self.connection.requeue_frame((next_msg, resolver))?;
                            Ok(())
                        }
                        e => {
                            error!("error generating frame: {:?}", e);
                            let error = Error::SerialisationError(Arc::new(e));
                            self.connection.set_error(error.clone())?;
                            Err(error)
                        }
                    }
                }
            }
        } else {
            Ok(())
        }
    }

    fn parse(&mut self) -> Result<()> {
        if self.can_parse() {
            if let Some(frame) = self.do_parse()? {
                self.connection.handle_frame(frame)?;
            }
        }
        Ok(())
    }

    fn do_parse(&mut self) -> Result<Option<AMQPFrame>> {
        match parse_frame(self.receive_buffer.data()) {
            Ok((i, f)) => {
                let consumed = self.receive_buffer.offset(i);
                self.receive_buffer.consume(consumed, false);
                Ok(Some(f))
            }
            Err(e) => {
                if e.is_incomplete() {
                    self.receive_buffer.shift();
                    Ok(None)
                } else {
                    error!("parse error: {:?}", e);
                    let error = Error::ParsingError(e);
                    self.connection.set_error(error.clone())?;
                    Err(error)
                }
            }
        }
    }
}
<|endoftext|>


<commit_before>pub mod config;
pub mod downloader;
pub mod socket;
pub mod database;<commit_msg>*socket deprecation<commit_after>pub mod config;
pub mod downloader;
//pub mod socket;
pub mod database;<|endoftext|>


<commit_before>use std::collections::HashMap;
use nameutil::split_namespace_name;

pub enum Transfer {
    None,
    Container,
    Full,
}

impl Transfer {
    pub fn by_name(name: &str) -> Option<Transfer> {
        use self::Transfer::*;
        match name {
            "none" => Some(None),
            "container" => Some(Container),
            "full" => Some(Full),
            _ => Option::None,
        }
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Fundamental {
    None,
    Boolean,
    Int8,
    UInt8,
    Int16,
    UInt16,
    Int32,
    UInt32,
    Int64,
    UInt64,
    Char,
    UChar,
    Int,
    UInt,
    Long,
    ULong,
    Size,
    SSize,
    Float,
    Double,
    Pointer,
    VarArgs,
    UniChar,
    Utf8,
    Filename,
    Type,
    Unsupported,
}

pub const FUNDAMENTAL: [(&'static str, Fundamental); 28] = [
    ("none", Fundamental::None),
    ("gboolean", Fundamental::Boolean),
    ("gint8", Fundamental::Int8),
    ("guint8", Fundamental::UInt8),
    ("gint16", Fundamental::Int16),
    ("guint16", Fundamental::UInt16),
    ("gint32", Fundamental::Int32),
    ("guint32", Fundamental::UInt32),
    ("gint64", Fundamental::Int64),
    ("guint64", Fundamental::UInt64),
    ("gchar", Fundamental::Char),
    ("guchar", Fundamental::UChar),
    ("gint", Fundamental::Int),
    ("guint", Fundamental::UInt),
    ("glong", Fundamental::Long),
    ("gulong", Fundamental::ULong),
    ("gsize", Fundamental::Size),
    ("gssize", Fundamental::SSize),
    ("gfloat", Fundamental::Float),
    ("gdouble", Fundamental::Double),
    ("long double", Fundamental::Unsupported),
    ("gunichar", Fundamental::UniChar),
    ("gpointer", Fundamental::Pointer),
    ("va_list", Fundamental::Unsupported),
    ("varargs", Fundamental::VarArgs),
    ("utf8", Fundamental::Utf8),
    ("filename", Fundamental::Filename),
    ("GType", Fundamental::Type),
];

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TypeId {
    ns_id: u16,
    id: u32,
}

pub struct Alias {
    pub name: String,
    pub c_identifier: String,
    pub typ: TypeId,
}

pub struct Constant {
    pub name: String,
    pub typ: TypeId,
    pub value: String,
}

pub struct Member {
    pub name: String,
    pub c_identifier: String,
    pub value: String,
}

pub struct Enumeration {
    pub name: String,
    pub glib_type_name: String,
    pub members: Vec<Member>,
    pub functions: Vec<Function>,
}

pub struct Bitfield {
    pub name: String,
    pub glib_type_name: String,
    pub members: Vec<Member>,
    pub functions: Vec<Function>,
}

pub struct Record {
    pub name: String,
    pub glib_type_name: String,
    pub functions: Vec<Function>,
}

pub struct Field {
    pub name: String,
    pub typ: TypeId,
}

pub struct Union {
    pub name: String,
    pub glib_type_name: String,
    pub fields: Vec<Field>,
    pub functions: Vec<Function>,
}

pub struct Parameter {
    pub name: String,
    pub typ: TypeId,
    pub transfer: Transfer,
}

pub struct Function {
    pub name: String,
    pub c_identifier: String,
    pub parameters: Vec<Parameter>,
    pub ret: Parameter,
}

pub struct Interface {
    pub name: String,
    pub glib_type_name: String,
    pub functions: Vec<Function>,
}

pub struct Class {
    pub name: String,
    pub glib_type_name: String,
    pub glib_get_type: String,
    pub functions: Vec<Function>,
    pub parent: Option<TypeId>,
    pub implements: Vec<TypeId>,
}

pub enum Type {
    Fundamental(Fundamental),
    Alias(Alias),
    Enumeration(Enumeration),
    Bitfield(Bitfield),
    Record(Record),
    Union(Union),
    Callback(Function),
    Interface(Interface),
    Class(Class),
    Array(TypeId),
    HashTable(TypeId, TypeId),
    List(TypeId),
    SList(TypeId),
}

impl Type {
    pub fn as_class(&self) -> Option<&Class> {
        if let &Type::Class(ref x) = self { Some(x) } else { None }
    }

    pub fn container(library: &mut Library, name: &str, mut inner: Vec<TypeId>) -> Option<TypeId> {
        match (name, inner.len()) {
            ("array", 1) => {
                let tid = inner.remove(0);
                Some((format!("array(#{:?})", tid), Type::Array(tid)))
            }
            ("GLib.HashTable", 2) => {
                let k_tid = inner.remove(0);
                let v_tid = inner.remove(0);
                Some((format!("HashTable(#{:?}, #{:?})", k_tid, v_tid), Type::HashTable(k_tid, v_tid)))
            }
            ("GLib.List", 1) => {
                let tid = inner.remove(0);
                Some((format!("List(#{:?})", tid), Type::List(tid)))
            }
            ("GLib.SList", 1) => {
                let tid = inner.remove(0);
                Some((format!("SList(#{:?})", tid), Type::SList(tid)))
            }
            _ => None,
        }.map(|(name, typ)| library.add_type(INTERNAL_NAMESPACE, &name, typ))
    }
}

pub trait AsArg {
    fn as_arg(&self, library: &Library) -> String;
}

impl AsArg for Fundamental {
    fn as_arg(&self, _: &Library) -> String {
        use self::Fundamental::*;
        match *self {
            Boolean => "gboolean",
            Int8 => "gint8",
            UInt8 => "guint8",
            Int16 => "gint16",
            UInt16 => "guint16",
            Int32 => "gint32",
            UInt32 => "guint32",
            Int64 => "gint64",
            UInt64 => "guint64",
            Char => "gchar",
            UChar => "guchar",
            Int => "gint",
            UInt => "guint",
            Long => "glong",
            ULong => "gulong",
            Size => "gsize",
            SSize => "gssize",
            Float => "gfloat",
            Double => "gdouble",
            UniChar => "gunichar",
            Pointer => "gpointer",
            VarArgs => "...",
            Utf8 => "*const c_char",
            Filename => "*const c_char",
            Type => "GType",
            None => "c_void",
            Unsupported => panic!("unsupported type"),
        }.into()
    }
}

impl AsArg for Type {
    fn as_arg(&self, library: &Library) -> String {
        use self::Type::*;
        match *self {
            Fundamental(ref x) => x.as_arg(library),
            Alias(ref x) => library.type_(x.typ).as_arg(library),
            Enumeration(ref x) => x.name.clone(),
            Bitfield(ref x) => x.name.clone(),
            Record(ref x) => format!("*mut {}", &x.name),
            Union(ref x) => format!("*mut {}", &x.name),
            Callback(_) => "TODO".into(),
            Interface(ref x) => format!("*mut {}", &x.name),
            Class(ref x) => format!("*mut {}", &x.name),
            Array(x) => format!("*mut {}", library.type_(x).as_arg(library)),
            HashTable(_, _)  => "*mut GHashTable".into(),
            List(_)  => "*mut GList".into(),
            SList(_)  => "*mut GSList".into(),
        }
    }
}

pub struct Namespace {
    pub name: String,
    pub types: Vec<Option<Type>>,
    pub index: HashMap<String, u32>,
    pub constants: Vec<Constant>,
    pub functions: Vec<Function>,
}

impl Namespace {
    fn new(name: &str) -> Namespace {
        Namespace {
            name: name.into(),
            types: Vec::new(),
            index: HashMap::new(),
            constants: Vec::new(),
            functions: Vec::new(),
        }
    }

    fn add_constant(&mut self, c: Constant) {
        self.constants.push(c);
    }

    fn add_function(&mut self, f: Function) {
        self.functions.push(f);
    }

    fn type_(&self, id: u32) -> &Type {
        self.types[id as usize].as_ref().unwrap()
    }

    fn add_type(&mut self, name: &str, typ: Option<Type>) -> u32 {
        if let Some(id) = self.find_type(name) {
            self.types[id as usize] = typ;
            id
        }
        else {
            let id = self.types.len() as u32;
            self.types.push(typ);
            self.index.insert(name.into(), id);
            id
        }
    }

    fn find_type(&self, name: &str) -> Option<u32> {
        self.index.get(name).cloned()
    }

    fn unresolved(&self) -> Vec<&str> {
        self.index.iter().filter_map(|(name, &id)| {
            if self.types[id as usize].is_none() {
                Some(&name[..])
            } else {
                None
            }
        }).collect()
    }
}

pub const INTERNAL_NAMESPACE_NAME: &'static str = "*";
pub const INTERNAL_NAMESPACE: u16 = 0;

pub struct Library {
    pub namespaces: Vec<Namespace>,
    pub index: HashMap<String, u16>,
}

impl Library {
    pub fn new() -> Library {
        let mut library = Library {
            namespaces: Vec::new(),
            index: HashMap::new(),
        };
        assert!(library.add_namespace(INTERNAL_NAMESPACE_NAME) == INTERNAL_NAMESPACE);
        library.namespace_mut(INTERNAL_NAMESPACE).name = INTERNAL_NAMESPACE_NAME.into();
        for &(name, t) in &FUNDAMENTAL {
            library.add_type(INTERNAL_NAMESPACE, name, Type::Fundamental(t));
        }
        library
    }

    pub fn namespace(&self, ns_id: u16) -> &Namespace {
        &self.namespaces[ns_id as usize]
    }

    pub fn namespace_mut(&mut self, ns_id: u16) -> &mut Namespace {
        &mut self.namespaces[ns_id as usize]
    }

    pub fn find_namespace(&self, name: &str) -> Option<u16> {
        self.index.get(name).cloned()
    }

    pub fn add_namespace(&mut self, name: &str) -> u16 {
        if let Some(&id) = self.index.get(name) {
            id
        }
        else {
            let id = self.namespaces.len() as u16;
            self.namespaces.push(Namespace::new(name));
            self.index.insert(name.into(), id);
            id
        }
    }

    pub fn add_constant(&mut self, ns_id: u16, c: Constant) {
        self.namespace_mut(ns_id).add_constant(c);
    }

    pub fn add_function(&mut self, ns_id: u16, f: Function) {
        self.namespace_mut(ns_id).add_function(f);
    }

    pub fn add_type(&mut self, ns_id: u16, name: &str, typ: Type) -> TypeId {
        TypeId { ns_id: ns_id, id: self.namespace_mut(ns_id).add_type(name, Some(typ)) }
    }

    pub fn find_type(&self, current_ns_id: u16, name: &str) -> Option<TypeId> {
        let (ns, name) = split_namespace_name(name);

        if let Some(ns) = ns {
            self.find_namespace(ns).and_then(|ns_id| {
                self.namespace(ns_id).find_type(name).map(|id| TypeId { ns_id: ns_id, id: id })
            })
        }
        else if let Some(id) = self.namespace(current_ns_id).find_type(name) {
            Some(TypeId { ns_id: current_ns_id, id: id })
        }
        else if let Some(id) = self.namespace(INTERNAL_NAMESPACE).find_type(name) {
            Some(TypeId { ns_id: INTERNAL_NAMESPACE, id: id })
        }
        else {
            None
        }
    }

    pub fn find_type_unwrapped(&self, current_ns_id: u16, name: &str, kind: &str) -> TypeId {
        self.find_type(current_ns_id, name)
            .unwrap_or_else(|| panic!("{} {} not found.", kind, name))
    }

    pub fn find_or_stub_type(&mut self, current_ns_id: u16, name: &str) -> TypeId {
        if let Some(tid) = self.find_type(current_ns_id, name) {
            return tid;
        }

        let (ns, name) = split_namespace_name(name);

        if let Some(ns) = ns {
            let ns_id = self.find_namespace(ns).unwrap_or_else(|| self.add_namespace(ns));
            let ns = self.namespace_mut(ns_id);
            let id = ns.find_type(name).unwrap_or_else(|| ns.add_type(name, None));
            return TypeId { ns_id: ns_id, id: id };
        }

        let id = self.namespace_mut(current_ns_id).add_type(name, None);
        TypeId { ns_id: current_ns_id, id: id }
    }

    pub fn type_(&self, tid: TypeId) -> &Type {
        self.namespace(tid.ns_id).type_(tid.id)
    }

    pub fn check_resolved(&self) {
        let list: Vec<_> = self.index.iter().flat_map(|(name, &id)| {
            let name = name.clone();
            self.namespace(id).unresolved().into_iter().map(move |s| format!("{}.{}", name, s))
        }).collect();

        if !list.is_empty() {
            panic!("Incomplete library, unresolved: {:?}", list);
        }
    }
}
<commit_msg>Add to_class and get_name to library::Type<commit_after>use std::collections::HashMap;
use nameutil::split_namespace_name;

pub enum Transfer {
    None,
    Container,
    Full,
}

impl Transfer {
    pub fn by_name(name: &str) -> Option<Transfer> {
        use self::Transfer::*;
        match name {
            "none" => Some(None),
            "container" => Some(Container),
            "full" => Some(Full),
            _ => Option::None,
        }
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Fundamental {
    None,
    Boolean,
    Int8,
    UInt8,
    Int16,
    UInt16,
    Int32,
    UInt32,
    Int64,
    UInt64,
    Char,
    UChar,
    Int,
    UInt,
    Long,
    ULong,
    Size,
    SSize,
    Float,
    Double,
    Pointer,
    VarArgs,
    UniChar,
    Utf8,
    Filename,
    Type,
    Unsupported,
}

pub const FUNDAMENTAL: [(&'static str, Fundamental); 28] = [
    ("none", Fundamental::None),
    ("gboolean", Fundamental::Boolean),
    ("gint8", Fundamental::Int8),
    ("guint8", Fundamental::UInt8),
    ("gint16", Fundamental::Int16),
    ("guint16", Fundamental::UInt16),
    ("gint32", Fundamental::Int32),
    ("guint32", Fundamental::UInt32),
    ("gint64", Fundamental::Int64),
    ("guint64", Fundamental::UInt64),
    ("gchar", Fundamental::Char),
    ("guchar", Fundamental::UChar),
    ("gint", Fundamental::Int),
    ("guint", Fundamental::UInt),
    ("glong", Fundamental::Long),
    ("gulong", Fundamental::ULong),
    ("gsize", Fundamental::Size),
    ("gssize", Fundamental::SSize),
    ("gfloat", Fundamental::Float),
    ("gdouble", Fundamental::Double),
    ("long double", Fundamental::Unsupported),
    ("gunichar", Fundamental::UniChar),
    ("gpointer", Fundamental::Pointer),
    ("va_list", Fundamental::Unsupported),
    ("varargs", Fundamental::VarArgs),
    ("utf8", Fundamental::Utf8),
    ("filename", Fundamental::Filename),
    ("GType", Fundamental::Type),
];

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TypeId {
    ns_id: u16,
    id: u32,
}

pub struct Alias {
    pub name: String,
    pub c_identifier: String,
    pub typ: TypeId,
}

pub struct Constant {
    pub name: String,
    pub typ: TypeId,
    pub value: String,
}

pub struct Member {
    pub name: String,
    pub c_identifier: String,
    pub value: String,
}

pub struct Enumeration {
    pub name: String,
    pub glib_type_name: String,
    pub members: Vec<Member>,
    pub functions: Vec<Function>,
}

pub struct Bitfield {
    pub name: String,
    pub glib_type_name: String,
    pub members: Vec<Member>,
    pub functions: Vec<Function>,
}

pub struct Record {
    pub name: String,
    pub glib_type_name: String,
    pub functions: Vec<Function>,
}

pub struct Field {
    pub name: String,
    pub typ: TypeId,
}

pub struct Union {
    pub name: String,
    pub glib_type_name: String,
    pub fields: Vec<Field>,
    pub functions: Vec<Function>,
}

pub struct Parameter {
    pub name: String,
    pub typ: TypeId,
    pub transfer: Transfer,
}

pub struct Function {
    pub name: String,
    pub c_identifier: String,
    pub parameters: Vec<Parameter>,
    pub ret: Parameter,
}

pub struct Interface {
    pub name: String,
    pub glib_type_name: String,
    pub functions: Vec<Function>,
}

pub struct Class {
    pub name: String,
    pub glib_type_name: String,
    pub glib_get_type: String,
    pub functions: Vec<Function>,
    pub parent: Option<TypeId>,
    pub implements: Vec<TypeId>,
}

pub enum Type {
    Fundamental(Fundamental),
    Alias(Alias),
    Enumeration(Enumeration),
    Bitfield(Bitfield),
    Record(Record),
    Union(Union),
    Callback(Function),
    Interface(Interface),
    Class(Class),
    Array(TypeId),
    HashTable(TypeId, TypeId),
    List(TypeId),
    SList(TypeId),
}

impl Type {
    pub fn as_class(&self) -> Option<&Class> {
        if let &Type::Class(ref x) = self { Some(x) } else { None }
    }
    pub fn to_class(&self) -> &Class {
        self.as_class()
            .unwrap_or_else(|| panic!("{} is not a class.", self.get_name()))
    }

    pub fn get_name(&self) -> String {
        match self {
            &Type::Fundamental(fund) => format!("{:?}", fund).into(),
            &Type::Alias(ref alias) => alias.name.clone(),
            &Type::Enumeration(ref enum_) => enum_.name.clone(),
            &Type::Bitfield(ref bit_field) => bit_field.name.clone(),
            &Type::Record(ref rec) => rec.name.clone(),
            &Type::Union(ref union) => union.name.clone(),
            &Type::Callback(ref func) => func.name.clone(),
            &Type::Interface(ref interface) => interface.name.clone(),
            &Type::Class(ref class) => class.name.clone(),
            &Type::Array(type_id) => format!("Array {:?}", type_id),
            &Type::HashTable(key_type_id, value_type_id) => format!("HashTable {:?}/{:?}", key_type_id, value_type_id),
            &Type::List(type_id) => format!("List {:?}", type_id),
            &Type::SList(type_id) => format!("SList {:?}", type_id),
        }
    }

    pub fn container(library: &mut Library, name: &str, mut inner: Vec<TypeId>) -> Option<TypeId> {
        match (name, inner.len()) {
            ("array", 1) => {
                let tid = inner.remove(0);
                Some((format!("array(#{:?})", tid), Type::Array(tid)))
            }
            ("GLib.HashTable", 2) => {
                let k_tid = inner.remove(0);
                let v_tid = inner.remove(0);
                Some((format!("HashTable(#{:?}, #{:?})", k_tid, v_tid), Type::HashTable(k_tid, v_tid)))
            }
            ("GLib.List", 1) => {
                let tid = inner.remove(0);
                Some((format!("List(#{:?})", tid), Type::List(tid)))
            }
            ("GLib.SList", 1) => {
                let tid = inner.remove(0);
                Some((format!("SList(#{:?})", tid), Type::SList(tid)))
            }
            _ => None,
        }.map(|(name, typ)| library.add_type(INTERNAL_NAMESPACE, &name, typ))
    }
}

pub trait AsArg {
    fn as_arg(&self, library: &Library) -> String;
}

impl AsArg for Fundamental {
    fn as_arg(&self, _: &Library) -> String {
        use self::Fundamental::*;
        match *self {
            Boolean => "gboolean",
            Int8 => "gint8",
            UInt8 => "guint8",
            Int16 => "gint16",
            UInt16 => "guint16",
            Int32 => "gint32",
            UInt32 => "guint32",
            Int64 => "gint64",
            UInt64 => "guint64",
            Char => "gchar",
            UChar => "guchar",
            Int => "gint",
            UInt => "guint",
            Long => "glong",
            ULong => "gulong",
            Size => "gsize",
            SSize => "gssize",
            Float => "gfloat",
            Double => "gdouble",
            UniChar => "gunichar",
            Pointer => "gpointer",
            VarArgs => "...",
            Utf8 => "*const c_char",
            Filename => "*const c_char",
            Type => "GType",
            None => "c_void",
            Unsupported => panic!("unsupported type"),
        }.into()
    }
}

impl AsArg for Type {
    fn as_arg(&self, library: &Library) -> String {
        use self::Type::*;
        match *self {
            Fundamental(ref x) => x.as_arg(library),
            Alias(ref x) => library.type_(x.typ).as_arg(library),
            Enumeration(ref x) => x.name.clone(),
            Bitfield(ref x) => x.name.clone(),
            Record(ref x) => format!("*mut {}", &x.name),
            Union(ref x) => format!("*mut {}", &x.name),
            Callback(_) => "TODO".into(),
            Interface(ref x) => format!("*mut {}", &x.name),
            Class(ref x) => format!("*mut {}", &x.name),
            Array(x) => format!("*mut {}", library.type_(x).as_arg(library)),
            HashTable(_, _)  => "*mut GHashTable".into(),
            List(_)  => "*mut GList".into(),
            SList(_)  => "*mut GSList".into(),
        }
    }
}

pub struct Namespace {
    pub name: String,
    pub types: Vec<Option<Type>>,
    pub index: HashMap<String, u32>,
    pub constants: Vec<Constant>,
    pub functions: Vec<Function>,
}

impl Namespace {
    fn new(name: &str) -> Namespace {
        Namespace {
            name: name.into(),
            types: Vec::new(),
            index: HashMap::new(),
            constants: Vec::new(),
            functions: Vec::new(),
        }
    }

    fn add_constant(&mut self, c: Constant) {
        self.constants.push(c);
    }

    fn add_function(&mut self, f: Function) {
        self.functions.push(f);
    }

    fn type_(&self, id: u32) -> &Type {
        self.types[id as usize].as_ref().unwrap()
    }

    fn add_type(&mut self, name: &str, typ: Option<Type>) -> u32 {
        if let Some(id) = self.find_type(name) {
            self.types[id as usize] = typ;
            id
        }
        else {
            let id = self.types.len() as u32;
            self.types.push(typ);
            self.index.insert(name.into(), id);
            id
        }
    }

    fn find_type(&self, name: &str) -> Option<u32> {
        self.index.get(name).cloned()
    }

    fn unresolved(&self) -> Vec<&str> {
        self.index.iter().filter_map(|(name, &id)| {
            if self.types[id as usize].is_none() {
                Some(&name[..])
            } else {
                None
            }
        }).collect()
    }
}

pub const INTERNAL_NAMESPACE_NAME: &'static str = "*";
pub const INTERNAL_NAMESPACE: u16 = 0;

pub struct Library {
    pub namespaces: Vec<Namespace>,
    pub index: HashMap<String, u16>,
}

impl Library {
    pub fn new() -> Library {
        let mut library = Library {
            namespaces: Vec::new(),
            index: HashMap::new(),
        };
        assert!(library.add_namespace(INTERNAL_NAMESPACE_NAME) == INTERNAL_NAMESPACE);
        library.namespace_mut(INTERNAL_NAMESPACE).name = INTERNAL_NAMESPACE_NAME.into();
        for &(name, t) in &FUNDAMENTAL {
            library.add_type(INTERNAL_NAMESPACE, name, Type::Fundamental(t));
        }
        library
    }

    pub fn namespace(&self, ns_id: u16) -> &Namespace {
        &self.namespaces[ns_id as usize]
    }

    pub fn namespace_mut(&mut self, ns_id: u16) -> &mut Namespace {
        &mut self.namespaces[ns_id as usize]
    }

    pub fn find_namespace(&self, name: &str) -> Option<u16> {
        self.index.get(name).cloned()
    }

    pub fn add_namespace(&mut self, name: &str) -> u16 {
        if let Some(&id) = self.index.get(name) {
            id
        }
        else {
            let id = self.namespaces.len() as u16;
            self.namespaces.push(Namespace::new(name));
            self.index.insert(name.into(), id);
            id
        }
    }

    pub fn add_constant(&mut self, ns_id: u16, c: Constant) {
        self.namespace_mut(ns_id).add_constant(c);
    }

    pub fn add_function(&mut self, ns_id: u16, f: Function) {
        self.namespace_mut(ns_id).add_function(f);
    }

    pub fn add_type(&mut self, ns_id: u16, name: &str, typ: Type) -> TypeId {
        TypeId { ns_id: ns_id, id: self.namespace_mut(ns_id).add_type(name, Some(typ)) }
    }

    pub fn find_type(&self, current_ns_id: u16, name: &str) -> Option<TypeId> {
        let (ns, name) = split_namespace_name(name);

        if let Some(ns) = ns {
            self.find_namespace(ns).and_then(|ns_id| {
                self.namespace(ns_id).find_type(name).map(|id| TypeId { ns_id: ns_id, id: id })
            })
        }
        else if let Some(id) = self.namespace(current_ns_id).find_type(name) {
            Some(TypeId { ns_id: current_ns_id, id: id })
        }
        else if let Some(id) = self.namespace(INTERNAL_NAMESPACE).find_type(name) {
            Some(TypeId { ns_id: INTERNAL_NAMESPACE, id: id })
        }
        else {
            None
        }
    }

    pub fn find_type_unwrapped(&self, current_ns_id: u16, name: &str, kind: &str) -> TypeId {
        self.find_type(current_ns_id, name)
            .unwrap_or_else(|| panic!("{} {} not found.", kind, name))
    }

    pub fn find_or_stub_type(&mut self, current_ns_id: u16, name: &str) -> TypeId {
        if let Some(tid) = self.find_type(current_ns_id, name) {
            return tid;
        }

        let (ns, name) = split_namespace_name(name);

        if let Some(ns) = ns {
            let ns_id = self.find_namespace(ns).unwrap_or_else(|| self.add_namespace(ns));
            let ns = self.namespace_mut(ns_id);
            let id = ns.find_type(name).unwrap_or_else(|| ns.add_type(name, None));
            return TypeId { ns_id: ns_id, id: id };
        }

        let id = self.namespace_mut(current_ns_id).add_type(name, None);
        TypeId { ns_id: current_ns_id, id: id }
    }

    pub fn type_(&self, tid: TypeId) -> &Type {
        self.namespace(tid.ns_id).type_(tid.id)
    }

    pub fn check_resolved(&self) {
        let list: Vec<_> = self.index.iter().flat_map(|(name, &id)| {
            let name = name.clone();
            self.namespace(id).unresolved().into_iter().map(move |s| format!("{}.{}", name, s))
        }).collect();

        if !list.is_empty() {
            panic!("Incomplete library, unresolved: {:?}", list);
        }
    }
}
<|endoftext|>


<commit_before>use super::wasm_base;
use super::{LinkArgs, LinkerFlavor, PanicStrategy, Target, TargetOptions};

pub fn target() -> Target {
    let mut options = wasm_base::options();

    let clang_args = options.pre_link_args.entry(LinkerFlavor::Gcc).or_default();

    // Rust really needs a way for users to specify exports and imports in
    // the source code. --export-dynamic isn't the right tool for this job,
    // however it does have the side effect of automatically exporting a lot
    // of symbols, which approximates what people want when compiling for
    // wasm32-unknown-unknown expect, so use it for now.
    clang_args.push("--export-dynamic".to_string());

    let mut post_link_args = LinkArgs::new();
    post_link_args.insert(
        LinkerFlavor::Em,
        vec![
            "-s".to_string(),
            "ERROR_ON_UNDEFINED_SYMBOLS=1".to_string(),
            "-s".to_string(),
            "ASSERTIONS=1".to_string(),
            "-s".to_string(),
            "ABORTING_MALLOC=0".to_string(),
            "-Wl,--fatal-warnings".to_string(),
        ],
    );

    let opts = TargetOptions {
        os: "emscripten".to_string(),
        linker_flavor: LinkerFlavor::Em,
        // emcc emits two files - a .js file to instantiate the wasm and supply platform
        // functionality, and a .wasm file.
        exe_suffix: ".js".to_string(),
        linker: None,
        is_like_emscripten: true,
        panic_strategy: PanicStrategy::Unwind,
        post_link_args,
        families: vec!["unix".to_string()],
        ..options
    };
    Target {
        llvm_target: "wasm32-unknown-emscripten".to_string(),
        pointer_width: 32,
        data_layout: "e-m:e-p:32:32-i64:64-f128:64-n32:64-S128-ni:1:10:20".to_string(),
        arch: "wasm32".to_string(),
        options: opts,
    }
}
<commit_msg>Add emscripten to the "wasm" family of targets<commit_after>use super::wasm_base;
use super::{LinkArgs, LinkerFlavor, PanicStrategy, Target, TargetOptions};

pub fn target() -> Target {
    let mut options = wasm_base::options();

    let clang_args = options.pre_link_args.entry(LinkerFlavor::Gcc).or_default();

    // Rust really needs a way for users to specify exports and imports in
    // the source code. --export-dynamic isn't the right tool for this job,
    // however it does have the side effect of automatically exporting a lot
    // of symbols, which approximates what people want when compiling for
    // wasm32-unknown-unknown expect, so use it for now.
    clang_args.push("--export-dynamic".to_string());

    let mut post_link_args = LinkArgs::new();
    post_link_args.insert(
        LinkerFlavor::Em,
        vec![
            "-s".to_string(),
            "ERROR_ON_UNDEFINED_SYMBOLS=1".to_string(),
            "-s".to_string(),
            "ASSERTIONS=1".to_string(),
            "-s".to_string(),
            "ABORTING_MALLOC=0".to_string(),
            "-Wl,--fatal-warnings".to_string(),
        ],
    );

    let opts = TargetOptions {
        os: "emscripten".to_string(),
        linker_flavor: LinkerFlavor::Em,
        // emcc emits two files - a .js file to instantiate the wasm and supply platform
        // functionality, and a .wasm file.
        exe_suffix: ".js".to_string(),
        linker: None,
        is_like_emscripten: true,
        panic_strategy: PanicStrategy::Unwind,
        post_link_args,
        families: vec!["unix".to_string(), "wasm".to_string()],
        ..options
    };
    Target {
        llvm_target: "wasm32-unknown-emscripten".to_string(),
        pointer_width: 32,
        data_layout: "e-m:e-p:32:32-i64:64-f128:64-n32:64-S128-ni:1:10:20".to_string(),
        arch: "wasm32".to_string(),
        options: opts,
    }
}
<|endoftext|>


<commit_before>extern crate git2;

use git2::*;
use std::process::Command;
use std::path::Path;
use std::path::PathBuf;

// FIXME: hardcoded path
const TMP_REPO_DIR: &'static str = "/home/christian/gerrit_testsite/tmp_automation_repo";

pub fn module_review_upload(project: &str,
                            newrev: &str,
                            remote_module_url: &Fn(&str) -> String,
                            central_name: &str)
                            -> Result<(), git2::Error> {

    let tmp_repo = try!(Repository::init_bare(TMP_REPO_DIR));
    try!(in_tmp_repo("fetch --all"));

    transfer_to_tmp(newrev);
    let mut parent_commit_oid: git2::Oid = try!(try!(tmp_repo.revparse_single(central_name))
        .as_commit()
        .map(|x| x.id())
        .ok_or(git2::Error::from_str("could not get commit from obj")));

    let module_name: &str = try!(Path::new(project)
        .components()
        .last()
        .map(|x| x.as_os_str().to_str().expect("not a valid unicode string"))
        .ok_or(git2::Error::from_str("needs to be valid name")));

    let object = try!(tmp_repo.revparse_single(&remote_ref_name(&module_name))
        .map(|x| x.id()));
    let oldrev = format!("{}", object);

    {
        let old = try!(tmp_repo.revparse_single(&oldrev).map(|x| x.id()));
        let new = try!(tmp_repo.revparse_single(&newrev).map(|x| x.id()));

        if !try!(tmp_repo.graph_descendant_of(new, old)) {
            println!(".");
            println!("===========================================================");
            println!("======== Commit not based on master, rebase first! ========");
            println!("===========================================================");
            return Ok(());
        }
    }

    let walk = {
        let mut walk = try!(tmp_repo.revwalk());
        walk.set_sorting(git2::SORT_REVERSE | git2::SORT_TIME);
        try!(walk.push_range(&format!("{}..{}", oldrev, newrev)));
        walk
    };

    println!("===== project path: {}", project);
    println!("===== Apply commits from {} to {}", oldrev, newrev);

    for rev in walk {
        let newrev = format!("{}", try!(rev));
        if oldrev == newrev {
            continue;
        }
        println!("===== Apply commit {}", newrev);

        let module_commit_obj = try!(tmp_repo.revparse_single(&newrev));
        let module_commit = try!(module_commit_obj.as_commit()
            .ok_or(git2::Error::from_str("object is not actually a commit")));
        let module_tree = try!(module_commit.tree());

        let parent_commit = try!(tmp_repo.find_commit(parent_commit_oid));

        let new_tree = {
            let master_tree: Tree = try!(parent_commit.tree());
            let new_tree_oid =
                try!(module_to_subfolder(Path::new(module_name), &module_tree, &master_tree));
            try!(tmp_repo.find_tree(new_tree_oid))
        };

        parent_commit_oid =
            try!(make_commit(&tmp_repo, &new_tree, module_commit, &vec![&parent_commit]));
    }

    println!("");
    println!("");
    println!("===================== Doing actual upload in central git ========================");
    let x = try!(push_from_tmp(&tmp_repo,
                               &try!(tmp_repo.find_commit(parent_commit_oid)),
                               central_name,
                               "refs/for/master",
                               &remote_module_url));
    println!("{}", x);
    println!("==== The review upload may have worked, even if it says error below. Look UP! ====");
    Ok(())
}

fn remote_ref_name(module_name: &str) -> String {
    format!("remotes/modules/{}/master", &module_name)
}

pub fn central_submit(remote_addr: &str,
                      newrev: &str,
                      remote_module_url: &Fn(&str) -> String,
                      check: &Fn(&str) -> Result<(), git2::Error>,
                      module_path_prefix: &str,
                      central_name: &str)
                      -> Result<(), git2::Error> {
    println!("central_submit");

    let module_names = try!(get_module_names(newrev));

    let tmp_repo = try!(setup_tmp_repo(&remote_addr,
                                       &module_names,
                                       &check,
                                       &remote_module_url,
                                       module_path_prefix));
    transfer_to_tmp(newrev);

    let central_commit_obj = try!(tmp_repo.revparse_single(newrev));
    let central_commit = try!(central_commit_obj.as_commit()
        .ok_or(git2::Error::from_str("could not get commit from obj")));
    let central_tree = try!(central_commit.tree());
    // create central_name branch in tmp_repo and point to the central_commit
    // is identical to master in central
    // marker for other hooks
    try!(tmp_repo.branch(central_name, central_commit, true));

    for module_name in module_names {
        // get master branches from module
        let module_master_commit_obj =
            try!(tmp_repo.revparse_single(&remote_ref_name(&module_name)));
        let module_master_commit = try!(module_master_commit_obj.as_commit()
            .ok_or(git2::Error::from_str("could not get commit from obj")));
        // maybe not needed: branch
        try!(tmp_repo.branch(&format!("modules/{}", module_name),
                             module_master_commit,
                             true));

        let parents = vec![module_master_commit];

        // get path for module in central repository
        let module_path = {
            let mut p = PathBuf::new();
            p.push("modules");
            p.push(&module_name);
            p
        };

        // new tree is sub-tree of complete central tree
        let new_tree_oid = try!(central_tree.get_path(&module_path)).id();
        let old_tree_oid = try!(module_master_commit.tree()).id();

        // if sha1's are equal the content is equal
        if new_tree_oid != old_tree_oid {
            // need to update module git

            let new_tree = try!(tmp_repo.find_tree(new_tree_oid));

            let module_commit = try!(make_commit(&tmp_repo, &new_tree, central_commit, &parents));
            // do the push to the module git
            let x = try!(push_from_tmp(&tmp_repo,
                                       &try!(tmp_repo.find_commit(module_commit)),
                                       &format!("{}/{}", module_path_prefix, module_name),
                                       "master",
                                       &remote_module_url));
            println!("{}", x);
        }
    }
    Ok(())
}

// force push of the new revision-object to temp repo
fn transfer_to_tmp(rev: &str) {
    // create tmp branch
    Command::new("git")
        .arg("branch")
        .arg("-f")
        .arg("tmp")
        .arg(rev)
        .output()
        .expect("failed to call git");

    // force push
    Command::new("git")
        .arg("push")
        .arg("--force")
        .arg(TMP_REPO_DIR)
        .arg("tmp")
        .output()
        .expect("failed to call git");

    // delete tmp branch
    Command::new("git")
        .arg("branch")
        .arg("-D")
        .arg("tmp")
        .output()
        .expect("failed to call git");
}

fn in_tmp_repo(cmd: &str) -> Result<String, git2::Error> {
    let args: Vec<&str> = cmd.split(" ").collect();
    match Command::new("git")
        .env("GIT_DIR", TMP_REPO_DIR)
        .args(&args)
        .output()
        .map(|output| format!("{}", String::from_utf8_lossy(&output.stderr))) {
        Ok(value) => Ok(value),
        Err(_) => Err(git2::Error::from_str("could not fire git command")),
    }
}

fn setup_tmp_repo(remote_addr: &str,
                  modules: &Vec<String>,
                  check: &Fn(&str) -> Result<(), git2::Error>,
                  remote_module_url: &Fn(&str) -> String,
                  module_path_prefix: &str)
                  -> Result<Repository, git2::Error> {
    let repo = try!(Repository::init_bare(TMP_REPO_DIR));

    // point remote to central
    if !repo.find_remote("central_repo").is_ok() {
        try!(repo.remote("central_repo", &remote_addr));
    }

    // create remote for each module
    for module in modules.iter() {
        try!(check(module));

        // create remote for each module
        let remote_name = format!("modules/{}", module);
        if !repo.find_remote(&remote_name).is_ok() {
            try!(repo.remote(&remote_name,
                             &remote_module_url(&format!("{}/{}.git",
                                                         module_path_prefix,
                                                         module))));
        }
    }

    // fetch all branches from remotes
    try!(in_tmp_repo("fetch --all"));

    Ok(repo)
}

fn module_to_subfolder(path: &Path,
                       module_tree: &Tree,
                       master_tree: &Tree)
                       -> Result<Oid, git2::Error> {
    let mpath = Path::new("modules");
    let modules_oid = try!(master_tree.get_path(mpath).map(|x| x.id()));
    let tmp_repo = try!(Repository::init_bare(TMP_REPO_DIR));

    let modules_tree = try!(tmp_repo.find_tree(modules_oid));
    let mut mbuilder = try!(tmp_repo.treebuilder(Some(&modules_tree)));
    try!(mbuilder.insert(path, module_tree.id(), 0o0040000)); // GIT_FILEMODE_TREE
    let mtree = try!(mbuilder.write());

    let mut builder = try!(tmp_repo.treebuilder(Some(master_tree)));
    try!(builder.insert(mpath, mtree, 0o0040000)); // GIT_FILEMODE_TREE
    let r = try!(builder.write());
    println!("module_to_subfolder {}", r);
    Ok(r)
}

fn get_module_names(rev: &str) -> Result<Vec<String>, git2::Error> {
    let central_repo = try!(Repository::open("."));

    let object = try!(central_repo.revparse_single(rev));
    let commit = try!(object.as_commit()
        .ok_or(git2::Error::from_str("could not get commit from obj")));
    let tree: git2::Tree = try!(commit.tree());

    let tree_object = try!(tree.get_path(&Path::new("modules")));
    let modules_o = try!(tree_object.to_object(&central_repo));
    let modules = try!(modules_o.as_tree()
        .ok_or(git2::Error::from_str("could not get tree from path")));

    let mut names = Vec::<String>::new();
    for module in modules.iter() {
        names.push(try!(module.name()
                .ok_or(git2::Error::from_str("could not get name for module")))
            .to_string());
    }
    Ok(names)
}

fn push_from_tmp(tmp_repo: &Repository,
                 commit: &Commit,
                 repo: &str,
                 to: &str,
                 remote_module_url: &Fn(&str) -> String)
                 -> Result<String, git2::Error> {
    try!(tmp_repo.set_head_detached(commit.id()));

    let remote_url = &remote_module_url(&format!("{}.git", repo));
    in_tmp_repo(&format!("push {} HEAD:{}", remote_url, to))
}

// takes everything from base except it's tree and replaces it with the tree given
fn make_commit(repo: &Repository,
               tree: &Tree,
               base: &Commit,
               parents: &[&Commit])
               -> Result<Oid, git2::Error> {
    if parents.len() != 0 {
        try!(repo.set_head_detached(parents[0].id()));
    }
    repo.commit(Some("HEAD"),
                &base.author(),
                &base.committer(),
                &base.message().unwrap_or("no message"),
                tree,
                parents)
}

#[cfg(test)]
mod tests {
    extern crate tempdir;
    extern crate git2;
    use super::central_submit;
    use self::tempdir::TempDir;
    use std::path::{Path, PathBuf};
    use std::fs::File;
    use std::io::Write;

    #[test]
    fn test_commit_to_central() {
        let td = TempDir::new("play").expect("folder play should be created");
        let repo = git2::Repository::init(td.path()).expect("init should succeed");

        let pb = Path::new(td.path());
        // Print a message to stdout like "git init" does
        let path = repo.workdir().unwrap().to_path_buf();
        println!("Initialized empty Git repository in {}", path.display());

        let file_name = "foo.txt";
        let foo_file = pb.join(file_name);
        create_dummy_file(&foo_file);
        let oid = commit_file(&repo, &Path::new(file_name), &[]);

        let parent_commit = repo.find_commit(oid).expect("find parent_commit");

        let file_name = "bar.txt";
        let bar_file = pb.join(file_name);
        create_dummy_file(&bar_file);
        commit_file(&repo, &Path::new(file_name), &vec![&parent_commit]);
        assert!(true);
    }

    fn commit_file(repo: &git2::Repository, file: &Path, parents: &[&git2::Commit]) -> git2::Oid {
        let mut index = repo.index().expect("get index of repo");
        index.add_path(file).expect("file should be added");
        index.write().expect("write index");
        let tree_id = index.write_tree().expect("got tree_id");
        let tree = repo.find_tree(tree_id).expect("got tree");
        let sig = git2::Signature::now("foo", "bar").expect("created signature");
        repo.commit(Some("HEAD"), &sig, &sig, "commit A", &tree, &parents)
            .expect("commit to repo")
    }

    fn create_dummy_file(f: &PathBuf) {
        let mut file = File::create(&f.as_path()).expect("create file");
        file.write_all("test content".as_bytes()).expect("write to file");
    }
}
<commit_msg>test setup<commit_after>extern crate git2;

use git2::*;
use std::process::Command;
use std::path::Path;
use std::path::PathBuf;

// FIXME: hardcoded path
const TMP_REPO_DIR: &'static str = "/home/christian/gerrit_testsite/tmp_automation_repo";

pub fn module_review_upload(project: &str,
                            newrev: &str,
                            remote_module_url: &Fn(&str) -> String,
                            central_name: &str)
                            -> Result<(), git2::Error> {

    let tmp_repo = try!(Repository::init_bare(TMP_REPO_DIR));
    try!(in_tmp_repo("fetch --all"));

    transfer_to_tmp(newrev);
    let mut parent_commit_oid: git2::Oid = try!(try!(tmp_repo.revparse_single(central_name))
        .as_commit()
        .map(|x| x.id())
        .ok_or(git2::Error::from_str("could not get commit from obj")));

    let module_name: &str = try!(Path::new(project)
        .components()
        .last()
        .map(|x| x.as_os_str().to_str().expect("not a valid unicode string"))
        .ok_or(git2::Error::from_str("needs to be valid name")));

    let object = try!(tmp_repo.revparse_single(&remote_ref_name(&module_name))
        .map(|x| x.id()));
    let oldrev = format!("{}", object);

    {
        let old = try!(tmp_repo.revparse_single(&oldrev).map(|x| x.id()));
        let new = try!(tmp_repo.revparse_single(&newrev).map(|x| x.id()));

        if !try!(tmp_repo.graph_descendant_of(new, old)) {
            println!(".");
            println!("===========================================================");
            println!("======== Commit not based on master, rebase first! ========");
            println!("===========================================================");
            return Ok(());
        }
    }

    let walk = {
        let mut walk = try!(tmp_repo.revwalk());
        walk.set_sorting(git2::SORT_REVERSE | git2::SORT_TIME);
        try!(walk.push_range(&format!("{}..{}", oldrev, newrev)));
        walk
    };

    println!("===== project path: {}", project);
    println!("===== Apply commits from {} to {}", oldrev, newrev);

    for rev in walk {
        let newrev = format!("{}", try!(rev));
        if oldrev == newrev {
            continue;
        }
        println!("===== Apply commit {}", newrev);

        let module_commit_obj = try!(tmp_repo.revparse_single(&newrev));
        let module_commit = try!(module_commit_obj.as_commit()
            .ok_or(git2::Error::from_str("object is not actually a commit")));
        let module_tree = try!(module_commit.tree());

        let parent_commit = try!(tmp_repo.find_commit(parent_commit_oid));

        let new_tree = {
            let master_tree: Tree = try!(parent_commit.tree());
            let new_tree_oid =
                try!(module_to_subfolder(Path::new(module_name), &module_tree, &master_tree));
            try!(tmp_repo.find_tree(new_tree_oid))
        };

        parent_commit_oid =
            try!(make_commit(&tmp_repo, &new_tree, module_commit, &vec![&parent_commit]));
    }

    println!("");
    println!("");
    println!("===================== Doing actual upload in central git ========================");
    let x = try!(push_from_tmp(&tmp_repo,
                               &try!(tmp_repo.find_commit(parent_commit_oid)),
                               central_name,
                               "refs/for/master",
                               &remote_module_url));
    println!("{}", x);
    println!("==== The review upload may have worked, even if it says error below. Look UP! ====");
    Ok(())
}

fn remote_ref_name(module_name: &str) -> String {
    format!("remotes/modules/{}/master", &module_name)
}

pub fn central_submit(remote_addr: &str,
                      newrev: &str,
                      remote_module_url: &Fn(&str) -> String,
                      check: &Fn(&str) -> Result<(), git2::Error>,
                      module_path_prefix: &str,
                      central_name: &str)
                      -> Result<(), git2::Error> {
    println!("central_submit");

    let module_names = try!(get_module_names(newrev));

    let tmp_repo = try!(setup_tmp_repo(&remote_addr,
                                       &module_names,
                                       &check,
                                       &remote_module_url,
                                       module_path_prefix));
    transfer_to_tmp(newrev);

    let central_commit_obj = try!(tmp_repo.revparse_single(newrev));
    let central_commit = try!(central_commit_obj.as_commit()
        .ok_or(git2::Error::from_str("could not get commit from obj")));
    let central_tree = try!(central_commit.tree());
    // create central_name branch in tmp_repo and point to the central_commit
    // is identical to master in central
    // marker for other hooks
    try!(tmp_repo.branch(central_name, central_commit, true));

    for module_name in module_names {
        // get master branches from module
        let module_master_commit_obj =
            try!(tmp_repo.revparse_single(&remote_ref_name(&module_name)));
        let module_master_commit = try!(module_master_commit_obj.as_commit()
            .ok_or(git2::Error::from_str("could not get commit from obj")));
        // maybe not needed: branch
        try!(tmp_repo.branch(&format!("modules/{}", module_name),
                             module_master_commit,
                             true));

        let parents = vec![module_master_commit];

        // get path for module in central repository
        let module_path = {
            let mut p = PathBuf::new();
            p.push("modules");
            p.push(&module_name);
            p
        };

        // new tree is sub-tree of complete central tree
        let new_tree_oid = try!(central_tree.get_path(&module_path)).id();
        let old_tree_oid = try!(module_master_commit.tree()).id();

        // if sha1's are equal the content is equal
        if new_tree_oid != old_tree_oid {
            // need to update module git

            let new_tree = try!(tmp_repo.find_tree(new_tree_oid));

            let module_commit = try!(make_commit(&tmp_repo, &new_tree, central_commit, &parents));
            // do the push to the module git
            let x = try!(push_from_tmp(&tmp_repo,
                                       &try!(tmp_repo.find_commit(module_commit)),
                                       &format!("{}/{}", module_path_prefix, module_name),
                                       "master",
                                       &remote_module_url));
            println!("{}", x);
        }
    }
    Ok(())
}

// force push of the new revision-object to temp repo
fn transfer_to_tmp(rev: &str) {
    // create tmp branch
    Command::new("git")
        .arg("branch")
        .arg("-f")
        .arg("tmp")
        .arg(rev)
        .output()
        .expect("failed to call git");

    // force push
    Command::new("git")
        .arg("push")
        .arg("--force")
        .arg(TMP_REPO_DIR)
        .arg("tmp")
        .output()
        .expect("failed to call git");

    // delete tmp branch
    Command::new("git")
        .arg("branch")
        .arg("-D")
        .arg("tmp")
        .output()
        .expect("failed to call git");
}

fn in_tmp_repo(cmd: &str) -> Result<String, git2::Error> {
    let args: Vec<&str> = cmd.split(" ").collect();
    match Command::new("git")
        .env("GIT_DIR", TMP_REPO_DIR)
        .args(&args)
        .output()
        .map(|output| format!("{}", String::from_utf8_lossy(&output.stderr))) {
        Ok(value) => Ok(value),
        Err(_) => Err(git2::Error::from_str("could not fire git command")),
    }
}

fn setup_tmp_repo(remote_addr: &str,
                  modules: &Vec<String>,
                  check: &Fn(&str) -> Result<(), git2::Error>,
                  remote_module_url: &Fn(&str) -> String,
                  module_path_prefix: &str)
                  -> Result<Repository, git2::Error> {
    let repo = try!(Repository::init_bare(TMP_REPO_DIR));

    // point remote to central
    if !repo.find_remote("central_repo").is_ok() {
        try!(repo.remote("central_repo", &remote_addr));
    }

    // create remote for each module
    for module in modules.iter() {
        try!(check(module));

        // create remote for each module
        let remote_name = format!("modules/{}", module);
        if !repo.find_remote(&remote_name).is_ok() {
            try!(repo.remote(&remote_name,
                             &remote_module_url(&format!("{}/{}.git",
                                                         module_path_prefix,
                                                         module))));
        }
    }

    // fetch all branches from remotes
    try!(in_tmp_repo("fetch --all"));

    Ok(repo)
}

fn module_to_subfolder(path: &Path,
                       module_tree: &Tree,
                       master_tree: &Tree)
                       -> Result<Oid, git2::Error> {
    let mpath = Path::new("modules");
    let modules_oid = try!(master_tree.get_path(mpath).map(|x| x.id()));
    let tmp_repo = try!(Repository::init_bare(TMP_REPO_DIR));

    let modules_tree = try!(tmp_repo.find_tree(modules_oid));
    let mut mbuilder = try!(tmp_repo.treebuilder(Some(&modules_tree)));
    try!(mbuilder.insert(path, module_tree.id(), 0o0040000)); // GIT_FILEMODE_TREE
    let mtree = try!(mbuilder.write());

    let mut builder = try!(tmp_repo.treebuilder(Some(master_tree)));
    try!(builder.insert(mpath, mtree, 0o0040000)); // GIT_FILEMODE_TREE
    let r = try!(builder.write());
    println!("module_to_subfolder {}", r);
    Ok(r)
}

fn get_module_names(rev: &str) -> Result<Vec<String>, git2::Error> {
    let central_repo = try!(Repository::open("."));

    let object = try!(central_repo.revparse_single(rev));
    let commit = try!(object.as_commit()
        .ok_or(git2::Error::from_str("could not get commit from obj")));
    let tree: git2::Tree = try!(commit.tree());

    let tree_object = try!(tree.get_path(&Path::new("modules")));
    let modules_o = try!(tree_object.to_object(&central_repo));
    let modules = try!(modules_o.as_tree()
        .ok_or(git2::Error::from_str("could not get tree from path")));

    let mut names = Vec::<String>::new();
    for module in modules.iter() {
        names.push(try!(module.name()
                .ok_or(git2::Error::from_str("could not get name for module")))
            .to_string());
    }
    Ok(names)
}

fn push_from_tmp(tmp_repo: &Repository,
                 commit: &Commit,
                 repo: &str,
                 to: &str,
                 remote_module_url: &Fn(&str) -> String)
                 -> Result<String, git2::Error> {
    try!(tmp_repo.set_head_detached(commit.id()));

    let remote_url = &remote_module_url(&format!("{}.git", repo));
    in_tmp_repo(&format!("push {} HEAD:{}", remote_url, to))
}

// takes everything from base except it's tree and replaces it with the tree given
fn make_commit(repo: &Repository,
               tree: &Tree,
               base: &Commit,
               parents: &[&Commit])
               -> Result<Oid, git2::Error> {
    if parents.len() != 0 {
        try!(repo.set_head_detached(parents[0].id()));
    }
    repo.commit(Some("HEAD"),
                &base.author(),
                &base.committer(),
                &base.message().unwrap_or("no message"),
                tree,
                parents)
}

#[cfg(test)]
mod tests {
    extern crate tempdir;
    extern crate git2;
    use super::central_submit;
    use self::tempdir::TempDir;
    use std::path::{Path, PathBuf};
    use std::fs::File;
    use std::fs;
    use std::io::Write;
    use std::process::Command;

    #[test]
    fn test_commit_to_central() {
        let td = TempDir::new("play").expect("folder play should be created");

        let _module_repos: Vec<git2::Repository> = vec!["moduleA", "moduleB", "moduleC"]
            .iter()
            .map(|m| {
                let x = td.path().join(&m);
                let repo = create_repository(&x);
                // commit_files(&repo, &Path::new(&x), &vec!["test/a.txt", "b.txt", "c.txt"]);
                // show_status(&repo.workdir().unwrap().to_path_buf());
                repo
            })
            .collect();
        let central_repo_dir = td.path().join("central");
        let central_repo = create_repository(&central_repo_dir);
        commit_files(&central_repo,
                     &Path::new(&central_repo_dir),
                     &vec!["moduleA/a.txt", "moduleB/b.txt", "moduleC/c.txt"]);
        call_command(&td.path().to_path_buf(), "tree", &["-L", "3"]);
        assert!(true);
    }

    fn commit_file(repo: &git2::Repository, file: &Path, parents: &[&git2::Commit]) -> git2::Oid {
        let mut index = repo.index().expect("get index of repo");
        index.add_path(file).expect("file should be added");
        index.write().expect("write index");
        let tree_id = index.write_tree().expect("got tree_id");
        let tree = repo.find_tree(tree_id).expect("got tree");
        let sig = git2::Signature::now("foo", "bar").expect("created signature");
        repo.commit(Some("HEAD"),
                    &sig,
                    &sig,
                    &format!("commit for {:?}", &file.as_os_str()),
                    &tree,
                    &parents)
            .expect("commit to repo")
    }

    fn commit_files(repo: &git2::Repository, pb: &Path, content: &Vec<&str>) {
        let mut parent_commit = None;
        for file_name in content {
            let foo_file = pb.join(file_name);
            create_dummy_file(&foo_file);
            let oid = match parent_commit {
                Some(parent) => commit_file(&repo, &Path::new(file_name), &[&parent]),
                None => commit_file(&repo, &Path::new(file_name), &[]),
            };
            parent_commit = repo.find_commit(oid).ok();
        }
    }

    fn create_repository(temp: &Path) -> git2::Repository {
        let repo = git2::Repository::init(temp).expect("init should succeed");
        let path = repo.workdir().unwrap().to_path_buf();
        println!("Initialized empty Git repository in {}", path.display());
        repo
    }

    fn show_status(path: &PathBuf) {
        call_command(&path, "git", &["status"]);
        call_command(&path,
                     "git",
                     &["log",
                       "--graph",
                       "--pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) \
                        %C(bold blue)<%an>%Creset'",
                       "--abbrev-commit",
                       "--date=relative"]);

    }
    fn call_command(path: &PathBuf, command: &str, args: &[&str]) {
        let output = Command::new(&command)
            .args(args)
            .current_dir(&path)
            .output()
            .unwrap_or_else(|e| panic!("failed to execute process: {}", e));
        println!("{}{}",
                 String::from_utf8_lossy(&output.stdout),
                 String::from_utf8_lossy(&output.stderr));
    }
    fn create_dummy_file(f: &PathBuf) {
        println!("trying to create {}", f.display());
        let parent_dir = f.as_path().parent().expect("need to get parent");
        fs::create_dir_all(parent_dir).unwrap();

        let mut file = File::create(&f.as_path()).expect("create file");
        file.write_all("test content".as_bytes()).expect("write to file");
    }
}
<|endoftext|>