<commit_before>use std::io;
use std::net::SocketAddr;
use std::time::{SystemTime, Duration};

use crypto::{PublicKey, SecretKey, Hash};
use events::{Events, Reactor, NetworkConfiguration, Event, EventsConfiguration, Channel, MioChannel,
             Network, EventLoop, Milliseconds, EventHandler, Result as EventsResult,
             Error as EventsError};
use blockchain::{Blockchain, Schema, GenesisConfig, Transaction};
use messages::{Connect, RawMessage};

pub use self::state::{State, Round, Height, RequestData, ValidatorId, TxPool, ValidatorState};

mod basic;
mod consensus;
mod requests;

pub mod state; // TODO: temporary solution to get access to WAIT consts
pub mod timeout_adjuster;

#[derive(Debug)]
pub enum ExternalMessage {
    Transaction(Box<Transaction>),
}

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum NodeTimeout {
    Status,
    Round(u64, u32),
    Request(RequestData, Option<PublicKey>),
    Propose(u64, u32),
    PeerExchange,
}

#[derive(Clone)]
pub struct TxSender<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    inner: S,
}

pub struct NodeHandler<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    pub state: State,
    pub channel: S,
    pub blockchain: Blockchain,
    // TODO: move this into peer exchange service
    pub peer_discovery: Vec<SocketAddr>,
    timeout_adjuster: Box<timeout_adjuster::Adjuster>
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ListenerConfig {
    pub public_key: PublicKey,
    pub secret_key: SecretKey,
    pub address: SocketAddr,
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct NodeConfig {
    pub genesis: GenesisConfig,
    pub listen_address: SocketAddr,
    pub network: NetworkConfiguration,
    pub peers: Vec<SocketAddr>,
    pub public_key: PublicKey,
    pub secret_key: SecretKey,
}

#[derive(Debug, Clone)]
pub struct Configuration {
    pub listener: ListenerConfig,
    pub events: EventsConfiguration,
    pub network: NetworkConfiguration,
    pub peer_discovery: Vec<SocketAddr>,
}

pub type NodeChannel = MioChannel<ExternalMessage, NodeTimeout>;

pub struct Node {
    reactor: Events<NodeHandler<NodeChannel>>,
}

impl<S> NodeHandler<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    pub fn new(blockchain: Blockchain, sender: S, config: Configuration) -> Self {
        let timeout_adjuster = Box::new(timeout_adjuster::Constant::default());
        Self::with_timeout_adjuster(blockchain, sender, config, timeout_adjuster)
    }

    pub fn with_timeout_adjuster(blockchain: Blockchain,
                                 sender: S,
                                 config: Configuration,
                                 mut timeout_adjuster: Box<timeout_adjuster::Adjuster>)
                                 -> Self {
        // FIXME: remove unwraps here, use FATAL log level instead
        let (last_hash, last_height) = {
            let block = blockchain.last_block().unwrap();
            (block.hash(), block.height() + 1)
        };

        let stored = Schema::new(&blockchain.view()).get_actual_configuration().unwrap();
        info!("Create node with config={:#?}", stored);

        let validator_id = stored.validators
            .iter()
            .position(|pk| pk == &config.listener.public_key)
            .map(|id| id as ValidatorId);
        info!("Validator={:#?}", validator_id);
        let connect = Connect::new(&config.listener.public_key,
                                   sender.address(),
                                   sender.get_time(),
                                   &config.listener.secret_key);

        let mut state = State::new(validator_id,
                                   config.listener.public_key,
                                   config.listener.secret_key,
                                   stored,
                                   connect,
                                   last_hash,
                                   last_height,
                                   sender.get_time());

        let timeout = timeout_adjuster.adjust_timeout(&state, blockchain.view());
        state.set_propose_timeout(timeout);

        NodeHandler {
            state: state,
            channel: sender,
            blockchain: blockchain,
            peer_discovery: config.peer_discovery,
            timeout_adjuster: timeout_adjuster,
        }
    }

    pub fn propose_timeout(&self) -> Milliseconds {
        self.state().consensus_config().propose_timeout
    }

    pub fn round_timeout(&self) -> Milliseconds {
        self.state().consensus_config().round_timeout
    }

    pub fn status_timeout(&self) -> Milliseconds {
        self.state().consensus_config().status_timeout
    }

    pub fn peers_timeout(&self) -> Milliseconds {
        self.state().consensus_config().peers_timeout
    }

    pub fn txs_block_limit(&self) -> u32 {
        self.state().consensus_config().txs_block_limit
    }

    pub fn state(&self) -> &State {
        &self.state
    }

    pub fn initialize(&mut self) {
        info!("Start listening address={}", self.channel.address());
        for address in &self.peer_discovery.clone() {
            if address == &self.channel.address() {
                continue;
            }
            self.connect(address);
            info!("Try to connect with peer {}", address);
        }

        let round = 1;
        self.state.jump_round(round);
        info!("Jump to round {}", round);

        self.add_round_timeout();
        self.add_status_timeout();
        self.add_peer_exchange_timeout();
    }

    pub fn send_to_validator(&mut self, id: u32, message: &RawMessage) {
        // TODO: check validator id
        let public_key = self.state.validators()[id as usize];
        self.send_to_peer(public_key, message);
    }

    pub fn send_to_peer(&mut self, public_key: PublicKey, message: &RawMessage) {
        if let Some(conn) = self.state.peers().get(&public_key) {
            trace!("Send to addr: {}", conn.addr());
            self.channel.send_to(&conn.addr(), message.clone());
        } else {
            warn!("Hasn't connection with peer {:?}", public_key);
        }
    }

    pub fn send_to_addr(&mut self, address: &SocketAddr, message: &RawMessage) {
        trace!("Send to addr: {}", address);
        self.channel.send_to(address, message.clone());
    }

    // TODO: use Into<RawMessage>
    pub fn broadcast(&mut self, message: &RawMessage) {
        for conn in self.state.peers().values() {
            trace!("Send to addr: {}", conn.addr());
            self.channel.send_to(&conn.addr(), message.clone());
        }
    }

    pub fn connect(&mut self, address: &SocketAddr) {
        self.channel.connect(address);
    }

    pub fn request(&mut self, data: RequestData, peer: PublicKey) {
        let is_new = self.state.request(data.clone(), peer);
        if is_new {
            self.add_request_timeout(data, None);
        }
    }

    pub fn add_round_timeout(&mut self) {
        let time = self.round_start_time(self.state.round() + 1);
        trace!("ADD ROUND TIMEOUT, time={:?}, height={}, round={}",
               time,
               self.state.height(),
               self.state.round());
        let timeout = NodeTimeout::Round(self.state.height(), self.state.round());
        self.channel.add_timeout(timeout, time);
    }

    pub fn add_propose_timeout(&mut self) {
        let adjusted_propose_timeout = self.state.propose_timeout();
        let time = self.round_start_time(self.state.round()) +
                   Duration::from_millis(adjusted_propose_timeout);

        trace!("ADD PROPOSE TIMEOUT, time={:?}, height={}, round={}",
               time,
               self.state.height(),
               self.state.round());
        let timeout = NodeTimeout::Propose(self.state.height(), self.state.round());
        self.channel.add_timeout(timeout, time);
    }

    pub fn add_status_timeout(&mut self) {
        let time = self.channel.get_time() + Duration::from_millis(self.status_timeout());
        self.channel.add_timeout(NodeTimeout::Status, time);
    }

    pub fn add_request_timeout(&mut self, data: RequestData, peer: Option<PublicKey>) {
        trace!("ADD REQUEST TIMEOUT");
        let time = self.channel.get_time() + data.timeout();
        self.channel.add_timeout(NodeTimeout::Request(data, peer), time);
    }

    pub fn add_peer_exchange_timeout(&mut self) {
        let time = self.channel.get_time() + Duration::from_millis(self.peers_timeout());
        self.channel.add_timeout(NodeTimeout::PeerExchange, time);
    }

    pub fn last_block_hash(&self) -> Hash {
        self.blockchain
            .last_block()
            .unwrap()
            .hash()
    }

    pub fn round_start_time(&self, round: Round) -> SystemTime {
        let ms = (round - 1) as u64 * self.round_timeout();
        self.state.height_start_time() + Duration::from_millis(ms)
    }
}

impl<S> EventHandler for NodeHandler<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    type Timeout = NodeTimeout;
    type ApplicationEvent = ExternalMessage;

    fn handle_event(&mut self, event: Event) {
        match event {
            Event::Connected(addr) => self.handle_connected(&addr),
            Event::Disconnected(addr) => self.handle_disconnected(&addr),
            Event::Incoming(raw) => self.handle_message(raw),
            Event::Error(_) => {}
        }
    }

    fn handle_application_event(&mut self, event: Self::ApplicationEvent) {
        match event {
            ExternalMessage::Transaction(tx) => {
                self.handle_incoming_tx(tx);
            }
        }
    }

    fn handle_timeout(&mut self, timeout: Self::Timeout) {
        match timeout {
            NodeTimeout::Round(height, round) => self.handle_round_timeout(height, round),
            NodeTimeout::Request(data, peer) => self.handle_request_timeout(data, peer),
            NodeTimeout::Status => self.handle_status_timeout(),
            NodeTimeout::PeerExchange => self.handle_peer_exchange_timeout(),
            NodeTimeout::Propose(height, round) => self.handle_propose_timeout(height, round),
        }
    }
}

pub trait TransactionSend: Send + Sync {
    fn send<T: Transaction>(&self, tx: T) -> EventsResult<()>;
}

impl<S> TxSender<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    pub fn new(inner: S) -> TxSender<S> {
        TxSender { inner: inner }
    }
}

impl<S> TransactionSend for TxSender<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    fn send<T: Transaction>(&self, tx: T) -> EventsResult<()> {
        // TODO remove double data convertation
        if !tx.verify() {
            return Err(EventsError::new("Unable to verify transaction"));
        }
        let msg = ExternalMessage::Transaction(Box::new(tx));
        self.inner.post_event(msg)
    }
}

impl Node {
    pub fn new(blockchain: Blockchain, node_cfg: NodeConfig) -> Node {
        blockchain.create_genesis_block(node_cfg.genesis.clone()).unwrap();

        let config = Configuration {
            listener: ListenerConfig {
                public_key: node_cfg.public_key,
                secret_key: node_cfg.secret_key,
                address: node_cfg.listen_address,
            },
            network: node_cfg.network,
            events: EventsConfiguration::default(),
            peer_discovery: node_cfg.peers,
        };
        let network = Network::with_config(node_cfg.listen_address, config.network);
        let event_loop = EventLoop::configured(config.events.clone()).unwrap();
        let channel = NodeChannel::new(node_cfg.listen_address, event_loop.channel());
        let worker = NodeHandler::new(blockchain, channel, config);
        Node { reactor: Events::with_event_loop(network, worker, event_loop) }
    }

    pub fn run(&mut self) -> io::Result<()> {
        self.reactor.bind()?;
        self.reactor.handler_mut().initialize();
        self.reactor.run()
    }

    pub fn state(&self) -> &State {
        self.reactor.handler().state()
    }

    pub fn channel(&self) -> TxSender<NodeChannel> {
        TxSender::new(self.reactor.handler().channel.clone())
    }
}
<commit_msg>Remove additional NodeHandler constructor<commit_after>use std::io;
use std::net::SocketAddr;
use std::time::{SystemTime, Duration};

use crypto::{PublicKey, SecretKey, Hash};
use events::{Events, Reactor, NetworkConfiguration, Event, EventsConfiguration, Channel, MioChannel,
             Network, EventLoop, Milliseconds, EventHandler, Result as EventsResult,
             Error as EventsError};
use blockchain::{Blockchain, Schema, GenesisConfig, Transaction};
use messages::{Connect, RawMessage};
use self::timeout_adjuster::Adjuster;

pub use self::state::{State, Round, Height, RequestData, ValidatorId, TxPool, ValidatorState};

mod basic;
mod consensus;
mod requests;

pub mod state; // TODO: temporary solution to get access to WAIT consts
pub mod timeout_adjuster;

#[derive(Debug)]
pub enum ExternalMessage {
    Transaction(Box<Transaction>),
}

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum NodeTimeout {
    Status,
    Round(u64, u32),
    Request(RequestData, Option<PublicKey>),
    Propose(u64, u32),
    PeerExchange,
}

#[derive(Clone)]
pub struct TxSender<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    inner: S,
}

pub struct NodeHandler<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    pub state: State,
    pub channel: S,
    pub blockchain: Blockchain,
    // TODO: move this into peer exchange service
    pub peer_discovery: Vec<SocketAddr>,
    timeout_adjuster: Box<Adjuster>
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ListenerConfig {
    pub public_key: PublicKey,
    pub secret_key: SecretKey,
    pub address: SocketAddr,
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct NodeConfig {
    pub genesis: GenesisConfig,
    pub listen_address: SocketAddr,
    pub network: NetworkConfiguration,
    pub peers: Vec<SocketAddr>,
    pub public_key: PublicKey,
    pub secret_key: SecretKey,
}

#[derive(Debug, Clone)]
pub struct Configuration {
    pub listener: ListenerConfig,
    pub events: EventsConfiguration,
    pub network: NetworkConfiguration,
    pub peer_discovery: Vec<SocketAddr>,
}

pub type NodeChannel = MioChannel<ExternalMessage, NodeTimeout>;

pub struct Node {
    reactor: Events<NodeHandler<NodeChannel>>,
}

impl<S> NodeHandler<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    pub fn new(blockchain: Blockchain, sender: S, config: Configuration) -> Self {
        // FIXME: remove unwraps here, use FATAL log level instead
        let (last_hash, last_height) = {
            let block = blockchain.last_block().unwrap();
            (block.hash(), block.height() + 1)
        };

        let stored = Schema::new(&blockchain.view()).get_actual_configuration().unwrap();
        info!("Create node with config={:#?}", stored);

        let validator_id = stored.validators
            .iter()
            .position(|pk| pk == &config.listener.public_key)
            .map(|id| id as ValidatorId);
        info!("Validator={:#?}", validator_id);
        let connect = Connect::new(&config.listener.public_key,
                                   sender.address(),
                                   sender.get_time(),
                                   &config.listener.secret_key);

        let mut state = State::new(validator_id,
                                   config.listener.public_key,
                                   config.listener.secret_key,
                                   stored,
                                   connect,
                                   last_hash,
                                   last_height,
                                   sender.get_time());

        let mut timeout_adjuster = Box::new(timeout_adjuster::Constant::default());
        let timeout = timeout_adjuster.adjust_timeout(&state, blockchain.view());
        state.set_propose_timeout(timeout);

        NodeHandler {
            state: state,
            channel: sender,
            blockchain: blockchain,
            peer_discovery: config.peer_discovery,
            timeout_adjuster: timeout_adjuster,
        }
    }

    pub fn propose_timeout(&self) -> Milliseconds {
        self.state().consensus_config().propose_timeout
    }

    pub fn round_timeout(&self) -> Milliseconds {
        self.state().consensus_config().round_timeout
    }

    pub fn status_timeout(&self) -> Milliseconds {
        self.state().consensus_config().status_timeout
    }

    pub fn peers_timeout(&self) -> Milliseconds {
        self.state().consensus_config().peers_timeout
    }

    pub fn txs_block_limit(&self) -> u32 {
        self.state().consensus_config().txs_block_limit
    }

    pub fn state(&self) -> &State {
        &self.state
    }

    pub fn initialize(&mut self) {
        info!("Start listening address={}", self.channel.address());
        for address in &self.peer_discovery.clone() {
            if address == &self.channel.address() {
                continue;
            }
            self.connect(address);
            info!("Try to connect with peer {}", address);
        }

        let round = 1;
        self.state.jump_round(round);
        info!("Jump to round {}", round);

        self.add_round_timeout();
        self.add_status_timeout();
        self.add_peer_exchange_timeout();
    }

    pub fn send_to_validator(&mut self, id: u32, message: &RawMessage) {
        // TODO: check validator id
        let public_key = self.state.validators()[id as usize];
        self.send_to_peer(public_key, message);
    }

    pub fn send_to_peer(&mut self, public_key: PublicKey, message: &RawMessage) {
        if let Some(conn) = self.state.peers().get(&public_key) {
            trace!("Send to addr: {}", conn.addr());
            self.channel.send_to(&conn.addr(), message.clone());
        } else {
            warn!("Hasn't connection with peer {:?}", public_key);
        }
    }

    pub fn send_to_addr(&mut self, address: &SocketAddr, message: &RawMessage) {
        trace!("Send to addr: {}", address);
        self.channel.send_to(address, message.clone());
    }

    // TODO: use Into<RawMessage>
    pub fn broadcast(&mut self, message: &RawMessage) {
        for conn in self.state.peers().values() {
            trace!("Send to addr: {}", conn.addr());
            self.channel.send_to(&conn.addr(), message.clone());
        }
    }

    pub fn connect(&mut self, address: &SocketAddr) {
        self.channel.connect(address);
    }

    pub fn request(&mut self, data: RequestData, peer: PublicKey) {
        let is_new = self.state.request(data.clone(), peer);
        if is_new {
            self.add_request_timeout(data, None);
        }
    }

    pub fn add_round_timeout(&mut self) {
        let time = self.round_start_time(self.state.round() + 1);
        trace!("ADD ROUND TIMEOUT, time={:?}, height={}, round={}",
               time,
               self.state.height(),
               self.state.round());
        let timeout = NodeTimeout::Round(self.state.height(), self.state.round());
        self.channel.add_timeout(timeout, time);
    }

    pub fn add_propose_timeout(&mut self) {
        let adjusted_propose_timeout = self.state.propose_timeout();
        let time = self.round_start_time(self.state.round()) +
                   Duration::from_millis(adjusted_propose_timeout);

        trace!("ADD PROPOSE TIMEOUT, time={:?}, height={}, round={}",
               time,
               self.state.height(),
               self.state.round());
        let timeout = NodeTimeout::Propose(self.state.height(), self.state.round());
        self.channel.add_timeout(timeout, time);
    }

    pub fn add_status_timeout(&mut self) {
        let time = self.channel.get_time() + Duration::from_millis(self.status_timeout());
        self.channel.add_timeout(NodeTimeout::Status, time);
    }

    pub fn add_request_timeout(&mut self, data: RequestData, peer: Option<PublicKey>) {
        trace!("ADD REQUEST TIMEOUT");
        let time = self.channel.get_time() + data.timeout();
        self.channel.add_timeout(NodeTimeout::Request(data, peer), time);
    }

    pub fn add_peer_exchange_timeout(&mut self) {
        let time = self.channel.get_time() + Duration::from_millis(self.peers_timeout());
        self.channel.add_timeout(NodeTimeout::PeerExchange, time);
    }

    pub fn last_block_hash(&self) -> Hash {
        self.blockchain
            .last_block()
            .unwrap()
            .hash()
    }

    pub fn round_start_time(&self, round: Round) -> SystemTime {
        let ms = (round - 1) as u64 * self.round_timeout();
        self.state.height_start_time() + Duration::from_millis(ms)
    }
}

impl<S> EventHandler for NodeHandler<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    type Timeout = NodeTimeout;
    type ApplicationEvent = ExternalMessage;

    fn handle_event(&mut self, event: Event) {
        match event {
            Event::Connected(addr) => self.handle_connected(&addr),
            Event::Disconnected(addr) => self.handle_disconnected(&addr),
            Event::Incoming(raw) => self.handle_message(raw),
            Event::Error(_) => {}
        }
    }

    fn handle_application_event(&mut self, event: Self::ApplicationEvent) {
        match event {
            ExternalMessage::Transaction(tx) => {
                self.handle_incoming_tx(tx);
            }
        }
    }

    fn handle_timeout(&mut self, timeout: Self::Timeout) {
        match timeout {
            NodeTimeout::Round(height, round) => self.handle_round_timeout(height, round),
            NodeTimeout::Request(data, peer) => self.handle_request_timeout(data, peer),
            NodeTimeout::Status => self.handle_status_timeout(),
            NodeTimeout::PeerExchange => self.handle_peer_exchange_timeout(),
            NodeTimeout::Propose(height, round) => self.handle_propose_timeout(height, round),
        }
    }
}

pub trait TransactionSend: Send + Sync {
    fn send<T: Transaction>(&self, tx: T) -> EventsResult<()>;
}

impl<S> TxSender<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    pub fn new(inner: S) -> TxSender<S> {
        TxSender { inner: inner }
    }
}

impl<S> TransactionSend for TxSender<S>
    where S: Channel<ApplicationEvent = ExternalMessage, Timeout = NodeTimeout>
{
    fn send<T: Transaction>(&self, tx: T) -> EventsResult<()> {
        // TODO remove double data convertation
        if !tx.verify() {
            return Err(EventsError::new("Unable to verify transaction"));
        }
        let msg = ExternalMessage::Transaction(Box::new(tx));
        self.inner.post_event(msg)
    }
}

impl Node {
    pub fn new(blockchain: Blockchain, node_cfg: NodeConfig) -> Node {
        blockchain.create_genesis_block(node_cfg.genesis.clone()).unwrap();

        let config = Configuration {
            listener: ListenerConfig {
                public_key: node_cfg.public_key,
                secret_key: node_cfg.secret_key,
                address: node_cfg.listen_address,
            },
            network: node_cfg.network,
            events: EventsConfiguration::default(),
            peer_discovery: node_cfg.peers,
        };
        let network = Network::with_config(node_cfg.listen_address, config.network);
        let event_loop = EventLoop::configured(config.events.clone()).unwrap();
        let channel = NodeChannel::new(node_cfg.listen_address, event_loop.channel());
        let worker = NodeHandler::new(blockchain, channel, config);
        Node { reactor: Events::with_event_loop(network, worker, event_loop) }
    }

    pub fn run(&mut self) -> io::Result<()> {
        self.reactor.bind()?;
        self.reactor.handler_mut().initialize();
        self.reactor.run()
    }

    pub fn state(&self) -> &State {
        self.reactor.handler().state()
    }

    pub fn channel(&self) -> TxSender<NodeChannel> {
        TxSender::new(self.reactor.handler().channel.clone())
    }
}
<|endoftext|>


<commit_before>use regex::Regex;
use std::collections::HashSet;
use std::fmt;
use std::fmt::Write as _;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::path::MAIN_SEPARATOR;
use std::str;
use tempfile::NamedTempFile;

use protobuf::descriptor::{
    DescriptorProto, EnumDescriptorProto, FileDescriptorProto, FileDescriptorSet,
};
use protobuf::RepeatedField;
use protobuf_test_common::build::copy_tests_v2_v3;
use protobuf_test_common::build::glob_simple;
use std::process::Command;
use std::process::Stdio;

fn to_paths(iter: impl IntoIterator<Item = impl Into<String>>) -> Vec<PathBuf> {
    iter.into_iter()
        .map(|item| item.into().replace(MAIN_SEPARATOR, "/").into())
        .collect()
}

#[derive(Default, Debug)]
struct TestStats {
    passed: u32,
    passed_marked_skip: u32,
    skipped: u32,
    failed: u32,
}

fn normalize_generated_file(contents: &str) -> String {
    let parsed_by = Regex::new("^// \\.proto file is parsed by.*").unwrap();

    let mut r = String::new();
    let mut inside_descriptor_data = false;
    for line in contents.lines() {
        let line = if inside_descriptor_data {
            if line == "\";" {
                inside_descriptor_data = false;
                line.to_owned()
            } else {
                continue;
            }
        } else if line.starts_with("static file_descriptor_proto_data") {
            inside_descriptor_data = true;
            line.to_owned()
        } else {
            parsed_by.replace(line, "").into_owned()
        };

        writeln!(&mut r, "{}", line).unwrap();
    }

    // sanity check
    assert!(!inside_descriptor_data);

    r
}

fn normalize_generated_file_in_place(path: &Path) {
    let contents = fs::read_to_string(path).unwrap();
    let contents = normalize_generated_file(&contents);
    fs::write(path, &contents).unwrap();
}

fn print_diff(dir: &Path, a: &Path, b: &Path) {
    if cfg!(windows) {
        // likely we don't have `diff` command on Windows
        return;
    }

    // sanity check
    let mut a_full = dir.to_path_buf();
    a_full.push(a);
    assert!(a_full.exists());
    let mut b_full = dir.to_path_buf();
    b_full.push(b);
    assert!(b_full.exists());

    // Not using difference crate because it's slow for large files
    let output = Command::new("diff")
        .current_dir(dir)
        .arg("-u")
        .arg(a)
        .arg(b)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .output()
        .unwrap();

    print!("{}", str::from_utf8(&output.stdout).unwrap());
    print!("{}", str::from_utf8(&output.stderr).unwrap());
}

fn protoc_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut temp_file = NamedTempFile::new().unwrap();
    protoc::Protoc::from_env_path()
        .descriptor_set_out_args()
        .out(temp_file.path())
        .includes(includes)
        .inputs(inputs)
        .write_descriptor_set()
        .unwrap();
    protobuf::parse_from_reader(&mut temp_file).unwrap()
}

// TODO: expose this utility from protobuf-codegen-pure crate.
fn pure_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut codegen = protobuf_codegen_pure::parse_and_typecheck(includes, inputs).unwrap();
    let relative_paths: HashSet<_> = codegen
        .relative_paths
        .iter()
        .map(|path| path.to_string_lossy())
        .collect();
    codegen
        .file_descriptors
        .retain(|fd| relative_paths.contains(fd.get_name()));
    let mut fds = FileDescriptorSet::new();
    fds.file = RepeatedField::from_vec(codegen.file_descriptors);
    fds
}

fn normalize_descriptor_set(fds: &mut FileDescriptorSet) {
    for desc in fds.file.iter_mut() {
        normalize_file_descriptor(desc)
    }
}

fn normalize_file_descriptor(desc: &mut FileDescriptorProto) {
    if !desc.has_syntax() {
        desc.set_syntax("proto2".into())
    }
    for desc in &mut *desc.message_type {
        normalize_descriptor(desc)
    }
    for desc in desc.enum_type.iter_mut() {
        normalize_enum_descriptor(desc)
    }

    // TODO: don't clear options.
    for desc in &mut *desc.extension {
        desc.options.clear();
    }
    desc.options.clear();

    // TODO: don't clear services.
    desc.service.clear();
}

fn normalize_enum_descriptor(desc: &mut EnumDescriptorProto) {
    // TODO: don't clear options.
    desc.options.clear();
    for value in desc.value.iter_mut() {
        value.options.clear();
    }
}

fn normalize_descriptor(desc: &mut DescriptorProto) {
    for desc in desc.nested_type.iter_mut() {
        normalize_descriptor(desc);
    }
    for desc in desc.enum_type.iter_mut() {
        normalize_enum_descriptor(desc);
    }

    // TODO: don't clear options.
    desc.options.clear();
    for field in desc.field.iter_mut() {
        field.options.clear();
    }
}

fn pretty_message<M: protobuf::Message>(message: &M) -> String {
    // TODO: pull this handy utility into the protobuf crate.
    struct FormatMessage<'a, M>(&'a M);

    impl<M: protobuf::Message> fmt::Display for FormatMessage<'_, M> {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            protobuf::text_format::fmt(self.0, f)
        }
    }

    format!("{:#}", FormatMessage(message))
}

fn test_diff_in<F>(root: &str, s: &str, include: &str, should_skip: F)
where
    F: Fn(&str) -> bool,
{
    let mut stats = TestStats {
        passed: 0,
        passed_marked_skip: 0,
        skipped: 0,
        failed: 0,
    };

    let include_full = format!("{}/{}", root, include);
    let s_full = format!("{}/{}", root, s);

    let inputs_glob = format!("{}/*.proto", s_full);
    let inputs = to_paths(glob_simple(&inputs_glob));
    assert!(!inputs.is_empty(), "glob is empty: {}", inputs_glob);
    let includes = to_paths(vec![include_full.as_str(), "../proto"]);

    let temp_dir = tempfile::Builder::new()
        .prefix("protobuf-codegen-identical-test")
        .tempdir()
        .unwrap();

    let protoc_dir = format!("{}/protoc", temp_dir.path().to_str().unwrap());
    let pure_dir = format!("{}/pure", temp_dir.path().to_str().unwrap());

    fs::create_dir(&protoc_dir).unwrap();
    fs::create_dir(&pure_dir).unwrap();

    protoc_rust::Codegen::new()
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&protoc_dir)
        .run()
        .unwrap();

    protobuf_codegen_pure::Codegen::new()
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&pure_dir)
        .run()
        .unwrap();

    for input in &inputs {
        let label = input.strip_prefix(root).unwrap().to_str().unwrap();
        let proto_name = input.file_name().unwrap().to_str().unwrap();
        let rs_name = protobuf_codegen::proto_name_to_rs(proto_name);
        let protoc_rs = format!("{}/{}", protoc_dir, rs_name);
        let pure_rs = format!("{}/{}", pure_dir, rs_name);

        normalize_generated_file_in_place(Path::new(&protoc_rs));
        normalize_generated_file_in_place(Path::new(&pure_rs));

        let protoc_rs_contents =
            fs::read_to_string(&protoc_rs).expect(&format!("while reading {}", protoc_rs));
        let pure_rs_contents =
            fs::read_to_string(&pure_rs).expect(&format!("while reading {}", pure_rs));

        let mut protoc_descriptors = protoc_descriptor_set(&includes, &inputs);
        let mut pure_descriptors = pure_descriptor_set(&includes, &inputs);
        normalize_descriptor_set(&mut protoc_descriptors);
        normalize_descriptor_set(&mut pure_descriptors);

        let skip = should_skip(input.to_str().unwrap());
        if protoc_rs_contents == pure_rs_contents && protoc_descriptors == pure_descriptors {
            if !skip {
                stats.passed += 1;
                println!("{}: PASSED", label);
            } else {
                stats.passed_marked_skip += 1;
                println!("{}: PASSED BUT MARKED SKIP", label);
            }
        } else {
            if skip {
                stats.skipped += 1;
                println!("{} SKIPPED", label);
            } else {
                stats.failed += 1;
                println!("{} FAILED", label);
            }

            print_diff(
                temp_dir.path(),
                Path::new(&protoc_rs).strip_prefix(temp_dir.path()).unwrap(),
                Path::new(&pure_rs).strip_prefix(temp_dir.path()).unwrap(),
            );

            fs::write(
                format!("{}/{}.descriptors", protoc_dir, proto_name),
                pretty_message(&protoc_descriptors),
            )
            .unwrap();
            fs::write(
                format!("{}/{}.descriptors", pure_dir, proto_name),
                pretty_message(&pure_descriptors),
            )
            .unwrap();
            print_diff(
                temp_dir.path(),
                Path::new(&format!("protoc/{}.descriptors", proto_name)),
                Path::new(&format!("pure/{}.descriptors", proto_name)),
            );
        }
    }

    println!("{:?}", stats);
    assert!(
        stats.passed != 0 || s == "src/google/protobuf",
        "sanity check"
    );
    assert!(stats.failed == 0, "at least one test failed");
}

fn should_skip_with_marker(path: &str) -> bool {
    fs::read_to_string(path)
        .unwrap()
        .contains("@skip-codegen-identical-test")
}

#[test]
fn common_v2() {
    test_diff_in(
        "../protobuf-test",
        "src/common/v2",
        "src/common/v2",
        should_skip_with_marker,
    );
}

#[test]
fn common_v3() {
    let common_v3_root = tempfile::Builder::new()
        .prefix("common-v3")
        .tempdir()
        .unwrap();
    fs::create_dir_all(format!(
        "{}/src/common/v3",
        common_v3_root.path().to_str().unwrap()
    ))
    .unwrap();

    copy_tests_v2_v3(
        "../protobuf-test/src/common/v2",
        &format!("{}/src/common/v3", common_v3_root.path().to_str().unwrap()),
    );

    test_diff_in(
        common_v3_root.path().to_str().unwrap(),
        "src/common/v3",
        "src/common/v3",
        should_skip_with_marker,
    );
}

#[test]
fn v2() {
    test_diff_in(
        "../protobuf-test",
        "src/v2",
        "src/v2",
        should_skip_with_marker,
    );
}

#[test]
fn v3() {
    test_diff_in(
        "../protobuf-test",
        "src/v3",
        "src/v3",
        should_skip_with_marker,
    );
}

#[test]
fn interop() {
    test_diff_in(
        "../protobuf-test",
        "../interop/cxx",
        "../interop/cxx",
        should_skip_with_marker,
    );
}

#[test]
fn google() {
    test_diff_in("../protobuf-test", "src/google/protobuf", "src", |_| true);
}
<commit_msg>Fix a couple of todos in identical tests<commit_after>use regex::Regex;
use std::collections::HashSet;
use std::fmt;
use std::fmt::Write as _;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::path::MAIN_SEPARATOR;
use std::str;
use tempfile::NamedTempFile;

use protobuf::descriptor::{
    DescriptorProto, EnumDescriptorProto, FileDescriptorProto, FileDescriptorSet,
};
use protobuf::RepeatedField;
use protobuf_test_common::build::copy_tests_v2_v3;
use protobuf_test_common::build::glob_simple;
use std::process::Command;
use std::process::Stdio;

fn to_paths(iter: impl IntoIterator<Item = impl Into<String>>) -> Vec<PathBuf> {
    iter.into_iter()
        .map(|item| item.into().replace(MAIN_SEPARATOR, "/").into())
        .collect()
}

#[derive(Default, Debug)]
struct TestStats {
    passed: u32,
    passed_marked_skip: u32,
    skipped: u32,
    failed: u32,
}

fn normalize_generated_file(contents: &str) -> String {
    let parsed_by = Regex::new("^// \\.proto file is parsed by.*").unwrap();

    let mut r = String::new();
    let mut inside_descriptor_data = false;
    for line in contents.lines() {
        let line = if inside_descriptor_data {
            if line == "\";" {
                inside_descriptor_data = false;
                line.to_owned()
            } else {
                continue;
            }
        } else if line.starts_with("static file_descriptor_proto_data") {
            inside_descriptor_data = true;
            line.to_owned()
        } else {
            parsed_by.replace(line, "").into_owned()
        };

        writeln!(&mut r, "{}", line).unwrap();
    }

    // sanity check
    assert!(!inside_descriptor_data);

    r
}

fn normalize_generated_file_in_place(path: &Path) {
    let contents = fs::read_to_string(path).unwrap();
    let contents = normalize_generated_file(&contents);
    fs::write(path, &contents).unwrap();
}

fn print_diff(dir: &Path, a: &Path, b: &Path) {
    if cfg!(windows) {
        // likely we don't have `diff` command on Windows
        return;
    }

    // sanity check
    let mut a_full = dir.to_path_buf();
    a_full.push(a);
    assert!(a_full.exists());
    let mut b_full = dir.to_path_buf();
    b_full.push(b);
    assert!(b_full.exists());

    // Not using difference crate because it's slow for large files
    let output = Command::new("diff")
        .current_dir(dir)
        .arg("-u")
        .arg(a)
        .arg(b)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .output()
        .unwrap();

    print!("{}", str::from_utf8(&output.stdout).unwrap());
    print!("{}", str::from_utf8(&output.stderr).unwrap());
}

fn protoc_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut temp_file = NamedTempFile::new().unwrap();
    protoc::Protoc::from_env_path()
        .descriptor_set_out_args()
        .out(temp_file.path())
        .includes(includes)
        .inputs(inputs)
        .write_descriptor_set()
        .unwrap();
    protobuf::parse_from_reader(&mut temp_file).unwrap()
}

// TODO: expose this utility from protobuf-codegen-pure crate.
fn pure_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut codegen = protobuf_codegen_pure::parse_and_typecheck(includes, inputs).unwrap();
    let relative_paths: HashSet<_> = codegen
        .relative_paths
        .iter()
        .map(|path| path.to_string_lossy())
        .collect();
    codegen
        .file_descriptors
        .retain(|fd| relative_paths.contains(fd.get_name()));
    let mut fds = FileDescriptorSet::new();
    fds.file = RepeatedField::from_vec(codegen.file_descriptors);
    fds
}

fn normalize_descriptor_set(fds: &mut FileDescriptorSet) {
    for desc in fds.file.iter_mut() {
        normalize_file_descriptor(desc)
    }
}

fn normalize_file_descriptor(desc: &mut FileDescriptorProto) {
    if !desc.has_syntax() {
        desc.set_syntax("proto2".into())
    }
    for desc in &mut *desc.message_type {
        normalize_descriptor(desc)
    }
    for desc in desc.enum_type.iter_mut() {
        normalize_enum_descriptor(desc)
    }

    // TODO: don't clear options.
    for desc in &mut *desc.extension {
        desc.options.clear();
    }
    desc.options.clear();

    // TODO: don't clear services.
    desc.service.clear();
}

fn normalize_enum_descriptor(desc: &mut EnumDescriptorProto) {
    desc.options.mut_or_default();

    for value in desc.value.iter_mut() {
        value.options.mut_or_default();
    }
}

fn normalize_descriptor(desc: &mut DescriptorProto) {
    for desc in desc.nested_type.iter_mut() {
        normalize_descriptor(desc);
    }
    for desc in desc.enum_type.iter_mut() {
        normalize_enum_descriptor(desc);
    }

    // TODO: don't clear options.
    desc.options.clear();
    for field in desc.field.iter_mut() {
        field.options.clear();
    }
}

fn pretty_message<M: protobuf::Message>(message: &M) -> String {
    // TODO: pull this handy utility into the protobuf crate.
    struct FormatMessage<'a, M>(&'a M);

    impl<M: protobuf::Message> fmt::Display for FormatMessage<'_, M> {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            protobuf::text_format::fmt(self.0, f)
        }
    }

    format!("{:#}", FormatMessage(message))
}

fn test_diff_in<F>(root: &str, s: &str, include: &str, should_skip: F)
where
    F: Fn(&str) -> bool,
{
    let mut stats = TestStats {
        passed: 0,
        passed_marked_skip: 0,
        skipped: 0,
        failed: 0,
    };

    let include_full = format!("{}/{}", root, include);
    let s_full = format!("{}/{}", root, s);

    let inputs_glob = format!("{}/*.proto", s_full);
    let inputs = to_paths(glob_simple(&inputs_glob));
    assert!(!inputs.is_empty(), "glob is empty: {}", inputs_glob);
    let includes = to_paths(vec![include_full.as_str(), "../proto"]);

    let temp_dir = tempfile::Builder::new()
        .prefix("protobuf-codegen-identical-test")
        .tempdir()
        .unwrap();

    let protoc_dir = format!("{}/protoc", temp_dir.path().to_str().unwrap());
    let pure_dir = format!("{}/pure", temp_dir.path().to_str().unwrap());

    fs::create_dir(&protoc_dir).unwrap();
    fs::create_dir(&pure_dir).unwrap();

    protoc_rust::Codegen::new()
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&protoc_dir)
        .run()
        .unwrap();

    protobuf_codegen_pure::Codegen::new()
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&pure_dir)
        .run()
        .unwrap();

    for input in &inputs {
        let label = input.strip_prefix(root).unwrap().to_str().unwrap();
        let proto_name = input.file_name().unwrap().to_str().unwrap();
        let rs_name = protobuf_codegen::proto_name_to_rs(proto_name);
        let protoc_rs = format!("{}/{}", protoc_dir, rs_name);
        let pure_rs = format!("{}/{}", pure_dir, rs_name);

        normalize_generated_file_in_place(Path::new(&protoc_rs));
        normalize_generated_file_in_place(Path::new(&pure_rs));

        let protoc_rs_contents =
            fs::read_to_string(&protoc_rs).expect(&format!("while reading {}", protoc_rs));
        let pure_rs_contents =
            fs::read_to_string(&pure_rs).expect(&format!("while reading {}", pure_rs));

        let mut protoc_descriptors = protoc_descriptor_set(&includes, &inputs);
        let mut pure_descriptors = pure_descriptor_set(&includes, &inputs);
        normalize_descriptor_set(&mut protoc_descriptors);
        normalize_descriptor_set(&mut pure_descriptors);

        let skip = should_skip(input.to_str().unwrap());
        if protoc_rs_contents == pure_rs_contents && protoc_descriptors == pure_descriptors {
            if !skip {
                stats.passed += 1;
                println!("{}: PASSED", label);
            } else {
                stats.passed_marked_skip += 1;
                println!("{}: PASSED BUT MARKED SKIP", label);
            }
        } else {
            if skip {
                stats.skipped += 1;
                println!("{} SKIPPED", label);
            } else {
                stats.failed += 1;
                println!("{} FAILED", label);
            }

            print_diff(
                temp_dir.path(),
                Path::new(&protoc_rs).strip_prefix(temp_dir.path()).unwrap(),
                Path::new(&pure_rs).strip_prefix(temp_dir.path()).unwrap(),
            );

            fs::write(
                format!("{}/{}.descriptors", protoc_dir, proto_name),
                pretty_message(&protoc_descriptors),
            )
            .unwrap();
            fs::write(
                format!("{}/{}.descriptors", pure_dir, proto_name),
                pretty_message(&pure_descriptors),
            )
            .unwrap();
            print_diff(
                temp_dir.path(),
                Path::new(&format!("protoc/{}.descriptors", proto_name)),
                Path::new(&format!("pure/{}.descriptors", proto_name)),
            );
        }
    }

    println!("{:?}", stats);
    assert!(
        stats.passed != 0 || s == "src/google/protobuf",
        "sanity check"
    );
    assert!(stats.failed == 0, "at least one test failed");
}

fn should_skip_with_marker(path: &str) -> bool {
    fs::read_to_string(path)
        .unwrap()
        .contains("@skip-codegen-identical-test")
}

#[test]
fn common_v2() {
    test_diff_in(
        "../protobuf-test",
        "src/common/v2",
        "src/common/v2",
        should_skip_with_marker,
    );
}

#[test]
fn common_v3() {
    let common_v3_root = tempfile::Builder::new()
        .prefix("common-v3")
        .tempdir()
        .unwrap();
    fs::create_dir_all(format!(
        "{}/src/common/v3",
        common_v3_root.path().to_str().unwrap()
    ))
    .unwrap();

    copy_tests_v2_v3(
        "../protobuf-test/src/common/v2",
        &format!("{}/src/common/v3", common_v3_root.path().to_str().unwrap()),
    );

    test_diff_in(
        common_v3_root.path().to_str().unwrap(),
        "src/common/v3",
        "src/common/v3",
        should_skip_with_marker,
    );
}

#[test]
fn v2() {
    test_diff_in(
        "../protobuf-test",
        "src/v2",
        "src/v2",
        should_skip_with_marker,
    );
}

#[test]
fn v3() {
    test_diff_in(
        "../protobuf-test",
        "src/v3",
        "src/v3",
        should_skip_with_marker,
    );
}

#[test]
fn interop() {
    test_diff_in(
        "../protobuf-test",
        "../interop/cxx",
        "../interop/cxx",
        should_skip_with_marker,
    );
}

#[test]
fn google() {
    test_diff_in("../protobuf-test", "src/google/protobuf", "src", |_| true);
}
<|endoftext|>


<commit_before>use slog;

use std::mem;
use std::fmt;

use store::{TxPtr, BlockHeaderPtr};
use store::FlatFilePtr;

use store::spent_tree::SpendingError;
use store::spent_tree::BlockPtr;
use store::spent_index::SpentIndex;

use store::flatfile::INITIAL_WRITEPOS;

// highest 2 bits are record-type
// 11 => start of block
// 10 => end of block
// 00 => transaction
// 01 => spent-output

const RECORD_TYPE:u64    = 0xC000_0000_0000_0000;
const START_OF_BLOCK:u64 = 0xC000_0000_0000_0000;
const END_OF_BLOCK:u64   = 0x8000_0000_0000_0000;
const TRANSACTION:u64    = 0x0000_0000_0000_0000;
const OUTPUT:u64         = 0x4000_0000_0000_0000;

// Record layout
// -------------
//
// START_OF_BLOCK;
// bits 0-61   fileoffset end of the previous block (in spent-tree)
//
// END_OF_BLOCK:
// bits 0 -31   fileoffset of blockheader
// bits 32-61   number of records in block
//
// TRANSACTION:
// bits 0 -31   fileoffset of transaction
// bits 32-47   filenumber of transaction
//
// OUTPUT:
// bits 0 -31   fileoffset of transaction
// bits 32-47   filenumber of transaction
// bits 48-61   output-index

// a orphan start of block is a start of block without a previous
const ORPHAN_START_OF_BLOCK:u64 = 0xC000_0000_0000_0000;

#[derive(Clone,Copy,PartialEq)]
pub struct Record(u64);

impl fmt::Debug for Record {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(fmt, "REC  {0:016X} ", self.0)
    }
}

impl Record {
    pub fn new_unmatched_input() -> Record {
        Record (0)
    }

    pub fn new_transaction(tx_ptr: TxPtr) -> Record {

        Record(
            (tx_ptr.get_file_number() as u64) << 32 |
            tx_ptr.get_file_offset()
        )
    }

    pub fn new_orphan_block_start() -> Record {

        Record(
            START_OF_BLOCK
        )
    }

    pub fn new_block_start(previous: BlockPtr) -> Record {

        Record(
            START_OF_BLOCK | (previous.start.to_index() + previous.length - 1)
        )

    }
    pub fn new_block_end(block: BlockHeaderPtr, size: usize) -> Record {

        debug_assert!(block.get_file_number() == 0);
        debug_assert!(size <= 0x3FFF_FFFF);

        Record(
            END_OF_BLOCK |
                (block.get_file_offset()) |
                ((size as u64) << 32)
        )
    }

    pub fn new_output(tx_ptr: TxPtr, output_index: u32) -> Record {
        assert!(output_index <= 0x3fff); // TODO: this might not be true; fallback is needed

        Record(
            OUTPUT |
                (output_index as u64) << 48 |
                (tx_ptr.get_file_number() as u64) << 32 |
                tx_ptr.get_file_offset()
        )
    }

    /// If called on block-record, returns the index of the block-record and the record-count
    /// of the previous block
    pub fn previous_block(self) -> (usize, usize) {
        unimplemented!()
    }

    pub fn is_transaction(self) -> bool {

        (self.0 & RECORD_TYPE) == TRANSACTION
    }

    pub fn get_transaction_ptr(self) -> TxPtr {

        debug_assert!(self.is_transaction() || self.is_output());

        TxPtr::new(
            ((self.0 & 0xFFFF_0000_0000) >> 32) as i16,
            self.0 & 0xFFFF_FFFF
        )
    }

    pub fn get_block_header_ptr(self) -> BlockHeaderPtr {

        debug_assert!((self.0 & RECORD_TYPE) == END_OF_BLOCK);

        BlockHeaderPtr::new(0, self.0 & 0xFFFF_FFFF)
    }

    pub fn is_output(self) -> bool {

        (self.0 & RECORD_TYPE) == OUTPUT
    }

    pub fn is_block_start(self) -> bool {

        (self.0 & RECORD_TYPE) == START_OF_BLOCK
    }

    pub fn is_block_end(self) -> bool {

        (self.0 & RECORD_TYPE) == END_OF_BLOCK
    }

    pub fn is_unmatched_input(self) -> bool {
        self.0 == 0
    }

    /// This creates a non-cryptographic but perfect "hash" of the transaction or output
    pub fn hash(self) -> u64 {

        debug_assert!(self.is_transaction() || self.is_output());

        // We drop 4 bits from the filen offset and, for an output  add 1 + the output-index
        // The result is just as unique but smaller; we just drop the info to find the transaction
        // or to find the transaction from an output
        // The resulting number is used for the spent-index

        ((self.0 & 0xFFFF_FFFF_FFFF) >> 4)          // file-offset and file-number
        + (self.0 >> 62)                            // the bit that indicates its an output
        + ((self.0 & 0x3FFF_0000_0000_0000) >> 48)  // output-index
    }

    fn to_transaction(self) -> Record {

        debug_assert!(self.is_output());

        Record(self.0 & 0x0000_FFFF_FFFF_FFFF)
    }

    // Test only as normally it makes no sense to mix up file_offsets from different record-types
    // in the same expression
    #[cfg(test)]
    pub fn get_file_offset(self) -> u32 {

        self.0 as u32
    }
}

/// A filepointer that points to a record in the SpentTree
#[derive(PartialEq, Copy, Clone)]
pub struct RecordPtr(u64);

impl FlatFilePtr for RecordPtr {
    fn new(file_number: i16, file_offset: u64) -> RecordPtr {
        assert_eq!(file_number, 0); // can only have one spent-tree file

        RecordPtr((file_offset as u64 - INITIAL_WRITEPOS)
                      / mem::size_of::<Record>() as u64)
    }

    fn get_file_number(self) -> i16 { 0 }
    fn get_file_offset(self) -> u64 {
        INITIAL_WRITEPOS + self.0 * mem::size_of::<Record>() as u64
    }
}

impl fmt::Debug for RecordPtr {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(fmt, "{:?}", self.0)
    }
}

impl RecordPtr {
    pub fn new(ptr: u64) -> Self {
        RecordPtr(ptr)
    }

    pub fn to_index(self) -> u64 {
        self.0
    }
}

impl Record {

    /// Checks if the output is not double-spent in the bit-index
    fn verify_spent_in_index(&mut self,
                             spent_index: &SpentIndex) -> Result<usize, SpendingError>
    {
        let seek_output      = *self;
        let seek_transaction = self.to_transaction();

        if spent_index.exists(seek_output.hash()) {

            return Err(SpendingError::OutputAlreadySpent);
        }
        else if !spent_index.exists(seek_transaction.hash()) {

            return Err(SpendingError::OutputNotFound);
        }
        else {

            return Ok(1);
        }

    }

    /// Verifies whether this this output is not already spent,
    /// and whether it is stored before self in the blockchain
    pub fn verify_spent(
        &mut self,
        spent_index: &SpentIndex,
        seek_idx: usize,
        records: &[Record],
        logger: &slog::Logger) -> Result<usize, SpendingError>

    {

        trace!(logger, format!("FL# Start search for {:?} {:?} {:?}", self, seek_idx, records[seek_idx]));

        if self.is_transaction() { return Ok(0) }

        let seek_output      = *self; // this may not be found
        let seek_transaction = self.to_transaction(); // this must be found

        debug_assert!(seek_output.is_output());
        debug_assert!(self.0 == records[seek_idx].0);

        let mut seek_idx = seek_idx as u64;

        let mut blocks = 0;

        seek_idx -= 1;
        loop {
            trace!(logger, format!("FL# Search  {:?} @ {:?}", self, seek_idx));

            // TODO: we need to be aware here of the chances of forking.
            // On initial load, the spent-index is always up-to-date after one block
            // so we should use 1 here
            if blocks >= 1 {

                return self.verify_spent_in_index(spent_index)
            }

            let seek_rec = records[seek_idx as usize];

            if seek_rec.0 == ORPHAN_START_OF_BLOCK {

                // Looks like we reached genesis
                return Err(SpendingError::OutputNotFound);
            }
            else if (seek_rec.0 & START_OF_BLOCK) == START_OF_BLOCK {

                // jump to previous block
                blocks += 1;
                seek_idx = seek_rec.0 & !START_OF_BLOCK;

                trace!(logger, format!("FL# Jump to {:?} @ {:?}", seek_rec, seek_idx));

            } else if seek_rec == seek_transaction {

                // Found tx before spent => all ok
                return Ok(1);
            }
            else if seek_rec == seek_output {

                return Err(SpendingError::OutputAlreadySpent);
            }
            else {

                seek_idx -= 1;
            }
        }

    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_size_of_record() {
        assert_eq!(::std::mem::size_of::<Record>(), 8);

    }
}
<commit_msg>Fix test<commit_after>use slog;

use std::mem;
use std::fmt;

use store::{TxPtr, BlockHeaderPtr};
use store::FlatFilePtr;

use store::spent_tree::SpendingError;
use store::spent_tree::BlockPtr;
use store::spent_index::SpentIndex;

use store::flatfile::INITIAL_WRITEPOS;

// highest 2 bits are record-type
// 11 => start of block
// 10 => end of block
// 00 => transaction
// 01 => spent-output

const RECORD_TYPE:u64    = 0xC000_0000_0000_0000;
const START_OF_BLOCK:u64 = 0xC000_0000_0000_0000;
const END_OF_BLOCK:u64   = 0x8000_0000_0000_0000;
const TRANSACTION:u64    = 0x0000_0000_0000_0000;
const OUTPUT:u64         = 0x4000_0000_0000_0000;

// Record layout
// -------------
//
// START_OF_BLOCK;
// bits 0-61   fileoffset end of the previous block (in spent-tree)
//
// END_OF_BLOCK:
// bits 0 -31   fileoffset of blockheader
// bits 32-61   number of records in block
//
// TRANSACTION:
// bits 0 -31   fileoffset of transaction
// bits 32-47   filenumber of transaction
//
// OUTPUT:
// bits 0 -31   fileoffset of transaction
// bits 32-47   filenumber of transaction
// bits 48-61   output-index

// a orphan start of block is a start of block without a previous
const ORPHAN_START_OF_BLOCK:u64 = 0xC000_0000_0000_0000;

#[derive(Clone,Copy,PartialEq)]
pub struct Record(u64);

impl fmt::Debug for Record {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(fmt, "REC  {0:016X} ", self.0)
    }
}

impl Record {
    pub fn new_unmatched_input() -> Record {
        Record (0)
    }

    pub fn new_transaction(tx_ptr: TxPtr) -> Record {

        Record(
            (tx_ptr.get_file_number() as u64) << 32 |
            tx_ptr.get_file_offset()
        )
    }

    pub fn new_orphan_block_start() -> Record {

        Record(
            START_OF_BLOCK
        )
    }

    pub fn new_block_start(previous: BlockPtr) -> Record {

        Record(
            START_OF_BLOCK | (previous.start.to_index() + previous.length - 1)
        )

    }
    pub fn new_block_end(block: BlockHeaderPtr, size: usize) -> Record {

        debug_assert!(block.get_file_number() == 0);
        debug_assert!(size <= 0x3FFF_FFFF);

        Record(
            END_OF_BLOCK |
                (block.get_file_offset()) |
                ((size as u64) << 32)
        )
    }

    pub fn new_output(tx_ptr: TxPtr, output_index: u32) -> Record {
        assert!(output_index <= 0x3fff); // TODO: this might not be true; fallback is needed

        Record(
            OUTPUT |
                (output_index as u64) << 48 |
                (tx_ptr.get_file_number() as u64) << 32 |
                tx_ptr.get_file_offset()
        )
    }

    /// If called on block-record, returns the index of the block-record and the record-count
    /// of the previous block
    pub fn previous_block(self) -> (usize, usize) {
        unimplemented!()
    }

    pub fn is_transaction(self) -> bool {

        (self.0 & RECORD_TYPE) == TRANSACTION
    }

    pub fn get_transaction_ptr(self) -> TxPtr {

        debug_assert!(self.is_transaction() || self.is_output());

        TxPtr::new(
            ((self.0 & 0xFFFF_0000_0000) >> 32) as i16,
            self.0 & 0xFFFF_FFFF
        )
    }

    pub fn get_block_header_ptr(self) -> BlockHeaderPtr {

        debug_assert!((self.0 & RECORD_TYPE) == END_OF_BLOCK);

        BlockHeaderPtr::new(0, self.0 & 0xFFFF_FFFF)
    }

    pub fn is_output(self) -> bool {

        (self.0 & RECORD_TYPE) == OUTPUT
    }

    pub fn is_block_start(self) -> bool {

        (self.0 & RECORD_TYPE) == START_OF_BLOCK
    }

    pub fn is_block_end(self) -> bool {

        (self.0 & RECORD_TYPE) == END_OF_BLOCK
    }

    pub fn is_unmatched_input(self) -> bool {
        self.0 == 0
    }

    /// This creates a non-cryptographic but perfect "hash" of the transaction or output
    pub fn hash(self) -> u64 {

        debug_assert!(self.is_transaction() || self.is_output());

        // We drop 4 bits from the filen offset and, for an output  add 1 + the output-index
        // The result is just as unique but smaller; we just drop the info to find the transaction
        // or to find the transaction from an output
        // The resulting number is used for the spent-index

        ((self.0 & 0xFFFF_FFFF_FFFF) >> 4)          // file-offset and file-number
        + (self.0 >> 62)                            // the bit that indicates its an output
        + ((self.0 & 0x3FFF_0000_0000_0000) >> 48)  // output-index
    }

    fn to_transaction(self) -> Record {

        debug_assert!(self.is_output());

        Record(self.0 & 0x0000_FFFF_FFFF_FFFF)
    }

    // Test only as normally it makes no sense to mix up file_offsets from different record-types
    // in the same expression
    #[cfg(test)]
    pub fn get_file_offset(self) -> u32 {

        self.0 as u32
    }
}

/// A filepointer that points to a record in the SpentTree
#[derive(PartialEq, Copy, Clone)]
pub struct RecordPtr(u64);

impl FlatFilePtr for RecordPtr {
    fn new(file_number: i16, file_offset: u64) -> RecordPtr {
        assert_eq!(file_number, 0); // can only have one spent-tree file

        RecordPtr((file_offset as u64 - INITIAL_WRITEPOS)
                      / mem::size_of::<Record>() as u64)
    }

    fn get_file_number(self) -> i16 { 0 }
    fn get_file_offset(self) -> u64 {
        INITIAL_WRITEPOS + self.0 * mem::size_of::<Record>() as u64
    }
}

impl fmt::Debug for RecordPtr {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
        write!(fmt, "{:?}", self.0)
    }
}

impl RecordPtr {
    pub fn new(ptr: u64) -> Self {
        RecordPtr(ptr)
    }

    pub fn to_index(self) -> u64 {
        self.0
    }
}

impl Record {

    /// Checks if the output is not double-spent in the bit-index
    fn verify_spent_in_index(&mut self,
                             spent_index: &SpentIndex) -> Result<usize, SpendingError>
    {
        let seek_output      = *self;
        let seek_transaction = self.to_transaction();

        if spent_index.exists(seek_output.hash()) {

            return Err(SpendingError::OutputAlreadySpent);
        }
        else if !spent_index.exists(seek_transaction.hash()) {

            return Err(SpendingError::OutputNotFound);
        }
        else {

            return Ok(1);
        }

    }

    /// Verifies whether this this output is not already spent,
    /// and whether it is stored before self in the blockchain
    pub fn verify_spent(
        &mut self,
        spent_index: &SpentIndex,
        seek_idx: usize,
        records: &[Record],
        logger: &slog::Logger) -> Result<usize, SpendingError>

    {

        trace!(logger, format!("FL# Start search for {:?} {:?} {:?}", self, seek_idx, records[seek_idx]));

        if self.is_transaction() { return Ok(0) }

        let seek_output      = *self; // this may not be found
        let seek_transaction = self.to_transaction(); // this must be found

        debug_assert!(seek_output.is_output());
        debug_assert!(self.0 == records[seek_idx].0);

        let mut seek_idx = seek_idx as u64;

        let mut blocks = 0;

        seek_idx -= 1;
        loop {
            trace!(logger, format!("FL# Search  {:?} @ {:?}", self, seek_idx));

            // TODO: we need to be aware here of the chances of forking.
            // On initial load, the spent-index is always up-to-date after one block
            // so we should use 1 here
            if blocks >= 3 {

                return self.verify_spent_in_index(spent_index)
            }

            let seek_rec = records[seek_idx as usize];

            if seek_rec.0 == ORPHAN_START_OF_BLOCK {

                // Looks like we reached genesis
                return Err(SpendingError::OutputNotFound);
            }
            else if (seek_rec.0 & START_OF_BLOCK) == START_OF_BLOCK {

                // jump to previous block
                blocks += 1;
                seek_idx = seek_rec.0 & !START_OF_BLOCK;

                trace!(logger, format!("FL# Jump to {:?} @ {:?}", seek_rec, seek_idx));

            } else if seek_rec == seek_transaction {

                // Found tx before spent => all ok
                return Ok(1);
            }
            else if seek_rec == seek_output {

                return Err(SpendingError::OutputAlreadySpent);
            }
            else {

                seek_idx -= 1;
            }
        }

    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_size_of_record() {
        assert_eq!(::std::mem::size_of::<Record>(), 8);

    }
}
<|endoftext|>


<commit_before>#![crate_name = "docopt_macros"]
#![crate_type = "dylib"]

#![feature(plugin_registrar, quote, rustc_private)]

//! This crate defines the `docopt!` macro. It is documented in the
//! documentation of the `docopt` crate.

extern crate syntax;
extern crate rustc_plugin;
extern crate docopt;

use std::borrow::Borrow;
use std::collections::HashMap;

use rustc_plugin::Registry;
use syntax::{ast, codemap, tokenstream};
use syntax::errors::DiagnosticBuilder;
use syntax::ext::base::{ExtCtxt, MacResult, MacEager, DummyResult};
use syntax::ext::build::AstBuilder;
use syntax::fold::Folder;
use syntax::parse::common::SeqSep;
use syntax::parse::parser::Parser;
use syntax::parse::token;
use syntax::print::pprust;
use syntax::ptr::P;
use syntax::symbol;
use syntax::util::small_vector::SmallVector;

use docopt::{Docopt, ArgvMap};
use docopt::parse::{Options, Atom, Positional, Zero, One};

type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>;

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_macro("docopt", expand);
}

fn expand(cx: &mut ExtCtxt, span: codemap::Span, tts: &[tokenstream::TokenTree])
         -> Box<MacResult+'static> {
    let parsed = match MacParser::new(cx, tts).parse() {
        Ok(parsed) => parsed,
        Err(_) => return DummyResult::any(span),
    };
    parsed.items(cx)
}

/// Parsed corresponds to the result of parsing a `docopt` macro call.
/// It can be used to write a corresponding struct.
struct Parsed {
    struct_info: StructInfo,
    doc: Docopt,
    /// Overrided type annotations for struct members. May be empty.
    /// When a type annotation for an atom doesn't exist, then one is
    /// inferred automatically. It is one of: `bool`, `usize`, `String` or
    /// `Vec<String>`.
    types: HashMap<Atom, P<ast::Ty>>,
}

impl Parsed {
    /// Returns a macro result suitable for expansion.
    /// Contains two items: one for the struct and one for the struct impls.
    fn items(&self, cx: &ExtCtxt) -> Box<MacResult+'static> {
        let mut its = vec!();
        its.push(self.struct_decl(cx));

        let struct_name = self.struct_info.name;
        let full_doc = &*self.doc.parser().full_doc;
        its.push(quote_item!(cx,
            impl $struct_name {
                #[allow(dead_code)]
                fn docopt() -> docopt::Docopt {
                    // The unwrap is justified here because this code
                    // gen only happens if the Docopt usage string is valid.
                    docopt::Docopt::new($full_doc).unwrap()
                }
            }
        ).unwrap());
        MacEager::items(SmallVector::many(its))
    }

    /// Returns an item for the struct definition.
    fn struct_decl(&self, cx: &ExtCtxt) -> P<ast::Item> {
        let name = self.struct_info.name.clone();
        let vis = if self.struct_info.public { ast::Visibility::Public }
                  else { ast::Visibility::Inherited };
        let def = ast::VariantData::Struct(
            self.struct_fields(cx), ast::DUMMY_NODE_ID);

        let mut traits = vec!["RustcDecodable".to_string()];
        traits.extend(self.struct_info.deriving.iter().cloned());
        let attrs = vec![attribute(cx, "allow", vec!["non_snake_case"]),
                         attribute(cx, "derive", traits)];

        let st = cx.item_struct(codemap::DUMMY_SP, name.clone(), def);
        cx.item(codemap::DUMMY_SP, name, attrs, st.node.clone()).map(|mut it| {
            it.vis = vis;
            it
        })
    }

    /// Returns a list of fields for the struct definition.
    /// Handles type annotations.
    fn struct_fields(&self, cx: &ExtCtxt) -> Vec<ast::StructField> {
        let mut fields: Vec<ast::StructField> = vec!();
        for (atom, opts) in self.doc.parser().descs.iter() {
            let name = ArgvMap::key_to_struct_field(&*atom.to_string());
            let ty = match self.types.get(atom) {
                None => self.pat_type(cx, atom, opts),
                Some(ty) => ty.clone(),
            };
            fields.push(self.mk_struct_field(&*name, ty));
        }
        fields
    }

    /// Returns an inferred type for a usage pattern.
    /// This is only invoked when a type annotation is not present.
    fn pat_type(&self, cx: &ExtCtxt, atom: &Atom, opts: &Options) -> P<ast::Ty> {
        let sp = codemap::DUMMY_SP;
        match (opts.repeats, &opts.arg) {
            (false, &Zero) => {
                match atom {
                    &Positional(_) => cx.ty_ident(sp, ident("String")),
                    _ => cx.ty_ident(sp, ident("bool")),
                }
            }
            (true, &Zero) => {
                match atom {
                    &Positional(_) => ty_vec_string(cx),
                    _ => cx.ty_ident(sp, ident("usize")),
                }
            }
            (false, &One(_)) => cx.ty_ident(sp, ident("String")),
            (true, &One(_)) => ty_vec_string(cx),
        }
    }

    /// Creates a struct field from a member name and type.
    fn mk_struct_field(&self, name: &str, ty: P<ast::Ty>) -> ast::StructField {
        ast::StructField {
            span: codemap::DUMMY_SP,
            ident: Some(ident(name)),
            vis: ast::Visibility::Public,
            id: ast::DUMMY_NODE_ID,
            ty: ty,
            attrs: vec!(),
        }
    }
}

/// State for parsing a `docopt` macro invocation.
struct MacParser<'a, 'b:'a> {
    cx: &'a mut ExtCtxt<'b>,
    p: Parser<'b>,
}

impl<'a, 'b> MacParser<'a, 'b> {
    fn new(cx: &'a mut ExtCtxt<'b>, tts: &[tokenstream::TokenTree]) -> MacParser<'a, 'b> {
        let p = cx.new_parser_from_tts(tts);
        MacParser { cx: cx, p: p }
    }

    /// Main entry point for parsing arguments to `docopt` macro.
    /// First looks for an identifier for the struct name.
    /// Second, a string containing the docopt usage patterns.
    /// Third, an optional list of type annotations.
    fn parse(&mut self) -> PResult<'b, Parsed> {
        if self.p.token == token::Eof {
            let err = self.cx.struct_span_err(
                self.cx.call_site(), "macro expects arguments");
            return Err(err);
        }
        let struct_info = try!(self.parse_struct_info());
        let docstr = try!(self.parse_str());

        let mut types = HashMap::new();
        if !self.p.check(&token::Eof) {
            let sep = SeqSep {
                sep: Some(token::Comma),
                trailing_sep_allowed: true,
            };
            types = self.p.parse_seq_to_before_end(
                &token::Eof, sep, |p| MacParser::parse_type_annotation(p)
            ).into_iter()
             .map(|(ident, ty)| {
                  let field_name = ident.to_string();
                  let key = ArgvMap::struct_field_to_key(&*field_name);
                  (Atom::new(&*key), ty)
              })
             .collect::<HashMap<Atom, P<ast::Ty>>>();
            try!(self.p.expect(&token::Eof));
        }

        // This config does not matter because we're only asking for the
        // usage patterns in the Docopt string. The configuration does not
        // affect the retrieval of usage patterns.
        let doc = match Docopt::new(docstr) {
            Ok(doc) => doc,
            Err(err) => {
                let err = self.cx.struct_span_err(
                    self.cx.call_site(),
                    &format!("Invalid Docopt usage: {}", err));
                return Err(err);
            }
        };
        Ok(Parsed {
            struct_info: struct_info,
            doc: doc,
            types: types,
        })
    }

    /// Parses a single string literal. On failure, an error is logged and
    /// unit is returned.
    fn parse_str(&mut self) -> PResult<'b, String> {
        fn lit_is_str(lit: &ast::Lit) -> bool {
            match lit.node {
                ast::LitKind::Str(_, _) => true,
                _ => false,
            }
        }
        fn lit_to_string(lit: &ast::Lit) -> String {
            match lit.node {
                ast::LitKind::Str(ref s, _) => s.to_string(),
                _ => panic!("BUG: expected string literal"),
            }
        }
        let exp = self.cx.expander().fold_expr(self.p.parse_expr().unwrap());
        let s = match exp.node {
            ast::ExprKind::Lit(ref lit) if lit_is_str(&**lit) => {
                lit_to_string(&**lit)
            }
            _ => {
                let err = format!("Expected string literal but got {}",
                                  pprust::expr_to_string(&*exp));
                let err = self.cx.struct_span_err(exp.span, &*err);
                return Err(err);
            }
        };
        self.p.bump();
        Ok(s)
    }

    /// Parses a type annotation in a `docopt` invocation of the form
    /// `ident: Ty`.
    /// Note that this is a static method as it is used as a HOF.
    fn parse_type_annotation(p: &mut Parser<'b>)
                             -> PResult<'b, (ast::Ident, P<ast::Ty>)> {
        let ident = try!(p.parse_ident());
        try!(p.expect(&token::Colon));
        let ty = p.parse_ty().unwrap();
        Ok((ident, ty))
    }

    /// Parses struct information, like visibility, name and deriving.
    fn parse_struct_info(&mut self) -> PResult<'b, StructInfo> {
        let public = self.p.eat_keyword(symbol::keywords::Pub);
        let mut info = StructInfo {
            name: try!(self.p.parse_ident()),
            public: public,
            deriving: vec![],
        };
        if self.p.eat(&token::Comma) { return Ok(info); }
        let deriving = try!(self.p.parse_ident());
        if *deriving.name.as_str() != *"derive" {
            let err = format!("Expected 'derive' keyword but got '{}'",
                              deriving);
            let err = self.cx.struct_span_err(self.cx.call_site(), &*err);
            return Err(err);
        }
        while !self.p.eat(&token::Comma) {
            info.deriving.push(
                try!(self.p.parse_ident()).name.to_string());
        }
        Ok(info)
    }
}

struct StructInfo {
    name: ast::Ident,
    public: bool,
    deriving: Vec<String>,
}

// Convenience functions for building intermediate values.

fn ident(s: &str) -> ast::Ident {
    ast::Ident::with_empty_ctxt(symbol::Symbol::intern(s))
}

fn attribute<S, T>(cx: &ExtCtxt, name: S, items: Vec<T>) -> ast::Attribute
            where S: Borrow<str>, T: Borrow<str> {
    let sp = codemap::DUMMY_SP;
    let its = items.into_iter().map(|s| meta_item(cx, s.borrow())).collect();
    let mi = cx.meta_list(sp, intern(name.borrow()), its);
    cx.attribute(sp, mi)
}

fn meta_item(cx: &ExtCtxt, s: &str) -> codemap::Spanned<ast::NestedMetaItemKind> {
    codemap::Spanned {
        node: ast::NestedMetaItemKind::MetaItem(cx.meta_word(codemap::DUMMY_SP, intern(s))),
        span: cx.call_site(),
    }
}

fn intern(s: &str) -> symbol::Symbol {
    symbol::Symbol::intern(s)
}

fn ty_vec_string(cx: &ExtCtxt) -> P<ast::Ty> {
    let sp = codemap::DUMMY_SP;
    let tystr = ast::AngleBracketedParameterData {
        lifetimes: vec![],
        types: P::from_vec(vec![cx.ty_ident(sp, ident("String"))]),
        bindings: P::new(),
    };
    cx.ty_path(ast::Path {
        span: sp,
        global: false,
        segments: vec![ast::PathSegment {
            identifier: ident("Vec"),
            parameters: ast::PathParameters::AngleBracketed(tystr),
        }]
    })
}
<commit_msg>Update for changes in ast::Path<commit_after>#![crate_name = "docopt_macros"]
#![crate_type = "dylib"]

#![feature(plugin_registrar, quote, rustc_private)]

//! This crate defines the `docopt!` macro. It is documented in the
//! documentation of the `docopt` crate.

extern crate syntax;
extern crate rustc_plugin;
extern crate docopt;

use std::borrow::Borrow;
use std::collections::HashMap;

use rustc_plugin::Registry;
use syntax::{ast, codemap, tokenstream};
use syntax::errors::DiagnosticBuilder;
use syntax::ext::base::{ExtCtxt, MacResult, MacEager, DummyResult};
use syntax::ext::build::AstBuilder;
use syntax::fold::Folder;
use syntax::parse::common::SeqSep;
use syntax::parse::parser::Parser;
use syntax::parse::token;
use syntax::print::pprust;
use syntax::ptr::P;
use syntax::symbol;
use syntax::util::small_vector::SmallVector;

use docopt::{Docopt, ArgvMap};
use docopt::parse::{Options, Atom, Positional, Zero, One};

type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>;

#[plugin_registrar]
pub fn plugin_registrar(reg: &mut Registry) {
    reg.register_macro("docopt", expand);
}

fn expand(cx: &mut ExtCtxt, span: codemap::Span, tts: &[tokenstream::TokenTree])
         -> Box<MacResult+'static> {
    let parsed = match MacParser::new(cx, tts).parse() {
        Ok(parsed) => parsed,
        Err(_) => return DummyResult::any(span),
    };
    parsed.items(cx)
}

/// Parsed corresponds to the result of parsing a `docopt` macro call.
/// It can be used to write a corresponding struct.
struct Parsed {
    struct_info: StructInfo,
    doc: Docopt,
    /// Overrided type annotations for struct members. May be empty.
    /// When a type annotation for an atom doesn't exist, then one is
    /// inferred automatically. It is one of: `bool`, `usize`, `String` or
    /// `Vec<String>`.
    types: HashMap<Atom, P<ast::Ty>>,
}

impl Parsed {
    /// Returns a macro result suitable for expansion.
    /// Contains two items: one for the struct and one for the struct impls.
    fn items(&self, cx: &ExtCtxt) -> Box<MacResult+'static> {
        let mut its = vec!();
        its.push(self.struct_decl(cx));

        let struct_name = self.struct_info.name;
        let full_doc = &*self.doc.parser().full_doc;
        its.push(quote_item!(cx,
            impl $struct_name {
                #[allow(dead_code)]
                fn docopt() -> docopt::Docopt {
                    // The unwrap is justified here because this code
                    // gen only happens if the Docopt usage string is valid.
                    docopt::Docopt::new($full_doc).unwrap()
                }
            }
        ).unwrap());
        MacEager::items(SmallVector::many(its))
    }

    /// Returns an item for the struct definition.
    fn struct_decl(&self, cx: &ExtCtxt) -> P<ast::Item> {
        let name = self.struct_info.name.clone();
        let vis = if self.struct_info.public { ast::Visibility::Public }
                  else { ast::Visibility::Inherited };
        let def = ast::VariantData::Struct(
            self.struct_fields(cx), ast::DUMMY_NODE_ID);

        let mut traits = vec!["RustcDecodable".to_string()];
        traits.extend(self.struct_info.deriving.iter().cloned());
        let attrs = vec![attribute(cx, "allow", vec!["non_snake_case"]),
                         attribute(cx, "derive", traits)];

        let st = cx.item_struct(codemap::DUMMY_SP, name.clone(), def);
        cx.item(codemap::DUMMY_SP, name, attrs, st.node.clone()).map(|mut it| {
            it.vis = vis;
            it
        })
    }

    /// Returns a list of fields for the struct definition.
    /// Handles type annotations.
    fn struct_fields(&self, cx: &ExtCtxt) -> Vec<ast::StructField> {
        let mut fields: Vec<ast::StructField> = vec!();
        for (atom, opts) in self.doc.parser().descs.iter() {
            let name = ArgvMap::key_to_struct_field(&*atom.to_string());
            let ty = match self.types.get(atom) {
                None => self.pat_type(cx, atom, opts),
                Some(ty) => ty.clone(),
            };
            fields.push(self.mk_struct_field(&*name, ty));
        }
        fields
    }

    /// Returns an inferred type for a usage pattern.
    /// This is only invoked when a type annotation is not present.
    fn pat_type(&self, cx: &ExtCtxt, atom: &Atom, opts: &Options) -> P<ast::Ty> {
        let sp = codemap::DUMMY_SP;
        match (opts.repeats, &opts.arg) {
            (false, &Zero) => {
                match atom {
                    &Positional(_) => cx.ty_ident(sp, ident("String")),
                    _ => cx.ty_ident(sp, ident("bool")),
                }
            }
            (true, &Zero) => {
                match atom {
                    &Positional(_) => ty_vec_string(cx),
                    _ => cx.ty_ident(sp, ident("usize")),
                }
            }
            (false, &One(_)) => cx.ty_ident(sp, ident("String")),
            (true, &One(_)) => ty_vec_string(cx),
        }
    }

    /// Creates a struct field from a member name and type.
    fn mk_struct_field(&self, name: &str, ty: P<ast::Ty>) -> ast::StructField {
        ast::StructField {
            span: codemap::DUMMY_SP,
            ident: Some(ident(name)),
            vis: ast::Visibility::Public,
            id: ast::DUMMY_NODE_ID,
            ty: ty,
            attrs: vec!(),
        }
    }
}

/// State for parsing a `docopt` macro invocation.
struct MacParser<'a, 'b:'a> {
    cx: &'a mut ExtCtxt<'b>,
    p: Parser<'b>,
}

impl<'a, 'b> MacParser<'a, 'b> {
    fn new(cx: &'a mut ExtCtxt<'b>, tts: &[tokenstream::TokenTree]) -> MacParser<'a, 'b> {
        let p = cx.new_parser_from_tts(tts);
        MacParser { cx: cx, p: p }
    }

    /// Main entry point for parsing arguments to `docopt` macro.
    /// First looks for an identifier for the struct name.
    /// Second, a string containing the docopt usage patterns.
    /// Third, an optional list of type annotations.
    fn parse(&mut self) -> PResult<'b, Parsed> {
        if self.p.token == token::Eof {
            let err = self.cx.struct_span_err(
                self.cx.call_site(), "macro expects arguments");
            return Err(err);
        }
        let struct_info = try!(self.parse_struct_info());
        let docstr = try!(self.parse_str());

        let mut types = HashMap::new();
        if !self.p.check(&token::Eof) {
            let sep = SeqSep {
                sep: Some(token::Comma),
                trailing_sep_allowed: true,
            };
            types = self.p.parse_seq_to_before_end(
                &token::Eof, sep, |p| MacParser::parse_type_annotation(p)
            ).into_iter()
             .map(|(ident, ty)| {
                  let field_name = ident.to_string();
                  let key = ArgvMap::struct_field_to_key(&*field_name);
                  (Atom::new(&*key), ty)
              })
             .collect::<HashMap<Atom, P<ast::Ty>>>();
            try!(self.p.expect(&token::Eof));
        }

        // This config does not matter because we're only asking for the
        // usage patterns in the Docopt string. The configuration does not
        // affect the retrieval of usage patterns.
        let doc = match Docopt::new(docstr) {
            Ok(doc) => doc,
            Err(err) => {
                let err = self.cx.struct_span_err(
                    self.cx.call_site(),
                    &format!("Invalid Docopt usage: {}", err));
                return Err(err);
            }
        };
        Ok(Parsed {
            struct_info: struct_info,
            doc: doc,
            types: types,
        })
    }

    /// Parses a single string literal. On failure, an error is logged and
    /// unit is returned.
    fn parse_str(&mut self) -> PResult<'b, String> {
        fn lit_is_str(lit: &ast::Lit) -> bool {
            match lit.node {
                ast::LitKind::Str(_, _) => true,
                _ => false,
            }
        }
        fn lit_to_string(lit: &ast::Lit) -> String {
            match lit.node {
                ast::LitKind::Str(ref s, _) => s.to_string(),
                _ => panic!("BUG: expected string literal"),
            }
        }
        let exp = self.cx.expander().fold_expr(self.p.parse_expr().unwrap());
        let s = match exp.node {
            ast::ExprKind::Lit(ref lit) if lit_is_str(&**lit) => {
                lit_to_string(&**lit)
            }
            _ => {
                let err = format!("Expected string literal but got {}",
                                  pprust::expr_to_string(&*exp));
                let err = self.cx.struct_span_err(exp.span, &*err);
                return Err(err);
            }
        };
        self.p.bump();
        Ok(s)
    }

    /// Parses a type annotation in a `docopt` invocation of the form
    /// `ident: Ty`.
    /// Note that this is a static method as it is used as a HOF.
    fn parse_type_annotation(p: &mut Parser<'b>)
                             -> PResult<'b, (ast::Ident, P<ast::Ty>)> {
        let ident = try!(p.parse_ident());
        try!(p.expect(&token::Colon));
        let ty = p.parse_ty().unwrap();
        Ok((ident, ty))
    }

    /// Parses struct information, like visibility, name and deriving.
    fn parse_struct_info(&mut self) -> PResult<'b, StructInfo> {
        let public = self.p.eat_keyword(symbol::keywords::Pub);
        let mut info = StructInfo {
            name: try!(self.p.parse_ident()),
            public: public,
            deriving: vec![],
        };
        if self.p.eat(&token::Comma) { return Ok(info); }
        let deriving = try!(self.p.parse_ident());
        if *deriving.name.as_str() != *"derive" {
            let err = format!("Expected 'derive' keyword but got '{}'",
                              deriving);
            let err = self.cx.struct_span_err(self.cx.call_site(), &*err);
            return Err(err);
        }
        while !self.p.eat(&token::Comma) {
            info.deriving.push(
                try!(self.p.parse_ident()).name.to_string());
        }
        Ok(info)
    }
}

struct StructInfo {
    name: ast::Ident,
    public: bool,
    deriving: Vec<String>,
}

// Convenience functions for building intermediate values.

fn ident(s: &str) -> ast::Ident {
    ast::Ident::with_empty_ctxt(symbol::Symbol::intern(s))
}

fn attribute<S, T>(cx: &ExtCtxt, name: S, items: Vec<T>) -> ast::Attribute
            where S: Borrow<str>, T: Borrow<str> {
    let sp = codemap::DUMMY_SP;
    let its = items.into_iter().map(|s| meta_item(cx, s.borrow())).collect();
    let mi = cx.meta_list(sp, intern(name.borrow()), its);
    cx.attribute(sp, mi)
}

fn meta_item(cx: &ExtCtxt, s: &str) -> codemap::Spanned<ast::NestedMetaItemKind> {
    codemap::Spanned {
        node: ast::NestedMetaItemKind::MetaItem(cx.meta_word(codemap::DUMMY_SP, intern(s))),
        span: cx.call_site(),
    }
}

fn intern(s: &str) -> symbol::Symbol {
    symbol::Symbol::intern(s)
}

fn ty_vec_string(cx: &ExtCtxt) -> P<ast::Ty> {
    let sp = codemap::DUMMY_SP;
    let tystr = ast::AngleBracketedParameterData {
        lifetimes: vec![],
        types: P::from_vec(vec![cx.ty_ident(sp, ident("String"))]),
        bindings: P::new(),
    };
    cx.ty_path(ast::Path {
        span: sp,
        segments: vec![ast::PathSegment {
            identifier: ident("Vec"),
            parameters: Some(P(ast::PathParameters::AngleBracketed(tystr))),
        }]
    })
}
<|endoftext|>