<commit_before>use std::io;
use std::iter::Iterator;
use std::slice;
use std::path::Component;

use flate2::read::GzDecoder;
use tar;
use time::Timespec;

use collections::{CollectionsStatus, SignatureChain, SignatureFile};
use backend::Backend;

pub struct BackupFiles {
    snapshots: Vec<Snapshot>
}

pub struct Snapshot {
    pub time: Timespec
}

pub struct File {
    pub name: String,
    pub last_modified: Timespec
}

/// Iterator over a list of backup snapshots.
pub type Snapshots<'a> = slice::Iter<'a, Snapshot>;

enum DiffType {
    Signature,
    Snapshot,
    Deleted
}

impl BackupFiles {
    pub fn new<B: Backend>(backend: &B) -> io::Result<BackupFiles> {
        let collection = {
            let filenames = try!(backend.get_file_names());
            CollectionsStatus::from_filenames(&filenames)
        };
        let chains = collection.signature_chains();
        for chain in chains {
            let file = try!(backend.open_file(chain.fullsig.file_name.as_ref()));
            if chain.fullsig.compressed {
                let gz_decoder = try!(GzDecoder::new(file));
                let mut tar = tar::Archive::new(gz_decoder);
                for tarfile in try!(tar.files_mut()) {
                    if let Ok(tarfile) = tarfile {
                        let header = tarfile.header();
                        let path = unwrap_or_continue!(header.path());
                        let mut pcomps = path.components();
                        // split the path in (first directory, the remaining path)
                        // the first is the type, the remaining is the real path
                        let pfirst = unwrap_opt_or_continue!(pcomps.next());
                        if let Component::Normal(strfirst) = pfirst {
                            let difftype = match strfirst.to_str() {
                                Some("signature") => DiffType::Signature,
                                Some("snapshot")  => DiffType::Snapshot,
                                Some("deleted")   => DiffType::Deleted,
                                _                 => { continue; }
                            };
                            let realpath = pcomps.as_path();
                        }
                    }
                }
            }
        }
        // TODO: go from signature chains to snapshots
        Ok(BackupFiles{ snapshots: Vec::new() })
    }

    pub fn snapshots(&self) -> Snapshots {
        self.snapshots.iter()
    }
}

<commit_msg>Refactoring signatures<commit_after>use std::io::{self, Read};
use std::iter::Iterator;
use std::slice;
use std::path::{Component, PathBuf};

use flate2::read::GzDecoder;
use tar;
use time::Timespec;

use backend::Backend;
use collections::{CollectionsStatus, SignatureChain, SignatureFile};

pub struct BackupFiles {
    chains: Vec<Chain>
}

pub struct Snapshot {
    pub time: Timespec
}

pub struct File {
    pub name: String,
    pub last_modified: Timespec
}

/// Iterator over a list of backup snapshots.
pub type Snapshots<'a> = slice::Iter<'a, Snapshot>;

enum DiffType {
    Signature,
    Snapshot,
    Deleted
}

/// Store separately informations about the signatures and informations about the paths in the
/// signatures. This allows to reuse informations between snapshots and avoid duplicating them.
struct Chain {
    timestamps: Vec<Timespec>,
    files: Vec<PathSnapshots>
}

struct PathSnapshots {
    // the directory or file path
    path: PathBuf,
    // all the snapshots for this path
    snapshots: Vec<PathSnapshot>
}

struct PathSnapshot {
    // info are None if the snapshot has deleted this path
    info: Option<PathInfo>,
    // the index of the snapshot in the chain
    index: u8
}

/// Informations about a path inside a snapshot.
struct PathInfo {
    mtime: Timespec
}

impl BackupFiles {
    pub fn new<B: Backend>(backend: &B) -> io::Result<BackupFiles> {
        let collection = {
            let filenames = try!(backend.get_file_names());
            CollectionsStatus::from_filenames(&filenames)
        };
        let mut chains: Vec<Chain> = Vec::new();
        let coll_chains = collection.signature_chains();
        for coll_chain in coll_chains {
            let mut chain = Chain{ timestamps: Vec::new(), files: Vec::new() };
            let file = try!(backend.open_file(coll_chain.fullsig.file_name.as_ref()));
            add_sigfile_to_chain(&mut chain, file, &coll_chain.fullsig);
            for inc in coll_chain.inclist.iter() {
                let file = try!(backend.open_file(inc.file_name.as_ref()));
                add_sigfile_to_chain(&mut chain, file, &inc);
            }
            chains.push(chain);
        }
        Ok(BackupFiles{ chains: chains })
    }

    pub fn snapshots(&self) -> Snapshots {
        unimplemented!()
    }
}

fn add_sigfile_to_chain<R: Read>(chain: &mut Chain, file: R, sigfile: &SignatureFile) -> io::Result<()> {
    if sigfile.compressed {
        let gz_decoder = try!(GzDecoder::new(file));
        let mut tar = tar::Archive::new(gz_decoder);
        for tarfile in try!(tar.files_mut()) {
            if let Ok(tarfile) = tarfile {
                let header = tarfile.header();
                let path = unwrap_or_continue!(header.path());
                let mut pcomps = path.components();
                // split the path in (first directory, the remaining path)
                // the first is the type, the remaining is the real path
                let pfirst = unwrap_opt_or_continue!(pcomps.next());
                if let Component::Normal(strfirst) = pfirst {
                    let difftype = match strfirst.to_str() {
                        Some("signature") => DiffType::Signature,
                        Some("snapshot")  => DiffType::Snapshot,
                        Some("deleted")   => DiffType::Deleted,
                        _                 => { continue; }
                    };
                    let realpath = pcomps.as_path();
                }
            }
        }
    }
    Ok(())
}

<|endoftext|>


<commit_before>// Copyright 2018 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::HashMap;
use std::fs::File;
use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream};
use std::sync::Arc;
use std::time::Duration;
use std::{io, thread};

use crate::lmdb;

use crate::core::core;
use crate::core::core::hash::Hash;
use crate::core::global;
use crate::core::pow::Difficulty;
use crate::handshake::Handshake;
use crate::peer::Peer;
use crate::peers::Peers;
use crate::store::PeerStore;
use crate::types::{Capabilities, ChainAdapter, Error, NetAdapter, P2PConfig, TxHashSetRead};
use crate::util::{Mutex, StopState};
use chrono::prelude::{DateTime, Utc};

/// P2P server implementation, handling bootstrapping to find and connect to
/// peers, receiving connections from other peers and keep track of all of them.
pub struct Server {
	pub config: P2PConfig,
	capabilities: Capabilities,
	handshake: Arc<Handshake>,
	pub peers: Arc<Peers>,
	stop_state: Arc<Mutex<StopState>>,
}

// TODO TLS
impl Server {
	/// Creates a new idle p2p server with no peers
	pub fn new(
		db_env: Arc<lmdb::Environment>,
		capab: Capabilities,
		config: P2PConfig,
		adapter: Arc<dyn ChainAdapter>,
		genesis: Hash,
		stop_state: Arc<Mutex<StopState>>,
	) -> Result<Server, Error> {
		Ok(Server {
			config: config.clone(),
			capabilities: capab,
			handshake: Arc::new(Handshake::new(genesis, config.clone())),
			peers: Arc::new(Peers::new(PeerStore::new(db_env)?, adapter, config)),
			stop_state,
		})
	}

	/// Starts a new TCP server and listen to incoming connections. This is a
	/// blocking call until the TCP server stops.
	pub fn listen(&self) -> Result<(), Error> {
		// start TCP listener and handle incoming connections
		let addr = SocketAddr::new(self.config.host, self.config.port);
		let listener = TcpListener::bind(addr)?;
		listener.set_nonblocking(true)?;

		let mut connected_sockets: HashMap<SocketAddr, TcpStream> = HashMap::new();

		let sleep_time = Duration::from_millis(1);
		loop {
			// Pause peer ingress connection request. Only for tests.
			if self.stop_state.lock().is_paused() {
				thread::sleep(Duration::from_secs(1));
				continue;
			}

			match listener.accept() {
				Ok((stream, peer_addr)) => {
					if !self.check_banned(&stream) {
						let sc = stream.try_clone();
						if let Err(e) = self.handle_new_peer(stream) {
							warn!("Error accepting peer {}: {:?}", peer_addr.to_string(), e);
						} else {
							if let Ok(s) = sc {
								connected_sockets.insert(peer_addr, s);
							}
						}
					}
					// if any active socket not in our peers list, close it
					self.clean_lost_sockets(&mut connected_sockets);
				}
				Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
					// nothing to do, will retry in next iteration
				}
				Err(e) => {
					warn!("Couldn't establish new client connection: {:?}", e);
				}
			}
			if self.stop_state.lock().is_stopped() {
				break;
			}
			thread::sleep(sleep_time);
		}
		Ok(())
	}

	/// Asks the server to connect to a new peer. Directly returns the peer if
	/// we're already connected to the provided address.
	pub fn connect(&self, addr: &SocketAddr) -> Result<Arc<Peer>, Error> {
		if Peer::is_denied(&self.config, &addr) {
			debug!("connect_peer: peer {} denied, not connecting.", addr);
			return Err(Error::ConnectionClose);
		}

		if global::is_production_mode() {
			let hs = self.handshake.clone();
			let addrs = hs.addrs.read();
			if addrs.contains(&addr) {
				debug!("connect: ignore connecting to PeerWithSelf, addr: {}", addr);
				return Err(Error::PeerWithSelf);
			}
		}

		if let Some(p) = self.peers.get_connected_peer(addr) {
			// if we're already connected to the addr, just return the peer
			trace!("connect_peer: already connected {}", addr);
			return Ok(p);
		}

		trace!(
			"connect_peer: on {}:{}. connecting to {}",
			self.config.host,
			self.config.port,
			addr
		);
		match TcpStream::connect_timeout(addr, Duration::from_secs(10)) {
			Ok(mut stream) => {
				let addr = SocketAddr::new(self.config.host, self.config.port);
				let total_diff = self.peers.total_difficulty();

				let mut peer = Peer::connect(
					&mut stream,
					self.capabilities,
					total_diff,
					addr,
					&self.handshake,
					self.peers.clone(),
				)?;
				peer.start(stream);
				let peer = Arc::new(peer);
				self.peers.add_connected(peer.clone())?;
				Ok(peer)
			}
			Err(e) => {
				trace!(
					"connect_peer: on {}:{}. Could not connect to {}: {:?}",
					self.config.host,
					self.config.port,
					addr,
					e
				);
				Err(Error::Connection(e))
			}
		}
	}

	fn handle_new_peer(&self, mut stream: TcpStream) -> Result<(), Error> {
		let total_diff = self.peers.total_difficulty();

		// accept the peer and add it to the server map
		let mut peer = Peer::accept(
			&mut stream,
			self.capabilities,
			total_diff,
			&self.handshake,
			self.peers.clone(),
		)?;
		peer.start(stream);
		self.peers.add_connected(Arc::new(peer))?;
		Ok(())
	}

	fn check_banned(&self, stream: &TcpStream) -> bool {
		// peer has been banned, go away!
		if let Ok(peer_addr) = stream.peer_addr() {
			if self.peers.is_banned(peer_addr) {
				debug!("Peer {} banned, refusing connection.", peer_addr);
				if let Err(e) = stream.shutdown(Shutdown::Both) {
					debug!("Error shutting down conn: {:?}", e);
				}
				return true;
			}
		}
		false
	}

	/// For all kinds of exception cases, the node could accepted / initiated a peer connection successfully but
	/// failed on the Handshake protocol communication, or a connected peer was closed but without a successful
	/// clean-up on its socket, that will cause this connected (on TcpStream) peer becomes so-called "invisible" peer!
	/// i.e. a peer not included in the 'self.peers.peers' hashmap. This "invisible" peer will cause some security
	/// concern because it still can send something to this node, but without enough visibility as other connected peers.
	/// Another impact is these connections could never be closed, which make the node fully occupied by all such
	/// kind of connections and become un-connectable.
	/// This function can help to clean the peer connections which is "invisible" for this node.
	fn clean_lost_sockets(&self, sockets: &mut HashMap<SocketAddr, TcpStream>) {
		let mut lost_sockets: Vec<SocketAddr> = vec![];
		for (socket, stream) in sockets.iter() {
			if !self.peers.is_known_ip(&socket) {
				if let Ok(_) = stream.shutdown(Shutdown::Both) {
					debug!(
						"clean_lost_sockets: {} cleaned which's not in peers list",
						socket
					);
				}
				lost_sockets.push(socket.clone());
			}
		}

		for socket in lost_sockets {
			sockets.remove(&socket);
		}
	}

	pub fn stop(&self) {
		self.stop_state.lock().stop();
		self.peers.stop();
	}

	/// Pause means: stop all the current peers connection, only for tests.
	/// Note:
	/// 1. must pause the 'seed' thread also, to avoid the new egress peer connection
	/// 2. must pause the 'p2p-server' thread also, to avoid the new ingress peer connection.
	pub fn pause(&self) {
		self.peers.stop();
	}
}

/// A no-op network adapter used for testing.
pub struct DummyAdapter {}

impl ChainAdapter for DummyAdapter {
	fn total_difficulty(&self) -> Difficulty {
		Difficulty::min()
	}
	fn total_height(&self) -> u64 {
		0
	}
	fn get_transaction(&self, _h: Hash) -> Option<core::Transaction> {
		None
	}
	fn tx_kernel_received(&self, _h: Hash, _addr: SocketAddr) {}
	fn transaction_received(&self, _: core::Transaction, _stem: bool) {}
	fn compact_block_received(&self, _cb: core::CompactBlock, _addr: SocketAddr) -> bool {
		true
	}
	fn header_received(&self, _bh: core::BlockHeader, _addr: SocketAddr) -> bool {
		true
	}
	fn block_received(&self, _: core::Block, _: SocketAddr) -> bool {
		true
	}
	fn headers_received(&self, _: &[core::BlockHeader], _: SocketAddr) -> bool {
		true
	}
	fn locate_headers(&self, _: &[Hash]) -> Vec<core::BlockHeader> {
		vec![]
	}
	fn get_block(&self, _: Hash) -> Option<core::Block> {
		None
	}
	fn txhashset_read(&self, _h: Hash) -> Option<TxHashSetRead> {
		unimplemented!()
	}

	fn txhashset_receive_ready(&self) -> bool {
		false
	}

	fn txhashset_write(&self, _h: Hash, _txhashset_data: File, _peer_addr: SocketAddr) -> bool {
		false
	}

	fn txhashset_download_update(
		&self,
		_start_time: DateTime<Utc>,
		_downloaded_size: u64,
		_total_size: u64,
	) -> bool {
		false
	}
}

impl NetAdapter for DummyAdapter {
	fn find_peer_addrs(&self, _: Capabilities) -> Vec<SocketAddr> {
		vec![]
	}
	fn peer_addrs_received(&self, _: Vec<SocketAddr>) {}
	fn peer_difficulty(&self, _: SocketAddr, _: Difficulty, _: u64) {}
	fn is_banned(&self, _: SocketAddr) -> bool {
		false
	}
}
<commit_msg>Break out of main peer loop on error<commit_after>// Copyright 2018 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::collections::HashMap;
use std::fs::File;
use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream};
use std::sync::Arc;
use std::time::Duration;
use std::{io, thread};

use crate::lmdb;

use crate::core::core;
use crate::core::core::hash::Hash;
use crate::core::global;
use crate::core::pow::Difficulty;
use crate::handshake::Handshake;
use crate::peer::Peer;
use crate::peers::Peers;
use crate::store::PeerStore;
use crate::types::{Capabilities, ChainAdapter, Error, NetAdapter, P2PConfig, TxHashSetRead};
use crate::util::{Mutex, StopState};
use chrono::prelude::{DateTime, Utc};

/// P2P server implementation, handling bootstrapping to find and connect to
/// peers, receiving connections from other peers and keep track of all of them.
pub struct Server {
	pub config: P2PConfig,
	capabilities: Capabilities,
	handshake: Arc<Handshake>,
	pub peers: Arc<Peers>,
	stop_state: Arc<Mutex<StopState>>,
}

// TODO TLS
impl Server {
	/// Creates a new idle p2p server with no peers
	pub fn new(
		db_env: Arc<lmdb::Environment>,
		capab: Capabilities,
		config: P2PConfig,
		adapter: Arc<dyn ChainAdapter>,
		genesis: Hash,
		stop_state: Arc<Mutex<StopState>>,
	) -> Result<Server, Error> {
		Ok(Server {
			config: config.clone(),
			capabilities: capab,
			handshake: Arc::new(Handshake::new(genesis, config.clone())),
			peers: Arc::new(Peers::new(PeerStore::new(db_env)?, adapter, config)),
			stop_state,
		})
	}

	/// Starts a new TCP server and listen to incoming connections. This is a
	/// blocking call until the TCP server stops.
	pub fn listen(&self) -> Result<(), Error> {
		// start TCP listener and handle incoming connections
		let addr = SocketAddr::new(self.config.host, self.config.port);
		let listener = TcpListener::bind(addr)?;
		listener.set_nonblocking(true)?;

		let mut connected_sockets: HashMap<SocketAddr, TcpStream> = HashMap::new();

		let sleep_time = Duration::from_millis(1);
		loop {
			// Pause peer ingress connection request. Only for tests.
			if self.stop_state.lock().is_paused() {
				thread::sleep(Duration::from_secs(1));
				continue;
			}

			match listener.accept() {
				Ok((stream, peer_addr)) => {
					if !self.check_banned(&stream) {
						let sc = stream.try_clone();
						if let Err(e) = self.handle_new_peer(stream) {
							warn!("Error accepting peer {}: {:?}", peer_addr.to_string(), e);
							break;
						} else {
							if let Ok(s) = sc {
								connected_sockets.insert(peer_addr, s);
							}
						}
					}
					// if any active socket not in our peers list, close it
					self.clean_lost_sockets(&mut connected_sockets);
				}
				Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {
					// nothing to do, will retry in next iteration
				}
				Err(e) => {
					warn!("Couldn't establish new client connection: {:?}", e);
					break;
				}
			}
			if self.stop_state.lock().is_stopped() {
				break;
			}
			thread::sleep(sleep_time);
		}
		Ok(())
	}

	/// Asks the server to connect to a new peer. Directly returns the peer if
	/// we're already connected to the provided address.
	pub fn connect(&self, addr: &SocketAddr) -> Result<Arc<Peer>, Error> {
		if Peer::is_denied(&self.config, &addr) {
			debug!("connect_peer: peer {} denied, not connecting.", addr);
			return Err(Error::ConnectionClose);
		}

		if global::is_production_mode() {
			let hs = self.handshake.clone();
			let addrs = hs.addrs.read();
			if addrs.contains(&addr) {
				debug!("connect: ignore connecting to PeerWithSelf, addr: {}", addr);
				return Err(Error::PeerWithSelf);
			}
		}

		if let Some(p) = self.peers.get_connected_peer(addr) {
			// if we're already connected to the addr, just return the peer
			trace!("connect_peer: already connected {}", addr);
			return Ok(p);
		}

		trace!(
			"connect_peer: on {}:{}. connecting to {}",
			self.config.host,
			self.config.port,
			addr
		);
		match TcpStream::connect_timeout(addr, Duration::from_secs(10)) {
			Ok(mut stream) => {
				let addr = SocketAddr::new(self.config.host, self.config.port);
				let total_diff = self.peers.total_difficulty();

				let mut peer = Peer::connect(
					&mut stream,
					self.capabilities,
					total_diff,
					addr,
					&self.handshake,
					self.peers.clone(),
				)?;
				peer.start(stream);
				let peer = Arc::new(peer);
				self.peers.add_connected(peer.clone())?;
				Ok(peer)
			}
			Err(e) => {
				trace!(
					"connect_peer: on {}:{}. Could not connect to {}: {:?}",
					self.config.host,
					self.config.port,
					addr,
					e
				);
				Err(Error::Connection(e))
			}
		}
	}

	fn handle_new_peer(&self, mut stream: TcpStream) -> Result<(), Error> {
		let total_diff = self.peers.total_difficulty();

		// accept the peer and add it to the server map
		let mut peer = Peer::accept(
			&mut stream,
			self.capabilities,
			total_diff,
			&self.handshake,
			self.peers.clone(),
		)?;
		peer.start(stream);
		self.peers.add_connected(Arc::new(peer))?;
		Ok(())
	}

	fn check_banned(&self, stream: &TcpStream) -> bool {
		// peer has been banned, go away!
		if let Ok(peer_addr) = stream.peer_addr() {
			if self.peers.is_banned(peer_addr) {
				debug!("Peer {} banned, refusing connection.", peer_addr);
				if let Err(e) = stream.shutdown(Shutdown::Both) {
					debug!("Error shutting down conn: {:?}", e);
				}
				return true;
			}
		}
		false
	}

	/// For all kinds of exception cases, the node could accepted / initiated a peer connection successfully but
	/// failed on the Handshake protocol communication, or a connected peer was closed but without a successful
	/// clean-up on its socket, that will cause this connected (on TcpStream) peer becomes so-called "invisible" peer!
	/// i.e. a peer not included in the 'self.peers.peers' hashmap. This "invisible" peer will cause some security
	/// concern because it still can send something to this node, but without enough visibility as other connected peers.
	/// Another impact is these connections could never be closed, which make the node fully occupied by all such
	/// kind of connections and become un-connectable.
	/// This function can help to clean the peer connections which is "invisible" for this node.
	fn clean_lost_sockets(&self, sockets: &mut HashMap<SocketAddr, TcpStream>) {
		let mut lost_sockets: Vec<SocketAddr> = vec![];
		for (socket, stream) in sockets.iter() {
			if !self.peers.is_known_ip(&socket) {
				if let Ok(_) = stream.shutdown(Shutdown::Both) {
					debug!(
						"clean_lost_sockets: {} cleaned which's not in peers list",
						socket
					);
				}
				lost_sockets.push(socket.clone());
			}
		}

		for socket in lost_sockets {
			sockets.remove(&socket);
		}
	}

	pub fn stop(&self) {
		self.stop_state.lock().stop();
		self.peers.stop();
	}

	/// Pause means: stop all the current peers connection, only for tests.
	/// Note:
	/// 1. must pause the 'seed' thread also, to avoid the new egress peer connection
	/// 2. must pause the 'p2p-server' thread also, to avoid the new ingress peer connection.
	pub fn pause(&self) {
		self.peers.stop();
	}
}

/// A no-op network adapter used for testing.
pub struct DummyAdapter {}

impl ChainAdapter for DummyAdapter {
	fn total_difficulty(&self) -> Difficulty {
		Difficulty::min()
	}
	fn total_height(&self) -> u64 {
		0
	}
	fn get_transaction(&self, _h: Hash) -> Option<core::Transaction> {
		None
	}
	fn tx_kernel_received(&self, _h: Hash, _addr: SocketAddr) {}
	fn transaction_received(&self, _: core::Transaction, _stem: bool) {}
	fn compact_block_received(&self, _cb: core::CompactBlock, _addr: SocketAddr) -> bool {
		true
	}
	fn header_received(&self, _bh: core::BlockHeader, _addr: SocketAddr) -> bool {
		true
	}
	fn block_received(&self, _: core::Block, _: SocketAddr) -> bool {
		true
	}
	fn headers_received(&self, _: &[core::BlockHeader], _: SocketAddr) -> bool {
		true
	}
	fn locate_headers(&self, _: &[Hash]) -> Vec<core::BlockHeader> {
		vec![]
	}
	fn get_block(&self, _: Hash) -> Option<core::Block> {
		None
	}
	fn txhashset_read(&self, _h: Hash) -> Option<TxHashSetRead> {
		unimplemented!()
	}

	fn txhashset_receive_ready(&self) -> bool {
		false
	}

	fn txhashset_write(&self, _h: Hash, _txhashset_data: File, _peer_addr: SocketAddr) -> bool {
		false
	}

	fn txhashset_download_update(
		&self,
		_start_time: DateTime<Utc>,
		_downloaded_size: u64,
		_total_size: u64,
	) -> bool {
		false
	}
}

impl NetAdapter for DummyAdapter {
	fn find_peer_addrs(&self, _: Capabilities) -> Vec<SocketAddr> {
		vec![]
	}
	fn peer_addrs_received(&self, _: Vec<SocketAddr>) {}
	fn peer_difficulty(&self, _: SocketAddr, _: Difficulty, _: u64) {}
	fn is_banned(&self, _: SocketAddr) -> bool {
		false
	}
}
<|endoftext|>


<commit_before>extern crate lalrpop;

fn main() {
    lalrpop::Configuration::new()
        .process_current_dir()
        .unwrap();
}
<commit_msg>fix: Avoid rebuilding the parser crate even if nothing has changed<commit_after>extern crate lalrpop;

fn main() {
    lalrpop::Configuration::new()
        .process_current_dir()
        .unwrap();
    println!("cargo:rerun-if-changed=src/grammar.lalrpop");
}
<|endoftext|>


<commit_before>/* kernel::sgash.rs */

use core::*;
use core::str::*;
use core::option::{Some, Option, None}; // Match statement
use core::iter::Iterator;
use core::vec::Vec;
use core::mem::Allocator;
use kernel::*;
use super::super::platform::*;

pub static mut buffer: cstr = cstr {
				p: 0 as *mut u8,
				p_cstr_i: 0,
				max: 0
			      };
pub fn putchar(key: char) {
    unsafe {
	/*
	 * We need to include a blank asm call to prevent rustc
	 * from optimizing this part out
	 */
	asm!("");
	io::write_char(key, io::UART0);
    }
}

#[lang = "exchange_free"]
fn putstr(msg: &str) {
    for c in slice::iter(as_bytes(msg)) {
	putchar(*c as char);
    }
}

pub unsafe fn drawstr(msg: &str) {
    let mut old_fg = super::super::io::FG_COLOR;
    let mut x: u32 = 0x6699AAFF;
    for c in slice::iter(as_bytes(msg)) {
	x = (x << 8) + (x >> 24);
	super::super::io::set_fg(x);
	drawchar(*c as char);
    }
    super::super::io::set_fg(old_fg);
}

pub unsafe fn putcstr(s: cstr)
{
    let mut p = s.p as uint;
    while *(p as *char) != '\0'
    {
	putchar(*(p as *char));
	p += 1;
    }
}

pub unsafe fn parsekey(x: char) {
	let x = x as u8;
	// Set this to false to learn the keycodes of various keys!
	// Key codes are printed backwards because life is hard

	if (true) {
		match x {
			13		=>	{
						io::CURSOR_Y += io::CURSOR_HEIGHT;
						io::CURSOR_X = 0u32;
						prompt(false);
			}
			127		=>	{
				if (buffer.delete_char()) {
					putchar('');
					putchar(' ');
					putchar('');
					io::CURSOR_X -= io::CURSOR_WIDTH;
				}
			}
			_		=>	{
				if (buffer.add_char(x)) {
					putchar(x as char);
					drawchar(x as char);
				}
			}
		}
	}
	else {
		keycode(x);
	}
}

unsafe fn drawchar(x: char)
{
    io::restore(640, 1024*1024);
    io::draw_char(x, 1024*1024);
    io::CURSOR_X += io::CURSOR_WIDTH;
    io::backup(640, 1024*1024);
    io::draw_cursor(640, 1024*1024);
}

fn keycode(x: u8) {
	let mut x = x;
	while ( x != 0 ) {
		putchar((x%10+ ('0' as u8) ) as char);
		x = x/10;
	}
	putchar(' ');
}

pub unsafe fn init() {
	buffer = cstr::new(256);
	prompt(true);
}

unsafe fn prompt(startup: bool) {
	/*
	if (buffer.streq(&"ls")) { putstr( &"\na\tb") };
	match buffer.getarg(' ', 0) {
	Some(y)	=> {
	if(y.streq(&"cat")) {
	match buffer.getarg(' ', 1) {
	Some(x)	=> {
	if(x.streq(&"a")) { putstr( &"\nHello"); }
	if(x.streq(&"b")) { putstr( &"\nworld!"); }
	}
	None	=> { }
	};
	}
	}
	None	=> { }
	};
	*/
	putstr(&"\nsgash > ");
	if !startup {drawstr(&"sgash > ");}

	buffer.reset();
}

/* BUFFER MODIFICATION FUNCTIONS */

struct cstr {
	p: *mut u8,
	p_cstr_i: uint,
	max: uint
}

impl cstr {
	pub unsafe fn new(size: uint) -> cstr {
		// Sometimes this doesn't allocate enough memory and gets stuck...
		let (x, y) = memory::allocator.alloc(size);
		let this = cstr {
			p: x,
			p_cstr_i: 0,
			max: y
		};
		*(((this.p as uint)+this.p_cstr_i) as *mut char) = '\0';
		this
	}

	unsafe fn from_str(s: &str) -> cstr {
		let mut this = cstr::new(256);
		for c in slice::iter(as_bytes(s)) {
			this.add_char(*c);
		};
		this
	}

	fn len(&self) -> uint { self.p_cstr_i }

	// HELP THIS DOESN'T WORK THERE IS NO GARBAGE COLLECTION!!!
	unsafe fn destroy(&self) { memory::allocator.free(self.p); }

	unsafe fn add_char(&mut self, x: u8) -> bool{
		if (self.p_cstr_i == self.max) { return false; }
		*(((self.p as uint)+self.p_cstr_i) as *mut u8) = x;
		self.p_cstr_i += 1;
		*(((self.p as uint)+self.p_cstr_i) as *mut char) = '\0';
		true
	}

	unsafe fn delete_char(&mut self) -> bool {
		if (self.p_cstr_i == 0) { return false; }
		self.p_cstr_i -= 1;
		*(((self.p as uint)+self.p_cstr_i) as *mut char) = '\0';
		true
	}

	unsafe fn reset(&mut self) {
		self.p_cstr_i = 0;
		*(self.p as *mut char) = '\0';
	}

	unsafe fn eq(&self, other: &cstr) -> bool {
		if (self.len() != other.len()) { return false; }
		else {
			let mut x = 0;
			let mut selfp: uint = self.p as uint;
			let mut otherp: uint = other.p as uint;
			while (x < self.len()) {
				if (*(selfp as *char) != *(otherp as *char)) { return false; }
				selfp += 1;
				otherp += 1;
				x += 1;
			}
			true
		}
	}

	unsafe fn streq(&self, other: &str) -> bool {
		let mut x = 0;
		let mut selfp: uint = self.p as uint;
		for c in slice::iter(as_bytes(other)) {
			if( *c != *(selfp as *u8) ) { return false; }
			selfp += 1;
		};
		*(selfp as *char) == '\0'
	}

	unsafe fn getarg(&self, delim: char, mut k: uint) -> Option<cstr> {
		let mut ind: uint = 0;
		let mut found = k == 0;
		let mut selfp: uint = self.p as uint;
		let mut s = cstr::new(256);
		loop {
			if (*(selfp as *char) == '\0') {
				// End of string
				if (found) { return Some(s); }
				else { return None; }
			};
			if (*(selfp as *u8) == delim as u8) {
				if (found) { return Some(s); }
				k -= 1;
			};
			if (found) {
				s.add_char(*(selfp as *u8));
			};
			found = k == 0;
			selfp += 1;
			ind += 1;
			if (ind == self.max) {
				putstr(&"\nSomething broke!");
				return None;
			}
		}
	}

	unsafe fn split(&self, delim: char) -> (cstr, cstr) {
		let mut selfp: uint = self.p as uint;
		let mut beg = cstr::new(256);
		let mut end = cstr::new(256);
		let mut found = false;
		loop {
			if (*(selfp as *char) == '\0') {
				return (beg, end);
			}
			else if (*(selfp as *u8) == delim as u8) {
				found = true;
			}
			else if (!found) {
				beg.add_char(*(selfp as *u8));
			}
			else if (found) {
				end.add_char(*(selfp as *u8));
			};
			selfp += 1;
		}
	}

}

<commit_msg>and now backspace works correctly.<commit_after>/* kernel::sgash.rs */

use core::*;
use core::str::*;
use core::option::{Some, Option, None}; // Match statement
use core::iter::Iterator;
use core::vec::Vec;
use core::mem::Allocator;
use kernel::*;
use super::super::platform::*;

pub static mut buffer: cstr = cstr {
				p: 0 as *mut u8,
				p_cstr_i: 0,
				max: 0
			      };
pub fn putchar(key: char) {
    unsafe {
	/*
	 * We need to include a blank asm call to prevent rustc
	 * from optimizing this part out
	 */
	asm!("");
	io::write_char(key, io::UART0);
    }
}

#[lang = "exchange_free"]
fn putstr(msg: &str) {
    for c in slice::iter(as_bytes(msg)) {
	putchar(*c as char);
    }
}

pub unsafe fn drawstr(msg: &str) {
    let mut old_fg = super::super::io::FG_COLOR;
    let mut x: u32 = 0x6699AAFF;
    for c in slice::iter(as_bytes(msg)) {
	x = (x << 8) + (x >> 24);
	super::super::io::set_fg(x);
	drawchar(*c as char);
    }
    super::super::io::set_fg(old_fg);
}

pub unsafe fn putcstr(s: cstr)
{
    let mut p = s.p as uint;
    while *(p as *char) != '\0'
    {
	putchar(*(p as *char));
	p += 1;
    }
}

pub unsafe fn parsekey(x: char) {
	let x = x as u8;
	// Set this to false to learn the keycodes of various keys!
	// Key codes are printed backwards because life is hard

	if (true) {
		match x {
			13		=>	{
						io::CURSOR_Y += io::CURSOR_HEIGHT;
						io::CURSOR_X = 0u32;
						prompt(false);
			}
			127		=>	{
				if (buffer.delete_char()) {
					putchar('');
					putchar(' ');
					putchar('');
					backspace();
				}
			}
			_		=>	{
				if (buffer.add_char(x)) {
					putchar(x as char);
					drawchar(x as char);
				}
			}
		}
	}
	else {
		keycode(x);
	}
}

unsafe fn drawchar(x: char)
{
    io::restore(640, 1024*1024);
    io::draw_char(x, 1024*1024);
    io::CURSOR_X += io::CURSOR_WIDTH;
    io::backup(640, 1024*1024);
    io::draw_cursor(640, 1024*1024);
}

unsafe fn backspace()
{
    io::restore(640, 1024*1024);
    io::CURSOR_X -= io::CURSOR_WIDTH;
    io::draw_char(' ', 1024*1024);
    io::backup(640, 1024*1024);
    io::draw_cursor(640, 1024*1024);
}

fn keycode(x: u8) {
	let mut x = x;
	while ( x != 0 ) {
		putchar((x%10+ ('0' as u8) ) as char);
		x = x/10;
	}
	putchar(' ');
}

pub unsafe fn init() {
	buffer = cstr::new(256);
	prompt(true);
}

unsafe fn prompt(startup: bool) {
	/*
	if (buffer.streq(&"ls")) { putstr( &"\na\tb") };
	match buffer.getarg(' ', 0) {
	Some(y)	=> {
	if(y.streq(&"cat")) {
	match buffer.getarg(' ', 1) {
	Some(x)	=> {
	if(x.streq(&"a")) { putstr( &"\nHello"); }
	if(x.streq(&"b")) { putstr( &"\nworld!"); }
	}
	None	=> { }
	};
	}
	}
	None	=> { }
	};
	*/
	putstr(&"\nsgash > ");
	if !startup {drawstr(&"sgash > ");}

	buffer.reset();
}

/* BUFFER MODIFICATION FUNCTIONS */

struct cstr {
	p: *mut u8,
	p_cstr_i: uint,
	max: uint
}

impl cstr {
	pub unsafe fn new(size: uint) -> cstr {
		// Sometimes this doesn't allocate enough memory and gets stuck...
		let (x, y) = memory::allocator.alloc(size);
		let this = cstr {
			p: x,
			p_cstr_i: 0,
			max: y
		};
		*(((this.p as uint)+this.p_cstr_i) as *mut char) = '\0';
		this
	}

	unsafe fn from_str(s: &str) -> cstr {
		let mut this = cstr::new(256);
		for c in slice::iter(as_bytes(s)) {
			this.add_char(*c);
		};
		this
	}

	fn len(&self) -> uint { self.p_cstr_i }

	// HELP THIS DOESN'T WORK THERE IS NO GARBAGE COLLECTION!!!
	unsafe fn destroy(&self) { memory::allocator.free(self.p); }

	unsafe fn add_char(&mut self, x: u8) -> bool{
		if (self.p_cstr_i == self.max) { return false; }
		*(((self.p as uint)+self.p_cstr_i) as *mut u8) = x;
		self.p_cstr_i += 1;
		*(((self.p as uint)+self.p_cstr_i) as *mut char) = '\0';
		true
	}

	unsafe fn delete_char(&mut self) -> bool {
		if (self.p_cstr_i == 0) { return false; }
		self.p_cstr_i -= 1;
		*(((self.p as uint)+self.p_cstr_i) as *mut char) = '\0';
		true
	}

	unsafe fn reset(&mut self) {
		self.p_cstr_i = 0;
		*(self.p as *mut char) = '\0';
	}

	unsafe fn eq(&self, other: &cstr) -> bool {
		if (self.len() != other.len()) { return false; }
		else {
			let mut x = 0;
			let mut selfp: uint = self.p as uint;
			let mut otherp: uint = other.p as uint;
			while (x < self.len()) {
				if (*(selfp as *char) != *(otherp as *char)) { return false; }
				selfp += 1;
				otherp += 1;
				x += 1;
			}
			true
		}
	}

	unsafe fn streq(&self, other: &str) -> bool {
		let mut x = 0;
		let mut selfp: uint = self.p as uint;
		for c in slice::iter(as_bytes(other)) {
			if( *c != *(selfp as *u8) ) { return false; }
			selfp += 1;
		};
		*(selfp as *char) == '\0'
	}

	unsafe fn getarg(&self, delim: char, mut k: uint) -> Option<cstr> {
		let mut ind: uint = 0;
		let mut found = k == 0;
		let mut selfp: uint = self.p as uint;
		let mut s = cstr::new(256);
		loop {
			if (*(selfp as *char) == '\0') {
				// End of string
				if (found) { return Some(s); }
				else { return None; }
			};
			if (*(selfp as *u8) == delim as u8) {
				if (found) { return Some(s); }
				k -= 1;
			};
			if (found) {
				s.add_char(*(selfp as *u8));
			};
			found = k == 0;
			selfp += 1;
			ind += 1;
			if (ind == self.max) {
				putstr(&"\nSomething broke!");
				return None;
			}
		}
	}

	unsafe fn split(&self, delim: char) -> (cstr, cstr) {
		let mut selfp: uint = self.p as uint;
		let mut beg = cstr::new(256);
		let mut end = cstr::new(256);
		let mut found = false;
		loop {
			if (*(selfp as *char) == '\0') {
				return (beg, end);
			}
			else if (*(selfp as *u8) == delim as u8) {
				found = true;
			}
			else if (!found) {
				beg.add_char(*(selfp as *u8));
			}
			else if (found) {
				end.add_char(*(selfp as *u8));
			};
			selfp += 1;
		}
	}

}

<|endoftext|>


<commit_before>extern crate env_logger;
extern crate git2;
#[macro_use]
extern crate log;
extern crate tools;
extern crate unicode_normalization;

use std::borrow::Borrow;
use std::collections::HashMap;
use std::env;
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};

use git2::{DiffFindOptions, ObjectType, Oid, Patch, Repository, Sort};
use tools::config::index_blame;
use unicode_normalization::UnicodeNormalization;

fn get_hg_rev(helper: &mut Child, git_oid: &Oid) -> Option<String> {
    write!(helper.stdin.as_mut().unwrap(), "git2hg {}\n", git_oid).unwrap();
    let mut reader = BufReader::new(helper.stdout.as_mut().unwrap());
    loop {
        let mut result = String::new();
        reader.read_line(&mut result).unwrap();
        if result.starts_with("changeset ") {
            return result.split(' ').nth(1).map(str::trim).map(str::to_string);
        }
        if result.trim().chars().all(|c| c == '0') {
            return None;
        }
        // else loop again. cinnabar-helper emits a bunch of other info that we don't care about
    }
}

fn start_cinnabar_helper(git_repo: &Repository) -> Child {
    Command::new("git")
        .arg("cinnabar-helper")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .current_dir(git_repo.path())
        .spawn()
        .unwrap()
}

fn count_lines(blob: &git2::Blob) -> usize {
    let data = blob.content();
    if data.is_empty() {
        return 0;
    }
    let mut linecount = 0;
    for b in data {
        if *b == b'\n' {
            linecount += 1;
        }
    }
    if data[data.len() - 1] != b'\n' {
        linecount += 1;
    }
    linecount
}

fn unmodified_lines(
    blob: &git2::Blob,
    parent_blob: &git2::Blob,
) -> Result<Vec<(usize, usize)>, git2::Error> {
    let mut unchanged = Vec::new();

    let patch = Patch::from_blobs(parent_blob, None, blob, None, None)?;

    if patch.delta().flags().is_binary() {
        return Ok(unchanged);
    }

    fn add_delta(lineno: usize, delta: i32) -> usize {
        ((lineno as i32) + delta) as usize
    }

    let mut latest_line: usize = 0;
    let mut delta: i32 = 0;

    for hunk_index in 0..patch.num_hunks() {
        for line_index in 0..patch.num_lines_in_hunk(hunk_index)? {
            let line = patch.line_in_hunk(hunk_index, line_index)?;

            if let Some(lineno) = line.new_lineno() {
                let lineno = lineno as usize;
                for i in latest_line..lineno - 1 {
                    unchanged.push((i, add_delta(i, delta)));
                }
                latest_line = (lineno - 1) + 1;
            }

            match line.origin() {
                '+' => delta -= 1,
                '-' => delta += 1,
                ' ' => {
                    assert_eq!(
                        line.old_lineno().unwrap() as usize,
                        add_delta(line.new_lineno().unwrap() as usize, delta)
                    );
                    unchanged.push((
                        (line.new_lineno().unwrap() - 1) as usize,
                        (line.old_lineno().unwrap() - 1) as usize,
                    ));
                }
                _ => (),
            };
        }
    }

    let linecount = count_lines(blob);
    for i in latest_line..linecount {
        unchanged.push((i, add_delta(i, delta)));
    }
    Ok(unchanged)
}

fn blame_for_path(
    file_movement: &HashMap<Oid, HashMap<Oid, PathBuf>>,
    git_repo: &git2::Repository,
    commit: &git2::Commit,
    blame_repo: &git2::Repository,
    blame_parents: &[git2::Commit],
    path: &Path,
) -> Result<String, git2::Error> {
    let blob = commit
        .tree()?
        .get_path(path)?
        .to_object(git_repo)?
        .peel_to_blob()?;
    let linecount = count_lines(&blob);
    // TODO: drop this author field entirely, I don't think it gets consumed by anything
    let commit_id = commit.id();
    let author = commit
        .author()
        .name()
        .map(|n| n.nfkd().filter(|c| c.is_ascii()).collect::<String>())
        .unwrap_or_default();
    let mut blame = Vec::with_capacity(linecount);
    for line in 1..=linecount {
        blame.push(format!("{}:%:{}:{}", commit_id, line, author));
    }

    for (parent, blame_parent) in commit.parents().zip(blame_parents.iter()).rev() {
        let parent_path = file_movement
            .get(&parent.id())
            .and_then(|m| m.get(&blob.id()))
            .map(|p| p.borrow())
            .unwrap_or(path);
        let parent_blob = match parent.tree()?.get_path(parent_path) {
            Ok(t) if t.kind() == Some(ObjectType::Blob) => t.to_object(git_repo)?.peel_to_blob()?,
            _ => {
                continue;
            }
        };
        let parent_blame_blob = match blame_parent.tree()?.get_path(parent_path) {
            Ok(entry) => entry.to_object(blame_repo)?.peel_to_blob()?,
            _ => continue,
        };
        let parent_blame = std::str::from_utf8(parent_blame_blob.content())
            .unwrap() // We only ever put ascii in the blame blob (for now)
            .lines()
            .collect::<Vec<&str>>();

        let path_unchanged = path == parent_path;
        for (lineno, parent_lineno) in unmodified_lines(&blob, &parent_blob)? {
            if path_unchanged {
                blame[lineno] = String::from(parent_blame[parent_lineno]);
                continue;
            }
            let mut pieces = parent_blame[parent_lineno].splitn(4, ':');
            let p_rev = pieces.next().unwrap();
            let mut p_fname = pieces.next().unwrap();
            let p_lineno = pieces.next().unwrap();
            let p_author = pieces.next().unwrap();
            if p_fname == "%" {
                p_fname = parent_path.to_str().unwrap();
            }
            blame[lineno] = format!("{}:{}:{}:{}", p_rev, p_fname, p_lineno, p_author);
        }
    }
    // Extra entry so the `join` call after adds a trailing newline
    blame.push(String::new());
    Ok(blame.join("\n"))
}

fn build_blame_tree(
    builder: &mut git2::TreeBuilder,
    file_movement: &HashMap<Oid, HashMap<Oid, PathBuf>>,
    git_repo: &git2::Repository,
    commit: &git2::Commit,
    blame_repo: &git2::Repository,
    blame_parents: &[git2::Commit],
    mut path: PathBuf,
) -> Result<(), git2::Error> {
    let tree_at_path = if path == PathBuf::new() {
        commit.tree()?
    } else {
        commit
            .tree()?
            .get_path(&path)?
            .to_object(git_repo)?
            .peel_to_tree()?
    };
    'outer: for entry in tree_at_path.iter() {
        path.push(entry.name().unwrap());
        for (i, parent) in commit.parents().enumerate() {
            if let Ok(parent_entry) = parent.tree()?.get_path(&path) {
                if parent_entry.id() == entry.id() {
                    // Item at `path` is the same in the tree for `commit` as in the tree
                    // for `parent`, so the blame must be the same too
                    let blame_parent_entry = blame_parents[i].tree()?.get_path(&path)?;
                    builder.insert(
                        entry.name().unwrap(),
                        blame_parent_entry.id(),
                        entry.filemode(),
                    )?;
                    path.pop();
                    continue 'outer;
                }
            }
        }

        match entry.kind() {
            Some(ObjectType::Blob) => {
                let blame_text = blame_for_path(
                    file_movement,
                    git_repo,
                    commit,
                    blame_repo,
                    blame_parents,
                    &path,
                )?;
                builder.insert(
                    entry.name().unwrap(),
                    blame_repo.blob(&blame_text.as_bytes())?,
                    entry.filemode(),
                )?;
            }
            Some(ObjectType::Commit) => {
                // This is a submodule, just treat it as an empty dir. We could
                // probably also skip over it entirely.
                let entry_builder = blame_repo.treebuilder(None)?;
                builder.insert(
                    entry.name().unwrap(),
                    entry_builder.write()?,
                    entry.filemode(),
                )?;
            }
            Some(ObjectType::Tree) => {
                let mut entry_builder = blame_repo.treebuilder(None)?;
                build_blame_tree(
                    &mut entry_builder,
                    file_movement,
                    git_repo,
                    commit,
                    blame_repo,
                    blame_parents,
                    path.clone(),
                )?;
                builder.insert(
                    entry.name().unwrap(),
                    entry_builder.write()?,
                    entry.filemode(),
                )?;
            }
            _ => {
                panic!(
                    "Unexpected entry kind {:?} found in tree for commit {:?} at path {:?}",
                    entry.kind(),
                    commit.id(),
                    path
                );
            }
        };

        path.pop();
    }

    Ok(())
}

fn main() {
    env_logger::init();

    let args: Vec<_> = env::args().collect();
    let git_repo = Repository::open(&args[1]).unwrap();
    let blame_repo = Repository::open(&args[2]).unwrap();
    let use_cinnabar = env::var("CINNABAR").map(|v| v == "1").unwrap_or(false);
    let mut hg_helper = if use_cinnabar {
        Some(start_cinnabar_helper(&git_repo))
    } else {
        None
    };
    let blame_ref = env::var("BLAME_REF").ok().unwrap_or("HEAD".to_string());

    info!("Reading existing blame map of ref {}...", blame_ref);
    let mut blame_map = if let Ok(oid) = blame_repo.refname_to_id(&blame_ref) {
        let (blame_map, _) = index_blame(&blame_repo, Some(oid));
        blame_map
    } else {
        HashMap::new()
    };

    let mut refobj = blame_repo
        .find_reference(&blame_ref)
        .ok()
        .and_then(|r| r.resolve().ok());

    let mut walk = git_repo.revwalk().unwrap();
    walk.set_sorting(Sort::TOPOLOGICAL | Sort::REVERSE).unwrap();
    walk.push(git_repo.refname_to_id(&blame_ref).unwrap())
        .unwrap();
    let revs_to_process = walk
        .map(|r| r.unwrap()) // walk produces Result<git2::Oid> so we unwrap to just the Oid
        .filter(|git_oid| !blame_map.contains_key(git_oid))
        .collect::<Vec<_>>();
    let rev_count = revs_to_process.len();

    info!("Transforming new commits...");

    let mut rev_done = 0;
    for git_oid in revs_to_process {
        rev_done += 1;

        let hg_rev = match hg_helper {
            Some(ref mut helper) => get_hg_rev(helper, &git_oid),
            None => None, // we don't support mapfiles any more.
        };

        info!(
            "Transforming {} (hg {:?}) progress {}/{}",
            git_oid, hg_rev, rev_done, rev_count
        );
        let commit = git_repo.find_commit(git_oid).unwrap();
        let blame_parents = commit
            .parent_ids()
            .map(|pid| blame_repo.find_commit(blame_map[&pid]).unwrap())
            .collect::<Vec<_>>();

        let mut file_movement = HashMap::new();
        if commit.parent_count() == 1 {
            let mut movement = HashMap::new();
            let mut diff = git_repo
                .diff_tree_to_tree(
                    Some(&commit.parent(0).unwrap().tree().unwrap()),
                    Some(&commit.tree().unwrap()),
                    None,
                )
                .unwrap();
            diff.find_similar(Some(
                DiffFindOptions::new().copies(true).rename_limit(1000000),
            ))
            .unwrap();
            for delta in diff.deltas() {
                if !delta.old_file().id().is_zero()
                    && !delta.new_file().id().is_zero()
                    && delta.old_file().path() != delta.new_file().path()
                {
                    movement.insert(
                        delta.new_file().id(),
                        delta.old_file().path().unwrap().to_path_buf(),
                    );
                }
            }
            file_movement.insert(commit.parent_id(0).unwrap(), movement);
        }

        let mut builder = blame_repo.treebuilder(None).unwrap();
        build_blame_tree(
            &mut builder,
            &file_movement,
            &git_repo,
            &commit,
            &blame_repo,
            &blame_parents,
            PathBuf::new(),
        )
        .unwrap();
        let tree_oid = builder.write().unwrap();

        let commit_msg = if let Some(hg_rev) = hg_rev {
            format!("git {}\nhg {}\n", git_oid, hg_rev)
        } else {
            format!("git {}\n", git_oid)
        };

        let commit_ref = if refobj.is_some() {
            None
        } else {
            // This should only happen on the first commit, if the blame_ref
            // doesn't exist yet in the destination repo
            Some(blame_ref.as_str())
        };
        let blame_oid = blame_repo
            .commit(
                commit_ref,
                &commit.author(),
                &commit.committer(),
                &commit_msg,
                &blame_repo.find_tree(tree_oid).unwrap(),
                &blame_parents.iter().collect::<Vec<_>>(),
            )
            .unwrap();

        if let Some(ref mut refobj) = refobj {
            *refobj = refobj.set_target(blame_oid, "").unwrap();
        } else {
            refobj = blame_repo
                .find_reference(&blame_ref)
                .ok()
                .and_then(|r| r.resolve().ok());
            assert!(refobj.is_some());
        }

        blame_map.insert(git_oid, blame_oid);
        info!("  -> {}", blame_oid);
    }

    if let Some(mut helper) = hg_helper {
        helper.kill().unwrap();
    }
}
<commit_msg>Bug 1627532 - Use cinnabar unless explicitly disabled.<commit_after>extern crate env_logger;
extern crate git2;
#[macro_use]
extern crate log;
extern crate tools;
extern crate unicode_normalization;

use std::borrow::Borrow;
use std::collections::HashMap;
use std::env;
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};

use git2::{DiffFindOptions, ObjectType, Oid, Patch, Repository, Sort};
use tools::config::index_blame;
use unicode_normalization::UnicodeNormalization;

fn get_hg_rev(helper: &mut Child, git_oid: &Oid) -> Option<String> {
    write!(helper.stdin.as_mut().unwrap(), "git2hg {}\n", git_oid).unwrap();
    let mut reader = BufReader::new(helper.stdout.as_mut().unwrap());
    loop {
        let mut result = String::new();
        reader.read_line(&mut result).unwrap();
        if result.starts_with("changeset ") {
            return result.split(' ').nth(1).map(str::trim).map(str::to_string);
        }
        if result.trim().chars().all(|c| c == '0') {
            return None;
        }
        // else loop again. cinnabar-helper emits a bunch of other info that we don't care about
    }
}

fn start_cinnabar_helper(git_repo: &Repository) -> Child {
    Command::new("git")
        .arg("cinnabar-helper")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .current_dir(git_repo.path())
        .spawn()
        .unwrap()
}

fn count_lines(blob: &git2::Blob) -> usize {
    let data = blob.content();
    if data.is_empty() {
        return 0;
    }
    let mut linecount = 0;
    for b in data {
        if *b == b'\n' {
            linecount += 1;
        }
    }
    if data[data.len() - 1] != b'\n' {
        linecount += 1;
    }
    linecount
}

fn unmodified_lines(
    blob: &git2::Blob,
    parent_blob: &git2::Blob,
) -> Result<Vec<(usize, usize)>, git2::Error> {
    let mut unchanged = Vec::new();

    let patch = Patch::from_blobs(parent_blob, None, blob, None, None)?;

    if patch.delta().flags().is_binary() {
        return Ok(unchanged);
    }

    fn add_delta(lineno: usize, delta: i32) -> usize {
        ((lineno as i32) + delta) as usize
    }

    let mut latest_line: usize = 0;
    let mut delta: i32 = 0;

    for hunk_index in 0..patch.num_hunks() {
        for line_index in 0..patch.num_lines_in_hunk(hunk_index)? {
            let line = patch.line_in_hunk(hunk_index, line_index)?;

            if let Some(lineno) = line.new_lineno() {
                let lineno = lineno as usize;
                for i in latest_line..lineno - 1 {
                    unchanged.push((i, add_delta(i, delta)));
                }
                latest_line = (lineno - 1) + 1;
            }

            match line.origin() {
                '+' => delta -= 1,
                '-' => delta += 1,
                ' ' => {
                    assert_eq!(
                        line.old_lineno().unwrap() as usize,
                        add_delta(line.new_lineno().unwrap() as usize, delta)
                    );
                    unchanged.push((
                        (line.new_lineno().unwrap() - 1) as usize,
                        (line.old_lineno().unwrap() - 1) as usize,
                    ));
                }
                _ => (),
            };
        }
    }

    let linecount = count_lines(blob);
    for i in latest_line..linecount {
        unchanged.push((i, add_delta(i, delta)));
    }
    Ok(unchanged)
}

fn blame_for_path(
    file_movement: &HashMap<Oid, HashMap<Oid, PathBuf>>,
    git_repo: &git2::Repository,
    commit: &git2::Commit,
    blame_repo: &git2::Repository,
    blame_parents: &[git2::Commit],
    path: &Path,
) -> Result<String, git2::Error> {
    let blob = commit
        .tree()?
        .get_path(path)?
        .to_object(git_repo)?
        .peel_to_blob()?;
    let linecount = count_lines(&blob);
    // TODO: drop this author field entirely, I don't think it gets consumed by anything
    let commit_id = commit.id();
    let author = commit
        .author()
        .name()
        .map(|n| n.nfkd().filter(|c| c.is_ascii()).collect::<String>())
        .unwrap_or_default();
    let mut blame = Vec::with_capacity(linecount);
    for line in 1..=linecount {
        blame.push(format!("{}:%:{}:{}", commit_id, line, author));
    }

    for (parent, blame_parent) in commit.parents().zip(blame_parents.iter()).rev() {
        let parent_path = file_movement
            .get(&parent.id())
            .and_then(|m| m.get(&blob.id()))
            .map(|p| p.borrow())
            .unwrap_or(path);
        let parent_blob = match parent.tree()?.get_path(parent_path) {
            Ok(t) if t.kind() == Some(ObjectType::Blob) => t.to_object(git_repo)?.peel_to_blob()?,
            _ => {
                continue;
            }
        };
        let parent_blame_blob = match blame_parent.tree()?.get_path(parent_path) {
            Ok(entry) => entry.to_object(blame_repo)?.peel_to_blob()?,
            _ => continue,
        };
        let parent_blame = std::str::from_utf8(parent_blame_blob.content())
            .unwrap() // We only ever put ascii in the blame blob (for now)
            .lines()
            .collect::<Vec<&str>>();

        let path_unchanged = path == parent_path;
        for (lineno, parent_lineno) in unmodified_lines(&blob, &parent_blob)? {
            if path_unchanged {
                blame[lineno] = String::from(parent_blame[parent_lineno]);
                continue;
            }
            let mut pieces = parent_blame[parent_lineno].splitn(4, ':');
            let p_rev = pieces.next().unwrap();
            let mut p_fname = pieces.next().unwrap();
            let p_lineno = pieces.next().unwrap();
            let p_author = pieces.next().unwrap();
            if p_fname == "%" {
                p_fname = parent_path.to_str().unwrap();
            }
            blame[lineno] = format!("{}:{}:{}:{}", p_rev, p_fname, p_lineno, p_author);
        }
    }
    // Extra entry so the `join` call after adds a trailing newline
    blame.push(String::new());
    Ok(blame.join("\n"))
}

fn build_blame_tree(
    builder: &mut git2::TreeBuilder,
    file_movement: &HashMap<Oid, HashMap<Oid, PathBuf>>,
    git_repo: &git2::Repository,
    commit: &git2::Commit,
    blame_repo: &git2::Repository,
    blame_parents: &[git2::Commit],
    mut path: PathBuf,
) -> Result<(), git2::Error> {
    let tree_at_path = if path == PathBuf::new() {
        commit.tree()?
    } else {
        commit
            .tree()?
            .get_path(&path)?
            .to_object(git_repo)?
            .peel_to_tree()?
    };
    'outer: for entry in tree_at_path.iter() {
        path.push(entry.name().unwrap());
        for (i, parent) in commit.parents().enumerate() {
            if let Ok(parent_entry) = parent.tree()?.get_path(&path) {
                if parent_entry.id() == entry.id() {
                    // Item at `path` is the same in the tree for `commit` as in the tree
                    // for `parent`, so the blame must be the same too
                    let blame_parent_entry = blame_parents[i].tree()?.get_path(&path)?;
                    builder.insert(
                        entry.name().unwrap(),
                        blame_parent_entry.id(),
                        entry.filemode(),
                    )?;
                    path.pop();
                    continue 'outer;
                }
            }
        }

        match entry.kind() {
            Some(ObjectType::Blob) => {
                let blame_text = blame_for_path(
                    file_movement,
                    git_repo,
                    commit,
                    blame_repo,
                    blame_parents,
                    &path,
                )?;
                builder.insert(
                    entry.name().unwrap(),
                    blame_repo.blob(&blame_text.as_bytes())?,
                    entry.filemode(),
                )?;
            }
            Some(ObjectType::Commit) => {
                // This is a submodule, just treat it as an empty dir. We could
                // probably also skip over it entirely.
                let entry_builder = blame_repo.treebuilder(None)?;
                builder.insert(
                    entry.name().unwrap(),
                    entry_builder.write()?,
                    entry.filemode(),
                )?;
            }
            Some(ObjectType::Tree) => {
                let mut entry_builder = blame_repo.treebuilder(None)?;
                build_blame_tree(
                    &mut entry_builder,
                    file_movement,
                    git_repo,
                    commit,
                    blame_repo,
                    blame_parents,
                    path.clone(),
                )?;
                builder.insert(
                    entry.name().unwrap(),
                    entry_builder.write()?,
                    entry.filemode(),
                )?;
            }
            _ => {
                panic!(
                    "Unexpected entry kind {:?} found in tree for commit {:?} at path {:?}",
                    entry.kind(),
                    commit.id(),
                    path
                );
            }
        };

        path.pop();
    }

    Ok(())
}

fn main() {
    env_logger::init();

    let args: Vec<_> = env::args().collect();
    let git_repo = Repository::open(&args[1]).unwrap();
    let blame_repo = Repository::open(&args[2]).unwrap();
    let use_cinnabar = env::var("CINNABAR").map_or(true, |v| v != "0");
    let mut hg_helper = if use_cinnabar {
        Some(start_cinnabar_helper(&git_repo))
    } else {
        None
    };
    let blame_ref = env::var("BLAME_REF").ok().unwrap_or("HEAD".to_string());

    info!("Reading existing blame map of ref {}...", blame_ref);
    let mut blame_map = if let Ok(oid) = blame_repo.refname_to_id(&blame_ref) {
        let (blame_map, _) = index_blame(&blame_repo, Some(oid));
        blame_map
    } else {
        HashMap::new()
    };

    let mut refobj = blame_repo
        .find_reference(&blame_ref)
        .ok()
        .and_then(|r| r.resolve().ok());

    let mut walk = git_repo.revwalk().unwrap();
    walk.set_sorting(Sort::TOPOLOGICAL | Sort::REVERSE).unwrap();
    walk.push(git_repo.refname_to_id(&blame_ref).unwrap())
        .unwrap();
    let revs_to_process = walk
        .map(|r| r.unwrap()) // walk produces Result<git2::Oid> so we unwrap to just the Oid
        .filter(|git_oid| !blame_map.contains_key(git_oid))
        .collect::<Vec<_>>();
    let rev_count = revs_to_process.len();

    info!("Transforming new commits...");

    let mut rev_done = 0;
    for git_oid in revs_to_process {
        rev_done += 1;

        let hg_rev = match hg_helper {
            Some(ref mut helper) => get_hg_rev(helper, &git_oid),
            None => None, // we don't support mapfiles any more.
        };

        info!(
            "Transforming {} (hg {:?}) progress {}/{}",
            git_oid, hg_rev, rev_done, rev_count
        );
        let commit = git_repo.find_commit(git_oid).unwrap();
        let blame_parents = commit
            .parent_ids()
            .map(|pid| blame_repo.find_commit(blame_map[&pid]).unwrap())
            .collect::<Vec<_>>();

        let mut file_movement = HashMap::new();
        if commit.parent_count() == 1 {
            let mut movement = HashMap::new();
            let mut diff = git_repo
                .diff_tree_to_tree(
                    Some(&commit.parent(0).unwrap().tree().unwrap()),
                    Some(&commit.tree().unwrap()),
                    None,
                )
                .unwrap();
            diff.find_similar(Some(
                DiffFindOptions::new().copies(true).rename_limit(1000000),
            ))
            .unwrap();
            for delta in diff.deltas() {
                if !delta.old_file().id().is_zero()
                    && !delta.new_file().id().is_zero()
                    && delta.old_file().path() != delta.new_file().path()
                {
                    movement.insert(
                        delta.new_file().id(),
                        delta.old_file().path().unwrap().to_path_buf(),
                    );
                }
            }
            file_movement.insert(commit.parent_id(0).unwrap(), movement);
        }

        let mut builder = blame_repo.treebuilder(None).unwrap();
        build_blame_tree(
            &mut builder,
            &file_movement,
            &git_repo,
            &commit,
            &blame_repo,
            &blame_parents,
            PathBuf::new(),
        )
        .unwrap();
        let tree_oid = builder.write().unwrap();

        let commit_msg = if let Some(hg_rev) = hg_rev {
            format!("git {}\nhg {}\n", git_oid, hg_rev)
        } else {
            format!("git {}\n", git_oid)
        };

        let commit_ref = if refobj.is_some() {
            None
        } else {
            // This should only happen on the first commit, if the blame_ref
            // doesn't exist yet in the destination repo
            Some(blame_ref.as_str())
        };
        let blame_oid = blame_repo
            .commit(
                commit_ref,
                &commit.author(),
                &commit.committer(),
                &commit_msg,
                &blame_repo.find_tree(tree_oid).unwrap(),
                &blame_parents.iter().collect::<Vec<_>>(),
            )
            .unwrap();

        if let Some(ref mut refobj) = refobj {
            *refobj = refobj.set_target(blame_oid, "").unwrap();
        } else {
            refobj = blame_repo
                .find_reference(&blame_ref)
                .ok()
                .and_then(|r| r.resolve().ok());
            assert!(refobj.is_some());
        }

        blame_map.insert(git_oid, blame_oid);
        info!("  -> {}", blame_oid);
    }

    if let Some(mut helper) = hg_helper {
        helper.kill().unwrap();
    }
}
<|endoftext|>


<commit_before>// Copyright 2021 Vector 35 Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use binaryninjacore_sys::*;

use std::result;

use crate::rc::*;
use crate::string::*;

pub type Result<R> = result::Result<R, ()>;

#[derive(PartialEq, Eq, Hash)]
pub struct BackgroundTask {
    pub(crate) handle: *mut BNBackgroundTask,
}

impl BackgroundTask {
    pub(crate) unsafe fn from_raw(handle: *mut BNBackgroundTask) -> Self {
        debug_assert!(!handle.is_null());

        Self { handle }
    }

    pub fn new<S: BnStrCompatible>(
        initial_text: S,
        can_cancel: bool
    ) -> Result<Ref<Self>> {
        let text = initial_text.as_bytes_with_nul();

        let handle = unsafe {
            BNBeginBackgroundTask(text.as_ref().as_ptr() as *mut _, can_cancel)
        };

        if handle.is_null() {
            return Err(());
        }

        unsafe { Ok(Ref::new(Self { handle })) }
    }

    pub fn can_cancel(&self) -> bool {
        unsafe { BNCanCancelBackgroundTask(self.handle) }
    }

    pub fn is_cancelled(&self) -> bool {
        unsafe { BNIsBackgroundTaskCancelled(self.handle) }
    }

    pub fn is_finished(&self) -> bool {
        unsafe { BNIsBackgroundTaskFinished(self.handle) }
    }

    pub fn get_progress_text(&self) -> BnString {
        unsafe { BnString::from_raw(BNGetBackgroundTaskProgressText(self.handle)) }
    }

    pub fn cancel(&mut self) {
        unsafe { BNCancelBackgroundTask(self.handle) }
    }

    pub fn finish(&self) {
        unsafe { BNFinishBackgroundTask(self.handle) }
    }

    pub fn set_progress_text<S: BnStrCompatible>(&self, text: S) {
        let progress_text = text.as_bytes_with_nul();

        unsafe {
            BNSetBackgroundTaskProgressText(
                self.handle,
                progress_text.as_ref().as_ptr() as *mut _
            )
        }
    }

    pub fn running_tasks() -> Array<BackgroundTask> {
        unsafe {
            let mut count = 0;
            let handles = BNGetRunningBackgroundTasks(&mut count);

            Array::new(handles, count, ())
        }
    }
}

unsafe impl RefCountable for BackgroundTask {
    unsafe fn inc_ref(handle: &Self) -> Ref<Self> {
        Ref::new(Self {
            handle: BNNewBackgroundTaskReference(handle.handle),
        })
    }

    unsafe fn dec_ref(handle: &Self) {
        BNFreeBackgroundTask(handle.handle);
    }
}

unsafe impl CoreOwnedArrayProvider for BackgroundTask {
    type Raw = *mut BNBackgroundTask;
    type Context = ();

    unsafe fn free(raw: *mut *mut BNBackgroundTask, count: usize, _context: &()) {
        BNFreeBackgroundTaskList(raw, count);
    }
}

unsafe impl<'a> CoreOwnedArrayWrapper<'a> for BackgroundTask {
    type Wrapped = Guard<'a, BackgroundTask>;

    unsafe fn wrap_raw(raw: &'a *mut BNBackgroundTask, context: &'a ()) -> Guard<'a, BackgroundTask> {
        Guard::new(BackgroundTask::from_raw(*raw), context)
    }
}

impl AsRef<BackgroundTask> for BackgroundTask {
    fn as_ref(&self) -> &Self {
        self
    }
}

impl ToOwned for BackgroundTask {
    type Owned = Ref<Self>;

    fn to_owned(&self) -> Self::Owned {
        unsafe { RefCountable::inc_ref(self) }
    }
}

unsafe impl Send for BackgroundTask {}
unsafe impl Sync for BackgroundTask {}<commit_msg>cargo fmt<commit_after>// Copyright 2021 Vector 35 Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use binaryninjacore_sys::*;

use std::result;

use crate::rc::*;
use crate::string::*;

pub type Result<R> = result::Result<R, ()>;

#[derive(PartialEq, Eq, Hash)]
pub struct BackgroundTask {
    pub(crate) handle: *mut BNBackgroundTask,
}

impl BackgroundTask {
    pub(crate) unsafe fn from_raw(handle: *mut BNBackgroundTask) -> Self {
        debug_assert!(!handle.is_null());

        Self { handle }
    }

    pub fn new<S: BnStrCompatible>(initial_text: S, can_cancel: bool) -> Result<Ref<Self>> {
        let text = initial_text.as_bytes_with_nul();

        let handle = unsafe { BNBeginBackgroundTask(text.as_ref().as_ptr() as *mut _, can_cancel) };

        if handle.is_null() {
            return Err(());
        }

        unsafe { Ok(Ref::new(Self { handle })) }
    }

    pub fn can_cancel(&self) -> bool {
        unsafe { BNCanCancelBackgroundTask(self.handle) }
    }

    pub fn is_cancelled(&self) -> bool {
        unsafe { BNIsBackgroundTaskCancelled(self.handle) }
    }

    pub fn is_finished(&self) -> bool {
        unsafe { BNIsBackgroundTaskFinished(self.handle) }
    }

    pub fn get_progress_text(&self) -> BnString {
        unsafe { BnString::from_raw(BNGetBackgroundTaskProgressText(self.handle)) }
    }

    pub fn cancel(&mut self) {
        unsafe { BNCancelBackgroundTask(self.handle) }
    }

    pub fn finish(&self) {
        unsafe { BNFinishBackgroundTask(self.handle) }
    }

    pub fn set_progress_text<S: BnStrCompatible>(&self, text: S) {
        let progress_text = text.as_bytes_with_nul();

        unsafe {
            BNSetBackgroundTaskProgressText(self.handle, progress_text.as_ref().as_ptr() as *mut _)
        }
    }

    pub fn running_tasks() -> Array<BackgroundTask> {
        unsafe {
            let mut count = 0;
            let handles = BNGetRunningBackgroundTasks(&mut count);

            Array::new(handles, count, ())
        }
    }
}

unsafe impl RefCountable for BackgroundTask {
    unsafe fn inc_ref(handle: &Self) -> Ref<Self> {
        Ref::new(Self {
            handle: BNNewBackgroundTaskReference(handle.handle),
        })
    }

    unsafe fn dec_ref(handle: &Self) {
        BNFreeBackgroundTask(handle.handle);
    }
}

unsafe impl CoreOwnedArrayProvider for BackgroundTask {
    type Raw = *mut BNBackgroundTask;
    type Context = ();

    unsafe fn free(raw: *mut *mut BNBackgroundTask, count: usize, _context: &()) {
        BNFreeBackgroundTaskList(raw, count);
    }
}

unsafe impl<'a> CoreOwnedArrayWrapper<'a> for BackgroundTask {
    type Wrapped = Guard<'a, BackgroundTask>;

    unsafe fn wrap_raw(
        raw: &'a *mut BNBackgroundTask,
        context: &'a (),
    ) -> Guard<'a, BackgroundTask> {
        Guard::new(BackgroundTask::from_raw(*raw), context)
    }
}

impl AsRef<BackgroundTask> for BackgroundTask {
    fn as_ref(&self) -> &Self {
        self
    }
}

impl ToOwned for BackgroundTask {
    type Owned = Ref<Self>;

    fn to_owned(&self) -> Self::Owned {
        unsafe { RefCountable::inc_ref(self) }
    }
}

unsafe impl Send for BackgroundTask {}
unsafe impl Sync for BackgroundTask {}
<|endoftext|>


<commit_before>//! Compiler plugin defining macros that create PHF data structures.
//!
//! # Example
//!
//! ```rust
//! #![feature(plugin, core)]
//! #![plugin(phf_macros)]
//!
//! extern crate phf;
//!
//! #[derive(Clone)]
//! pub enum Keyword {
//!     Loop,
//!     Continue,
//!     Break,
//!     Fn,
//!     Extern,
//! }
//!
//! static KEYWORDS: phf::Map<&'static str, Keyword> = phf_map! {
//!     "loop" => Keyword::Loop,
//!     "continue" => Keyword::Continue,
//!     "break" => Keyword::Break,
//!     "fn" => Keyword::Fn,
//!     "extern" => Keyword::Extern,
//! };
//!
//! pub fn parse_keyword(keyword: &str) -> Option<Keyword> {
//!     KEYWORDS.get(keyword).cloned()
//! }
//! # fn main() {}
//! ```
#![doc(html_root_url="http://sfackler.github.io/rust-phf/doc/v0.7.4")]
#![feature(plugin_registrar, quote, rustc_private)]

extern crate syntax;
#[cfg(feature = "stats")]
extern crate time;
extern crate rustc;
extern crate phf_shared;
extern crate phf_generator;

use std::collections::HashMap;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use syntax::ast::{self, TokenTree, LitStr, LitBinary, LitByte, LitChar, Expr, ExprLit, ExprVec};
use syntax::codemap::{Span, Spanned};
use syntax::ext::base::{DummyResult,
                        ExtCtxt,
                        MacResult};
use syntax::fold::Folder;
use syntax::parse;
use syntax::parse::token::{InternedString, Comma, Eof, FatArrow};
use syntax::print::pprust;
use rustc::plugin::Registry;
use phf_generator::HashState;
use std::env;

use util::{Entry, Key};
use util::{create_map, create_set, create_ordered_map, create_ordered_set};

pub mod util;
mod macros;

#[plugin_registrar]
#[doc(hidden)]
pub fn macro_registrar(reg: &mut Registry) {
    reg.register_macro("phf_map", expand_phf_map);
    reg.register_macro("phf_set", expand_phf_set);
    reg.register_macro("phf_ordered_map", expand_phf_ordered_map);
    reg.register_macro("phf_ordered_set", expand_phf_ordered_set);
}

fn generate_hash(cx: &mut ExtCtxt, sp: Span, entries: &[Entry]) -> HashState {
    #[cfg(feature = "stats")]
    use time::precise_time_s;
    #[cfg(not(feature = "stats"))]
    fn precise_time_s() -> f64 { 0. }

    let start = precise_time_s();
    let state = phf_generator::generate_hash(entries);
    let time = precise_time_s() - start;

    if cfg!(feature = "stats") && env::var_os("PHF_STATS").is_some() {
        cx.span_note(sp, &format!("PHF generation took {} seconds", time));
    }

    state
}

fn expand_phf_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult+'static> {
    let entries = match parse_map(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, &*entries) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, &*entries);

    create_map(cx, sp, entries, state)
}

fn expand_phf_set(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult+'static> {
    let entries = match parse_set(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, &*entries) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, &*entries);

    create_set(cx, sp, entries, state)
}

fn expand_phf_ordered_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult+'static> {
    let entries = match parse_map(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp),
    };

    if has_duplicates(cx, sp, &*entries) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, &*entries);

    create_ordered_map(cx, sp, entries, state)
}

fn expand_phf_ordered_set(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult+'static> {
    let entries = match parse_set(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, &*entries) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, &*entries);

    create_ordered_set(cx, sp, entries, state)
}

fn parse_map(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Option<Vec<Entry>> {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(), tts.to_vec());
    let mut entries = Vec::new();

    let mut bad = false;
    while parser.token != Eof {
        let key = cx.expander().fold_expr(parser.parse_expr());
        let key_contents = parse_key(cx, &*key).unwrap_or_else(|| {
            bad = true;
            Key::Str(InternedString::new(""))
        });

        if !parser.eat(&FatArrow).ok().unwrap() {
            cx.span_err(parser.span, "expected `=>`");
            return None;
        }

        let value = parser.parse_expr();

        entries.push(Entry {
            key_contents: key_contents,
            key: key,
            value: value
        });

        if !parser.eat(&Comma).ok().unwrap() && parser.token != Eof {
            cx.span_err(parser.span, "expected `,`");
            return None;
        }
    }

    if bad {
        return None;
    }

    Some(entries)
}

fn parse_set(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Option<Vec<Entry>> {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(), tts.to_vec());
    let mut entries = Vec::new();
    let value = quote_expr!(&*cx, ());

    let mut bad = false;
    while parser.token != Eof {
        let key = cx.expander().fold_expr(parser.parse_expr());
        let key_contents = parse_key(cx, &*key).unwrap_or_else(|| {
            bad = true;
            Key::Str(InternedString::new(""))
        });

        entries.push(Entry {
            key_contents: key_contents,
            key: key,
            value: value.clone(),
        });

        if !parser.eat(&Comma).ok().unwrap() && parser.token != Eof {
            cx.span_err(parser.span, "expected `,`");
            return None;
        }
    }

    if bad {
        return None;
    }

    Some(entries)
}

fn parse_key(cx: &mut ExtCtxt, e: &Expr) -> Option<Key> {
    match e.node {
        ExprLit(ref lit) => {
            match lit.node {
                ast::LitStr(ref s, _) => Some(Key::Str(s.clone())),
                ast::LitBinary(ref b) => Some(Key::Binary(b.clone())),
                ast::LitByte(b) => Some(Key::U8(b)),
                ast::LitChar(c) => Some(Key::Char(c)),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI8, ast::Plus)) => Some(Key::I8(i as i8)),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI8, ast::Minus)) => Some(Key::I8(-(i as i8))),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI16, ast::Plus)) => Some(Key::I16(i as i16)),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI16, ast::Minus)) => Some(Key::I16(-(i as i16))),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI32, ast::Plus)) => Some(Key::I32(i as i32)),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI32, ast::Minus)) => Some(Key::I32(-(i as i32))),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI64, ast::Plus)) => Some(Key::I64(i as i64)),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI64, ast::Minus)) => Some(Key::I64(-(i as i64))),
                ast::LitInt(i, ast::UnsignedIntLit(ast::TyU8)) => Some(Key::U8(i as u8)),
                ast::LitInt(i, ast::UnsignedIntLit(ast::TyU16)) => Some(Key::U16(i as u16)),
                ast::LitInt(i, ast::UnsignedIntLit(ast::TyU32)) => Some(Key::U32(i as u32)),
                ast::LitInt(i, ast::UnsignedIntLit(ast::TyU64)) => Some(Key::U64(i as u64)),
                ast::LitBool(b) => Some(Key::Bool(b)),
                _ => {
                    cx.span_err(e.span, "unsupported literal type");
                    None
                }
            }
        }
        ExprVec(ref v) => {
            let bytes: Vec<Option<u8>> = v.iter().map(|expr|
                if let ExprLit(ref p) = expr.node {
                    match **p {
                        Spanned {node: ast::LitInt(val, ast::UnsignedIntLit(ast::UintTy::TyU8)), ..} if val < 256 => Some(val as u8),
                        Spanned {node: ast::LitInt(val, ast::UnsuffixedIntLit(ast::Plus)), ..} if val < 256 => Some(val as u8),
                        _ => None,
                    }
                } else {
                    None
            }).collect();
            if bytes.iter().all(|x| x.is_some()) {
                Some(Key::Binary(std::rc::Rc::new(bytes.iter().map(|x| x.unwrap()).collect())))
            } else {
                cx.span_err(e.span, "not all elements of an expected u8 array literal were u8 literals");
                None
            }
        }
        _ => {
            cx.span_err(e.span, "expected a literal");
            None
        }
    }
}

fn has_duplicates(cx: &mut ExtCtxt, sp: Span, entries: &[Entry]) -> bool {
    let mut dups = false;
    let mut strings = HashMap::new();
    for entry in entries.iter() {
        let &mut (ref mut spans, _) = match strings.entry(&entry.key_contents) {
            Occupied(e) => e.into_mut(),
            Vacant(e) => e.insert((vec![], &entry.key)),
        };
        spans.push(entry.key.span);
    }

    for &(ref spans, key) in strings.values() {
        if spans.len() == 1 {
            continue;
        }

        dups = true;
        cx.span_err(sp, &*format!("duplicate key {}", pprust::expr_to_string(&**key)));
        for span in spans.iter() {
            cx.span_note(*span, "one occurrence here");
        }
    }

    dups
}
<commit_msg>Fix: Update code for changes in Rust<commit_after>//! Compiler plugin defining macros that create PHF data structures.
//!
//! # Example
//!
//! ```rust
//! #![feature(plugin, core)]
//! #![plugin(phf_macros)]
//!
//! extern crate phf;
//!
//! #[derive(Clone)]
//! pub enum Keyword {
//!     Loop,
//!     Continue,
//!     Break,
//!     Fn,
//!     Extern,
//! }
//!
//! static KEYWORDS: phf::Map<&'static str, Keyword> = phf_map! {
//!     "loop" => Keyword::Loop,
//!     "continue" => Keyword::Continue,
//!     "break" => Keyword::Break,
//!     "fn" => Keyword::Fn,
//!     "extern" => Keyword::Extern,
//! };
//!
//! pub fn parse_keyword(keyword: &str) -> Option<Keyword> {
//!     KEYWORDS.get(keyword).cloned()
//! }
//! # fn main() {}
//! ```
#![doc(html_root_url="http://sfackler.github.io/rust-phf/doc/v0.7.4")]
#![feature(plugin_registrar, quote, rustc_private)]

extern crate syntax;
#[cfg(feature = "stats")]
extern crate time;
extern crate rustc;
extern crate phf_shared;
extern crate phf_generator;

use std::collections::HashMap;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use syntax::ast::{self, TokenTree, LitStr, LitByteStr, LitByte, LitChar, Expr, ExprLit, ExprVec};
use syntax::codemap::{Span, Spanned};
use syntax::ext::base::{DummyResult,
                        ExtCtxt,
                        MacResult};
use syntax::fold::Folder;
use syntax::parse;
use syntax::parse::token::{InternedString, Comma, Eof, FatArrow};
use syntax::print::pprust;
use rustc::plugin::Registry;
use phf_generator::HashState;
use std::env;

use util::{Entry, Key};
use util::{create_map, create_set, create_ordered_map, create_ordered_set};

pub mod util;
mod macros;

#[plugin_registrar]
#[doc(hidden)]
pub fn macro_registrar(reg: &mut Registry) {
    reg.register_macro("phf_map", expand_phf_map);
    reg.register_macro("phf_set", expand_phf_set);
    reg.register_macro("phf_ordered_map", expand_phf_ordered_map);
    reg.register_macro("phf_ordered_set", expand_phf_ordered_set);
}

fn generate_hash(cx: &mut ExtCtxt, sp: Span, entries: &[Entry]) -> HashState {
    #[cfg(feature = "stats")]
    use time::precise_time_s;
    #[cfg(not(feature = "stats"))]
    fn precise_time_s() -> f64 { 0. }

    let start = precise_time_s();
    let state = phf_generator::generate_hash(entries);
    let time = precise_time_s() - start;

    if cfg!(feature = "stats") && env::var_os("PHF_STATS").is_some() {
        cx.span_note(sp, &format!("PHF generation took {} seconds", time));
    }

    state
}

fn expand_phf_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult+'static> {
    let entries = match parse_map(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, &*entries) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, &*entries);

    create_map(cx, sp, entries, state)
}

fn expand_phf_set(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult+'static> {
    let entries = match parse_set(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, &*entries) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, &*entries);

    create_set(cx, sp, entries, state)
}

fn expand_phf_ordered_map(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult+'static> {
    let entries = match parse_map(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp),
    };

    if has_duplicates(cx, sp, &*entries) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, &*entries);

    create_ordered_map(cx, sp, entries, state)
}

fn expand_phf_ordered_set(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult+'static> {
    let entries = match parse_set(cx, tts) {
        Some(entries) => entries,
        None => return DummyResult::expr(sp)
    };

    if has_duplicates(cx, sp, &*entries) {
        return DummyResult::expr(sp);
    }

    let state = generate_hash(cx, sp, &*entries);

    create_ordered_set(cx, sp, entries, state)
}

fn parse_map(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Option<Vec<Entry>> {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(), tts.to_vec());
    let mut entries = Vec::new();

    let mut bad = false;
    while parser.token != Eof {
        let key = cx.expander().fold_expr(parser.parse_expr());
        let key_contents = parse_key(cx, &*key).unwrap_or_else(|| {
            bad = true;
            Key::Str(InternedString::new(""))
        });

        if !parser.eat(&FatArrow).ok().unwrap() {
            cx.span_err(parser.span, "expected `=>`");
            return None;
        }

        let value = parser.parse_expr();

        entries.push(Entry {
            key_contents: key_contents,
            key: key,
            value: value
        });

        if !parser.eat(&Comma).ok().unwrap() && parser.token != Eof {
            cx.span_err(parser.span, "expected `,`");
            return None;
        }
    }

    if bad {
        return None;
    }

    Some(entries)
}

fn parse_set(cx: &mut ExtCtxt, tts: &[TokenTree]) -> Option<Vec<Entry>> {
    let mut parser = parse::new_parser_from_tts(cx.parse_sess(), cx.cfg(), tts.to_vec());
    let mut entries = Vec::new();
    let value = quote_expr!(&*cx, ());

    let mut bad = false;
    while parser.token != Eof {
        let key = cx.expander().fold_expr(parser.parse_expr());
        let key_contents = parse_key(cx, &*key).unwrap_or_else(|| {
            bad = true;
            Key::Str(InternedString::new(""))
        });

        entries.push(Entry {
            key_contents: key_contents,
            key: key,
            value: value.clone(),
        });

        if !parser.eat(&Comma).ok().unwrap() && parser.token != Eof {
            cx.span_err(parser.span, "expected `,`");
            return None;
        }
    }

    if bad {
        return None;
    }

    Some(entries)
}

fn parse_key(cx: &mut ExtCtxt, e: &Expr) -> Option<Key> {
    match e.node {
        ExprLit(ref lit) => {
            match lit.node {
                ast::LitStr(ref s, _) => Some(Key::Str(s.clone())),
                ast::LitByteStr(ref b) => Some(Key::Binary(b.clone())),
                ast::LitByte(b) => Some(Key::U8(b)),
                ast::LitChar(c) => Some(Key::Char(c)),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI8, ast::Plus)) => Some(Key::I8(i as i8)),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI8, ast::Minus)) => Some(Key::I8(-(i as i8))),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI16, ast::Plus)) => Some(Key::I16(i as i16)),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI16, ast::Minus)) => Some(Key::I16(-(i as i16))),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI32, ast::Plus)) => Some(Key::I32(i as i32)),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI32, ast::Minus)) => Some(Key::I32(-(i as i32))),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI64, ast::Plus)) => Some(Key::I64(i as i64)),
                ast::LitInt(i, ast::SignedIntLit(ast::TyI64, ast::Minus)) => Some(Key::I64(-(i as i64))),
                ast::LitInt(i, ast::UnsignedIntLit(ast::TyU8)) => Some(Key::U8(i as u8)),
                ast::LitInt(i, ast::UnsignedIntLit(ast::TyU16)) => Some(Key::U16(i as u16)),
                ast::LitInt(i, ast::UnsignedIntLit(ast::TyU32)) => Some(Key::U32(i as u32)),
                ast::LitInt(i, ast::UnsignedIntLit(ast::TyU64)) => Some(Key::U64(i as u64)),
                ast::LitBool(b) => Some(Key::Bool(b)),
                _ => {
                    cx.span_err(e.span, "unsupported literal type");
                    None
                }
            }
        }
        ExprVec(ref v) => {
            let bytes: Vec<Option<u8>> = v.iter().map(|expr|
                if let ExprLit(ref p) = expr.node {
                    match **p {
                        Spanned {node: ast::LitInt(val, ast::UnsignedIntLit(ast::UintTy::TyU8)), ..} if val < 256 => Some(val as u8),
                        Spanned {node: ast::LitInt(val, ast::UnsuffixedIntLit(ast::Plus)), ..} if val < 256 => Some(val as u8),
                        _ => None,
                    }
                } else {
                    None
            }).collect();
            if bytes.iter().all(|x| x.is_some()) {
                Some(Key::Binary(std::rc::Rc::new(bytes.iter().map(|x| x.unwrap()).collect())))
            } else {
                cx.span_err(e.span, "not all elements of an expected u8 array literal were u8 literals");
                None
            }
        }
        _ => {
            cx.span_err(e.span, "expected a literal");
            None
        }
    }
}

fn has_duplicates(cx: &mut ExtCtxt, sp: Span, entries: &[Entry]) -> bool {
    let mut dups = false;
    let mut strings = HashMap::new();
    for entry in entries.iter() {
        let &mut (ref mut spans, _) = match strings.entry(&entry.key_contents) {
            Occupied(e) => e.into_mut(),
            Vacant(e) => e.insert((vec![], &entry.key)),
        };
        spans.push(entry.key.span);
    }

    for &(ref spans, key) in strings.values() {
        if spans.len() == 1 {
            continue;
        }

        dups = true;
        cx.span_err(sp, &*format!("duplicate key {}", pprust::expr_to_string(&**key)));
        for span in spans.iter() {
            cx.span_note(*span, "one occurrence here");
        }
    }

    dups
}
<|endoftext|>