<commit_before>// Copyright 2018 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Mining plugin manager, using the cuckoo-miner crate to provide
//! a mining worker implementation
//!

use chrono::prelude::Utc;
use chrono::{Duration, MIN_DATE};
use rand::{thread_rng, Rng};
use std::net::{SocketAddr, ToSocketAddrs};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{mpsc, Arc};
use std::{cmp, io, str, thread, time};

use p2p;
use p2p::ChainAdapter;
use pool::DandelionConfig;
use util::LOGGER;

// DNS Seeds with contact email associated
const DNS_SEEDS: &'static [&'static str] = &[
	"t3.seed.grin-tech.org",    // igno.peverell@protonmail.com
	"seed.grin.lesceller.com",  // q.lesceller@gmail.com
	"t3.grin-seed.prokapi.com", // info@prokapi.com
];

pub fn connect_and_monitor(
	p2p_server: Arc<p2p::Server>,
	capabilities: p2p::Capabilities,
	dandelion_config: DandelionConfig,
	seed_list: Box<Fn() -> Vec<SocketAddr> + Send>,
	preferred_peers: Option<Vec<SocketAddr>>,
	stop: Arc<AtomicBool>,
) {
	let _ = thread::Builder::new()
		.name("seed".to_string())
		.spawn(move || {
			let peers = p2p_server.peers.clone();

			// open a channel with a listener that connects every peer address sent below
			// max peer count
			let (tx, rx) = mpsc::channel();

			// check seeds first
			connect_to_seeds_and_preferred_peers(
				peers.clone(),
				tx.clone(),
				seed_list,
				preferred_peers.clone(),
			);

			let mut prev = MIN_DATE.and_hms(0, 0, 0);
			let mut prev_ping = Utc::now();
			let mut start_attempt = 0;

			while !stop.load(Ordering::Relaxed) {
				// make several attempts to get peers as quick as possible
				// with exponential backoff
				if Utc::now() - prev > Duration::seconds(cmp::min(20, 1 << start_attempt)) {
					// try to connect to any address sent to the channel
					listen_for_addrs(peers.clone(), p2p_server.clone(), capabilities, &rx);

					// monitor additional peers if we need to add more
					monitor_peers(
						peers.clone(),
						p2p_server.config.clone(),
						capabilities,
						tx.clone(),
						preferred_peers.clone(),
					);

					update_dandelion_relay(peers.clone(), dandelion_config.clone());

					prev = Utc::now();
					start_attempt = cmp::min(6, start_attempt + 1);
				}

				// Ping connected peers on every 10s to monitor peers.
				if Utc::now() - prev_ping > Duration::seconds(10) {
					let total_diff = peers.total_difficulty();
					let total_height = peers.total_height();
					peers.check_all(total_diff, total_height);
					prev_ping = Utc::now();
				}

				thread::sleep(time::Duration::from_secs(1));
			}
		});
}

fn monitor_peers(
	peers: Arc<p2p::Peers>,
	config: p2p::P2PConfig,
	capabilities: p2p::Capabilities,
	tx: mpsc::Sender<SocketAddr>,
	preferred_peers_list: Option<Vec<SocketAddr>>,
) {
	// regularly check if we need to acquire more peers  and if so, gets
	// them from db
	let total_count = peers.all_peers().len();
	let mut healthy_count = 0;
	let mut banned_count = 0;
	let mut defuncts = vec![];
	for x in peers.all_peers() {
		match x.flags {
			p2p::State::Banned => {
				let interval = Utc::now().timestamp() - x.last_banned;
				// Unban peer
				if interval >= config.ban_window() {
					peers.unban_peer(&x.addr);
					debug!(
						LOGGER,
						"monitor_peers: unbanned {} after {} seconds", x.addr, interval
					);
				} else {
					banned_count += 1;
				}
			}
			p2p::State::Healthy => healthy_count += 1,
			p2p::State::Defunct => defuncts.push(x),
		}
	}

	debug!(
		LOGGER,
		"monitor_peers: on {}:{}, {} connected ({} most_work). \
		 all {} = {} healthy + {} banned + {} defunct",
		config.host,
		config.port,
		peers.peer_count(),
		peers.most_work_peers().len(),
		total_count,
		healthy_count,
		banned_count,
		defuncts.len(),
	);

	// maintenance step first, clean up p2p server peers
	peers.clean_peers(config.peer_max_count() as usize);

	// not enough peers, getting more from db
	if peers.peer_count() >= config.peer_min_preferred_count() {
		return;
	}

	// loop over connected peers
	// ask them for their list of peers
	let mut connected_peers: Vec<SocketAddr> = vec![];
	for p in peers.connected_peers() {
		debug!(
			LOGGER,
			"monitor_peers: {}:{} ask {} for more peers", config.host, config.port, p.info.addr,
		);
		let _ = p.send_peer_request(capabilities);
		connected_peers.push(p.info.addr)
	}

	// Attempt to connect to preferred peers if there is some
	match preferred_peers_list {
		Some(preferred_peers) => {
			for mut p in preferred_peers {
				if !connected_peers.is_empty() {
					if !connected_peers.contains(&p) {
						tx.send(p).unwrap();
					}
				} else {
					tx.send(p).unwrap();
				}
			}
		}
		None => debug!(LOGGER, "monitor_peers: no preferred peers"),
	}

	// take a random defunct peer and mark it healthy: over a long period any
	// peer will see another as defunct eventually, gives us a chance to retry
	if defuncts.len() > 0 {
		thread_rng().shuffle(&mut defuncts);
		let _ = peers.update_state(defuncts[0].addr, p2p::State::Healthy);
	}

	// find some peers from our db
	// and queue them up for a connection attempt
	let new_peers = peers.find_peers(
		p2p::State::Healthy,
		p2p::Capabilities::UNKNOWN,
		config.peer_max_count() as usize,
	);
	for p in new_peers.iter().filter(|p| !peers.is_known(&p.addr)) {
		debug!(
			LOGGER,
			"monitor_peers: on {}:{}, queue to soon try {}", config.host, config.port, p.addr,
		);
		tx.send(p.addr).unwrap();
	}
}

fn update_dandelion_relay(peers: Arc<p2p::Peers>, dandelion_config: DandelionConfig) {
	// Dandelion Relay Updater
	let dandelion_relay = peers.get_dandelion_relay();
	if dandelion_relay.is_empty() {
		debug!(LOGGER, "monitor_peers: no dandelion relay updating");
		peers.update_dandelion_relay();
	} else {
		for last_added in dandelion_relay.keys() {
			let dandelion_interval = Utc::now().timestamp() - last_added;
			if dandelion_interval >= dandelion_config.relay_secs.unwrap() as i64 {
				debug!(LOGGER, "monitor_peers: updating expired dandelion relay");
				peers.update_dandelion_relay();
			}
		}
	}
}

// Check if we have any pre-existing peer in db. If so, start with those,
// otherwise use the seeds provided.
fn connect_to_seeds_and_preferred_peers(
	peers: Arc<p2p::Peers>,
	tx: mpsc::Sender<SocketAddr>,
	seed_list: Box<Fn() -> Vec<SocketAddr>>,
	peers_preferred_list: Option<Vec<SocketAddr>>,
) {
	// check if we have some peers in db
	let peers = peers.find_peers(p2p::State::Healthy, p2p::Capabilities::FULL_HIST, 100);

	// if so, get their addresses, otherwise use our seeds
	let mut peer_addrs = if peers.len() > 3 {
		peers.iter().map(|p| p.addr).collect::<Vec<_>>()
	} else {
		seed_list()
	};

	// If we have preferred peers add them to the connection
	match peers_preferred_list {
		Some(mut peers_preferred) => peer_addrs.append(&mut peers_preferred),
		None => debug!(LOGGER, "No preferred peers"),
	};

	if peer_addrs.len() == 0 {
		warn!(LOGGER, "No seeds were retrieved.");
	}

	// connect to this first set of addresses
	for addr in peer_addrs {
		tx.send(addr).unwrap();
	}
}

/// Regularly poll a channel receiver for new addresses and initiate a
/// connection if the max peer count isn't exceeded. A request for more
/// peers is also automatically sent after connection.
fn listen_for_addrs(
	peers: Arc<p2p::Peers>,
	p2p: Arc<p2p::Server>,
	capab: p2p::Capabilities,
	rx: &mpsc::Receiver<SocketAddr>,
) {
	if peers.peer_count() >= p2p.config.peer_max_count() {
		// clean the rx messages to avoid accumulating
		for _ in rx.try_iter() {}
		return;
	}

	for addr in rx.try_iter() {
		let peers_c = peers.clone();
		let p2p_c = p2p.clone();
		let _ = thread::Builder::new()
			.name("peer_connect".to_string())
			.spawn(move || {
				// connect and retry on fail, but for 3 times at most
				for _ in 0..3 {
					match p2p_c.connect(&addr) {
						Ok(p) => {
							let _ = p.send_peer_request(capab);
							let _ = peers_c.update_state(addr, p2p::State::Healthy);
							break;
						}
						Err(e) => {
							let _ = peers_c.update_state(addr, p2p::State::Defunct);

							// don't retry if connection refused or PeerWithSelf
							match e {
								p2p::Error::Connection(io_err) => {
									if io::ErrorKind::ConnectionRefused == io_err.kind() {
										break;
									}
								}
								p2p::Error::PeerWithSelf => break,
								_ => (), // allow to retry on any other error
							}
						}
					}

					thread::sleep(time::Duration::from_secs(1));
				}
			});
	}
}

pub fn dns_seeds() -> Box<Fn() -> Vec<SocketAddr> + Send> {
	Box::new(|| {
		let mut addresses: Vec<SocketAddr> = vec![];
		for dns_seed in DNS_SEEDS {
			let temp_addresses = addresses.clone();
			debug!(LOGGER, "Retrieving seed nodes from dns {}", dns_seed);
			match (dns_seed.to_owned(), 0).to_socket_addrs() {
				Ok(addrs) => addresses.append(
					&mut (addrs
						.map(|mut addr| {
							addr.set_port(13414);
							addr
						}).filter(|addr| !temp_addresses.contains(addr))
						.collect()),
				),
				Err(e) => debug!(
					LOGGER,
					"Failed to resolve seed {:?} got error {:?}", dns_seed, e
				),
			}
		}
		debug!(LOGGER, "Retrieved seed addresses: {:?}", addresses);
		addresses
	})
}

/// Convenience function when the seed list is immediately known. Mostly used
/// for tests.
pub fn predefined_seeds(addrs_str: Vec<String>) -> Box<Fn() -> Vec<SocketAddr> + Send> {
	Box::new(move || {
		addrs_str
			.iter()
			.map(|s| s.parse().unwrap())
			.collect::<Vec<_>>()
	})
}

/// Convenience function when the seed list is immediately known. Mostly used
/// for tests.
pub fn preferred_peers(addrs_str: Vec<String>) -> Option<Vec<SocketAddr>> {
	if addrs_str.is_empty() {
		None
	} else {
		Some(
			addrs_str
				.iter()
				.map(|s| s.parse().unwrap())
				.collect::<Vec<_>>(),
		)
	}
}
<commit_msg>Updated seed<commit_after>// Copyright 2018 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Mining plugin manager, using the cuckoo-miner crate to provide
//! a mining worker implementation
//!

use chrono::prelude::Utc;
use chrono::{Duration, MIN_DATE};
use rand::{thread_rng, Rng};
use std::net::{SocketAddr, ToSocketAddrs};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{mpsc, Arc};
use std::{cmp, io, str, thread, time};

use p2p;
use p2p::ChainAdapter;
use pool::DandelionConfig;
use util::LOGGER;

// DNS Seeds with contact email associated
const DNS_SEEDS: &'static [&'static str] = &[
	"t4.seed.grin-tech.org",    // igno.peverell@protonmail.com
];

pub fn connect_and_monitor(
	p2p_server: Arc<p2p::Server>,
	capabilities: p2p::Capabilities,
	dandelion_config: DandelionConfig,
	seed_list: Box<Fn() -> Vec<SocketAddr> + Send>,
	preferred_peers: Option<Vec<SocketAddr>>,
	stop: Arc<AtomicBool>,
) {
	let _ = thread::Builder::new()
		.name("seed".to_string())
		.spawn(move || {
			let peers = p2p_server.peers.clone();

			// open a channel with a listener that connects every peer address sent below
			// max peer count
			let (tx, rx) = mpsc::channel();

			// check seeds first
			connect_to_seeds_and_preferred_peers(
				peers.clone(),
				tx.clone(),
				seed_list,
				preferred_peers.clone(),
			);

			let mut prev = MIN_DATE.and_hms(0, 0, 0);
			let mut prev_ping = Utc::now();
			let mut start_attempt = 0;

			while !stop.load(Ordering::Relaxed) {
				// make several attempts to get peers as quick as possible
				// with exponential backoff
				if Utc::now() - prev > Duration::seconds(cmp::min(20, 1 << start_attempt)) {
					// try to connect to any address sent to the channel
					listen_for_addrs(peers.clone(), p2p_server.clone(), capabilities, &rx);

					// monitor additional peers if we need to add more
					monitor_peers(
						peers.clone(),
						p2p_server.config.clone(),
						capabilities,
						tx.clone(),
						preferred_peers.clone(),
					);

					update_dandelion_relay(peers.clone(), dandelion_config.clone());

					prev = Utc::now();
					start_attempt = cmp::min(6, start_attempt + 1);
				}

				// Ping connected peers on every 10s to monitor peers.
				if Utc::now() - prev_ping > Duration::seconds(10) {
					let total_diff = peers.total_difficulty();
					let total_height = peers.total_height();
					peers.check_all(total_diff, total_height);
					prev_ping = Utc::now();
				}

				thread::sleep(time::Duration::from_secs(1));
			}
		});
}

fn monitor_peers(
	peers: Arc<p2p::Peers>,
	config: p2p::P2PConfig,
	capabilities: p2p::Capabilities,
	tx: mpsc::Sender<SocketAddr>,
	preferred_peers_list: Option<Vec<SocketAddr>>,
) {
	// regularly check if we need to acquire more peers  and if so, gets
	// them from db
	let total_count = peers.all_peers().len();
	let mut healthy_count = 0;
	let mut banned_count = 0;
	let mut defuncts = vec![];
	for x in peers.all_peers() {
		match x.flags {
			p2p::State::Banned => {
				let interval = Utc::now().timestamp() - x.last_banned;
				// Unban peer
				if interval >= config.ban_window() {
					peers.unban_peer(&x.addr);
					debug!(
						LOGGER,
						"monitor_peers: unbanned {} after {} seconds", x.addr, interval
					);
				} else {
					banned_count += 1;
				}
			}
			p2p::State::Healthy => healthy_count += 1,
			p2p::State::Defunct => defuncts.push(x),
		}
	}

	debug!(
		LOGGER,
		"monitor_peers: on {}:{}, {} connected ({} most_work). \
		 all {} = {} healthy + {} banned + {} defunct",
		config.host,
		config.port,
		peers.peer_count(),
		peers.most_work_peers().len(),
		total_count,
		healthy_count,
		banned_count,
		defuncts.len(),
	);

	// maintenance step first, clean up p2p server peers
	peers.clean_peers(config.peer_max_count() as usize);

	// not enough peers, getting more from db
	if peers.peer_count() >= config.peer_min_preferred_count() {
		return;
	}

	// loop over connected peers
	// ask them for their list of peers
	let mut connected_peers: Vec<SocketAddr> = vec![];
	for p in peers.connected_peers() {
		debug!(
			LOGGER,
			"monitor_peers: {}:{} ask {} for more peers", config.host, config.port, p.info.addr,
		);
		let _ = p.send_peer_request(capabilities);
		connected_peers.push(p.info.addr)
	}

	// Attempt to connect to preferred peers if there is some
	match preferred_peers_list {
		Some(preferred_peers) => {
			for mut p in preferred_peers {
				if !connected_peers.is_empty() {
					if !connected_peers.contains(&p) {
						tx.send(p).unwrap();
					}
				} else {
					tx.send(p).unwrap();
				}
			}
		}
		None => debug!(LOGGER, "monitor_peers: no preferred peers"),
	}

	// take a random defunct peer and mark it healthy: over a long period any
	// peer will see another as defunct eventually, gives us a chance to retry
	if defuncts.len() > 0 {
		thread_rng().shuffle(&mut defuncts);
		let _ = peers.update_state(defuncts[0].addr, p2p::State::Healthy);
	}

	// find some peers from our db
	// and queue them up for a connection attempt
	let new_peers = peers.find_peers(
		p2p::State::Healthy,
		p2p::Capabilities::UNKNOWN,
		config.peer_max_count() as usize,
	);
	for p in new_peers.iter().filter(|p| !peers.is_known(&p.addr)) {
		debug!(
			LOGGER,
			"monitor_peers: on {}:{}, queue to soon try {}", config.host, config.port, p.addr,
		);
		tx.send(p.addr).unwrap();
	}
}

fn update_dandelion_relay(peers: Arc<p2p::Peers>, dandelion_config: DandelionConfig) {
	// Dandelion Relay Updater
	let dandelion_relay = peers.get_dandelion_relay();
	if dandelion_relay.is_empty() {
		debug!(LOGGER, "monitor_peers: no dandelion relay updating");
		peers.update_dandelion_relay();
	} else {
		for last_added in dandelion_relay.keys() {
			let dandelion_interval = Utc::now().timestamp() - last_added;
			if dandelion_interval >= dandelion_config.relay_secs.unwrap() as i64 {
				debug!(LOGGER, "monitor_peers: updating expired dandelion relay");
				peers.update_dandelion_relay();
			}
		}
	}
}

// Check if we have any pre-existing peer in db. If so, start with those,
// otherwise use the seeds provided.
fn connect_to_seeds_and_preferred_peers(
	peers: Arc<p2p::Peers>,
	tx: mpsc::Sender<SocketAddr>,
	seed_list: Box<Fn() -> Vec<SocketAddr>>,
	peers_preferred_list: Option<Vec<SocketAddr>>,
) {
	// check if we have some peers in db
	let peers = peers.find_peers(p2p::State::Healthy, p2p::Capabilities::FULL_HIST, 100);

	// if so, get their addresses, otherwise use our seeds
	let mut peer_addrs = if peers.len() > 3 {
		peers.iter().map(|p| p.addr).collect::<Vec<_>>()
	} else {
		seed_list()
	};

	// If we have preferred peers add them to the connection
	match peers_preferred_list {
		Some(mut peers_preferred) => peer_addrs.append(&mut peers_preferred),
		None => debug!(LOGGER, "No preferred peers"),
	};

	if peer_addrs.len() == 0 {
		warn!(LOGGER, "No seeds were retrieved.");
	}

	// connect to this first set of addresses
	for addr in peer_addrs {
		tx.send(addr).unwrap();
	}
}

/// Regularly poll a channel receiver for new addresses and initiate a
/// connection if the max peer count isn't exceeded. A request for more
/// peers is also automatically sent after connection.
fn listen_for_addrs(
	peers: Arc<p2p::Peers>,
	p2p: Arc<p2p::Server>,
	capab: p2p::Capabilities,
	rx: &mpsc::Receiver<SocketAddr>,
) {
	if peers.peer_count() >= p2p.config.peer_max_count() {
		// clean the rx messages to avoid accumulating
		for _ in rx.try_iter() {}
		return;
	}

	for addr in rx.try_iter() {
		let peers_c = peers.clone();
		let p2p_c = p2p.clone();
		let _ = thread::Builder::new()
			.name("peer_connect".to_string())
			.spawn(move || {
				// connect and retry on fail, but for 3 times at most
				for _ in 0..3 {
					match p2p_c.connect(&addr) {
						Ok(p) => {
							let _ = p.send_peer_request(capab);
							let _ = peers_c.update_state(addr, p2p::State::Healthy);
							break;
						}
						Err(e) => {
							let _ = peers_c.update_state(addr, p2p::State::Defunct);

							// don't retry if connection refused or PeerWithSelf
							match e {
								p2p::Error::Connection(io_err) => {
									if io::ErrorKind::ConnectionRefused == io_err.kind() {
										break;
									}
								}
								p2p::Error::PeerWithSelf => break,
								_ => (), // allow to retry on any other error
							}
						}
					}

					thread::sleep(time::Duration::from_secs(1));
				}
			});
	}
}

pub fn dns_seeds() -> Box<Fn() -> Vec<SocketAddr> + Send> {
	Box::new(|| {
		let mut addresses: Vec<SocketAddr> = vec![];
		for dns_seed in DNS_SEEDS {
			let temp_addresses = addresses.clone();
			debug!(LOGGER, "Retrieving seed nodes from dns {}", dns_seed);
			match (dns_seed.to_owned(), 0).to_socket_addrs() {
				Ok(addrs) => addresses.append(
					&mut (addrs
						.map(|mut addr| {
							addr.set_port(13414);
							addr
						}).filter(|addr| !temp_addresses.contains(addr))
						.collect()),
				),
				Err(e) => debug!(
					LOGGER,
					"Failed to resolve seed {:?} got error {:?}", dns_seed, e
				),
			}
		}
		debug!(LOGGER, "Retrieved seed addresses: {:?}", addresses);
		addresses
	})
}

/// Convenience function when the seed list is immediately known. Mostly used
/// for tests.
pub fn predefined_seeds(addrs_str: Vec<String>) -> Box<Fn() -> Vec<SocketAddr> + Send> {
	Box::new(move || {
		addrs_str
			.iter()
			.map(|s| s.parse().unwrap())
			.collect::<Vec<_>>()
	})
}

/// Convenience function when the seed list is immediately known. Mostly used
/// for tests.
pub fn preferred_peers(addrs_str: Vec<String>) -> Option<Vec<SocketAddr>> {
	if addrs_str.is_empty() {
		None
	} else {
		Some(
			addrs_str
				.iter()
				.map(|s| s.parse().unwrap())
				.collect::<Vec<_>>(),
		)
	}
}
<|endoftext|>


<commit_before>use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

use support::{database, project};

#[test]
fn run_infer_schema_without_docs() {
    test_print_schema("print_schema_simple_without_docs", vec![]);
}

#[test]
fn run_infer_schema() {
    test_print_schema("print_schema_simple", vec!["--with-docs"]);
}

#[test]
fn run_infer_schema_whitelist() {
    test_print_schema(
        "print_schema_whitelist",
        vec!["--with-docs", "-w", "users1"],
    );
}

#[test]
fn run_infer_schema_blacklist() {
    test_print_schema(
        "print_schema_blacklist",
        vec!["--with-docs", "-b", "users1"],
    );
}

#[test]
fn run_infer_schema_order() {
    test_print_schema("print_schema_order", vec!["--with-docs"]);
}

#[test]
fn run_infer_schema_compound_primary_key() {
    test_print_schema("print_schema_compound_primary_key", vec!["--with-docs"]);
}

#[test]
#[cfg(feature = "postgres")]
fn print_schema_specifying_schema_name() {
    test_print_schema(
        "print_schema_specifying_schema_name",
        vec!["--with-docs", "--schema", "custom_schema"],
    )
}

#[test]
#[cfg(feature = "postgres")]
fn print_schema_specifying_schema_name_with_foreign_keys() {
    test_print_schema(
        "print_schema_specifying_schema_name_with_foreign_keys",
        vec!["--with-docs", "--schema", "custom_schema"],
    )
}

#[test]
#[cfg(feature = "postgres")]
fn print_schema_with_foreign_keys() {
    test_print_schema("print_schema_with_foreign_keys", vec!["--with-docs"]);
}

#[test]
fn print_schema_column_renaming() {
    test_print_schema("print_schema_column_renaming", vec!["--with-docs"]);
}

#[cfg(feature = "sqlite")]
const BACKEND: &str = "sqlite";
#[cfg(feature = "postgres")]
const BACKEND: &str = "postgres";
#[cfg(feature = "mysql")]
const BACKEND: &str = "mysql";

fn test_print_schema(test_name: &str, args: Vec<&str>) {
    let test_path = Path::new(file!())
        .parent()
        .unwrap()
        .join("print_schema")
        .join(test_name)
        .join(BACKEND);
    let p = project(test_name).build();
    let db = database(&p.database_url());

    p.command("setup").run();

    let schema = read_file(&test_path.join("schema.sql"));
    db.execute(&schema);

    let result = p.command("print-schema").args(args).run();

    assert!(result.is_success(), "Result was unsuccessful {:?}", result);
    let expected = read_file(&test_path.join("expected.rs"));

    assert_diff!(&expected, result.stdout(), "\n", 0);
}

fn read_file(path: &Path) -> String {
    println!("{}", path.display());
    let mut file = File::open(path).expect(&format!("Could not open {}", path.display()));
    let mut string = String::new();
    file.read_to_string(&mut string).unwrap();
    string
}
<commit_msg>Partially fix our test suite on beta/nightly<commit_after>use std::fs::File;
use std::io::prelude::*;
use std::path::Path;

use support::{database, project};

#[test]
fn run_infer_schema_without_docs() {
    test_print_schema("print_schema_simple_without_docs", vec![]);
}

#[test]
fn run_infer_schema() {
    test_print_schema("print_schema_simple", vec!["--with-docs"]);
}

#[test]
fn run_infer_schema_whitelist() {
    test_print_schema(
        "print_schema_whitelist",
        vec!["--with-docs", "-w", "users1"],
    );
}

#[test]
fn run_infer_schema_blacklist() {
    test_print_schema(
        "print_schema_blacklist",
        vec!["--with-docs", "-b", "users1"],
    );
}

#[test]
fn run_infer_schema_order() {
    test_print_schema("print_schema_order", vec!["--with-docs"]);
}

#[test]
fn run_infer_schema_compound_primary_key() {
    test_print_schema("print_schema_compound_primary_key", vec!["--with-docs"]);
}

#[test]
#[cfg(feature = "postgres")]
fn print_schema_specifying_schema_name() {
    test_print_schema(
        "print_schema_specifying_schema_name",
        vec!["--with-docs", "--schema", "custom_schema"],
    )
}

#[test]
#[cfg(feature = "postgres")]
fn print_schema_specifying_schema_name_with_foreign_keys() {
    test_print_schema(
        "print_schema_specifying_schema_name_with_foreign_keys",
        vec!["--with-docs", "--schema", "custom_schema"],
    )
}

#[test]
#[cfg(feature = "postgres")]
fn print_schema_with_foreign_keys() {
    test_print_schema("print_schema_with_foreign_keys", vec!["--with-docs"]);
}

#[test]
fn print_schema_column_renaming() {
    test_print_schema("print_schema_column_renaming", vec!["--with-docs"]);
}

#[cfg(feature = "sqlite")]
const BACKEND: &str = "sqlite";
#[cfg(feature = "postgres")]
const BACKEND: &str = "postgres";
#[cfg(feature = "mysql")]
const BACKEND: &str = "mysql";

fn test_print_schema(test_name: &str, args: Vec<&str>) {
    let test_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("tests")
        .join("print_schema")
        .join(test_name)
        .join(BACKEND);
    let p = project(test_name).build();
    let db = database(&p.database_url());

    p.command("setup").run();

    let schema = read_file(&test_path.join("schema.sql"));
    db.execute(&schema);

    let result = p.command("print-schema").args(args).run();

    assert!(result.is_success(), "Result was unsuccessful {:?}", result);
    let expected = read_file(&test_path.join("expected.rs"));

    assert_diff!(&expected, result.stdout(), "\n", 0);
}

fn read_file(path: &Path) -> String {
    println!("{}", path.display());
    let mut file = File::open(path).expect(&format!("Could not open {}", path.display()));
    let mut string = String::new();
    file.read_to_string(&mut string).unwrap();
    string
}
<|endoftext|>


<commit_before>use af;
use af::{Dim4, Array};
use af::MatProp;

use activations;
use initializations;
use layer::{Layer, Input};

pub enum LSTMIndex {
  Input,
  Forget,
  Output,
  CellTilda,
}

pub enum ActivationIndex {
  Input,
  Inner,
  Outer,
}

pub enum DataIndex {
  Input,
  Recurrence,
}

pub struct LSTM {
  weights: Vec<Array>,
  recurrent_weights: Vec<Array>,
  bias: Vec<Array>,
  inner_activation: &'static str,
  outer_activation: &'static str,
  return_sequences: bool,
  delta: Array,
  inputs: Input,
}

impl LSTM {
  pub fn new(input_size: u64, output_size: u64
             , outer_activation: &'static str
             , inner_activation: &'static str
             , w_init: &'static str
             , w_inner_init: &'static str
             , bias_init: &'static str
             , forget_bias_init: &'static str
             , return_sequences: bool) -> LSTM
  {
    LSTM{
      weights: vec![initializations::get_initialization(w_init, Dim4::new(&[output_size, input_size, 1, 1])).unwrap()],
      recurrent_weights: vec![initializations::get_initialization(w_inner_init, Dim4::new(&[output_size, output_size, 1, 1])).unwrap(); 4],
      bias: vec![initializations::get_initialization(b_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(forget_bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()],
      inner_activation: inner_activation,
      outer_activation: outer_activation,
      return_sequences: return_sequences,
    }
  }
}

impl RTRL for LSTM {
  pub fn rtrl(&self, dW_tm1: &mut Array  // previous W derivatives for [I, F, Ct]
              , dU_tm1: &mut Array       // previous U derivatives for [I, F, Ct]
              , db_tm1: &mut Array       // previous b derivatives for [I, F, Ct]
              , z_t: &Array              // current time activation
              , inputs: &Input)          // x_t & h_{t-1}
  {
    let block_size = z_t.dims().unwrap()[0];
    assert!(block_size as f32 % 5.0f32 == 0); // there are 5 data pieces we need
    let chunk_size = block_size / 5;
    // chunk out i, f, ct, c_tm1
    let if_t = activations::get_activation(self.inner_activation,
                                            , &af::rows(z_t, 0, 2*chunk_size).unwrap).unwrap();
    let ct_t = activations::get_activation(self.outer_activation,
                                           , &af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap).unwrap();
    let c_tm1 = af::rows(z_t, 4*chunk_size, 5*chunk_size).unwrap();

    // diff(z_i), diff(z_f), diff(z_ct)
    let dz = vec![&activations::get_activation_derivative(self.inner_activation, &af::rows(z_t, 0, chunk_size).unwrap()).unwrap() //d(z)
                  , &activations::get_activation_derivative(self.inner_activation, &af::rows(z_t, chunk_size, 2*chunk_size).unwrap()).unwrap()
                  , &activations::get_activation_derivative(self.outer_activation, &af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap()).unwrap()];
    let ct_ctm1_i = vec![&ct_t, &c_tm1, &af::rows(if_t, 0, chunk_size).unwrap()];

    // [Ct_t; C_{t-1}; i_t] * dz
    let dzprod = af::mul(&af::join_many(0, ct_ctm1_i).unwrap()
                         , af::join_many(0, dzvec).unwrap(), false).unwrap();

    // dC_t/dWi  = (dC_{t-1}/dWi  * f_t) + ct_t  * inner_activation(z_i) * x_t
    // dC_t/dWf  = (dC_{t-1}/dWf  * f_t) + c_tm1 * inner_activation(z_f) * x_t
    // dC_t/dWct = (dC_{t-1}/dWct * f_t) + i_t   * outer_activation(Ct)  * x_t
    let w_lhs = af::mul(dW_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/dW * f_t
    let w_rhs = af::mul(&dzprod, &inputs.data[DataIndex::Input]);
    dW_tm1 = af::add(&w_lhs, &w_rhs).unwrap();

    // dC_t/dUi = (dC_{t-1}/dUi * f_t) + ct_t * inner_activation(z_i) * h_{t-1}
    // dC_t/dUf = (dC_{t-1}/dUf * f_t) + c_tm1 * inner_activation(z_f) * h_{t-1}
    // dC_t/dUct = (dC_{t-1}/dUct * f_t) + outer_activation(Ct) * x_t * h_{t-1}
    let u_lhs = af::mul(dU_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/dU * f_t
    let u_rhs = af::mul(&dzprod, &inputs.data[DataIndex::Recurrence]);
    dU_tm1 = af::add(&u_lhs, &u_rhs).unwrap();

    // dC_t/dbi = (dC_{t-1}/dbi * f_t) + ct_t * inner_activation(z_i)
    // dC_t/dbf = (dC_{t-1}/dbf * f_t) + c_tm1 * inner_activation(z_f)
    // dC_t/dbct = (dC_{t-1}/dbct * f_t) + outer_activation(Ct)
    let b_lhs = af::mul(db_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/db * f_t
    dW_tm1 = af::add(&b_lhs, &dzprod).unwrap();
  }
}

impl Layer for LSTM {

  pub fn forward(&mut self, inputs:& Input) {
    // keep previous layer's outputs
    self.inputs = inputs.clone();

    // apply the activation to the previous layer [Optimization: Memory saving]
    let activated_input = activations::get_activation(inputs.activation[ActivationIndex::Inner]
                                                      , &inputs.data[DataIndex::Input]).unwrap();

    // extract the sub-block of each gate [i_tm1, f_tm1, o_tm1, ct_tm1, c_tm2]
    let block_size = inputs.data[DataIndex::Recurrence].dims().unwrap()[0];
    assert!(block_size as f32 % 5.0f32 == 0); // there are 5 data pieces we need
    let chunk_size = block_size / 5;
    let ifo_tm1 = activations::get_activation(inputs.activation[ActivationIndex::Inner]
                                              , &af::rows(&inputs.data[DataIndex::Recurrence], 0, 3 * chunk_size).unwrap).unwrap();
    let ct_tm1 = activations::get_activation(inputs.activation[ActivationIndex::Outer]
                                             , &af::rows(&inputs.data[DataIndex::Recurrence], 3 * chunk_size, 4 * chunk_size).unwrap).unwrap();
    let c_tm2 = af::rows(&inputs.data[DataIndex::Recurrence], 4 * chunk_size, 5 * chunk_size).unwrap();

    // calculate c_tm1 & h_tm1
    let c_tm1 = af::add(&af::mul(&af::rows(&ifo_tm1, 0, chunk_size).unwrap(), &ct_tm1, false).unwrap()
                        , &af::mul(&af::rows(&ifo_tm1, chunk_size, 2 * chunk_size).unwrap(), &c_tm2, false).unwrap()
                        , false).unwrap();
    let h_tm1 = af::mul(&o_tm1, activations::get_activation(inputs.activation[ActivationIndex::Outer]
                                                            , &c_tm1), false).unwrap();

    // forward pass in a batch for performance
    let weights_ref    = vec![&self.weights[LSTMIndex::Input]
                              , &self.weights[LSTMIndex::Forget]
                              , &self.weights[LSTMIndex::Output]
                              , &self.weights[LSTMIndex::CellTilda]];
    let recurrents_ref = vec![&self.recurrent_weights[LSTMIndex::Input]
                              , &self.recurrent_weights[LSTMIndex::Forget]
                              , &self.recurrent_weights[LSTMIndex::Output]
                              , &self.recurrent_weights[LSTMIndex::CellTilda]];
    let bias_ref       = vec![&self.bias[LSTMIndex::Input]
                              , &self.bias[LSTMIndex::Forget]
                              , &self.bias[LSTMIndex::Output]
                              , &self.bias[LSTMIndex::CellTilda]];
    // [ifo_ct] = W*x + U*h_tm1 + b
    let z_t = af::add(&af::add(&af::matmul(&af::join_many(0, weights_ref).unwrap(), &activated_input).unwrap()
                               , &af::matmul(&af::join_many(0, recurrents_ref).unwrap(), &h_tm1).unwrap(), false).unwrap()
                      , &af::join_many(0, bias_ref).unwrap(), true).unwrap();
    rtrl(&d_tm1, &z_t, )
      if self.return_sequences {
        Input { data: af::join_many(0, vec![&z_t, &c_tm1]).unwrap()
                , activation: vec![self.inner_activation, self.outer_activation] }
      }else { //TODO: Fix this
        Input { data: af::join_many(0, vec![&ifo_tm1, &ct_tm1, &c_tm1]).unwrap()
                , activation: vec![self.inner_activation, self.outer_activation] }
      }
  }

  fn backward(&mut self, delta: &Array) -> Array {
  self.delta = delta.clone();

  let activation_prev = activations::get_activation(self.inputs.activation[0], &self.inputs.data[DataIndex::Input]).unwrap();
  let d_activation_prev = activations::get_activation_derivative(self.inputs.activation[0], &activation_prev).unwrap();
  let delta_prev = af::mul(&af::matmul(&self.weights[0], delta, af::MatProp::TRANS, af::MatProp::NONE).unwrap()
                           , &d_activation_prev, false).unwrap();
  delta_prev
}

fn get_delta(&self) -> Array {
self.delta.clone()
  }

  fn get_weights(&self) -> Vec<Array> {
    self.recurrent_weights.extend(self.weights.iter().cloned()).clone()
  }

  fn set_weights(&mut self, weights: &Array, index: usize) {
    self.weights[index] = weights.clone();
  }

  fn get_bias(&self) -> Vec<Array> {
    self.bias.clone()
  }

  fn set_bias(&mut self, bias: &Array, index: usize) {
    self.bias[index] = bias.clone();
  }

  fn get_bias_dims(&self) -> Vec<Dim4> {
    let mut dims = Vec::new();
    for b in &self.bias {
      dims.push(b.dims().unwrap().clone())
    }
    dims
  }

  fn get_weight_dims(&self) -> Vec<Dim4> {
    let mut dims = Vec::new();
    for w in &self.weights {
      dims.push(w.dims().unwrap().clone())
    }
    for w in &self.recurrent_weights {
      dims.push(w.dims().unwrap().clone())
    }
    dims
  }

  fn get_input(&self) -> Input {
    self.inputs.clone()
  }

  fn output_size(&self) -> u64 {
    let weight_dims = self.get_weight_dims();
    weight_dims[weight_dims.len() - 1][1]
  }

  fn input_size(&self) -> u64 {
    let weight_dims = self.get_weight_dims();
    weight_dims[0][0]
  }

  fn get_activation_type(&self) -> &'static str {
    &self.activation
  }
}
<commit_msg>add internal rtrl call in forward pass since we have our z values<commit_after>use af;
use af::{Dim4, Array};
use af::MatProp;

use activations;
use initializations;
use layer::{Layer, Input};

pub enum LSTMIndex {
  Input,
  Forget,
  Output,
  CellTilda,
}

pub enum ActivationIndex {
  Input,
  Inner,
  Outer,
}

pub enum DataIndex {
  Input,
  Recurrence,
}

pub struct LSTM {
  weights: Vec<Array>,
  recurrent_weights: Vec<Array>,
  bias: Vec<Array>,
  inner_activation: &'static str,
  outer_activation: &'static str,
  return_sequences: bool,
  delta: Array,
  inputs: Input,
}

impl LSTM {
  pub fn new(input_size: u64, output_size: u64
             , outer_activation: &'static str
             , inner_activation: &'static str
             , w_init: &'static str
             , w_inner_init: &'static str
             , bias_init: &'static str
             , forget_bias_init: &'static str
             , return_sequences: bool) -> LSTM
  {
    LSTM{
      weights: vec![initializations::get_initialization(w_init, Dim4::new(&[output_size, input_size, 1, 1])).unwrap()],
      recurrent_weights: vec![initializations::get_initialization(w_inner_init, Dim4::new(&[output_size, output_size, 1, 1])).unwrap(); 4],
      bias: vec![initializations::get_initialization(b_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()
                 , initializations::get_initialization(forget_bias_init, Dim4::new(&[output_size, 1, 1, 1])).unwrap()],
      inner_activation: inner_activation,
      outer_activation: outer_activation,
      return_sequences: return_sequences,
    }
  }
}

impl RTRL for LSTM {
  pub fn rtrl(&self, dW_tm1: &mut Array  // previous W derivatives for [I, F, Ct]
              , dU_tm1: &mut Array       // previous U derivatives for [I, F, Ct]
              , db_tm1: &mut Array       // previous b derivatives for [I, F, Ct]
              , z_t: &Array              // current time activation
              , inputs: &Input)          // x_t & h_{t-1}
  {
    let block_size = z_t.dims().unwrap()[0];
    assert!(block_size as f32 % 5.0f32 == 0); // there are 5 data pieces we need
    let chunk_size = block_size / 5;
    // chunk out i, f, ct, c_tm1
    let if_t = activations::get_activation(self.inner_activation,
                                            , &af::rows(z_t, 0, 2*chunk_size).unwrap).unwrap();
    let ct_t = activations::get_activation(self.outer_activation,
                                           , &af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap).unwrap();
    let c_tm1 = af::rows(z_t, 4*chunk_size, 5*chunk_size).unwrap();

    // diff(z_i), diff(z_f), diff(z_ct)
    let dz = vec![&activations::get_activation_derivative(self.inner_activation, &af::rows(z_t, 0, chunk_size).unwrap()).unwrap() //d(z)
                  , &activations::get_activation_derivative(self.inner_activation, &af::rows(z_t, chunk_size, 2*chunk_size).unwrap()).unwrap()
                  , &activations::get_activation_derivative(self.outer_activation, &af::rows(z_t, 3*chunk_size, 4*chunk_size).unwrap()).unwrap()];
    let ct_ctm1_i = vec![&ct_t, &c_tm1, &af::rows(if_t, 0, chunk_size).unwrap()];

    // [Ct_t; C_{t-1}; i_t] * dz
    let dzprod = af::mul(&af::join_many(0, ct_ctm1_i).unwrap()
                         , af::join_many(0, dzvec).unwrap(), false).unwrap();

    // dC_t/dWi  = (dC_{t-1}/dWi  * f_t) + ct_t  * inner_activation(z_i) * x_t
    // dC_t/dWf  = (dC_{t-1}/dWf  * f_t) + c_tm1 * inner_activation(z_f) * x_t
    // dC_t/dWct = (dC_{t-1}/dWct * f_t) + i_t   * outer_activation(Ct)  * x_t
    let w_lhs = af::mul(dW_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/dW * f_t
    let w_rhs = af::mul(&dzprod, &inputs.data[DataIndex::Input]);
    dW_tm1 = af::add(&w_lhs, &w_rhs).unwrap();

    // dC_t/dUi = (dC_{t-1}/dUi * f_t) + ct_t * inner_activation(z_i) * h_{t-1}
    // dC_t/dUf = (dC_{t-1}/dUf * f_t) + c_tm1 * inner_activation(z_f) * h_{t-1}
    // dC_t/dUct = (dC_{t-1}/dUct * f_t) + outer_activation(Ct) * x_t * h_{t-1}
    let u_lhs = af::mul(dU_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/dU * f_t
    let u_rhs = af::mul(&dzprod, &inputs.data[DataIndex::Recurrence]);
    dU_tm1 = af::add(&u_lhs, &u_rhs).unwrap();

    // dC_t/dbi = (dC_{t-1}/dbi * f_t) + ct_t * inner_activation(z_i)
    // dC_t/dbf = (dC_{t-1}/dbf * f_t) + c_tm1 * inner_activation(z_f)
    // dC_t/dbct = (dC_{t-1}/dbct * f_t) + outer_activation(Ct)
    let b_lhs = af::mul(db_tm1, &af::rows(if_t, chunk_size, 2*chunk_size).unwrap(), true).unwrap(); // dC_{t-1}/db * f_t
    dW_tm1 = af::add(&b_lhs, &dzprod).unwrap();
  }
}

impl Layer for LSTM {

  pub fn forward(&mut self, inputs:& Input) {
    // keep previous layer's outputs
    self.inputs = inputs.clone();

    // apply the activation to the previous layer [Optimization: Memory saving]
    let activated_input = activations::get_activation(inputs.activation[ActivationIndex::Inner]
                                                      , &inputs.data[DataIndex::Input]).unwrap();

    // extract the sub-block of each gate [i_tm1, f_tm1, o_tm1, ct_tm1, c_tm2]
    let block_size = inputs.data[DataIndex::Recurrence].dims().unwrap()[0];
    assert!(block_size as f32 % 5.0f32 == 0); // there are 5 data pieces we need
    let chunk_size = block_size / 5;
    let ifo_tm1 = activations::get_activation(inputs.activation[ActivationIndex::Inner]
                                              , &af::rows(&inputs.data[DataIndex::Recurrence], 0, 3 * chunk_size).unwrap).unwrap();
    let ct_tm1 = activations::get_activation(inputs.activation[ActivationIndex::Outer]
                                             , &af::rows(&inputs.data[DataIndex::Recurrence], 3 * chunk_size, 4 * chunk_size).unwrap).unwrap();
    let c_tm2 = af::rows(&inputs.data[DataIndex::Recurrence], 4 * chunk_size, 5 * chunk_size).unwrap();

    // calculate c_tm1 & h_tm1
    let c_tm1 = af::add(&af::mul(&af::rows(&ifo_tm1, 0, chunk_size).unwrap(), &ct_tm1, false).unwrap()
                        , &af::mul(&af::rows(&ifo_tm1, chunk_size, 2 * chunk_size).unwrap(), &c_tm2, false).unwrap()
                        , false).unwrap();
    let h_tm1 = af::mul(&o_tm1, activations::get_activation(inputs.activation[ActivationIndex::Outer]
                                                            , &c_tm1), false).unwrap();

    // forward pass in a batch for performance
    let weights_ref    = vec![&self.weights[LSTMIndex::Input]
                              , &self.weights[LSTMIndex::Forget]
                              , &self.weights[LSTMIndex::Output]
                              , &self.weights[LSTMIndex::CellTilda]];
    let recurrents_ref = vec![&self.recurrent_weights[LSTMIndex::Input]
                              , &self.recurrent_weights[LSTMIndex::Forget]
                              , &self.recurrent_weights[LSTMIndex::Output]
                              , &self.recurrent_weights[LSTMIndex::CellTilda]];
    let bias_ref       = vec![&self.bias[LSTMIndex::Input]
                              , &self.bias[LSTMIndex::Forget]
                              , &self.bias[LSTMIndex::Output]
                              , &self.bias[LSTMIndex::CellTilda]];
    // [ifo_ct] = W*x + U*h_tm1 + b
    let z_t = af::add(&af::add(&af::matmul(&af::join_many(0, weights_ref).unwrap(), &activated_input).unwrap()
                               , &af::matmul(&af::join_many(0, recurrents_ref).unwrap(), &h_tm1).unwrap(), false).unwrap()
                      , &af::join_many(0, bias_ref).unwrap(), true).unwrap();
    self.rtrl(&self.dW, &self.dU, &self.db, &z_t, inputs);
    // since we are RTRL'ing I, F, Ct w.r.t. C we just need to shoot out O
    if self.return_sequences {
      Input { data: af::join_many(0, vec![&z_t, &c_tm1]).unwrap()
              , activation: vec![self.inner_activation, self.outer_activation] }
    }else { //TODO: Fix this
      Input { data: af::join_many(0, vec![&ifo_tm1, &ct_tm1, &c_tm1]).unwrap()
              , activation: vec![self.inner_activation, self.outer_activation] }
    }
  }

  fn backward(&mut self, delta: &Array) -> Array {
  self.delta = delta.clone();

  let activation_prev = activations::get_activation(self.inputs.activation[0], &self.inputs.data[DataIndex::Input]).unwrap();
  let d_activation_prev = activations::get_activation_derivative(self.inputs.activation[0], &activation_prev).unwrap();
  let delta_prev = af::mul(&af::matmul(&self.weights[0], delta, af::MatProp::TRANS, af::MatProp::NONE).unwrap()
                           , &d_activation_prev, false).unwrap();
  delta_prev
}

fn get_delta(&self) -> Array {
self.delta.clone()
  }

  fn get_weights(&self) -> Vec<Array> {
    self.recurrent_weights.extend(self.weights.iter().cloned()).clone()
  }

  fn set_weights(&mut self, weights: &Array, index: usize) {
    self.weights[index] = weights.clone();
  }

  fn get_bias(&self) -> Vec<Array> {
    self.bias.clone()
  }

  fn set_bias(&mut self, bias: &Array, index: usize) {
    self.bias[index] = bias.clone();
  }

  fn get_bias_dims(&self) -> Vec<Dim4> {
    let mut dims = Vec::new();
    for b in &self.bias {
      dims.push(b.dims().unwrap().clone())
    }
    dims
  }

  fn get_weight_dims(&self) -> Vec<Dim4> {
    let mut dims = Vec::new();
    for w in &self.weights {
      dims.push(w.dims().unwrap().clone())
    }
    for w in &self.recurrent_weights {
      dims.push(w.dims().unwrap().clone())
    }
    dims
  }

  fn get_input(&self) -> Input {
    self.inputs.clone()
  }

  fn output_size(&self) -> u64 {
    let weight_dims = self.get_weight_dims();
    weight_dims[weight_dims.len() - 1][1]
  }

  fn input_size(&self) -> u64 {
    let weight_dims = self.get_weight_dims();
    weight_dims[0][0]
  }

  fn get_activation_type(&self) -> &'static str {
    &self.activation
  }
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#[macro_use]
extern crate lazy_static;
#[cfg(feature = "bindgen")]
extern crate bindgen;
#[cfg(feature = "bindgen")]
extern crate log;
#[cfg(feature = "bindgen")]
extern crate regex;
#[cfg(feature = "bindgen")]
extern crate toml;
extern crate walkdir;

use std::env;
use std::path::Path;
use std::process::{Command, exit};
use walkdir::WalkDir;

#[cfg(feature = "gecko")]
mod build_gecko;

#[cfg(not(feature = "gecko"))]
mod build_gecko {
    pub fn generate() {}
}

#[cfg(windows)]
fn find_python() -> String {
    if Command::new("python2.7.exe").arg("--version").output().is_ok() {
        return "python2.7.exe".to_owned();
    }

    if Command::new("python27.exe").arg("--version").output().is_ok() {
        return "python27.exe".to_owned();
    }

    if Command::new("python.exe").arg("--version").output().is_ok() {
        return "python.exe".to_owned();
    }

    panic!(concat!("Can't find python (tried python2.7.exe, python27.exe, and python.exe)! ",
                   "Try fixing PATH or setting the PYTHON env var"));
}

#[cfg(not(windows))]
fn find_python() -> String {
    if Command::new("python2.7").arg("--version").output().unwrap().status.success() {
        "python2.7"
    } else {
        "python"
    }.to_owned()
}

lazy_static! {
    pub static ref PYTHON: String = env::var("PYTHON").ok().unwrap_or_else(find_python);
}

fn generate_properties() {
    for entry in WalkDir::new("properties") {
        let entry = entry.unwrap();
        match entry.path().extension().and_then(|e| e.to_str()) {
            Some("mako") | Some("rs") | Some("py") | Some("zip") => {
                println!("cargo:rerun-if-changed={}", entry.path().display());
            }
            _ => {}
        }
    }

    let script = Path::new(file!()).parent().unwrap().join("properties").join("build.py");
    let product = if cfg!(feature = "gecko") { "gecko" } else { "servo" };
    let status = Command::new(&*PYTHON)
        .arg(&script)
        .arg(product)
        .arg("style-crate")
        .status()
        .unwrap();
    if !status.success() {
        exit(1)
    }
}

fn main() {
    println!("cargo:rerun-if-changed=build.rs");
    println!("cargo:out_dir={}", env::var("OUT_DIR").unwrap());
    generate_properties();
    build_gecko::generate();
}
<commit_msg>Auto merge of #18363 - paulrouget:fix_style_build, r=SimonSapin<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#[macro_use]
extern crate lazy_static;
#[cfg(feature = "bindgen")]
extern crate bindgen;
#[cfg(feature = "bindgen")]
extern crate log;
#[cfg(feature = "bindgen")]
extern crate regex;
#[cfg(feature = "bindgen")]
extern crate toml;
extern crate walkdir;

use std::env;
use std::path::Path;
use std::process::{Command, exit};
use walkdir::WalkDir;

#[cfg(feature = "gecko")]
mod build_gecko;

#[cfg(not(feature = "gecko"))]
mod build_gecko {
    pub fn generate() {}
}

#[cfg(windows)]
fn find_python() -> String {
    if Command::new("python2.7.exe").arg("--version").output().is_ok() {
        return "python2.7.exe".to_owned();
    }

    if Command::new("python27.exe").arg("--version").output().is_ok() {
        return "python27.exe".to_owned();
    }

    if Command::new("python.exe").arg("--version").output().is_ok() {
        return "python.exe".to_owned();
    }

    panic!(concat!("Can't find python (tried python2.7.exe, python27.exe, and python.exe)! ",
                   "Try fixing PATH or setting the PYTHON env var"));
}

#[cfg(not(windows))]
fn find_python() -> String {
    if Command::new("python2.7").arg("--version").output().unwrap().status.success() {
        "python2.7"
    } else {
        "python"
    }.to_owned()
}

lazy_static! {
    pub static ref PYTHON: String = env::var("PYTHON").ok().unwrap_or_else(find_python);
}

fn generate_properties() {
    for entry in WalkDir::new("properties") {
        let entry = entry.unwrap();
        match entry.path().extension().and_then(|e| e.to_str()) {
            Some("mako") | Some("rs") | Some("py") | Some("zip") => {
                println!("cargo:rerun-if-changed={}", entry.path().display());
            }
            _ => {}
        }
    }

    let script = Path::new(&env::var("CARGO_MANIFEST_DIR").unwrap())
        .join("properties").join("build.py");
    let product = if cfg!(feature = "gecko") { "gecko" } else { "servo" };
    let status = Command::new(&*PYTHON)
        .arg(&script)
        .arg(product)
        .arg("style-crate")
        .status()
        .unwrap();
    if !status.success() {
        exit(1)
    }
}

fn main() {
    println!("cargo:rerun-if-changed=build.rs");
    println!("cargo:out_dir={}", env::var("OUT_DIR").unwrap());
    generate_properties();
    build_gecko::generate();
}
<|endoftext|>


<commit_before>#![allow(dead_code, unused_variables)]

use std::collections::HashMap;
use std::fmt;

#[derive(Debug, PartialEq)]
pub struct Source {
    nick: String,
    ident: Option<String>,
    host: Option<String>,
    is_server: bool,
}

impl fmt::Display for Source {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        if self.is_server {
            write!(f, ":")?;
        }

        write!(f, "{}", self.nick)?;

        if let Some(ref ident) = self.ident {
            write!(f, "!{}", ident)?;
        }
        if let Some(ref host) = self.host {
            write!(f, "@{}", host)?;
        }

        Ok(())
    }
}

//  <prefix>   ::= <nick> [ '!' <ident>  [ '@' <host> ] ]
impl Source {
    fn parse(s: &str) -> Option<Source> {
        if s.is_empty() {
            return None;
        }

        let (nick, rest) = if let Some(index) = s.find('!') {
            (&s[..index], Some(&s[index + 1..]))
        } else {
            (s, None)
        };

        let (ident, host) = rest
            .map(|r| {
                if let Some(index) = r.find('@') {
                    (Some(&r[..index]), Some(&r[index + 1..]))
                } else {
                    (rest, None)
                }
            })
            .unwrap_or((None, None));

        Some(Source {
            nick: nick.trim_start_matches(':').to_string(),
            ident: ident.map(str::to_string),
            host: host.map(str::to_string),
            is_server: nick.starts_with(':'),
        })
    }
}

#[derive(Debug, PartialEq)]
pub enum MessageKind {
    Away,
    Capability,
    Error,
    Invite,
    Join,
    Kick,
    Mode,
    Nick,
    Notice,
    Numeric(u16),
    Part,
    Pass,
    Ping,
    Pong,
    Privmsg,
    Quit,
    Topic,
    User,
    Unknown,
    Wallops,
}

impl MessageKind {
    pub fn from(msg: &RawMessage) -> MessageKind {
        if let Ok(code) = msg.command.parse::<u16>() {
            return MessageKind::Numeric(code);
        }

        match msg.command.as_str() {
            "CAP" => Self::Capability,
            "JOIN" => Self::Join,
            "KICK" => Self::Kick,
            "NOTICE" => Self::Notice,
            "PART" => Self::Part,
            "PASS" => Self::Pass,
            "PING" => Self::Ping,
            "PONG" => Self::Pong,
            "PRIVMSG" => Self::Privmsg,
            "USER" => Self::User,

            // TODO: Map remaining types
            _ => Self::Unknown,
        }
    }
}

type MessageTags = HashMap<String, MessageTag>;

#[derive(Debug, PartialEq)]
pub struct MessageTag {
    key: String,
    value: Option<String>,
}

#[derive(Debug, PartialEq)]
pub struct RawMessage {
    pub source: Option<Source>,
    pub command: String,
    params: Vec<String>,

    pub timestamp: u64, // TODO: datetime
    pub tags: MessageTags,
}

/// Split `msg` at first space character or end of word if no spaces remain.
fn split_message(msg: &str) -> Option<(&str, &str)> {
    if msg.is_empty() {
        return None;
    }

    match msg.find(' ') {
        Some(index) => Some((&msg[..index], &msg[index + 1..])),
        None => Some((msg, "")),
    }
}

impl RawMessage {
    pub fn new(command: &str, params: &[&str]) -> RawMessage {
        RawMessage {
            command: command.to_string(),
            params: params.iter().map(|s| s.to_string()).collect(),

            source: None,
            timestamp: 0,
            tags: HashMap::new(),
        }
    }

    /// Expects that any trailing CR, LF have been removed.
    ///
    ///  <message>  ::= [':' <prefix> <SPACE> ] <command> <params> <crlf>
    ///  <prefix>   ::= <servername> | <nick> [ '!' <user> ] [ '@' <host> ]
    ///  <command>  ::= <letter> { <letter> } | <number> <number> <number>
    ///  <SPACE>    ::= ' ' { ' ' }
    ///  <params>   ::= <SPACE> [ ':' <trailing> | <middle> <params> ]
    ///  <middle>   ::= <Any *non-empty* sequence of octets not including SPACE
    ///                 or NUL or CR or LF, the first of which may not be ':'>
    ///  <trailing> ::= <Any, possibly *empty*, sequence of octets not including
    ///                   NUL or CR or LF>
    pub fn parse(line: &str) -> Option<Self> {
        let mut line: &str = line;
        let mut msg: Self;

        let tags = if line.starts_with('@') {
            let (tags, rest) = split_message(line)?;
            line = rest;
            // TODO: parse tags
            HashMap::new()
        } else {
            HashMap::new()
        };

        let nick = if line.starts_with(':') {
            let (nick, rest) = split_message(line)?;
            line = rest;
            Some(Source::parse(nick)?)
        } else {
            None
        };

        let (cmd, mut line) = split_message(line)?;
        // line = rest;

        let mut params: Vec<String> = Vec::new();
        while let Some((param, rest)) = split_message(line) {
            // Trailing param, this will be the end, regardless of
            // remaining whitespace.
            if param.starts_with(':') {
                params.push(line[1..].to_string());
                break;
            } else {
                params.push(param.to_string());
            }

            line = rest;
        }

        Some(Self {
            source: nick,
            command: cmd.to_string(),
            params,
            tags,

            // TODO: pull this out of the tags, if possible
            timestamp: 0,
        })
    }

    pub fn param(&self, i: usize) -> Option<&str> {
        self.params.get(i).map(String::as_ref)
    }

    pub fn trailing(&self) -> Option<&str> {
        self.params.last().map(String::as_ref)
    }
}

impl fmt::Display for RawMessage {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        // TOOD: tags

        if let Some(source) = &self.source {
            write!(f, "{} ", source)?;
        }

        write!(f, "{}", self.command)?;

        for (i, param) in self.params.iter().enumerate() {
            // Last is a special case
            if param.contains(' ') {
                write!(f, " :{}", param)?;
            } else {
                write!(f, " {}", param)?;
            }
        }

        Ok(())
    }
}

/// IRC capabilities we support
#[derive(Debug, Hash, PartialEq, Eq)]
pub enum Capability {
    ServerTime,
    Sasl(Vec<String>),
}

impl Capability {
    pub fn from(s: &str) -> Option<Self> {
        let split: Vec<&str> = s.splitn(2, '=').collect();

        let cap = *split.get(0)?;
        let args = split.get(1).unwrap_or(&"").split(',');

        match cap {
            "server-time" => Some(Self::ServerTime),
            "sasl" => {
                let set = args.map(str::to_uppercase).collect();
                Some(Self::Sasl(set))
            }
            _ => None,
        }
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_parse_caps() {
        assert_eq!(
            Capability::from("server-time"),
            Some(Capability::ServerTime)
        );

        assert_eq!(
            Capability::from("sasl=plain,EXTERNAL"),
            Some(Capability::Sasl(vec![
                "PLAIN".to_string(),
                "EXTERNAL".to_string()
            ]))
        );
    }

    #[test]
    fn test_parse_source() {
        assert_eq!(
            Source::parse("nick!~ident@host").unwrap(),
            Source {
                nick: "nick".to_string(),
                ident: Some("~ident".to_string()),
                host: Some("host".to_string()),
                is_server: false,
            }
        );

        assert_eq!(
            Source::parse("nick!~user").unwrap(),
            Source {
                nick: "nick".to_string(),
                ident: Some("~user".to_string()),
                host: None,
                is_server: false,
            }
        );

        assert_eq!(
            Source::parse("nick").unwrap(),
            Source {
                nick: "nick".to_string(),
                ident: None,
                host: None,
                is_server: false,
            }
        );

        assert_eq!(
            Source::parse(":irc.server!~ident@host").unwrap(),
            Source {
                nick: "irc.server".to_string(),
                ident: Some("~ident".to_string()),
                host: Some("host".to_string()),
                is_server: true,
            }
        );
    }

    #[test]
    fn test_display_source() {
        let cases = vec![
            "nick",
            "nick!~ident",
            "nick!~ident@host",
            ":irc.server!~ident@host",
        ];

        for case in cases {
            let round_trip = format!("{}", Source::parse(case).unwrap());
            assert_eq!(round_trip, case);
        }
    }

    #[test]
    fn test_parse_messages() {
        let line = "@foo=bar :server 001 abc :hello, world";
        let msg = RawMessage::parse(line).expect("failed to parse");

        assert_eq!(msg.command, "001");
        assert_eq!(msg.source.unwrap().nick, "server");
        assert_eq!(msg.params, vec!["abc", "hello, world"]);
        // TODO: wire up assert
    }

    #[test]
    fn test_split_message() {
        assert_eq!(split_message("foo bar baz"), Some(("foo", "bar baz")));
        assert_eq!(split_message("foo bar"), Some(("foo", "bar")));
        assert_eq!(split_message("foo"), Some(("foo", "")));
        assert_eq!(split_message(""), None);
        assert_eq!(split_message("trailing "), Some(("trailing", "")));
        assert_eq!(split_message(" leading"), Some(("", "leading")));
    }
}
<commit_msg>Parse MODE strings<commit_after>#![allow(dead_code, unused_variables)]

use std::collections::HashMap;
use std::fmt;

#[derive(Debug, PartialEq)]
pub struct Source {
    nick: String,
    ident: Option<String>,
    host: Option<String>,
    is_server: bool,
}

impl fmt::Display for Source {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        if self.is_server {
            write!(f, ":")?;
        }
        write!(f, "{}", self.nick)?;
        if let Some(ref ident) = self.ident {
            write!(f, "!{}", ident)?;
        }
        if let Some(ref host) = self.host {
            write!(f, "@{}", host)?;
        }

        Ok(())
    }
}

//  <prefix>   ::= <nick> [ '!' <ident>  [ '@' <host> ] ]
impl Source {
    fn parse(s: &str) -> Option<Source> {
        if s.is_empty() {
            return None;
        }

        let (nick, rest) = if let Some(index) = s.find('!') {
            (&s[..index], Some(&s[index + 1..]))
        } else {
            (s, None)
        };

        let (ident, host) = rest
            .map(|r| {
                if let Some(index) = r.find('@') {
                    (Some(&r[..index]), Some(&r[index + 1..]))
                } else {
                    (rest, None)
                }
            })
            .unwrap_or((None, None));

        Some(Source {
            nick: nick.trim_start_matches(':').to_string(),
            ident: ident.map(str::to_string),
            host: host.map(str::to_string),
            is_server: nick.starts_with(':'),
        })
    }
}

#[derive(Debug, PartialEq)]
pub enum MessageKind {
    Away,
    Capability,
    Error,
    Invite,
    Join,
    Kick,
    Mode,
    Nick,
    Notice,
    Numeric(u16),
    Part,
    Pass,
    Ping,
    Pong,
    Privmsg,
    Quit,
    Topic,
    User,
    Unknown,
    Wallops,
}

impl MessageKind {
    pub fn from(msg: &RawMessage) -> MessageKind {
        if let Ok(code) = msg.command.parse::<u16>() {
            return MessageKind::Numeric(code);
        }

        match msg.command.as_str() {
            "CAP" => Self::Capability,
            "JOIN" => Self::Join,
            "KICK" => Self::Kick,
            "NOTICE" => Self::Notice,
            "PART" => Self::Part,
            "PASS" => Self::Pass,
            "PING" => Self::Ping,
            "PONG" => Self::Pong,
            "PRIVMSG" => Self::Privmsg,
            "USER" => Self::User,

            // TODO: Map remaining types
            _ => Self::Unknown,
        }
    }
}

type MessageTags = HashMap<String, MessageTag>;

#[derive(Debug, PartialEq)]
pub struct MessageTag {
    key: String,
    value: Option<String>,
}

#[derive(Debug, PartialEq)]
pub struct RawMessage {
    pub source: Option<Source>,
    pub command: String,
    params: Vec<String>,

    pub timestamp: u64, // TODO: datetime
    pub tags: MessageTags,
}

/// Split `msg` at first space character or end of word if no spaces remain.
fn split_message(msg: &str) -> Option<(&str, &str)> {
    if msg.is_empty() {
        return None;
    }

    match msg.find(' ') {
        Some(index) => Some((&msg[..index], &msg[index + 1..])),
        None => Some((msg, "")),
    }
}

impl RawMessage {
    pub fn new(command: &str, params: &[&str]) -> RawMessage {
        RawMessage {
            command: command.to_string(),
            params: params.iter().map(|s| s.to_string()).collect(),

            source: None,
            timestamp: 0,
            tags: HashMap::new(),
        }
    }

    /// Expects that any trailing CR, LF have been removed.
    ///
    ///  <message>  ::= [':' <prefix> <SPACE> ] <command> <params> <crlf>
    ///  <prefix>   ::= <servername> | <nick> [ '!' <user> ] [ '@' <host> ]
    ///  <command>  ::= <letter> { <letter> } | <number> <number> <number>
    ///  <SPACE>    ::= ' ' { ' ' }
    ///  <params>   ::= <SPACE> [ ':' <trailing> | <middle> <params> ]
    ///  <middle>   ::= <Any *non-empty* sequence of octets not including SPACE
    ///                 or NUL or CR or LF, the first of which may not be ':'>
    ///  <trailing> ::= <Any, possibly *empty*, sequence of octets not including
    ///                   NUL or CR or LF>
    pub fn parse(line: &str) -> Option<Self> {
        let mut line: &str = line;
        let mut msg: Self;

        let tags = if line.starts_with('@') {
            let (tags, rest) = split_message(line)?;
            line = rest;
            // TODO: parse tags
            HashMap::new()
        } else {
            HashMap::new()
        };

        let nick = if line.starts_with(':') {
            let (nick, rest) = split_message(line)?;
            line = rest;
            Some(Source::parse(nick)?)
        } else {
            None
        };

        let (cmd, mut line) = split_message(line)?;

        let mut params: Vec<String> = Vec::new();
        while let Some((param, rest)) = split_message(line) {
            // Trailing param, this will be the end, regardless of
            // remaining whitespace.
            if param.starts_with(':') {
                params.push(line[1..].to_string());
                break;
            } else {
                params.push(param.to_string());
            }

            line = rest;
        }

        Some(Self {
            source: nick,
            command: cmd.to_string(),
            params,
            tags,

            // TODO: pull this out of the tags, if possible
            timestamp: 0,
        })
    }

    pub fn param(&self, i: usize) -> Option<&str> {
        self.params.get(i).map(String::as_ref)
    }

    pub fn trailing(&self) -> Option<&str> {
        self.params.last().map(String::as_ref)
    }
}

impl fmt::Display for RawMessage {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        // TOOD: tags

        if let Some(source) = &self.source {
            write!(f, "{} ", source)?;
        }

        write!(f, "{}", self.command)?;

        for (i, param) in self.params.iter().enumerate() {
            // Last is a special case
            if param.contains(' ') {
                write!(f, " :{}", param)?;
            } else {
                write!(f, " {}", param)?;
            }
        }

        Ok(())
    }
}

/// IRC capabilities we support
#[derive(Debug, Hash, PartialEq, Eq)]
pub enum Capability {
    ServerTime,
    Sasl(Vec<String>),
}

impl Capability {
    pub fn from(s: &str) -> Option<Self> {
        let split: Vec<&str> = s.splitn(2, '=').collect();

        let cap = *split.get(0)?;
        let args = split.get(1).unwrap_or(&"").split(',');

        match cap {
            "server-time" => Some(Self::ServerTime),
            "sasl" => {
                let set = args.map(str::to_uppercase).collect();
                Some(Self::Sasl(set))
            }
            _ => None,
        }
    }
}

#[derive(PartialEq, Debug)]
pub struct ModeSet {
    set: HashMap<char, bool>,
}

impl ModeSet {
    pub fn from(s: &str) -> Option<Self> {
        let mut value: Option<bool> = None;
        let mut modes = ModeSet {
            set: HashMap::new(),
        };

        for ch in s.chars() {
            match ch {
                '+' => value = Some(true),
                '-' => value = Some(false),
                _ => {
                    if let Some(value) = value {
                        modes.set.insert(ch, value);
                    } else {
                        // Invalid string format
                        return None;
                    }
                }
            }
        }

        Some(modes)
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_parse_mode_string() {
        let modes = ModeSet::from("-bB+xy");
        let expected = [('b', false), ('B', false), ('x', true), ('y', true)]
            .iter()
            .cloned()
            .collect();

        assert_eq!(modes, Some(ModeSet { set: expected }));
    }

    #[test]
    fn test_parse_caps() {
        assert_eq!(
            Capability::from("server-time"),
            Some(Capability::ServerTime)
        );

        assert_eq!(
            Capability::from("sasl=plain,EXTERNAL"),
            Some(Capability::Sasl(vec![
                "PLAIN".to_string(),
                "EXTERNAL".to_string()
            ]))
        );
    }

    #[test]
    fn test_parse_source() {
        assert_eq!(
            Source::parse("nick!~ident@host").unwrap(),
            Source {
                nick: "nick".to_string(),
                ident: Some("~ident".to_string()),
                host: Some("host".to_string()),
                is_server: false,
            }
        );

        assert_eq!(
            Source::parse("nick!~user").unwrap(),
            Source {
                nick: "nick".to_string(),
                ident: Some("~user".to_string()),
                host: None,
                is_server: false,
            }
        );

        assert_eq!(
            Source::parse("nick").unwrap(),
            Source {
                nick: "nick".to_string(),
                ident: None,
                host: None,
                is_server: false,
            }
        );

        assert_eq!(
            Source::parse(":irc.server!~ident@host").unwrap(),
            Source {
                nick: "irc.server".to_string(),
                ident: Some("~ident".to_string()),
                host: Some("host".to_string()),
                is_server: true,
            }
        );
    }

    #[test]
    fn test_display_source() {
        let cases = vec![
            "nick",
            "nick!~ident",
            "nick!~ident@host",
            ":irc.server!~ident@host",
        ];

        for case in cases {
            let round_trip = format!("{}", Source::parse(case).unwrap());
            assert_eq!(round_trip, case);
        }
    }

    #[test]
    fn test_parse_messages() {
        let line = "@foo=bar :server 001 abc :hello, world";
        let msg = RawMessage::parse(line).expect("failed to parse");

        assert_eq!(msg.command, "001");
        assert_eq!(msg.source.unwrap().nick, "server");
        assert_eq!(msg.params, vec!["abc", "hello, world"]);
        // TODO: wire up assert
    }

    #[test]
    fn test_split_message() {
        assert_eq!(split_message("foo bar baz"), Some(("foo", "bar baz")));
        assert_eq!(split_message("foo bar"), Some(("foo", "bar")));
        assert_eq!(split_message("foo"), Some(("foo", "")));
        assert_eq!(split_message(""), None);
        assert_eq!(split_message("trailing "), Some(("trailing", "")));
        assert_eq!(split_message(" leading"), Some(("", "leading")));
    }
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

extern crate capnp;
pub mod addressbook_capnp {
  include!(concat!(env!("OUT_DIR"), "/addressbook_capnp.rs"));
}

use capnp::message::{Builder, HeapAllocator, TypedReader};
use std::sync::mpsc;
use std::thread;

pub mod addressbook {
    use addressbook_capnp::{address_book, person};
    use capnp::message::{Builder, HeapAllocator, TypedReader};

    pub fn build_address_book() -> TypedReader<Builder<HeapAllocator>, address_book::Owned> {
        let mut message = Builder::new_default();
        {
            let address_book = message.init_root::<address_book::Builder>();

            let mut people = address_book.init_people(2);

            {
                let mut alice = people.borrow().get(0);
                alice.set_id(123);
                alice.set_name("Alice");
                alice.set_email("alice@example.com");
                {
                    let mut alice_phones = alice.borrow().init_phones(1);
                    alice_phones.borrow().get(0).set_number("555-1212");
                    alice_phones.borrow().get(0).set_type(person::phone_number::Type::Mobile);
                }
                alice.get_employment().set_school("MIT");
            }

            {
                let mut bob = people.get(1);
                bob.set_id(456);
                bob.set_name("Bob");
                bob.set_email("bob@example.com");
                {
                    let mut bob_phones = bob.borrow().init_phones(2);
                    bob_phones.borrow().get(0).set_number("555-4567");
                    bob_phones.borrow().get(0).set_type(person::phone_number::Type::Home);
                    bob_phones.borrow().get(1).set_number("555-7654");
                    bob_phones.borrow().get(1).set_type(person::phone_number::Type::Work);
                }
                bob.get_employment().set_unemployed(());
            }
        }

        // There are two ways to get a TypedReader from our `message`:
        //
        // Option 1: Go through the full process manually
        //  message.into_reader::<address_book::Owned>().into_typed()
        //
        //  Option 2: Use the "Into" trait defined on the builder
        //   message.into()
        //
        // Option 3: Use the "From" trait defined on the builder
        TypedReader::from(message)
    }
}

pub fn main() {

    let book = addressbook::build_address_book();

    let (tx_book, rx_book) = mpsc::channel::<TypedReader<Builder<HeapAllocator>, addressbook_capnp::address_book::Owned>>();
    let (tx_id, rx_id) = mpsc::channel::<u32>();

    thread::spawn(move || {
        let addressbook_reader = rx_book.recv().unwrap();
        let addressbook = addressbook_reader.get().unwrap();
        let first_person = addressbook.get_people().unwrap().get(0);
        let first_id = first_person.get_id();
        tx_id.send(first_id)
    });

    tx_book.send(book).unwrap();
    let first_id = rx_id.recv().unwrap();
    assert_eq!(first_id, 123);
}
<commit_msg>touch up comment<commit_after>// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

extern crate capnp;
pub mod addressbook_capnp {
  include!(concat!(env!("OUT_DIR"), "/addressbook_capnp.rs"));
}

use capnp::message::{Builder, HeapAllocator, TypedReader};
use std::sync::mpsc;
use std::thread;

pub mod addressbook {
    use addressbook_capnp::{address_book, person};
    use capnp::message::{Builder, HeapAllocator, TypedReader};

    pub fn build_address_book() -> TypedReader<Builder<HeapAllocator>, address_book::Owned> {
        let mut message = Builder::new_default();
        {
            let address_book = message.init_root::<address_book::Builder>();

            let mut people = address_book.init_people(2);

            {
                let mut alice = people.borrow().get(0);
                alice.set_id(123);
                alice.set_name("Alice");
                alice.set_email("alice@example.com");
                {
                    let mut alice_phones = alice.borrow().init_phones(1);
                    alice_phones.borrow().get(0).set_number("555-1212");
                    alice_phones.borrow().get(0).set_type(person::phone_number::Type::Mobile);
                }
                alice.get_employment().set_school("MIT");
            }

            {
                let mut bob = people.get(1);
                bob.set_id(456);
                bob.set_name("Bob");
                bob.set_email("bob@example.com");
                {
                    let mut bob_phones = bob.borrow().init_phones(2);
                    bob_phones.borrow().get(0).set_number("555-4567");
                    bob_phones.borrow().get(0).set_type(person::phone_number::Type::Home);
                    bob_phones.borrow().get(1).set_number("555-7654");
                    bob_phones.borrow().get(1).set_type(person::phone_number::Type::Work);
                }
                bob.get_employment().set_unemployed(());
            }
        }

        // There are two ways to get a TypedReader from our `message`:
        //
        // Option 1: Go through the full process manually
        //  message.into_reader().into_typed()
        //
        // Option 2: Use the "Into" trait defined on the builder
        //   message.into()
        //
        // Option 3: Use the "From" trait defined on the builder
        TypedReader::from(message)
    }
}

pub fn main() {

    let book = addressbook::build_address_book();

    let (tx_book, rx_book) = mpsc::channel::<TypedReader<Builder<HeapAllocator>, addressbook_capnp::address_book::Owned>>();
    let (tx_id, rx_id) = mpsc::channel::<u32>();

    thread::spawn(move || {
        let addressbook_reader = rx_book.recv().unwrap();
        let addressbook = addressbook_reader.get().unwrap();
        let first_person = addressbook.get_people().unwrap().get(0);
        let first_id = first_person.get_id();
        tx_id.send(first_id)
    });

    tx_book.send(book).unwrap();
    let first_id = rx_id.recv().unwrap();
    assert_eq!(first_id, 123);
}
<|endoftext|>


<commit_before>// Simple API example, ported from http://lua-users.org/wiki/SimpleLuaApiExample

// This is a simple introductory example of how to interface to Lua from Rust.
// The Rust program loads a Lua script file, sets some Lua variables, runs the
// Lua script, and reads back the return value.

extern mod lua;

use std::{io, os};
use std::iter::range_inclusive;

fn main() {
    let mut L = lua::State::new();
    L.openlibs(); // Load Lua libraries

    // Load the file containing the script we are going to run
    let path = Path::new("simpleapi.lua");
    match L.loadfile(Some(&path)) {
        Ok(_) => (),
        Err(_) => {
            // If something went wrong, error message is at the top of the stack
            writeln!(&mut io::stderr() as &mut io::Writer,
                     "Couldn't load file: {}", L.tostring(-1).unwrap_or_default());
            os::set_exit_status(1);
            return;
        }
    }

    /*
     * Ok, now here we go: We pass data to the lua script on the stack.
     * That is, we first have to prepare Lua's virtual stack the way we
     * want the script to receive it, then ask Lua to run it.
     */
    L.newtable(); // We will pass a table

    /*
     * To put values into the table, we first push the index, then the
     * value, and then call rawset() with the index of the table in the
     * stack. Let's see why it's -3: In Lua, the value -1 always refers to
     * the top of the stack. When you create the table with newtable(),
     * the table gets pushed into the top of the stack. When you push the
     * index and then the cell value, the stack looks like:
     *
     * - [stack bottom] -- table, index, value [top]
     *
     * So the -1 will refer to the cell value, thus -3 is used to refer to
     * the table itself. Note that rawset() pops the last two elements
     * of the stack, so that after it has been called, the table is at the
     * top of the stack.
     */
    for i in range_inclusive(1, 5) {
        L.pushinteger(i);   // Push the table index
        L.pushinteger(i*2); // Push the cell value
        L.rawset(-3);       // Stores the pair in the table
    }

    // By what name is the script going to reference our table?
    L.setglobal("foo");

    // Ask Lua to run our little script
    match L.pcall(0, lua::MULTRET, 0) {
        Ok(()) => (),
        Err(_) => {
            writeln!(&mut io::stderr() as &mut io::Writer,
                     "Failed to run script: {}", L.tostring(-1).unwrap_or_default());
            os::set_exit_status(1);
            return;
        }
    }

    // Get the returned value at the to of the stack (index -1)
    let sum = L.tonumber(-1);

    println!("Script returned: {}", sum);

    L.pop(1); // Take the returned value out of the stack

    // L's destructor will close the state for us
}
<commit_msg>Squash unused_result warning in simpleapi example<commit_after>// Simple API example, ported from http://lua-users.org/wiki/SimpleLuaApiExample

// This is a simple introductory example of how to interface to Lua from Rust.
// The Rust program loads a Lua script file, sets some Lua variables, runs the
// Lua script, and reads back the return value.

extern mod lua;

use std::{io, os};
use std::iter::range_inclusive;

fn main() {
    let mut L = lua::State::new();
    L.openlibs(); // Load Lua libraries

    // Load the file containing the script we are going to run
    let path = Path::new("simpleapi.lua");
    match L.loadfile(Some(&path)) {
        Ok(_) => (),
        Err(_) => {
            // If something went wrong, error message is at the top of the stack
            let _ = writeln!(&mut io::stderr(),
                             "Couldn't load file: {}", L.tostring(-1).unwrap_or_default());
            os::set_exit_status(1);
            return;
        }
    }

    /*
     * Ok, now here we go: We pass data to the lua script on the stack.
     * That is, we first have to prepare Lua's virtual stack the way we
     * want the script to receive it, then ask Lua to run it.
     */
    L.newtable(); // We will pass a table

    /*
     * To put values into the table, we first push the index, then the
     * value, and then call rawset() with the index of the table in the
     * stack. Let's see why it's -3: In Lua, the value -1 always refers to
     * the top of the stack. When you create the table with newtable(),
     * the table gets pushed into the top of the stack. When you push the
     * index and then the cell value, the stack looks like:
     *
     * - [stack bottom] -- table, index, value [top]
     *
     * So the -1 will refer to the cell value, thus -3 is used to refer to
     * the table itself. Note that rawset() pops the last two elements
     * of the stack, so that after it has been called, the table is at the
     * top of the stack.
     */
    for i in range_inclusive(1, 5) {
        L.pushinteger(i);   // Push the table index
        L.pushinteger(i*2); // Push the cell value
        L.rawset(-3);       // Stores the pair in the table
    }

    // By what name is the script going to reference our table?
    L.setglobal("foo");

    // Ask Lua to run our little script
    match L.pcall(0, lua::MULTRET, 0) {
        Ok(()) => (),
        Err(_) => {
            let _ = writeln!(&mut io::stderr(),
                             "Failed to run script: {}", L.tostring(-1).unwrap_or_default());
            os::set_exit_status(1);
            return;
        }
    }

    // Get the returned value at the to of the stack (index -1)
    let sum = L.tonumber(-1);

    println!("Script returned: {}", sum);

    L.pop(1); // Take the returned value out of the stack

    // L's destructor will close the state for us
}
<|endoftext|>