<commit_before>// Copyright 2017 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Internal module responsible for job delegation, creating hashes
//! and sending them to the plugin's internal queues. Used internally
//!
//!

use std::sync::{Arc, RwLock};
use std::{thread, time};
use std::mem::transmute;

use rand::{self, Rng};
use byteorder::{ByteOrder, BigEndian};
use blake2::blake2b::Blake2b;
use env_logger;

use cuckoo_sys::manager::PluginLibrary;
use error::error::CuckooMinerError;
use CuckooMinerJobHandle;
use CuckooMinerSolution;

/// From grin
/// The target is the 8-bytes hash block hashes must be lower than.
const MAX_TARGET: [u8; 8] = [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];

type JobSharedDataType = Arc<RwLock<JobSharedData>>;
type JobControlDataType = Arc<RwLock<JobControlData>>;
type PluginLibrariesDataType = Arc<RwLock<Vec<PluginLibrary>>>;

/// Data intended to be shared across threads
pub struct JobSharedData {
	/// ID of the current running job (not currently used)
	pub job_id: u32,

	/// The part of the header before the nonce, which this
	/// module will mutate in search of a solution
	pub pre_nonce: String,

	/// The part of the header after the nonce
	pub post_nonce: String,

	/// The target difficulty. Only solutions >= this
	/// target will be put into the output queue
	pub difficulty: u64,

	/// Output solutions
	pub solutions: Vec<CuckooMinerSolution>,
}

impl Default for JobSharedData {
	fn default() -> JobSharedData {
		JobSharedData {
			job_id: 0,
			pre_nonce: String::from(""),
			post_nonce: String::from(""),
			difficulty: 0,
			solutions: Vec::new(),
		}
	}
}

impl JobSharedData {
	pub fn new(job_id: u32, pre_nonce: &str, post_nonce: &str, difficulty: u64) -> JobSharedData {
		JobSharedData {
			job_id: job_id,
			pre_nonce: String::from(pre_nonce),
			post_nonce: String::from(post_nonce),
			difficulty: difficulty,
			solutions: Vec::new(),
		}
	}
}

/// an internal structure to flag job control

pub struct JobControlData {
	/// Whether the mining job is running
	pub stop_flag: bool,

	/// Whether all plugins have stopped
	pub has_stopped: bool,
}

impl Default for JobControlData {
	fn default() -> JobControlData {
		JobControlData {
			stop_flag: false,
			has_stopped: false,
		}
	}
}

/// Internal structure which controls and runs processing jobs.
///
///

pub struct Delegator {
	/// Data which is shared across all threads
	shared_data: JobSharedDataType,

	/// Job control flags which are shared across threads
	control_data: JobControlDataType,

	/// Loaded Plugin Library
	libraries: PluginLibrariesDataType,
}

impl Delegator {
	/// Create a new job delegator

	pub fn new(job_id: u32, pre_nonce: &str, post_nonce: &str, difficulty: u64, libraries:Vec<PluginLibrary>) -> Delegator {
		Delegator {
			shared_data: Arc::new(RwLock::new(JobSharedData::new(
				job_id,
				pre_nonce,
				post_nonce,
				difficulty,
			))),
			control_data: Arc::new(RwLock::new(JobControlData::default())),
			libraries: Arc::new(RwLock::new(libraries)),
		}
	}

	/// Starts the job loop, and initialises the internal plugin

	pub fn start_job_loop(self) -> Result<CuckooMinerJobHandle, CuckooMinerError> {
		let _=env_logger::init();
		// this will block, waiting until previous job is cleared
		// call_cuckoo_stop_processing();

		let shared_data = self.shared_data.clone();
		let control_data = self.control_data.clone();
		let jh_library = self.libraries.clone();

		thread::spawn(move || {
			let result = self.job_loop();
			if let Err(e) = result {
				error!("Error in job loop: {:?}", e);
			}
		});
		Ok(CuckooMinerJobHandle {
			shared_data: shared_data,
			control_data: control_data,
			library: jh_library,
		})
	}

	/// Helper to convert a hex string

	fn from_hex_string(&self, in_str: &str) -> Vec<u8> {
		let mut bytes = Vec::new();
		for i in 0..(in_str.len() / 2) {
			let res = u8::from_str_radix(&in_str[2 * i..2 * i + 2], 16);
			match res {
				Ok(v) => bytes.push(v),
				Err(e) => println!("Problem with hex: {}", e),
			}
		}
		bytes
	}

	/// Helper that returns a hash generated by the header parts and a
	/// nonce

	fn get_hash(&self, pre_nonce: &str, post_nonce: &str, nonce: u64) -> [u8; 32] {
		// Turn input strings into vectors
		let mut pre_vec = self.from_hex_string(pre_nonce);
		let mut post_vec = self.from_hex_string(post_nonce);

		// println!("nonce: {}", nonce);
		let mut nonce_bytes = [0; 8];
		BigEndian::write_u64(&mut nonce_bytes, nonce);
		let mut nonce_vec = nonce_bytes.to_vec();

		// Generate new header
		pre_vec.append(&mut nonce_vec);
		pre_vec.append(&mut post_vec);

		// println!("pre-vec: {:?}", pre_vec);

		// Hash
		let mut blake2b = Blake2b::new(32);
		blake2b.update(&pre_vec);

		let mut ret = [0; 32];
		ret.copy_from_slice(blake2b.finalize().as_bytes());
		ret
	}

	/// helper that generates a nonce and returns a header

	fn get_next_hash(&self, pre_nonce: &str, post_nonce: &str) -> (u64, [u8; 32]) {
		// Generate new nonce
		let nonce: u64 = rand::OsRng::new().unwrap().gen();
		(nonce, self.get_hash(pre_nonce, post_nonce, nonce))
	}

	/// Helper to determing whether a solution meets a target difficulty
	/// based on same algorithm from grin

	fn meets_difficulty(&self, in_difficulty: u64, sol: CuckooMinerSolution) -> bool {
		let max_target = BigEndian::read_u64(&MAX_TARGET);
		let num = BigEndian::read_u64(&sol.hash()[0..8]);
		max_target / num >= in_difficulty
	}

	/// The main job loop. Pushes hashes to the plugin and reads solutions
	/// from the queue, putting them into the job's output queue. Continues
	/// until another thread sets the is_running flag to false

	fn job_loop(self) -> Result<(), CuckooMinerError> {
		// keep some unchanging data here, can move this out of shared
		// object later if it's not needed anywhere else
		let pre_nonce: String;
		let post_nonce: String;
		let difficulty;
		{
			let s = self.shared_data.read().unwrap();
			pre_nonce = s.pre_nonce.clone();
			post_nonce = s.post_nonce.clone();
			difficulty = s.difficulty;
		}
		debug!(
			"Cuckoo-miner: Searching for solution >= difficulty {}",
			difficulty
		);

		for l in self.libraries.read().unwrap().iter() {
			l.call_cuckoo_start_processing();
		}

		debug!("Cuckoo Miner Job loop processing");
		let mut solution = CuckooMinerSolution::new();

		loop {
			// Check if it's time to stop
			{
				let s = self.control_data.read().unwrap();
				if s.stop_flag {
					break;
				}
			}
			for l in self.libraries.read().unwrap().iter() {
				while l.call_cuckoo_is_queue_under_limit() == 1 {
					let (nonce, hash) = self.get_next_hash(&pre_nonce, &post_nonce);
					// println!("Hash thread 1: {:?}", hash);
					// TODO: make this a serialise operation instead
					let nonce_bytes: [u8; 8] = unsafe { transmute(nonce.to_be()) };
					l.call_cuckoo_push_to_input_queue(&hash, &nonce_bytes);
				}
			}

			let mut plugin_index=0;
			for l in self.libraries.read().unwrap().iter() {
				while l.call_cuckoo_read_from_output_queue(
					&mut solution.solution_nonces,
					&mut solution.nonce,
				) != 0
				{
					// TODO: make this a serialise operation instead
					let nonce = unsafe { transmute::<[u8; 8], u64>(solution.nonce) }.to_be();

					if self.meets_difficulty(difficulty, solution) {
						debug!(
							"Cuckoo-miner plugin[{}]: Solution Found for Nonce:({}), {:?}",
							plugin_index,
							nonce,
							solution
						);
						let mut s = self.shared_data.write().unwrap();
						s.solutions.push(solution.clone());
						plugin_index+=1;
					}

				}
			}
			//avoid busy wait
			let sleep_dur = time::Duration::from_millis(100);
			thread::sleep(sleep_dur);
		}

		// Do any cleanup
		for l in self.libraries.read().unwrap().iter() {
			l.call_cuckoo_stop_processing();
		}
		for l in self.libraries.read().unwrap().iter() {
			//wait for internal processing to finish
			while l.call_cuckoo_has_processing_stopped()==0{
				thread::sleep(time::Duration::from_millis(1));
			};
			l.call_cuckoo_reset_processing();
		}
		let mut s = self.control_data.write().unwrap();
		s.has_stopped=true;
		Ok(())
	}
}
<commit_msg>remove redundant hashing<commit_after>// Copyright 2017 The Grin Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Internal module responsible for job delegation, creating hashes
//! and sending them to the plugin's internal queues. Used internally
//!
//!

use std::sync::{Arc, RwLock};
use std::{thread, time};
use std::mem::transmute;

use rand::{self, Rng};
use byteorder::{ByteOrder, BigEndian};
use env_logger;

use cuckoo_sys::manager::PluginLibrary;
use error::error::CuckooMinerError;
use CuckooMinerJobHandle;
use CuckooMinerSolution;

/// From grin
/// The target is the 8-bytes hash block hashes must be lower than.
const MAX_TARGET: [u8; 8] = [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff];

type JobSharedDataType = Arc<RwLock<JobSharedData>>;
type JobControlDataType = Arc<RwLock<JobControlData>>;
type PluginLibrariesDataType = Arc<RwLock<Vec<PluginLibrary>>>;

/// Data intended to be shared across threads
pub struct JobSharedData {
	/// ID of the current running job (not currently used)
	pub job_id: u32,

	/// The part of the header before the nonce, which this
	/// module will mutate in search of a solution
	pub pre_nonce: String,

	/// The part of the header after the nonce
	pub post_nonce: String,

	/// The target difficulty. Only solutions >= this
	/// target will be put into the output queue
	pub difficulty: u64,

	/// Output solutions
	pub solutions: Vec<CuckooMinerSolution>,
}

impl Default for JobSharedData {
	fn default() -> JobSharedData {
		JobSharedData {
			job_id: 0,
			pre_nonce: String::from(""),
			post_nonce: String::from(""),
			difficulty: 0,
			solutions: Vec::new(),
		}
	}
}

impl JobSharedData {
	pub fn new(job_id: u32, pre_nonce: &str, post_nonce: &str, difficulty: u64) -> JobSharedData {
		JobSharedData {
			job_id: job_id,
			pre_nonce: String::from(pre_nonce),
			post_nonce: String::from(post_nonce),
			difficulty: difficulty,
			solutions: Vec::new(),
		}
	}
}

/// an internal structure to flag job control

pub struct JobControlData {
	/// Whether the mining job is running
	pub stop_flag: bool,

	/// Whether all plugins have stopped
	pub has_stopped: bool,
}

impl Default for JobControlData {
	fn default() -> JobControlData {
		JobControlData {
			stop_flag: false,
			has_stopped: false,
		}
	}
}

/// Internal structure which controls and runs processing jobs.
///
///

pub struct Delegator {
	/// Data which is shared across all threads
	shared_data: JobSharedDataType,

	/// Job control flags which are shared across threads
	control_data: JobControlDataType,

	/// Loaded Plugin Library
	libraries: PluginLibrariesDataType,
}

impl Delegator {
	/// Create a new job delegator

	pub fn new(job_id: u32, pre_nonce: &str, post_nonce: &str, difficulty: u64, libraries:Vec<PluginLibrary>) -> Delegator {
		Delegator {
			shared_data: Arc::new(RwLock::new(JobSharedData::new(
				job_id,
				pre_nonce,
				post_nonce,
				difficulty,
			))),
			control_data: Arc::new(RwLock::new(JobControlData::default())),
			libraries: Arc::new(RwLock::new(libraries)),
		}
	}

	/// Starts the job loop, and initialises the internal plugin

	pub fn start_job_loop(self) -> Result<CuckooMinerJobHandle, CuckooMinerError> {
		let _=env_logger::init();
		// this will block, waiting until previous job is cleared
		// call_cuckoo_stop_processing();

		let shared_data = self.shared_data.clone();
		let control_data = self.control_data.clone();
		let jh_library = self.libraries.clone();

		thread::spawn(move || {
			let result = self.job_loop();
			if let Err(e) = result {
				error!("Error in job loop: {:?}", e);
			}
		});
		Ok(CuckooMinerJobHandle {
			shared_data: shared_data,
			control_data: control_data,
			library: jh_library,
		})
	}

	/// Helper to convert a hex string

	fn from_hex_string(&self, in_str: &str) -> Vec<u8> {
		let mut bytes = Vec::new();
		for i in 0..(in_str.len() / 2) {
			let res = u8::from_str_radix(&in_str[2 * i..2 * i + 2], 16);
			match res {
				Ok(v) => bytes.push(v),
				Err(e) => println!("Problem with hex: {}", e),
			}
		}
		bytes
	}

	/// Return header data in the format expected by the plugin
	fn header_data(&self, pre_nonce: &str, post_nonce: &str, nonce: u64) -> Vec<u8> {
		// Turn input strings into vectors
		let mut pre_vec = self.from_hex_string(pre_nonce);
		let mut post_vec = self.from_hex_string(post_nonce);

		let mut nonce_bytes = [0; 8];
		BigEndian::write_u64(&mut nonce_bytes, nonce);
		let mut nonce_vec = nonce_bytes.to_vec();

		// Generate new header
		pre_vec.append(&mut nonce_vec);
		pre_vec.append(&mut post_vec);

		pre_vec
	}

	/// Retrieve next potential chunk of header data
	fn get_next_header_data(&self, pre_nonce: &str, post_nonce: &str) -> (u64, Vec<u8>) {
		let nonce: u64 = rand:: OsRng::new().unwrap().gen();
		(nonce, self.header_data(pre_nonce, post_nonce, nonce))
	}

	/// Helper to determing whether a solution meets a target difficulty
	/// based on same algorithm from grin

	fn meets_difficulty(&self, in_difficulty: u64, sol: CuckooMinerSolution) -> bool {
		let max_target = BigEndian::read_u64(&MAX_TARGET);
		let num = BigEndian::read_u64(&sol.hash()[0..8]);
		max_target / num >= in_difficulty
	}

	/// The main job loop. Pushes hashes to the plugin and reads solutions
	/// from the queue, putting them into the job's output queue. Continues
	/// until another thread sets the is_running flag to false

	fn job_loop(self) -> Result<(), CuckooMinerError> {
		// keep some unchanging data here, can move this out of shared
		// object later if it's not needed anywhere else
		let pre_nonce: String;
		let post_nonce: String;
		let difficulty;
		{
			let s = self.shared_data.read().unwrap();
			pre_nonce = s.pre_nonce.clone();
			post_nonce = s.post_nonce.clone();
			difficulty = s.difficulty;
		}
		debug!(
			"Cuckoo-miner: Searching for solution >= difficulty {}",
			difficulty
		);

		for l in self.libraries.read().unwrap().iter() {
			l.call_cuckoo_start_processing();
		}

		debug!("Cuckoo Miner Job loop processing");
		let mut solution = CuckooMinerSolution::new();

		loop {
			// Check if it's time to stop
			{
				let s = self.control_data.read().unwrap();
				if s.stop_flag {
					break;
				}
			}
			for l in self.libraries.read().unwrap().iter() {
				while l.call_cuckoo_is_queue_under_limit() == 1 {
					let (nonce, data) = self.get_next_header_data(&pre_nonce, &post_nonce);
					// println!("Hash thread 1: {:?}", hash);
					// TODO: make this a serialise operation instead
					let nonce_bytes: [u8; 8] = unsafe { transmute(nonce.to_be()) };
					l.call_cuckoo_push_to_input_queue(&data, &nonce_bytes);
				}
			}

			let mut plugin_index=0;
			for l in self.libraries.read().unwrap().iter() {
				while l.call_cuckoo_read_from_output_queue(
					&mut solution.solution_nonces,
					&mut solution.nonce,
				) != 0
				{
					// TODO: make this a serialise operation instead
					let nonce = unsafe { transmute::<[u8; 8], u64>(solution.nonce) }.to_be();

					if self.meets_difficulty(difficulty, solution) {
						debug!(
							"Cuckoo-miner plugin[{}]: Solution Found for Nonce:({}), {:?}",
							plugin_index,
							nonce,
							solution
						);
						let mut s = self.shared_data.write().unwrap();
						s.solutions.push(solution.clone());
						plugin_index+=1;
					}

				}
			}
			//avoid busy wait
			let sleep_dur = time::Duration::from_millis(100);
			thread::sleep(sleep_dur);
		}

		// Do any cleanup
		for l in self.libraries.read().unwrap().iter() {
			l.call_cuckoo_stop_processing();
		}
		for l in self.libraries.read().unwrap().iter() {
			//wait for internal processing to finish
			while l.call_cuckoo_has_processing_stopped()==0{
				thread::sleep(time::Duration::from_millis(1));
			};
			l.call_cuckoo_reset_processing();
		}
		let mut s = self.control_data.write().unwrap();
		s.has_stopped=true;
		Ok(())
	}
}
<|endoftext|>


<commit_before>use piston_window::{Context, Graphics, Texture, Transformed};
use piston_window::character::CharacterCache;

pub struct Score {
    score: u32,
    location: (f64, f64),
    color: [f32; 4],
    font_size: u32,
}

impl Score {
    pub fn new() -> Score {
        Score {
            score: 0,
            location: (320.0, 29.0),
            color: [0.0, 0.0, 0.0, 1.0],
            font_size: 16,
        }
    }

    pub fn add(&mut self, value: u32) {
        self.score = self.score + value;
    }

    pub fn render<C, G>(&self, cache: &mut C, context: Context, gfx: &mut G)
        where C: CharacterCache,
              G: Graphics<Texture = <C as CharacterCache>::Texture>
    {
        let transform = context.transform.trans(self.location.0 as f64, self.location.1 as f64);
        let text = format!("Score: {}", self.score);

        ::piston_window::Text::new_color(self.color, self.font_size)
            .draw(&text[..], cache, &context.draw_state, transform, gfx);
    }
}<commit_msg>Updated score rendering<commit_after>use piston_window::{Context, Graphics, Texture, Transformed};
use piston_window::character::CharacterCache;

pub struct Score {
    score: u32,
    location: (f64, f64),
    color: [f32; 4],
    font_size: u32,
}

impl Score {
    pub fn new() -> Score {
        Score {
            score: 0,
            location: (320.0, 29.0),
            color: [0.0, 0.0, 0.0, 1.0],
            font_size: 16,
        }
    }

    pub fn add(&mut self, value: u32) {
        self.score = self.score + value;
    }

    pub fn render<C, G>(&self, cache: &mut C, context: Context, gfx: &mut G)
        where C: CharacterCache,
              G: Graphics<Texture = <C as CharacterCache>::Texture>
    {
        let score_label_transform = context.transform
            .trans(self.location.0 as f64, self.location.1 as f64);

        let score_number_transform = context.transform
            .trans(self.location.0 as f64,
                   self.location.1 + (self.font_size + (self.font_size / 3)) as f64);

        let score = format!("{}", self.score);

        ::piston_window::Text::new_color(self.color, self.font_size - (self.font_size / 3))
            .draw("Score",
                  cache,
                  &context.draw_state,
                  score_label_transform,
                  gfx);

        ::piston_window::Text::new_color(self.color, self.font_size).draw(&score[..],
                                                                          cache,
                                                                          &context.draw_state,
                                                                          score_number_transform,
                                                                          gfx);
    }
}<|endoftext|>


<commit_before>extern crate rand;

pub struct Repeatedly<F> {
    pub f: F,
}

impl<T, F: FnMut() -> T> Iterator for Repeatedly<F> {
    type Item = T;

    fn next(&mut self) -> Option<T> {
        let mut f = &mut self.f;
        Some(f())
    }
}

/// takes a function `f` of no args, presumably with side effects, and
/// returns an iterator that endlessly calls `f` and yields the return value
pub fn repeatedly<T, F: FnMut() -> T>(f: F) -> Repeatedly<F> {
    Repeatedly { f: f }
}

#[test]
fn test_repeatedly() {
    use self::rand::{StdRng, SeedableRng};
    use helpers::random01;

    assert_eq!(vec![3, 3, 3, 3],
               repeatedly(|| 3).take(4).collect::<Vec<usize>>());

    let mut i = 0;
    assert_eq!(vec![1, 2, 3, 4],
               repeatedly(|| {
                   i += 1;
                   i
               })
                   .take(4)
                   .collect::<Vec<usize>>());
    assert_eq!(i, 4);

    let seed: &[_] = &[1, 2, 3, 4];
    let mut rng: StdRng = SeedableRng::from_seed(seed);
    assert_eq!(vec![0.5162139860908154, 0.13628294371987984, 0.21635575241586105, 0.10006169673911681],
               repeatedly(|| random01(&mut rng))
                   .take(4)
                   .collect::<Vec<f64>>());
}
<commit_msg>improve docstring<commit_after>extern crate rand;

pub struct Repeatedly<F> {
    pub f: F,
}

impl<T, F: FnMut() -> T> Iterator for Repeatedly<F> {
    type Item = T;

    fn next(&mut self) -> Option<T> {
        let mut f = &mut self.f;
        Some(f())
    }
}

/// takes a function `f` of no args, presumably with side effects, and
/// returns an iterator that endlessly calls `f` and yields what it returns
pub fn repeatedly<T, F: FnMut() -> T>(f: F) -> Repeatedly<F> {
    Repeatedly { f: f }
}

#[test]
fn test_repeatedly() {
    use self::rand::{StdRng, SeedableRng};
    use helpers::random01;

    assert_eq!(vec![3, 3, 3, 3],
               repeatedly(|| 3).take(4).collect::<Vec<usize>>());

    let mut i = 0;
    assert_eq!(vec![1, 2, 3, 4],
               repeatedly(|| {
                   i += 1;
                   i
               })
                   .take(4)
                   .collect::<Vec<usize>>());
    assert_eq!(i, 4);

    let seed: &[_] = &[1, 2, 3, 4];
    let mut rng: StdRng = SeedableRng::from_seed(seed);
    assert_eq!(vec![0.5162139860908154, 0.13628294371987984, 0.21635575241586105, 0.10006169673911681],
               repeatedly(|| random01(&mut rng))
                   .take(4)
                   .collect::<Vec<f64>>());
}
<|endoftext|>


<commit_before>use std::rc::Rc;
use std::cell::RefCell;
use std::default::Default;

use progress::frontier::{MutableAntichain, Antichain};
use progress::{Scope, Timestamp};
use progress::nested::subgraph::Source::{ScopeOutput};
use progress::count_map::CountMap;
use progress::timestamp::RootTimestamp;
use progress::nested::product::Product;

use communication::{Communicator, Data, Message, Observer};
use communication::observer::{Tee, Counter};

use construction::stream::Stream;
use construction::builder::*;

// TODO : This is an exogenous input, but it would be nice to wrap a Subgraph in something
// TODO : more like a harness, with direct access to its inputs.

// NOTE : This only takes a &self, not a &mut self, which works but is a bit weird.
// NOTE : Experiments with &mut indicate that the borrow of 'a lives for too long.
// NOTE : Might be able to fix with another lifetime parameter, say 'c: 'a.

// returns both an input scope and a stream representing its output.
pub trait InputExtensionTrait<C: Communicator, T: Timestamp+Ord> {
    fn new_input<D:Data>(&self) -> (InputHelper<T, D>, Stream<SubgraphBuilder<GraphRoot<C>, T>, D>);
}

impl<C: Communicator, T: Timestamp+Ord> InputExtensionTrait<C, T> for SubgraphBuilder<GraphRoot<C>, T> {
    fn new_input<D:Data>(&self) -> (InputHelper<T, D>, Stream<SubgraphBuilder<GraphRoot<C>, T>, D>) {

        let (output, registrar) = Tee::<Product<RootTimestamp, T>, D>::new();
        let produced = Rc::new(RefCell::new(CountMap::new()));

        let helper = InputHelper::new(Counter::new(output, produced.clone()));
        // {
        //     frontier: Rc::new(RefCell::new(MutableAntichain::new_bottom(Default::default()))),
        //     progress: Rc::new(RefCell::new(CountMap::new())),
        //     output:   Counter::new(output, produced.clone()),
        //     now_at:   Default::default(),
        //     closed:   false,
        // };

        let copies = self.peers();

        let index = self.add_scope(InputScope {
            frontier: helper.frontier.clone(),
            progress: helper.progress.clone(),
            messages: produced.clone(),
            copies:   copies,
        });

        return (helper, Stream::new(ScopeOutput(index, 0), registrar, self.clone()));
    }
}

pub struct InputScope<T:Timestamp+Ord> {
    frontier:   Rc<RefCell<MutableAntichain<Product<RootTimestamp, T>>>>,   // times available for sending
    progress:   Rc<RefCell<CountMap<Product<RootTimestamp, T>>>>,           // times closed since last asked
    messages:   Rc<RefCell<CountMap<Product<RootTimestamp, T>>>>,           // messages sent since last asked
    copies:     u64,
}

impl<T:Timestamp+Ord> Scope<Product<RootTimestamp, T>> for InputScope<T> {
    fn name(&self) -> String { format!("Input") }
    fn inputs(&self) -> u64 { 0 }
    fn outputs(&self) -> u64 { 1 }

    fn get_internal_summary(&mut self) -> (Vec<Vec<Antichain<<Product<RootTimestamp, T> as Timestamp>::Summary>>>,
                                           Vec<CountMap<Product<RootTimestamp, T>>>) {
        let mut map = CountMap::new();
        for x in self.frontier.borrow().elements().iter() {
            map.update(x, self.copies as i64);
        }
        (Vec::new(), vec![map])
    }

    fn pull_internal_progress(&mut self, frontier_progress: &mut [CountMap<Product<RootTimestamp, T>>],
                                        _messages_consumed: &mut [CountMap<Product<RootTimestamp, T>>],
                                         messages_produced: &mut [CountMap<Product<RootTimestamp, T>>]) -> bool
    {
        self.messages.borrow_mut().drain_into(&mut messages_produced[0]);
        self.progress.borrow_mut().drain_into(&mut frontier_progress[0]);
        return false;
    }

    fn notify_me(&self) -> bool { false }
}

/// Manages the movement of data into the dataflow from the outside world.
pub struct InputHelper<T: Timestamp+Ord, D: Data> {
    frontier: Rc<RefCell<MutableAntichain<Product<RootTimestamp, T>>>>,   // times available for sending
    progress: Rc<RefCell<CountMap<Product<RootTimestamp, T>>>>,           // times closed since last asked
    observer: Counter<Tee<Product<RootTimestamp, T>, D>>,
    buffer: Vec<D>,
    now_at: Option<Product<RootTimestamp, T>>,
}

// an input helper's state is either uninitialized, with now_at == None, or at some specific time.
// if now_at == None it has a hold on Default::default(), else it has a hold on the specific time.
// if now_at == None the observer has not been opened, else it is open with the specific time.

impl<T:Timestamp+Ord, D: Data> InputHelper<T, D> {
    fn new(observer: Counter<Tee<Product<RootTimestamp, T>, D>>) -> InputHelper<T, D> {
        InputHelper {
            frontier: Rc::new(RefCell::new(MutableAntichain::new_bottom(Default::default()))),
            progress: Rc::new(RefCell::new(CountMap::new())),
            observer: observer,
            buffer: Vec::with_capacity(4096),
            now_at: None,
        }
    }

    // flushes any data we are sitting on. may need to initialize self.now_at if no one has yet.
    fn flush(&mut self) {
        // if we haven't yet we should initialize now_at, to Default::default() because
        // we haven't been told anything better.
        if self.now_at.is_none() {
            self.now_at = Some(Default::default());
            self.observer.open(&Default::default());
            // we skip the frontier update because we would be adding and subtracting from the same
            // time: Default::default().
        }

        let mut message = Message::from_typed(&mut self.buffer);
        self.observer.give(&mut message);
        self.buffer = message.into_typed(4096);
        self.buffer.clear();
    }

    // closes the current epoch, flushing if needed, shutting if needed, and updating the frontier.
    fn close_epoch(&mut self) {
        // if we are sitting on any data, clear it out.
        if self.buffer.len() > 0 { self.flush(); }

        // shut things down if needed; release hold.
        if let Some(old_time) = self.now_at {
            self.observer.shut(&old_time);
            self.frontier.borrow_mut().update_weight(&old_time, -1, &mut (*self.progress.borrow_mut()));
        }
        else {
            let old_time = Default::default();
            self.frontier.borrow_mut().update_weight(&old_time, -1, &mut (*self.progress.borrow_mut()));
        }
    }

    #[inline(always)]
    pub fn give(&mut self, data: D) {
        self.buffer.push(data);
        if self.buffer.len() == self.buffer.capacity() {
            self.flush();
        }
    }

    pub fn advance_to(&mut self, next: T) {
        // attempting to reverse time's arrow is not supported.
        if let Some(time) = self.now_at { assert!(next >= time.inner); }

        self.close_epoch();

        // open up new epoch
        let new_time = Product::new(RootTimestamp, next);
        self.now_at = Some(new_time);
        self.observer.open(&new_time);
        self.frontier.borrow_mut().update_weight(&new_time,  1, &mut (*self.progress.borrow_mut()));
    }

    pub fn close(self) { }

    pub fn epoch(&mut self) -> &T {
        if self.now_at.is_none() {
            self.now_at = Some(Default::default());
            self.observer.open(&Default::default());
        }

        &self.now_at.as_ref().unwrap().inner
    }
}

impl<T:Timestamp+Ord, D: Data> Drop for InputHelper<T, D> {
    fn drop(&mut self) {
        self.close_epoch();
    }
}
<commit_msg>tidying up unused code<commit_after>use std::rc::Rc;
use std::cell::RefCell;
use std::default::Default;

use progress::frontier::{MutableAntichain, Antichain};
use progress::{Scope, Timestamp};
use progress::nested::subgraph::Source::{ScopeOutput};
use progress::count_map::CountMap;
use progress::timestamp::RootTimestamp;
use progress::nested::product::Product;

use communication::{Communicator, Data, Message, Observer};
use communication::observer::{Tee, Counter};

use construction::stream::Stream;
use construction::builder::*;

// TODO : This is an exogenous input, but it would be nice to wrap a Subgraph in something
// TODO : more like a harness, with direct access to its inputs.

// NOTE : This only takes a &self, not a &mut self, which works but is a bit weird.
// NOTE : Experiments with &mut indicate that the borrow of 'a lives for too long.
// NOTE : Might be able to fix with another lifetime parameter, say 'c: 'a.

// returns both an input scope and a stream representing its output.
pub trait InputExtensionTrait<C: Communicator, T: Timestamp+Ord> {
    fn new_input<D:Data>(&self) -> (InputHelper<T, D>, Stream<SubgraphBuilder<GraphRoot<C>, T>, D>);
}

impl<C: Communicator, T: Timestamp+Ord> InputExtensionTrait<C, T> for SubgraphBuilder<GraphRoot<C>, T> {
    fn new_input<D:Data>(&self) -> (InputHelper<T, D>, Stream<SubgraphBuilder<GraphRoot<C>, T>, D>) {

        let (output, registrar) = Tee::<Product<RootTimestamp, T>, D>::new();
        let produced = Rc::new(RefCell::new(CountMap::new()));
        let helper = InputHelper::new(Counter::new(output, produced.clone()));
        let copies = self.peers();

        let index = self.add_scope(InputScope {
            frontier: helper.frontier.clone(),
            progress: helper.progress.clone(),
            messages: produced.clone(),
            copies:   copies,
        });

        return (helper, Stream::new(ScopeOutput(index, 0), registrar, self.clone()));
    }
}

pub struct InputScope<T:Timestamp+Ord> {
    frontier:   Rc<RefCell<MutableAntichain<Product<RootTimestamp, T>>>>,   // times available for sending
    progress:   Rc<RefCell<CountMap<Product<RootTimestamp, T>>>>,           // times closed since last asked
    messages:   Rc<RefCell<CountMap<Product<RootTimestamp, T>>>>,           // messages sent since last asked
    copies:     u64,
}

impl<T:Timestamp+Ord> Scope<Product<RootTimestamp, T>> for InputScope<T> {
    fn name(&self) -> String { format!("Input") }
    fn inputs(&self) -> u64 { 0 }
    fn outputs(&self) -> u64 { 1 }

    fn get_internal_summary(&mut self) -> (Vec<Vec<Antichain<<Product<RootTimestamp, T> as Timestamp>::Summary>>>,
                                           Vec<CountMap<Product<RootTimestamp, T>>>) {
        let mut map = CountMap::new();
        for x in self.frontier.borrow().elements().iter() {
            map.update(x, self.copies as i64);
        }
        (Vec::new(), vec![map])
    }

    fn pull_internal_progress(&mut self, frontier_progress: &mut [CountMap<Product<RootTimestamp, T>>],
                                        _messages_consumed: &mut [CountMap<Product<RootTimestamp, T>>],
                                         messages_produced: &mut [CountMap<Product<RootTimestamp, T>>]) -> bool
    {
        self.messages.borrow_mut().drain_into(&mut messages_produced[0]);
        self.progress.borrow_mut().drain_into(&mut frontier_progress[0]);
        return false;
    }

    fn notify_me(&self) -> bool { false }
}

/// Manages the movement of data into the dataflow from the outside world.
pub struct InputHelper<T: Timestamp+Ord, D: Data> {
    frontier: Rc<RefCell<MutableAntichain<Product<RootTimestamp, T>>>>,   // times available for sending
    progress: Rc<RefCell<CountMap<Product<RootTimestamp, T>>>>,           // times closed since last asked
    observer: Counter<Tee<Product<RootTimestamp, T>, D>>,
    buffer: Vec<D>,
    now_at: Option<Product<RootTimestamp, T>>,
}

// an input helper's state is either uninitialized, with now_at == None, or at some specific time.
// if now_at == None it has a hold on Default::default(), else it has a hold on the specific time.
// if now_at == None the observer has not been opened, else it is open with the specific time.

impl<T:Timestamp+Ord, D: Data> InputHelper<T, D> {
    fn new(observer: Counter<Tee<Product<RootTimestamp, T>, D>>) -> InputHelper<T, D> {
        InputHelper {
            frontier: Rc::new(RefCell::new(MutableAntichain::new_bottom(Default::default()))),
            progress: Rc::new(RefCell::new(CountMap::new())),
            observer: observer,
            buffer: Vec::with_capacity(4096),
            now_at: None,
        }
    }

    // flushes any data we are sitting on. may need to initialize self.now_at if no one has yet.
    fn flush(&mut self) {
        // if we haven't yet we should initialize now_at, to Default::default() because
        // we haven't been told anything better.
        if self.now_at.is_none() {
            self.now_at = Some(Default::default());
            self.observer.open(&Default::default());
            // we skip the frontier update because we would be adding and subtracting from the same
            // time: Default::default().
        }

        let mut message = Message::from_typed(&mut self.buffer);
        self.observer.give(&mut message);
        self.buffer = message.into_typed(4096);
        self.buffer.clear();
    }

    // closes the current epoch, flushing if needed, shutting if needed, and updating the frontier.
    fn close_epoch(&mut self) {
        // if we are sitting on any data, clear it out.
        if self.buffer.len() > 0 { self.flush(); }

        // shut things down if needed; release hold.
        if let Some(old_time) = self.now_at {
            self.observer.shut(&old_time);
            self.frontier.borrow_mut().update_weight(&old_time, -1, &mut (*self.progress.borrow_mut()));
        }
        else {
            let old_time = Default::default();
            self.frontier.borrow_mut().update_weight(&old_time, -1, &mut (*self.progress.borrow_mut()));
        }
    }

    #[inline(always)]
    pub fn give(&mut self, data: D) {
        self.buffer.push(data);
        if self.buffer.len() == self.buffer.capacity() {
            self.flush();
        }
    }

    pub fn advance_to(&mut self, next: T) {
        // attempting to reverse time's arrow is not supported.
        if let Some(time) = self.now_at { assert!(next >= time.inner); }

        self.close_epoch();

        // open up new epoch
        let new_time = Product::new(RootTimestamp, next);
        self.now_at = Some(new_time);
        self.observer.open(&new_time);
        self.frontier.borrow_mut().update_weight(&new_time,  1, &mut (*self.progress.borrow_mut()));
    }

    pub fn close(self) { }

    pub fn epoch(&mut self) -> &T {
        if self.now_at.is_none() {
            self.now_at = Some(Default::default());
            self.observer.open(&Default::default());
        }

        &self.now_at.as_ref().unwrap().inner
    }
}

impl<T:Timestamp+Ord, D: Data> Drop for InputHelper<T, D> {
    fn drop(&mut self) {
        self.close_epoch();
    }
}
<|endoftext|>


<commit_before>//! Group records by a key, and apply a reduction function.
//!
//! The `group` operators act on data that can be viewed as pairs `(key, val)`. They group records
//! with the same key, and apply user supplied functions to the key and a list of values, which are
//! expected to populate a list of output values.
//!
//! Several variants of `group` exist which allow more precise control over how grouping is done.
//! For example, the `_by` suffixed variants take arbitrary data, but require a key-value selector
//! to be applied to each record. The `_u` suffixed variants use unsigned integers as keys, and
//! will use a dense array rather than a `HashMap` to store their keys.
//!
//! The list of values are presented as an iterator which internally merges sorted lists of values.
//! This ordering can be exploited in several cases to avoid computation when only the first few
//! elements are required.
//!
//! #Examples
//!
//! This example groups a stream of `(key,val)` pairs by `key`, and yields only the most frequently
//! occurring value for each key.
//!
//! ```ignore
//! stream.group(|key, vals, output| {
//!     let (mut max_val, mut max_wgt) = vals.next().unwrap();
//!     for (val, wgt) in vals {
//!         if wgt > max_wgt {
//!             max_wgt = wgt;
//!             max_val = val;
//!         }
//!     }
//!     output.push((max_val.clone(), max_wgt));
//! })
//! ```

use rc::Rc;
use std::cell::RefCell;
use std::default::Default;
use std::collections::HashMap;

use itertools::Itertools;
use linear_map::LinearMap;
use vec_map::VecMap;

use ::{Data, Collection, Delta};
use timely::dataflow::*;
use timely::dataflow::operators::{Map, Unary};
use timely::dataflow::channels::pact::Pipeline;
use timely_sort::Unsigned;

use collection::{LeastUpperBound, Lookup, Trace, BasicTrace, Offset};
use collection::trace::CollectionIterator;
use collection::basic::DifferenceIterator;
use collection::compact::Compact;

use iterators::coalesce::Coalesce;

use operators::arrange::{Arranged, ArrangeByKey};

/// Extension trait for the `group` differential dataflow method
pub trait Group<G: Scope, K: Data, V: Data>// : GroupByCore<G, (K,V)>
    where G::Timestamp: LeastUpperBound {

    /// Groups records by their first field, and applies reduction logic to the associated values.
    ///
    /// It would be nice for this to be generic over possible arrangements of data, but it seems that Rust
    /// ICEs when I try this. I'm not exactly sure how one would specify the arrangement (the type is used
    /// in `logic`, by way of its associated iterator types, but not clearly enough to drive type inference),
    /// but we don't have that problem yet anyhow.
    ///
    /// In any case, it would be great if the same implementation could handle `(K,V)` pairs and `K` elements
    /// treated as `(K, ())` pairs.
    fn group<L, V2: Data>(&self, logic: L) -> Collection<G, (K,V2)>
        where L: Fn(&K, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, Delta)>)+'static;
}

impl<G: Scope, K: Data+Default, V: Data+Default> Group<G, K, V> for Collection<G, (K,V)>
where G::Timestamp: LeastUpperBound
{
    fn group<L, V2: Data>(&self, logic: L) -> Collection<G, (K,V2)>
        where L: Fn(&K, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, Delta)>)+'static {
            self.arrange_by_key(|k| k.hashed(), |_| HashMap::new())
                .group(|k| k.hashed(), |_| HashMap::new(), logic)
                .as_collection()
    }
}

/// Extension trait for the `group_u` differential dataflow method.
pub trait GroupUnsigned<G: Scope, U: Unsigned+Data+Default, V: Data>
    where G::Timestamp: LeastUpperBound {
    /// Groups records by their first field, when this field implements `Unsigned`.
    ///
    /// This method uses a `Vec<Option<_>>` as its internal storage, allocating
    /// enough memory to directly index based on the first fields. This can be
    /// very useful when these fields are small integers, but it can be expensive
    /// if they are large and sparse.
    fn group_u<L, V2: Data>(&self, logic: L) -> Collection<G, (U, V2)>
        where L: Fn(&U, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, i32)>)+'static;
}

impl<G: Scope, U: Unsigned+Data+Default, V: Data> GroupUnsigned<G, U, V> for Collection<G, (U,V)>
where G::Timestamp: LeastUpperBound {
    fn group_u<L, V2: Data>(&self, logic: L) -> Collection<G, (U, V2)>
        where L: Fn(&U, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, i32)>)+'static {
            self.arrange_by_key(|k| k.as_u64(), |x| (VecMap::new(), x))
                .group(|k| k.as_u64(), |x| (VecMap::new(), x), logic)
                .as_collection()
        }
}

/// Extension trait for the `group` operator on `Arrange<_>` data.
pub trait GroupArranged<G: Scope, K: Data, V: Data> {
    /// Groups arranged data using a key hash function, a lookup generator, and user logic.
    ///
    /// This method is used by `group` and `group_u` as defined on `Collection`, and it can
    /// also be called directly by user code that wants access to the arranged output. This
    /// can be helpful when the resulting data are re-used immediately with the same keys.
    fn group<V2, U, KH, Look, LookG, Logic>(&self, key_h: KH, look: LookG, logic: Logic) -> Arranged<G, BasicTrace<K,G::Timestamp,V2,Look>>
    where
        G::Timestamp: LeastUpperBound,
        V2:    Data,
        U:     Unsigned+Default,
        KH:    Fn(&K)->U+'static,
        Look:  Lookup<K, Offset>+'static,
        LookG: Fn(u64)->Look,
        Logic: Fn(&K, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, i32)>)+'static;
}

impl<G, K, V, L> GroupArranged<G, K, V> for Arranged<G, BasicTrace<K, G::Timestamp, V, L>>
where
    G: Scope,
    K: Data,
    V: Data,
    L: Lookup<K, Offset>+'static,
    G::Timestamp: LeastUpperBound {

    fn group<V2, U, KH, Look, LookG, Logic>(&self, key_h: KH, look: LookG, logic: Logic) -> Arranged<G, BasicTrace<K,G::Timestamp,V2,Look>>
    where
        V2:    Data,
        U:     Unsigned+Default,
        KH:    Fn(&K)->U+'static,
        Look:  Lookup<K, Offset>+'static,
        LookG: Fn(u64)->Look,
        Logic: Fn(&K, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, i32)>)+'static {

        let peers = self.stream.scope().peers();
        let mut log_peers = 0;
        while (1 << (log_peers + 1)) <= peers {
            log_peers += 1;
        }

        let source = self.trace.clone();
        let result = Rc::new(RefCell::new(BasicTrace::new(look(log_peers))));
        let target = result.clone();

        // A map from times to received (key, val, wgt) triples.
        let mut inputs = LinearMap::new();

        // A map from times to a list of keys that need processing at that time.
        let mut to_do = LinearMap::new();

        // temporary storage for operator implementations to populate
        let mut buffer = vec![];

        // fabricate a data-parallel operator using the `unary_notify` pattern.
        let stream = self.stream.unary_notify(Pipeline, "GroupArranged", vec![], move |input, output, notificator| {

            // 1. read each input, and stash it in our staging area.
            // only stash the keys, because vals etc are in self.trace
            while let Some((time, data)) = input.next() {
                inputs.entry_or_insert(time.time(), || { notificator.notify_at(time); data.drain(..).next().unwrap().0 });
            }

            // 2. go through each time of interest that has reached completion
            // times are interesting either because we received data, or because we conclude
            // in the processing of a time that a future time will be interesting.
            while let Some((capability, _count)) = notificator.next() {

                let time = capability.time();

                // 2a. If we received any keys, determine the interesting times for each.
                //     We then enqueue the keys at the corresponding time, for use later.
                if let Some(queue) = inputs.remove_key(&time) {

                    let source = source.borrow();
                    let mut stash = Vec::new();
                    for key in queue {
                        if source.get_difference(&key, &time).is_some() {
                            stash.push(capability.time());
                            source.interesting_times(&key, &time, &mut stash);
                            for new_time in &stash {
                                to_do.entry_or_insert(time.clone(), || { notificator.notify_at(capability.delayed(new_time)); Vec::new() })
                                     .push(key.clone());
                            }
                            stash.clear();
                        }
                    }
                }

                // 2b. Process any interesting keys at this time.
                if let Some(mut keys) = to_do.remove_key(&time) {

                    // We would like these keys in a particular order.
                    // We also want to de-duplicate them, in case there are dupes.
                    keys.sort_by(|x,y| (key_h(&x), x).cmp(&(key_h(&y), y)));
                    keys.dedup();

                    // accumulations for installation into result
                    let mut accumulation = Compact::new(0,0);

                    // borrow `source` to avoid constant re-borrowing.
                    let mut source_borrow = source.borrow_mut();

                    for key in keys {

                        // acquire an iterator over the collection at `time`.
                        let mut input = source_borrow.get_collection(&key, &time);

                        // if we have some input data, invoke logic to populate buffer.
                        if input.peek().is_some() { logic(&key, &mut input, &mut buffer); }

                        // sort the buffer, because we can't trust user code to do that.
                        buffer.sort_by(|x,y| x.0.cmp(&y.0));

                        // push differences in to Compact.
                        let mut compact = accumulation.session();
                        for (val, wgt) in Coalesce::coalesce(target.borrow_mut()
                                                                   .get_collection(&key, &time)
                                                                   .map(|(v, w)| (v,-w))
                                                                   .merge_by(buffer.iter().map(|&(ref v, w)| (v, w)), |x,y| {
                                                                        x.0 <= y.0
                                                                   }))
                        {
                            compact.push(val.clone(), wgt);
                        }
                        compact.done(key);
                        buffer.clear();
                    }

                    if accumulation.vals.len() > 0 {
                        output.session(&capability).give((accumulation.keys.clone(), accumulation.cnts.clone(), accumulation.vals.clone()));
                        target.borrow_mut().set_difference(time, accumulation);
                    }
                }
            }
        });

        Arranged { stream: stream, trace: result }
    }
}
<commit_msg>tidying<commit_after>//! Group records by a key, and apply a reduction function.
//!
//! The `group` operators act on data that can be viewed as pairs `(key, val)`. They group records
//! with the same key, and apply user supplied functions to the key and a list of values, which are
//! expected to populate a list of output values.
//!
//! Several variants of `group` exist which allow more precise control over how grouping is done.
//! For example, the `_by` suffixed variants take arbitrary data, but require a key-value selector
//! to be applied to each record. The `_u` suffixed variants use unsigned integers as keys, and
//! will use a dense array rather than a `HashMap` to store their keys.
//!
//! The list of values are presented as an iterator which internally merges sorted lists of values.
//! This ordering can be exploited in several cases to avoid computation when only the first few
//! elements are required.
//!
//! #Examples
//!
//! This example groups a stream of `(key,val)` pairs by `key`, and yields only the most frequently
//! occurring value for each key.
//!
//! ```ignore
//! stream.group(|key, vals, output| {
//!     let (mut max_val, mut max_wgt) = vals.next().unwrap();
//!     for (val, wgt) in vals {
//!         if wgt > max_wgt {
//!             max_wgt = wgt;
//!             max_val = val;
//!         }
//!     }
//!     output.push((max_val.clone(), max_wgt));
//! })
//! ```

use rc::Rc;
use std::cell::RefCell;
use std::default::Default;
use std::collections::HashMap;

use itertools::Itertools;
use linear_map::LinearMap;
use vec_map::VecMap;

use ::{Data, Collection, Delta};
use timely::dataflow::*;
use timely::dataflow::operators::{Map, Unary};
use timely::dataflow::channels::pact::Pipeline;
use timely_sort::Unsigned;

use collection::{LeastUpperBound, Lookup, Trace, BasicTrace, Offset};
use collection::trace::CollectionIterator;
use collection::basic::DifferenceIterator;
use collection::compact::Compact;

use iterators::coalesce::Coalesce;

use operators::arrange::{Arranged, ArrangeByKey};

/// Extension trait for the `group` differential dataflow method
pub trait Group<G: Scope, K: Data, V: Data>
    where G::Timestamp: LeastUpperBound {

    /// Groups records by their first field, and applies reduction logic to the associated values.
    ///
    /// It would be nice for this to be generic over possible arrangements of data, but it seems that Rust
    /// ICEs when I try this. I'm not exactly sure how one would specify the arrangement (the type is used
    /// in `logic`, by way of its associated iterator types, but not clearly enough to drive type inference),
    /// but we don't have that problem yet anyhow.
    ///
    /// In any case, it would be great if the same implementation could handle `(K,V)` pairs and `K` elements
    /// treated as `(K, ())` pairs.
    fn group<L, V2: Data>(&self, logic: L) -> Collection<G, (K,V2)>
        where L: Fn(&K, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, Delta)>)+'static;
}

impl<G: Scope, K: Data+Default, V: Data+Default> Group<G, K, V> for Collection<G, (K,V)>
where G::Timestamp: LeastUpperBound
{
    fn group<L, V2: Data>(&self, logic: L) -> Collection<G, (K,V2)>
        where L: Fn(&K, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, Delta)>)+'static {
            self.arrange_by_key(|k| k.hashed(), |_| HashMap::new())
                .group(|k| k.hashed(), |_| HashMap::new(), logic)
                .as_collection()
    }
}

/// Extension trait for the `group_u` differential dataflow method.
pub trait GroupUnsigned<G: Scope, U: Unsigned+Data+Default, V: Data>
    where G::Timestamp: LeastUpperBound {
    /// Groups records by their first field, when this field implements `Unsigned`.
    ///
    /// This method uses a `Vec<Option<_>>` as its internal storage, allocating
    /// enough memory to directly index based on the first fields. This can be
    /// very useful when these fields are small integers, but it can be expensive
    /// if they are large and sparse.
    fn group_u<L, V2: Data>(&self, logic: L) -> Collection<G, (U, V2)>
        where L: Fn(&U, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, i32)>)+'static;
}

impl<G: Scope, U: Unsigned+Data+Default, V: Data> GroupUnsigned<G, U, V> for Collection<G, (U,V)>
where G::Timestamp: LeastUpperBound {
    fn group_u<L, V2: Data>(&self, logic: L) -> Collection<G, (U, V2)>
        where L: Fn(&U, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, i32)>)+'static {
            self.arrange_by_key(|k| k.as_u64(), |x| (VecMap::new(), x))
                .group(|k| k.as_u64(), |x| (VecMap::new(), x), logic)
                .as_collection()
        }
}

/// Extension trait for the `group` operator on `Arrange<_>` data.
pub trait GroupArranged<G: Scope, K: Data, V: Data> {
    /// Groups arranged data using a key hash function, a lookup generator, and user logic.
    ///
    /// This method is used by `group` and `group_u` as defined on `Collection`, and it can
    /// also be called directly by user code that wants access to the arranged output. This
    /// can be helpful when the resulting data are re-used immediately with the same keys.
    fn group<V2, U, KH, Look, LookG, Logic>(&self, key_h: KH, look: LookG, logic: Logic)
        -> Arranged<G, BasicTrace<K,G::Timestamp,V2,Look>>
    where
        G::Timestamp: LeastUpperBound,
        V2:    Data,
        U:     Unsigned+Default,
        KH:    Fn(&K)->U+'static,
        Look:  Lookup<K, Offset>+'static,
        LookG: Fn(u64)->Look,
        Logic: Fn(&K, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, i32)>)+'static;
}

impl<G, K, V, L> GroupArranged<G, K, V> for Arranged<G, BasicTrace<K, G::Timestamp, V, L>>
where
    G: Scope,
    K: Data,
    V: Data,
    L: Lookup<K, Offset>+'static,
    G::Timestamp: LeastUpperBound {

    fn group<V2, U, KH, Look, LookG, Logic>(&self, key_h: KH, look: LookG, logic: Logic)
        -> Arranged<G, BasicTrace<K,G::Timestamp,V2,Look>>
    where
        V2:    Data,
        U:     Unsigned+Default,
        KH:    Fn(&K)->U+'static,
        Look:  Lookup<K, Offset>+'static,
        LookG: Fn(u64)->Look,
        Logic: Fn(&K, &mut CollectionIterator<DifferenceIterator<V>>, &mut Vec<(V2, i32)>)+'static {

        let peers = self.stream.scope().peers();
        let mut log_peers = 0;
        while (1 << (log_peers + 1)) <= peers {
            log_peers += 1;
        }

        let source = self.trace.clone();
        let result = Rc::new(RefCell::new(BasicTrace::new(look(log_peers))));
        let target = result.clone();

        // A map from times to received (key, val, wgt) triples.
        let mut inputs = LinearMap::new();

        // A map from times to a list of keys that need processing at that time.
        let mut to_do = LinearMap::new();

        // temporary storage for operator implementations to populate
        let mut buffer = vec![];

        // fabricate a data-parallel operator using the `unary_notify` pattern.
        let stream = self.stream.unary_notify(Pipeline, "GroupArranged", vec![], move |input, output, notificator| {

            // 1. read each input, and stash it in our staging area.
            // only stash the keys, because vals etc are in self.trace
            while let Some((time, data)) = input.next() {
                inputs.entry_or_insert(time.time(), || { notificator.notify_at(time); data.drain(..).next().unwrap().0 });
            }

            // 2. go through each time of interest that has reached completion
            // times are interesting either because we received data, or because we conclude
            // in the processing of a time that a future time will be interesting.
            while let Some((capability, _count)) = notificator.next() {

                let time = capability.time();

                // 2a. If we received any keys, determine the interesting times for each.
                //     We then enqueue the keys at the corresponding time, for use later.
                if let Some(queue) = inputs.remove_key(&time) {

                    let source = source.borrow();
                    let mut stash = Vec::new();
                    for key in queue {
                        if source.get_difference(&key, &time).is_some() {
                            stash.push(capability.time());
                            source.interesting_times(&key, &time, &mut stash);
                            for new_time in &stash {
                                to_do.entry_or_insert(time.clone(), || { notificator.notify_at(capability.delayed(new_time)); Vec::new() })
                                     .push(key.clone());
                            }
                            stash.clear();
                        }
                    }
                }

                // 2b. Process any interesting keys at this time.
                if let Some(mut keys) = to_do.remove_key(&time) {

                    // We would like these keys in a particular order.
                    // We also want to de-duplicate them, in case there are dupes.
                    keys.sort_by(|x,y| (key_h(&x), x).cmp(&(key_h(&y), y)));
                    keys.dedup();

                    // accumulations for installation into result
                    let mut accumulation = Compact::new(0,0);

                    // borrow `source` to avoid constant re-borrowing.
                    let mut source_borrow = source.borrow_mut();

                    for key in keys {

                        // acquire an iterator over the collection at `time`.
                        let mut input = source_borrow.get_collection(&key, &time);

                        // if we have some input data, invoke logic to populate buffer.
                        if input.peek().is_some() { logic(&key, &mut input, &mut buffer); }

                        // sort the buffer, because we can't trust user code to do that.
                        buffer.sort_by(|x,y| x.0.cmp(&y.0));

                        // push differences in to Compact.
                        let mut compact = accumulation.session();
                        for (val, wgt) in Coalesce::coalesce(target.borrow_mut()
                                                                   .get_collection(&key, &time)
                                                                   .map(|(v, w)| (v,-w))
                                                                   .merge_by(buffer.iter().map(|&(ref v, w)| (v, w)), |x,y| {
                                                                        x.0 <= y.0
                                                                   }))
                        {
                            compact.push(val.clone(), wgt);
                        }
                        compact.done(key);
                        buffer.clear();
                    }

                    if accumulation.vals.len() > 0 {
                        output.session(&capability).give((accumulation.keys.clone(), accumulation.cnts.clone(), accumulation.vals.clone()));
                        target.borrow_mut().set_difference(time, accumulation);
                    }
                }
            }
        });

        Arranged { stream: stream, trace: result }
    }
}
<|endoftext|>


<commit_before>use core::intrinsics::{breakpoint, volatile_load, volatile_store};
use core::mem;
use core::raw::{Repr,Slice};

use common::{RingBuffer, Queue};

use container;

#[allow(improper_ctypes)]
extern {
    pub fn switch_to_user(user_stack: *const u8, mem_base: *const u8) -> *mut u8;
}

/// Size of each processes's memory region in bytes
pub const PROC_MEMORY_SIZE : usize = 8192;
pub const NUM_PROCS : usize = 2;

static mut FREE_MEMORY_IDX: usize = 0;

#[link_section = ".app_memory"]
static mut MEMORIES: [[u8; PROC_MEMORY_SIZE]; NUM_PROCS] = [[0; PROC_MEMORY_SIZE]; NUM_PROCS];

pub static mut PROCS : [Option<Process<'static>>; NUM_PROCS] = [None, None];

pub fn schedule(callback: Callback, appid: ::AppId) -> bool {
    let procs = unsafe { &mut PROCS };
    let idx = appid.idx();
    if idx >= procs.len() {
        return false
    }

    match procs[idx] {
        None => false,
        Some(ref mut p) => {
            // TODO(alevy): validate appid liveness
            p.callbacks.enqueue(callback)
        }
    }
}

#[derive(Copy,Clone,PartialEq,Eq)]
pub enum Error {
    NoSuchApp,
    OutOfMemory,
    AddressOutOfBounds
}

#[derive(Copy,Clone,PartialEq,Eq)]
pub enum State {
    Running,
    Waiting
}

#[derive(Copy,Clone,Debug)]
pub struct Callback {
    pub r0: usize,
    pub r1: usize,
    pub r2: usize,
    pub r3: usize,
    pub pc: usize
}

#[repr(C,packed)]
struct LoadInfo {
    rel_data_size: usize,
    entry_loc: usize,        /* Entry point for user application */
    init_data_loc: usize,    /* Data initialization information in flash */
    init_data_size: usize,    /* Size of initialization information */
    got_start_offset: usize,  /* Offset to start of GOT */
    got_end_offset: usize,    /* Offset to end of GOT */
    bss_start_offset: usize,  /* Offset to start of BSS */
    bss_end_offset: usize    /* Offset to end of BSS */
}

pub struct Process<'a> {
    /// The process's memory.
    memory: Slice<u8>,

    app_memory_break: *const u8,
    kernel_memory_break: *const u8,

    /// Process text segment
    text: Slice<u8>,

    /// The offset in `memory` to use for the process stack.
    cur_stack: *const u8,

    wait_pc: usize,
    psr: usize,

    pub state: State,

    pub callbacks: RingBuffer<'a, Callback>
}

#[inline(never)]
pub unsafe fn load_processes(mut start_addr: *const usize) ->
        &'static mut [Option<Process<'static>>] {
    for op in PROCS.iter_mut() {
        if *start_addr != 0 {
            let prog_start = start_addr.offset(1);
            let length = *start_addr as isize;
            start_addr = (start_addr as *const u8).offset(length) as *const usize;

            *op = Process::create(prog_start);
        } else {
            *op = None;
        }
    }
    &mut PROCS
}

impl<'a> Process<'a> {
    pub const fn mem_start(&self) -> *const u8 {
        self.memory.data
    }

    pub fn mem_end(&self) -> *const u8 {
        unsafe {
            self.memory.data.offset(self.memory.len as isize)
        }
    }

    pub unsafe fn create(start_addr: *const usize) -> Option<Process<'a>> {
        let cur_idx = FREE_MEMORY_IDX;
        if cur_idx <= MEMORIES.len() {
            FREE_MEMORY_IDX += 1;
            let memory = MEMORIES[cur_idx].repr();

            let mut kernel_memory_break = {
                // make room for container pointers
                let psz = mem::size_of::<*const usize>();
                let num_ctrs = volatile_load(&container::CONTAINER_COUNTER);
                let container_ptrs_size = num_ctrs * psz;
                let res = memory.data.offset((memory.len - container_ptrs_size) as isize);
                // set all ptrs to null
                let opts : &mut [*const usize] = mem::transmute(Slice {
                    data: res as *mut *const usize,
                    len: num_ctrs
                });
                for opt in opts.iter_mut() {
                    *opt = ::core::ptr::null()
                }
                res
            };

            // Take callback buffer from of memory
            let callback_size = mem::size_of::<Option<Callback>>();
            let callback_len = 10;
            let callback_offset = callback_len * callback_size;
            // Set kernel break to beginning of callback buffer
            kernel_memory_break =
                kernel_memory_break.offset(-(callback_offset as isize));
            let callback_buf = mem::transmute(Slice {
                data: kernel_memory_break as *const Option<Callback>,
                len: callback_len
            });

            let callbacks = RingBuffer::new(callback_buf);

            let load_result = load(start_addr, memory.data);

            let stack_bottom = load_result.app_mem_start.offset(512);

            let mut process = Process {
                memory: memory,
                app_memory_break: stack_bottom,
                kernel_memory_break: kernel_memory_break,
                text: Slice {
                    data: load_result.text_start,
                    len: load_result.text_len },
                cur_stack: stack_bottom,
                wait_pc: 0,
                psr: 0x01000000,
                state: State::Waiting,
                callbacks: callbacks
            };

            process.callbacks.enqueue(Callback {
                pc: load_result.init_fn,
                r0: load_result.app_mem_start as usize,
                r1: process.app_memory_break as usize,
                r2: process.kernel_memory_break as usize,
                r3: 0
            });

            Some(process)
        } else {
            None
        }
    }

    pub fn sbrk(&mut self, increment: isize) -> Result<*const u8, Error> {
        let new_break = unsafe { self.app_memory_break.offset(increment) };
        self.brk(new_break)
    }

    pub fn brk(&mut self, new_break: *const u8) -> Result<*const u8, Error> {
        if new_break < self.mem_start() || new_break >= self.mem_end() {
            Err(Error::AddressOutOfBounds)
        } else if new_break > self.kernel_memory_break {
            Err(Error::OutOfMemory)
        } else {
            let old_break = self.app_memory_break;
            self.app_memory_break = new_break;
            Ok(old_break)
        }
    }

    pub fn in_exposed_bounds(&self, buf_start_addr: *const u8, size: usize)
            -> bool {

        let buf_end_addr = ((buf_start_addr as usize) + size) as *const u8;

        let mem_end =
            ((self.memory.data as usize) + self.memory.len) as *const u8;

        buf_start_addr >= self.memory.data && buf_end_addr <= mem_end
    }

    pub unsafe fn alloc(&mut self, size: usize) -> Option<&mut [u8]> {
        let new_break = self.kernel_memory_break.offset(-(size as isize));
        if new_break < self.app_memory_break {
            None
        } else {
            self.kernel_memory_break = new_break;
            Some(mem::transmute(Slice {
                data: new_break as *mut u8,
                len: size
            }))
        }
    }

    pub unsafe fn free<T>(&mut self, _: *mut T) {}

    pub unsafe fn container_for<T>(&mut self, container_num: usize)
            -> *mut *mut T {
        let container_num = container_num as isize;
        let ptr = (self.mem_end() as *mut usize)
                        .offset(-(container_num + 1));
        ptr as *mut *mut T
    }

    pub unsafe fn container_for_or_alloc<T: Default>(&mut self,
                                                     container_num: usize)
            -> Option<*mut T> {
        let ctr_ptr = self.container_for::<T>(container_num);
        if (*ctr_ptr).is_null() {
            self.alloc(mem::size_of::<T>()).map(|root_arr| {
                let root_ptr = root_arr.repr().data as *mut T;
                *root_ptr = Default::default();
                volatile_store(ctr_ptr, root_ptr);
                root_ptr
            })
        } else {
            Some(*ctr_ptr)
        }
    }

    pub fn pop_syscall_stack(&mut self) {
        let pspr = self.cur_stack as *const usize;
        unsafe {
            self.wait_pc = volatile_load(pspr.offset(6));
            self.psr = volatile_load(pspr.offset(7));
            self.cur_stack =
                (self.cur_stack as *mut usize).offset(8) as *mut u8;
        }
    }

    /// Context switch to the process.
    pub unsafe fn switch_to_callback(&mut self, callback: Callback) {
        // Fill in initial stack expected by SVC handler
        // Top minus 8 u32s for r0-r3, r12, lr, pc and xPSR
        let stack_bottom = (self.cur_stack as *mut usize).offset(-8);
        volatile_store(stack_bottom.offset(7), self.psr);
        volatile_store(stack_bottom.offset(6), callback.pc | 1);
        // Set the LR register to the saved PC so the callback returns to
        // wherever wait was called. Set lowest bit to one because of THUMB
        // instruction requirements.
        volatile_store(stack_bottom.offset(5), self.wait_pc | 0x1);
        volatile_store(stack_bottom, callback.r0);
        volatile_store(stack_bottom.offset(1), callback.r1);
        volatile_store(stack_bottom.offset(2), callback.r2);
        volatile_store(stack_bottom.offset(3), callback.r3);

        self.cur_stack = stack_bottom as *mut u8;
        self.switch_to();
    }

    /// Context switch to the process.
    pub unsafe fn switch_to(&mut self) {
        if self.cur_stack < self.memory.data {
            breakpoint();
        }
        let psp = switch_to_user(self.cur_stack, self.memory.data);
        self.cur_stack = psp;
    }

    pub fn svc_number(&self) -> Option<u8> {
        let psp = self.cur_stack as *const *const u16;
        unsafe {
            let pcptr = volatile_load((psp as *const *const u16).offset(6));
            let svc_instr = volatile_load(pcptr.offset(-1));
            Some((svc_instr & 0xff) as u8)
        }
    }

    pub fn lr(&self) -> usize {
        let pspr = self.cur_stack as *const usize;
        unsafe { volatile_load(pspr.offset(5)) }
    }

    pub fn r0(&self) -> usize {
        let pspr = self.cur_stack as *const usize;
        unsafe { volatile_load(pspr) }
    }

    pub fn set_r0(&mut self, val: isize) {
        let pspr = self.cur_stack as *mut isize;
        unsafe { volatile_store(pspr, val) }
    }

    pub fn r1(&self) -> usize {
        let pspr = self.cur_stack as *const usize;
        unsafe { volatile_load(pspr.offset(1)) }
    }

    pub fn r2(&self) -> usize {
        let pspr = self.cur_stack as *const usize;
        unsafe { volatile_load(pspr.offset(2)) }
    }

    pub fn r3(&self) -> usize {
        let pspr = self.cur_stack as *const usize;
        unsafe { volatile_load(pspr.offset(3)) }
    }

}

struct LoadResult {
    text_start: *const u8,
    text_len: usize,
    init_fn: usize,
    app_mem_start: *const u8
}

unsafe fn load(start_addr: *const usize, mem_base: *const u8) -> LoadResult {
    let mut result = LoadResult {
        text_start: 0 as *const u8,
        text_len: 0,
        init_fn: 0,
        app_mem_start: 0 as *const u8
    };

    let mut start_addr = start_addr as *const u8;
    let load_info : &LoadInfo = mem::transmute(start_addr);
    start_addr = start_addr.offset(mem::size_of::<LoadInfo>() as isize);

    let rel_data : &mut [usize] = mem::transmute(Slice{
        data: start_addr,
        len: load_info.rel_data_size / 4
    });
    start_addr = start_addr.offset(load_info.rel_data_size as isize);

    // Update text location in self
    result.text_start = start_addr;
    result.text_len = load_info.init_data_loc;

    // Zero out BSS
    ::core::intrinsics::write_bytes(
        mem_base.offset(load_info.bss_start_offset as isize) as *mut u8,
        0,
        load_info.bss_end_offset - load_info.bss_start_offset);

    // Copy data into Data section
    let init_data : &[u8] = mem::transmute(Slice{
        data: (load_info.init_data_loc + start_addr as usize) as *mut u8,
        len: load_info.init_data_size
    });

    let target_data : &mut [u8] = mem::transmute(Slice{
        data: mem_base,
        len: load_info.init_data_size
    });

    target_data.clone_from_slice(init_data);

    let fixup = |addr: *mut usize| {
        let entry = *addr;
        if (entry & 0x80000000) == 0 {
            // Regular data (memory relative)
            *addr = entry + (mem_base as usize);
        } else {
            // rodata or function pointer (code relative)
            *addr = (entry ^ 0x80000000) + (start_addr as usize);
        }
    };

    // Fixup Global Offset Table
    let mut got_cur = mem_base.offset(
        load_info.got_start_offset as isize) as *mut usize;
    let got_end = mem_base.offset(
        load_info.got_end_offset as isize) as *mut usize;
    while got_cur != got_end {
        fixup(got_cur);
        got_cur = got_cur.offset(1);
    }

    // Fixup relocation data
    for (i, addr) in rel_data.iter().enumerate() {
        if i % 2 == 0 { // Only the first of every 2 entries is an address
            fixup(mem_base.offset(*addr as isize) as *mut usize);
        }
    }

    // Entry point is offset from app code
    result.init_fn = start_addr as usize + load_info.entry_loc;

    let mut aligned_mem_start = load_info.bss_end_offset as isize;
    aligned_mem_start += (8 - (aligned_mem_start % 8)) % 8;
    result.app_mem_start = mem_base.offset(aligned_mem_start);

    result
}

<commit_msg>Derive `debug` for process to help with debug<commit_after>use core::intrinsics::{breakpoint, volatile_load, volatile_store};
use core::mem;
use core::raw::{Repr,Slice};

use common::{RingBuffer, Queue};

use container;

#[allow(improper_ctypes)]
extern {
    pub fn switch_to_user(user_stack: *const u8, mem_base: *const u8) -> *mut u8;
}

/// Size of each processes's memory region in bytes
pub const PROC_MEMORY_SIZE : usize = 8192;
pub const NUM_PROCS : usize = 2;

static mut FREE_MEMORY_IDX: usize = 0;

#[link_section = ".app_memory"]
static mut MEMORIES: [[u8; PROC_MEMORY_SIZE]; NUM_PROCS] = [[0; PROC_MEMORY_SIZE]; NUM_PROCS];

pub static mut PROCS : [Option<Process<'static>>; NUM_PROCS] = [None, None];

pub fn schedule(callback: Callback, appid: ::AppId) -> bool {
    let procs = unsafe { &mut PROCS };
    let idx = appid.idx();
    if idx >= procs.len() {
        return false
    }

    match procs[idx] {
        None => false,
        Some(ref mut p) => {
            // TODO(alevy): validate appid liveness
            p.callbacks.enqueue(callback)
        }
    }
}

#[derive(Copy,Clone,PartialEq,Eq)]
pub enum Error {
    NoSuchApp,
    OutOfMemory,
    AddressOutOfBounds
}

#[derive(Copy,Clone,PartialEq,Eq)]
pub enum State {
    Running,
    Waiting
}

#[derive(Copy,Clone,Debug)]
pub struct Callback {
    pub r0: usize,
    pub r1: usize,
    pub r2: usize,
    pub r3: usize,
    pub pc: usize
}

#[repr(C,packed)]
struct LoadInfo {
    rel_data_size: usize,
    entry_loc: usize,        /* Entry point for user application */
    init_data_loc: usize,    /* Data initialization information in flash */
    init_data_size: usize,    /* Size of initialization information */
    got_start_offset: usize,  /* Offset to start of GOT */
    got_end_offset: usize,    /* Offset to end of GOT */
    bss_start_offset: usize,  /* Offset to start of BSS */
    bss_end_offset: usize    /* Offset to end of BSS */
}

pub struct Process<'a> {
    /// The process's memory.
    memory: Slice<u8>,

    app_memory_break: *const u8,
    kernel_memory_break: *const u8,

    /// Process text segment
    text: Slice<u8>,

    /// The offset in `memory` to use for the process stack.
    cur_stack: *const u8,

    wait_pc: usize,
    psr: usize,

    pub state: State,

    pub callbacks: RingBuffer<'a, Callback>
}

#[inline(never)]
pub unsafe fn load_processes(mut start_addr: *const usize) ->
        &'static mut [Option<Process<'static>>] {
    for op in PROCS.iter_mut() {
        if *start_addr != 0 {
            let prog_start = start_addr.offset(1);
            let length = *start_addr as isize;
            start_addr = (start_addr as *const u8).offset(length) as *const usize;

            *op = Process::create(prog_start);
        } else {
            *op = None;
        }
    }
    &mut PROCS
}

impl<'a> Process<'a> {
    pub const fn mem_start(&self) -> *const u8 {
        self.memory.data
    }

    pub fn mem_end(&self) -> *const u8 {
        unsafe {
            self.memory.data.offset(self.memory.len as isize)
        }
    }

    pub unsafe fn create(start_addr: *const usize) -> Option<Process<'a>> {
        let cur_idx = FREE_MEMORY_IDX;
        if cur_idx <= MEMORIES.len() {
            FREE_MEMORY_IDX += 1;
            let memory = MEMORIES[cur_idx].repr();

            let mut kernel_memory_break = {
                // make room for container pointers
                let psz = mem::size_of::<*const usize>();
                let num_ctrs = volatile_load(&container::CONTAINER_COUNTER);
                let container_ptrs_size = num_ctrs * psz;
                let res = memory.data.offset((memory.len - container_ptrs_size) as isize);
                // set all ptrs to null
                let opts : &mut [*const usize] = mem::transmute(Slice {
                    data: res as *mut *const usize,
                    len: num_ctrs
                });
                for opt in opts.iter_mut() {
                    *opt = ::core::ptr::null()
                }
                res
            };

            // Take callback buffer from of memory
            let callback_size = mem::size_of::<Option<Callback>>();
            let callback_len = 10;
            let callback_offset = callback_len * callback_size;
            // Set kernel break to beginning of callback buffer
            kernel_memory_break =
                kernel_memory_break.offset(-(callback_offset as isize));
            let callback_buf = mem::transmute(Slice {
                data: kernel_memory_break as *const Option<Callback>,
                len: callback_len
            });

            let callbacks = RingBuffer::new(callback_buf);

            let load_result = load(start_addr, memory.data);

            let stack_bottom = load_result.app_mem_start.offset(512);

            let mut process = Process {
                memory: memory,
                app_memory_break: stack_bottom,
                kernel_memory_break: kernel_memory_break,
                text: Slice {
                    data: load_result.text_start,
                    len: load_result.text_len },
                cur_stack: stack_bottom,
                wait_pc: 0,
                psr: 0x01000000,
                state: State::Waiting,
                callbacks: callbacks
            };

            process.callbacks.enqueue(Callback {
                pc: load_result.init_fn,
                r0: load_result.app_mem_start as usize,
                r1: process.app_memory_break as usize,
                r2: process.kernel_memory_break as usize,
                r3: 0
            });

            Some(process)
        } else {
            None
        }
    }

    pub fn sbrk(&mut self, increment: isize) -> Result<*const u8, Error> {
        let new_break = unsafe { self.app_memory_break.offset(increment) };
        self.brk(new_break)
    }

    pub fn brk(&mut self, new_break: *const u8) -> Result<*const u8, Error> {
        if new_break < self.mem_start() || new_break >= self.mem_end() {
            Err(Error::AddressOutOfBounds)
        } else if new_break > self.kernel_memory_break {
            Err(Error::OutOfMemory)
        } else {
            let old_break = self.app_memory_break;
            self.app_memory_break = new_break;
            Ok(old_break)
        }
    }

    pub fn in_exposed_bounds(&self, buf_start_addr: *const u8, size: usize)
            -> bool {

        let buf_end_addr = ((buf_start_addr as usize) + size) as *const u8;

        let mem_end =
            ((self.memory.data as usize) + self.memory.len) as *const u8;

        buf_start_addr >= self.memory.data && buf_end_addr <= mem_end
    }

    pub unsafe fn alloc(&mut self, size: usize) -> Option<&mut [u8]> {
        let new_break = self.kernel_memory_break.offset(-(size as isize));
        if new_break < self.app_memory_break {
            None
        } else {
            self.kernel_memory_break = new_break;
            Some(mem::transmute(Slice {
                data: new_break as *mut u8,
                len: size
            }))
        }
    }

    pub unsafe fn free<T>(&mut self, _: *mut T) {}

    pub unsafe fn container_for<T>(&mut self, container_num: usize)
            -> *mut *mut T {
        let container_num = container_num as isize;
        let ptr = (self.mem_end() as *mut usize)
                        .offset(-(container_num + 1));
        ptr as *mut *mut T
    }

    pub unsafe fn container_for_or_alloc<T: Default>(&mut self,
                                                     container_num: usize)
            -> Option<*mut T> {
        let ctr_ptr = self.container_for::<T>(container_num);
        if (*ctr_ptr).is_null() {
            self.alloc(mem::size_of::<T>()).map(|root_arr| {
                let root_ptr = root_arr.repr().data as *mut T;
                *root_ptr = Default::default();
                volatile_store(ctr_ptr, root_ptr);
                root_ptr
            })
        } else {
            Some(*ctr_ptr)
        }
    }

    pub fn pop_syscall_stack(&mut self) {
        let pspr = self.cur_stack as *const usize;
        unsafe {
            self.wait_pc = volatile_load(pspr.offset(6));
            self.psr = volatile_load(pspr.offset(7));
            self.cur_stack =
                (self.cur_stack as *mut usize).offset(8) as *mut u8;
        }
    }

    /// Context switch to the process.
    pub unsafe fn switch_to_callback(&mut self, callback: Callback) {
        // Fill in initial stack expected by SVC handler
        // Top minus 8 u32s for r0-r3, r12, lr, pc and xPSR
        let stack_bottom = (self.cur_stack as *mut usize).offset(-8);
        volatile_store(stack_bottom.offset(7), self.psr);
        volatile_store(stack_bottom.offset(6), callback.pc | 1);
        // Set the LR register to the saved PC so the callback returns to
        // wherever wait was called. Set lowest bit to one because of THUMB
        // instruction requirements.
        volatile_store(stack_bottom.offset(5), self.wait_pc | 0x1);
        volatile_store(stack_bottom, callback.r0);
        volatile_store(stack_bottom.offset(1), callback.r1);
        volatile_store(stack_bottom.offset(2), callback.r2);
        volatile_store(stack_bottom.offset(3), callback.r3);

        self.cur_stack = stack_bottom as *mut u8;
        self.switch_to();
    }

    /// Context switch to the process.
    pub unsafe fn switch_to(&mut self) {
        if self.cur_stack < self.memory.data {
            breakpoint();
        }
        let psp = switch_to_user(self.cur_stack, self.memory.data);
        self.cur_stack = psp;
    }

    pub fn svc_number(&self) -> Option<u8> {
        let psp = self.cur_stack as *const *const u16;
        unsafe {
            let pcptr = volatile_load((psp as *const *const u16).offset(6));
            let svc_instr = volatile_load(pcptr.offset(-1));
            Some((svc_instr & 0xff) as u8)
        }
    }

    pub fn lr(&self) -> usize {
        let pspr = self.cur_stack as *const usize;
        unsafe { volatile_load(pspr.offset(5)) }
    }

    pub fn r0(&self) -> usize {
        let pspr = self.cur_stack as *const usize;
        unsafe { volatile_load(pspr) }
    }

    pub fn set_r0(&mut self, val: isize) {
        let pspr = self.cur_stack as *mut isize;
        unsafe { volatile_store(pspr, val) }
    }

    pub fn r1(&self) -> usize {
        let pspr = self.cur_stack as *const usize;
        unsafe { volatile_load(pspr.offset(1)) }
    }

    pub fn r2(&self) -> usize {
        let pspr = self.cur_stack as *const usize;
        unsafe { volatile_load(pspr.offset(2)) }
    }

    pub fn r3(&self) -> usize {
        let pspr = self.cur_stack as *const usize;
        unsafe { volatile_load(pspr.offset(3)) }
    }

}

#[derive(Debug)]
struct LoadResult {
    text_start: *const u8,
    text_len: usize,
    init_fn: usize,
    app_mem_start: *const u8
}

unsafe fn load(start_addr: *const usize, mem_base: *const u8) -> LoadResult {
    let mut result = LoadResult {
        text_start: 0 as *const u8,
        text_len: 0,
        init_fn: 0,
        app_mem_start: 0 as *const u8
    };

    let mut start_addr = start_addr as *const u8;
    let load_info : &LoadInfo = mem::transmute(start_addr);
    start_addr = start_addr.offset(mem::size_of::<LoadInfo>() as isize);

    let rel_data : &mut [usize] = mem::transmute(Slice{
        data: start_addr,
        len: load_info.rel_data_size / 4
    });
    start_addr = start_addr.offset(load_info.rel_data_size as isize);

    // Update text location in self
    result.text_start = start_addr;
    result.text_len = load_info.init_data_loc;

    // Zero out BSS
    ::core::intrinsics::write_bytes(
        mem_base.offset(load_info.bss_start_offset as isize) as *mut u8,
        0,
        load_info.bss_end_offset - load_info.bss_start_offset);

    // Copy data into Data section
    let init_data : &[u8] = mem::transmute(Slice{
        data: (load_info.init_data_loc + start_addr as usize) as *mut u8,
        len: load_info.init_data_size
    });

    let target_data : &mut [u8] = mem::transmute(Slice{
        data: mem_base,
        len: load_info.init_data_size
    });

    target_data.clone_from_slice(init_data);

    let fixup = |addr: *mut usize| {
        let entry = *addr;
        if (entry & 0x80000000) == 0 {
            // Regular data (memory relative)
            *addr = entry + (mem_base as usize);
        } else {
            // rodata or function pointer (code relative)
            *addr = (entry ^ 0x80000000) + (start_addr as usize);
        }
    };

    // Fixup Global Offset Table
    let mut got_cur = mem_base.offset(
        load_info.got_start_offset as isize) as *mut usize;
    let got_end = mem_base.offset(
        load_info.got_end_offset as isize) as *mut usize;
    while got_cur != got_end {
        fixup(got_cur);
        got_cur = got_cur.offset(1);
    }

    // Fixup relocation data
    for (i, addr) in rel_data.iter().enumerate() {
        if i % 2 == 0 { // Only the first of every 2 entries is an address
            fixup(mem_base.offset(*addr as isize) as *mut usize);
        }
    }

    // Entry point is offset from app code
    result.init_fn = start_addr as usize + load_info.entry_loc;

    let mut aligned_mem_start = load_info.bss_end_offset as isize;
    aligned_mem_start += (8 - (aligned_mem_start % 8)) % 8;
    result.app_mem_start = mem_base.offset(aligned_mem_start);

    result
}

<|endoftext|>