<commit_before>#![feature(phase)]

extern crate regex;
#[phase(plugin)] extern crate regex_macros;

use std::io::{BufferedReader, File, IoError, OtherIoError};
use std::os::{getenv, setenv, self_exe_path};

#[deriving(Show, Clone)]
pub struct ParseError {
	line: String
}

#[deriving(Show, Clone)]
pub enum DotenvError {
	Parse(ParseError),
	Io(IoError)
}

fn parse_line(line: String) -> Result<Option<(String, String)>, ParseError> {
	let line_regex = regex!(concat!(r"^(\s*(",
		r"#.*|", // A comment, or...
		r"\s*|", // ...an empty string, or...
		r"(export\s+)?", // ...(optionally preceded by "export")...
		r"(?P<key>[A-Za-z_][A-Za-z0-9_]*)", // ...a key,...
		r"\s*=\s*", // ...then an equal sign,...
		r"(?P<value>.+?)", // ...and then its corresponding value.
	r")\s*)[\r\n]*$"));

	line_regex.captures(line.as_slice()).map_or(
		Err(ParseError{line: line.clone()}),
		|captures| {
			let key = captures.name("key");
			let value = captures.name("value");

			if key == "" || value == "" {
				// If there's no key and value, but capturing did not fail,
				// then this means we're dealing with a comment or an empty
				// string.
				Ok(None)
			} else {
				Ok(Some((key.to_string(), value.to_string())))
			}
		}
	)
}

fn parse_line_iter<T: Iterator<String>>(lines: T) -> Result<Vec<(String, String)>, ParseError> {
	let parsed_lines: Vec<Result<Option<(String, String)>, ParseError>> = lines.map(parse_line).collect();
	let failure = parsed_lines.iter().find(|line| line.is_err());

	if failure.is_some() {
		return Err(failure.unwrap().clone().err().unwrap());
	}

	Ok(parsed_lines.iter().filter_map(|line| {
		line.clone().unwrap()
	}).collect())
}

fn lines_to_env(lines: Vec<(String, String)>) {
	for (key, value) in lines.into_iter() {
		if getenv(key.as_slice()).is_none() {
			setenv(key.as_slice(), value);
		}
	}
}

fn from_file(file: File) -> Result<(), DotenvError> {
	let mut reader = BufferedReader::new(file);
	let lines = reader.lines();

	parse_line_iter(lines.filter_map(|result| {
		result.ok()
	})).map(lines_to_env).map_err(|err| {
		DotenvError::Parse(err)
	})
}

pub fn from_path(path: &Path) -> Result<(), DotenvError> {
	match File::open(path) {
		Ok(file) => from_file(file),
		Err(err) => Err(DotenvError::Io(err))
	}
}

pub fn from_filename(filename: &str) -> Result<(), DotenvError> {
	self_exe_path().as_mut().map(|path| {
		path.push(filename);
		from_path(path)
	}).unwrap_or(Err(DotenvError::Io(IoError{
		kind: OtherIoError,
		desc: "Could not fetch the path of this executable",
		detail: None
	})))
}

pub fn dotenv() -> Result<(), DotenvError> {
	from_filename(".env")
}

#[test]
fn test_parse_line_env() {
	let input_iter = vec![
		"THIS_IS_KEY=hi this is value",
		"   many_spaces  =   wow a  maze   ",
		"export   SHELL_LOVER=1"
	].into_iter().map(|input| input.to_string());
	let actual_iter = input_iter.map(|input| parse_line(input));

	let expected_iter = vec![
		("THIS_IS_KEY", "hi this is value"),
		("many_spaces", "wow a  maze"),
		("SHELL_LOVER", "1")
	].into_iter().map(|(key, value)| (key.to_string(), value.to_string()));

	for (expected, actual) in expected_iter.zip(actual_iter) {
		assert!(actual.is_ok());
		assert!(actual.clone().ok().unwrap().is_some())
		assert_eq!(expected, actual.ok().unwrap().unwrap())
	}
}

#[test]
fn test_parse_line_comment() {
	let input_iter = vec![
		"# foo=bar",
		"    #    "
	].into_iter().map(|input| input.to_string());
	let mut actual_iter = input_iter.map(|input| parse_line(input));

	for actual in actual_iter {
		assert!(actual.is_ok());
		assert!(actual.ok().unwrap().is_none());
	}
}

#[test]
fn test_parse_line_invalid() {
	let input_iter = vec![
		"  invalid    ",
		"very bacon = yes indeed",
		"key=",
		"=value"
	].into_iter().map(|input| input.to_string());
	let mut actual_iter = input_iter.map(|input| parse_line(input));

	for actual in actual_iter {
		assert!(actual.is_err());
	}
}

#[test]
fn test_from_line_iter_valid() {
	let input = vec![
		"test_env_one=1",
		"# a comment",
		"test_env_two=2"
	].into_iter().map(|line| line.to_string());
	let actual = parse_line_iter(input);

	assert!(actual.is_ok())
}

#[test]
fn test_from_line_iter_invalid() {
	let input = vec![
		"test_env_one=1",
		"# a comment",
		"not valid"
	].into_iter().map(|line| line.to_string());
	let actual = parse_line_iter(input);

	assert!(actual.is_err())
}
<commit_msg>Added type aliases (readability counts!)<commit_after>#![feature(phase)]

extern crate regex;
#[phase(plugin)] extern crate regex_macros;

use std::io::{BufferedReader, File, IoError, OtherIoError};
use std::os::{getenv, setenv, self_exe_path};

#[deriving(Show, Clone)]
pub struct ParseError {
	line: String
}

#[deriving(Show, Clone)]
pub enum DotenvError {
	Parse(ParseError),
	Io(IoError)
}

// for readability's sake
type ParsedLine = Result<Option<(String, String)>, ParseError>;
type ParsedLines = Result<Vec<(String, String)>, ParseError>;

fn parse_line(line: String) -> ParsedLine {
	let line_regex = regex!(concat!(r"^(\s*(",
		r"#.*|", // A comment, or...
		r"\s*|", // ...an empty string, or...
		r"(export\s+)?", // ...(optionally preceded by "export")...
		r"(?P<key>[A-Za-z_][A-Za-z0-9_]*)", // ...a key,...
		r"\s*=\s*", // ...then an equal sign,...
		r"(?P<value>.+?)", // ...and then its corresponding value.
	r")\s*)[\r\n]*$"));

	line_regex.captures(line.as_slice()).map_or(
		Err(ParseError{line: line.clone()}),
		|captures| {
			let key = captures.name("key");
			let value = captures.name("value");

			if key == "" || value == "" {
				// If there's no key and value, but capturing did not fail,
				// then this means we're dealing with a comment or an empty
				// string.
				Ok(None)
			} else {
				Ok(Some((key.to_string(), value.to_string())))
			}
		}
	)
}

fn parse_line_iter<T: Iterator<String>>(lines: T) -> ParsedLines {
	let parsed_lines: Vec<ParsedLine> =
			lines.map(parse_line).collect();
	let failure = parsed_lines.iter().find(|line| line.is_err());

	if failure.is_some() {
		return Err(failure.unwrap().clone().err().unwrap());
	}

	Ok(parsed_lines.iter().filter_map(|line| {
		line.clone().unwrap()
	}).collect())
}

fn lines_to_env(lines: Vec<(String, String)>) {
	for (key, value) in lines.into_iter() {
		if getenv(key.as_slice()).is_none() {
			setenv(key.as_slice(), value);
		}
	}
}

fn from_file(file: File) -> Result<(), DotenvError> {
	let mut reader = BufferedReader::new(file);
	let lines = reader.lines();

	parse_line_iter(lines.filter_map(|result| {
		result.ok()
	})).map(lines_to_env).map_err(|err| {
		DotenvError::Parse(err)
	})
}

pub fn from_path(path: &Path) -> Result<(), DotenvError> {
	match File::open(path) {
		Ok(file) => from_file(file),
		Err(err) => Err(DotenvError::Io(err))
	}
}

pub fn from_filename(filename: &str) -> Result<(), DotenvError> {
	self_exe_path().as_mut().map(|path| {
		path.push(filename);
		from_path(path)
	}).unwrap_or(Err(DotenvError::Io(IoError{
		kind: OtherIoError,
		desc: "Could not fetch the path of this executable",
		detail: None
	})))
}

pub fn dotenv() -> Result<(), DotenvError> {
	from_filename(".env")
}

#[test]
fn test_parse_line_env() {
	let input_iter = vec![
		"THIS_IS_KEY=hi this is value",
		"   many_spaces  =   wow a  maze   ",
		"export   SHELL_LOVER=1"
	].into_iter().map(|input| input.to_string());
	let actual_iter = input_iter.map(|input| parse_line(input));

	let expected_iter = vec![
		("THIS_IS_KEY", "hi this is value"),
		("many_spaces", "wow a  maze"),
		("SHELL_LOVER", "1")
	].into_iter().map(|(key, value)| (key.to_string(), value.to_string()));

	for (expected, actual) in expected_iter.zip(actual_iter) {
		assert!(actual.is_ok());
		assert!(actual.clone().ok().unwrap().is_some())
		assert_eq!(expected, actual.ok().unwrap().unwrap())
	}
}

#[test]
fn test_parse_line_comment() {
	let input_iter = vec![
		"# foo=bar",
		"    #    "
	].into_iter().map(|input| input.to_string());
	let mut actual_iter = input_iter.map(|input| parse_line(input));

	for actual in actual_iter {
		assert!(actual.is_ok());
		assert!(actual.ok().unwrap().is_none());
	}
}

#[test]
fn test_parse_line_invalid() {
	let input_iter = vec![
		"  invalid    ",
		"very bacon = yes indeed",
		"key=",
		"=value"
	].into_iter().map(|input| input.to_string());
	let mut actual_iter = input_iter.map(|input| parse_line(input));

	for actual in actual_iter {
		assert!(actual.is_err());
	}
}

#[test]
fn test_from_line_iter_valid() {
	let input = vec![
		"test_env_one=1",
		"# a comment",
		"test_env_two=2"
	].into_iter().map(|line| line.to_string());
	let actual = parse_line_iter(input);

	assert!(actual.is_ok())
}

#[test]
fn test_from_line_iter_invalid() {
	let input = vec![
		"test_env_one=1",
		"# a comment",
		"not valid"
	].into_iter().map(|line| line.to_string());
	let actual = parse_line_iter(input);

	assert!(actual.is_err())
}
<|endoftext|>


<commit_before>//! Provides a simple and unified API to run fast and highly parallel computations on different
//! devices such as CPUs and GPUs, accross different computation languages such as OpenCL and
//! CUDA and allows you to swap your backend on run-time.
//!
//! Collenchyma was started at [Autumn][autumn] to create an easy and performant abstraction over
//! different backends for the Machine Intelligence Framework [Leaf][leaf], with no hard
//! dependency on any driver or libraries so that it can easily be used without the need for a
//! long and painful build process.
//!
//! ## Abstract
//!
//! Code often is executed on the native CPU, but could be executed on other devices such as GPUs
//! and Accelerators as well. These devices are accessable through frameworks like OpenCL and CUDA
//! but have a more complicated interfaces than your every-day native CPU
//! which makes the use of these devices a painful experience. Some of the pain points, when
//! writing such device code, are:
//!
//! * non-portable: frameworks have different interfaces, devices support different versions and
//! machines might have different hardware - all this leads to code that will be executable only on
//! a very specific set of machines and platforms.
//! * steep learning curve: executing code on a device through a framework is quite different to
//! running code on the native CPU and comes with a lot of hurdles. OpenCLs 1.2 specification for
//! example has close to 400 pages.
//! * custom code: integrating support for devices into your project, requires the need for writing
//! a lot of custom code e.g. kernels, memory management, genereal business logic.
//!
//! But writing code for devices would often be a good choice as these devices can execute many
//! operations a lot faster than the native CPUs. GPUs for example can execute operations roughly
//! one to two orders of magnitudes faster, thanks to better support of parallising operations.
//! OpenCL and CUDA make parallising operations super easy.
//!
//! With Collenchyma we eleminate the pain points of writing device code, so you can run your code
//! like any other Rust code, don't need to learn about kernels, events, or memory
//! synchronization, and can deploy your code with ease to servers, desktops or mobiles and
//! your code will make full use of the underlying hardware.
//!
//! ## Architecture
//!
//! The single entry point of Collenchyma is a [Backend][backend]. A Backend is agnostic over the [Device][device] it
//! runs [Operations][operation] on. In order to be agnostic over the Device, such as native host CPU, GPUs,
//! Accelerators or other types of [Hardware][hardware], the Backend needs to be agnostic over the
//! [Framework][framework] as well. A Framework is a computation language such as OpenCL, Cuda or the native programming
//! language. The Framework is important, as it provides us with the interface to turn Hardware into Devices and
//! therefore, among other things, execute Operations on the created Device. With a Framework, we get access to Hardware
//! as long as the Hardware supports the Framework. As different vendors of Hardware use different
//! Frameworks, it becomes important that the Backend is agnostic over the Framework, which allows us, that we can
//! really run computations on any machine such as servers, desktops and mobiles without the need to worry about what
//! Hardware is available on the machine. That gives us the freedom to write code once and deploy it on different
//! machines where it will execute on the most potent Hardware by default.
//!
//! Operations get introduced by a [Library][library]. A Library provides which operations are available and provides
//! the finished implementation of an Operation. This allows us, to implement the Library directly on the Backend, which
//! gives us a native Rust interface to execute our Operations. The interface is like any other interface e.g. to
//! execute the dot product operation we can simply call `backend.dot(...)`. If the dot Operation is executed on e.g.
//! one or many GPUs or CPUs depends solely on how you configured the Backend or if you did not further specify which
//! Framework and Hardware to use, solely on the machine you execute the dot Operation on. A Library exposes the
//! Operation traits, which define one `computation` method, that is further defined by the Framework, where the
//! exact Operation execution is stated. In the field of Operation there is one more component
//! - the [Binary][binary]. As - different to executing code on the native CPU - devices need to compile and build the
//! Operation, which is a significant part of a Framework, each Framework exposes a Binary representation, which mounts
//! the Operations exposed by a Library.
//!
//! The last peace of Collenchyma is the [Memory][memory]. A Operation happens over data, but this data needs to be
//! accessable by the device on which the Operation is executed. The process is therefore often, that memory space needs
//! to be allocated on the device and then in a later step, synced from the host to the device or from
//! the device back to the host. Thanks to [SharedMemory][shared-memory] we do not have to care about memory management
//! between devices for the execution of Operations. SharedMemory tracks and automatically manages data and it's memory
//! accross devices, which is often the host and the Device. But it can also be passed around to different Backends.
//! Operations take as arguments SharedMemory and handle the synchronization and allocation for you.
//!
//! ## Examples
//!
//! Backend with custom defined Framework and Device.
//!
//! ```
//! extern crate collenchyma as co;
//! use co::framework::IFramework;
//! use co::backend::{Backend, BackendConfig};
//! use co::frameworks::Native;
//! fn main() {
//!     let framework = Native::new(); // Initialize the Framework
//!     let hardwares = framework.hardwares(); // Now you can obtain a list of available hardware for that Framework.
//!     // Create the custom Backend by providing a Framework and one or many Hardwares.
//!     let backend_config = BackendConfig::new(framework, hardwares);
//!     let backend = Backend::new(backend_config);
//!     // You can now execute all the operations available, e.g.
//!     // backend.dot(x, y);
//! }
//! ```
//!
//! Machine-agnostic Backend.
//!
//! ```
//! extern crate collenchyma as co;
//! use co::framework::IFramework;
//! use co::backend::{Backend, BackendConfig};
//! use co::frameworks::Native;
//! fn main() {
//!     // Not yet implemented.
//!     // No need to provide a Backend Configuration.
//!     // let backend = Backend::new(None);
//!     // You can now execute all the operations available, e.g.
//!     // backend.dot(x, y);
//! }
//! ```
//!
//! ## Development
//!
//! At the moment Collenchyma itself will provide Rust APIs for the important frameworks - OpenCL
//! and CUDA. One step we are looking out for is to seperate OpenCL and CUDA into their own crate.
//! Something similar to [Glium][glium].
//!
//! Every operation exposed via a library and implemented on the backend, should take as the last argument an
//! `Option<OperationConfig>` to specify custom parallelisation behaviour and tracking the operation via events.
//!
//! When initializing a new Backend from a BackendConfig you might not want to specify the Framework, which is currently
//! mandatory. Leaving it blank, the Backend would try to use the most potent Framework given the underlying hardware,
//! which would be probably in this order Cuda -> OpenCL -> Native. The setup might take longer, as every framework
//! needs to be checked, and devices be loaded in order to identify the best setup. But this would allow, that you
//! really could deploy a Collenchyma-backed application to almost any hardware - server, desktops, mobiles.
//!
//! [autumn]: http://autumnai.com
//! [leaf]: https://github.com/autumnai/leaf
//! [glium]: https://github.com/tomaka/glium
//! [backend]: ./backend/index.html
//! [device]: ./device/index.html
//! [binary]: ./binary/index.html
//! [operation]: ./operation/index.html
//! [hardware]: ./hardware/index.html
//! [framework]: ./framework/index.html
//! [library]: ./libraries/index.html
//! [memory]: ./memory/index.html
//! [shared-memory]: ./shared-memory/index.html
#![cfg_attr(lint, feature(plugin))]
#![cfg_attr(lint, plugin(clippy))]
#![allow(dead_code)]
#![feature(associated_type_defaults)]
#![feature(unboxed_closures)]
#![feature(static_mutex)]
#![deny(missing_docs,
        missing_debug_implementations, missing_copy_implementations,
        trivial_casts, trivial_numeric_casts,
        unused_import_braces, unused_qualifications)]

extern crate libc;
#[macro_use]
extern crate bitflags;
#[macro_use]
extern crate enum_primitive;
extern crate num;
extern crate byteorder;
extern crate rblas as blas;

pub mod backend;
pub mod device;
pub mod hardware;
pub mod framework;
pub mod frameworks;
pub mod memory;
pub mod libraries;
pub mod shared_memory;
pub mod operation;
pub mod binary;
pub mod error;
<commit_msg>refactor/features: remove associated_type_defaults<commit_after>//! Provides a simple and unified API to run fast and highly parallel computations on different
//! devices such as CPUs and GPUs, accross different computation languages such as OpenCL and
//! CUDA and allows you to swap your backend on run-time.
//!
//! Collenchyma was started at [Autumn][autumn] to create an easy and performant abstraction over
//! different backends for the Machine Intelligence Framework [Leaf][leaf], with no hard
//! dependency on any driver or libraries so that it can easily be used without the need for a
//! long and painful build process.
//!
//! ## Abstract
//!
//! Code often is executed on the native CPU, but could be executed on other devices such as GPUs
//! and Accelerators as well. These devices are accessable through frameworks like OpenCL and CUDA
//! but have a more complicated interfaces than your every-day native CPU
//! which makes the use of these devices a painful experience. Some of the pain points, when
//! writing such device code, are:
//!
//! * non-portable: frameworks have different interfaces, devices support different versions and
//! machines might have different hardware - all this leads to code that will be executable only on
//! a very specific set of machines and platforms.
//! * steep learning curve: executing code on a device through a framework is quite different to
//! running code on the native CPU and comes with a lot of hurdles. OpenCLs 1.2 specification for
//! example has close to 400 pages.
//! * custom code: integrating support for devices into your project, requires the need for writing
//! a lot of custom code e.g. kernels, memory management, genereal business logic.
//!
//! But writing code for devices would often be a good choice as these devices can execute many
//! operations a lot faster than the native CPUs. GPUs for example can execute operations roughly
//! one to two orders of magnitudes faster, thanks to better support of parallising operations.
//! OpenCL and CUDA make parallising operations super easy.
//!
//! With Collenchyma we eleminate the pain points of writing device code, so you can run your code
//! like any other Rust code, don't need to learn about kernels, events, or memory
//! synchronization, and can deploy your code with ease to servers, desktops or mobiles and
//! your code will make full use of the underlying hardware.
//!
//! ## Architecture
//!
//! The single entry point of Collenchyma is a [Backend][backend]. A Backend is agnostic over the [Device][device] it
//! runs [Operations][operation] on. In order to be agnostic over the Device, such as native host CPU, GPUs,
//! Accelerators or other types of [Hardware][hardware], the Backend needs to be agnostic over the
//! [Framework][framework] as well. A Framework is a computation language such as OpenCL, Cuda or the native programming
//! language. The Framework is important, as it provides us with the interface to turn Hardware into Devices and
//! therefore, among other things, execute Operations on the created Device. With a Framework, we get access to Hardware
//! as long as the Hardware supports the Framework. As different vendors of Hardware use different
//! Frameworks, it becomes important that the Backend is agnostic over the Framework, which allows us, that we can
//! really run computations on any machine such as servers, desktops and mobiles without the need to worry about what
//! Hardware is available on the machine. That gives us the freedom to write code once and deploy it on different
//! machines where it will execute on the most potent Hardware by default.
//!
//! Operations get introduced by a [Library][library]. A Library provides which operations are available and provides
//! the finished implementation of an Operation. This allows us, to implement the Library directly on the Backend, which
//! gives us a native Rust interface to execute our Operations. The interface is like any other interface e.g. to
//! execute the dot product operation we can simply call `backend.dot(...)`. If the dot Operation is executed on e.g.
//! one or many GPUs or CPUs depends solely on how you configured the Backend or if you did not further specify which
//! Framework and Hardware to use, solely on the machine you execute the dot Operation on. A Library exposes the
//! Operation traits, which define one `computation` method, that is further defined by the Framework, where the
//! exact Operation execution is stated. In the field of Operation there is one more component
//! - the [Binary][binary]. As - different to executing code on the native CPU - devices need to compile and build the
//! Operation, which is a significant part of a Framework, each Framework exposes a Binary representation, which mounts
//! the Operations exposed by a Library.
//!
//! The last peace of Collenchyma is the [Memory][memory]. A Operation happens over data, but this data needs to be
//! accessable by the device on which the Operation is executed. The process is therefore often, that memory space needs
//! to be allocated on the device and then in a later step, synced from the host to the device or from
//! the device back to the host. Thanks to [SharedMemory][shared-memory] we do not have to care about memory management
//! between devices for the execution of Operations. SharedMemory tracks and automatically manages data and it's memory
//! accross devices, which is often the host and the Device. But it can also be passed around to different Backends.
//! Operations take as arguments SharedMemory and handle the synchronization and allocation for you.
//!
//! ## Examples
//!
//! Backend with custom defined Framework and Device.
//!
//! ```
//! extern crate collenchyma as co;
//! use co::framework::IFramework;
//! use co::backend::{Backend, BackendConfig};
//! use co::frameworks::Native;
//! fn main() {
//!     let framework = Native::new(); // Initialize the Framework
//!     let hardwares = framework.hardwares(); // Now you can obtain a list of available hardware for that Framework.
//!     // Create the custom Backend by providing a Framework and one or many Hardwares.
//!     let backend_config = BackendConfig::new(framework, hardwares);
//!     let backend = Backend::new(backend_config);
//!     // You can now execute all the operations available, e.g.
//!     // backend.dot(x, y);
//! }
//! ```
//!
//! Machine-agnostic Backend.
//!
//! ```
//! extern crate collenchyma as co;
//! use co::framework::IFramework;
//! use co::backend::{Backend, BackendConfig};
//! use co::frameworks::Native;
//! fn main() {
//!     // Not yet implemented.
//!     // No need to provide a Backend Configuration.
//!     // let backend = Backend::new(None);
//!     // You can now execute all the operations available, e.g.
//!     // backend.dot(x, y);
//! }
//! ```
//!
//! ## Development
//!
//! At the moment Collenchyma itself will provide Rust APIs for the important frameworks - OpenCL
//! and CUDA. One step we are looking out for is to seperate OpenCL and CUDA into their own crate.
//! Something similar to [Glium][glium].
//!
//! Every operation exposed via a library and implemented on the backend, should take as the last argument an
//! `Option<OperationConfig>` to specify custom parallelisation behaviour and tracking the operation via events.
//!
//! When initializing a new Backend from a BackendConfig you might not want to specify the Framework, which is currently
//! mandatory. Leaving it blank, the Backend would try to use the most potent Framework given the underlying hardware,
//! which would be probably in this order Cuda -> OpenCL -> Native. The setup might take longer, as every framework
//! needs to be checked, and devices be loaded in order to identify the best setup. But this would allow, that you
//! really could deploy a Collenchyma-backed application to almost any hardware - server, desktops, mobiles.
//!
//! [autumn]: http://autumnai.com
//! [leaf]: https://github.com/autumnai/leaf
//! [glium]: https://github.com/tomaka/glium
//! [backend]: ./backend/index.html
//! [device]: ./device/index.html
//! [binary]: ./binary/index.html
//! [operation]: ./operation/index.html
//! [hardware]: ./hardware/index.html
//! [framework]: ./framework/index.html
//! [library]: ./libraries/index.html
//! [memory]: ./memory/index.html
//! [shared-memory]: ./shared-memory/index.html
#![cfg_attr(lint, feature(plugin))]
#![cfg_attr(lint, plugin(clippy))]
#![allow(dead_code)]
#![feature(unboxed_closures)]
#![feature(static_mutex)]
#![deny(missing_docs,
        missing_debug_implementations, missing_copy_implementations,
        trivial_casts, trivial_numeric_casts,
        unused_import_braces, unused_qualifications)]

extern crate libc;
#[macro_use]
extern crate bitflags;
#[macro_use]
extern crate enum_primitive;
extern crate num;
extern crate byteorder;
extern crate rblas as blas;

pub mod backend;
pub mod device;
pub mod hardware;
pub mod framework;
pub mod frameworks;
pub mod memory;
pub mod libraries;
pub mod shared_memory;
pub mod operation;
pub mod binary;
pub mod error;
<|endoftext|>


<commit_before>extern crate hyper;
extern crate url;
extern crate serde;
extern crate serde_json;

use std::error::Error as StdError;
use std::fmt::{self, Display, Formatter};
use std::io::{Read, Write};
use std::io::Error as IoError;
use std::time::Duration;

use serde::de::Deserialize;
use hyper::client::Request as HyperRequest;
use hyper::client::Response as HyperResponse;
use hyper::method::Method;
use hyper::net::Fresh;

#[doc(no_inline)]
pub use hyper::header;
#[doc(no_inline)]
pub use url::Url;
#[doc(no_inline)]
pub use hyper::error::Error as HyperError;
#[doc(no_inline)]
pub use hyper::status::StatusCode;

#[derive(Debug)]
pub enum Error {
    UnsuccessfulResponse(Response),
    Json(serde_json::error::Error),
    Hyper(HyperError),
}

impl Display for Error {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        match *self {
            Error::UnsuccessfulResponse(ref rsp) => write!(f, "server response failed: {:?}", rsp),
            Error::Json(ref error) => write!(f, "json parse error: {}", error),
            Error::Hyper(ref error) => write!(f, "HTTP communication error: {}", error),
        }
    }
}

impl StdError for Error {
    fn description(&self) -> &str {
        "ease error"
    }

    fn cause(&self) -> Option<&StdError> {
        match *self {
            Error::Json(ref error) => Some(error),
            Error::Hyper(ref error) => Some(error),
            _ => None,
        }
    }
}

impl From<HyperError> for Error {
    fn from(h: HyperError) -> Error {
        Error::Hyper(h)
    }
}

impl From<IoError> for Error {
    fn from(i: IoError) -> Error {
        Error::Hyper(HyperError::Io(i))
    }
}

#[derive(Debug)]
pub struct Response {
    pub hyper_response: HyperResponse,
    pub body: String,
}

impl Response {
    fn from_hyper_response(mut hyper_response: HyperResponse) -> Result<Response, IoError> {
        let mut body = String::new();
        hyper_response.read_to_string(&mut body).map(|_| {
            Response {
                hyper_response: hyper_response,
                body: body,
            }
        })
    }

    /// Deserializes the body of the response from JSON into
    /// a `T`.
    pub fn from_json<T: Deserialize>(&self) -> Result<T, Error> {
        serde_json::from_str(&*self.body).map_err(|e| Error::Json(e))
    }
}

#[derive(Clone)]
pub struct Request<'a> {
    url: Url,
    params: Option<Vec<(&'a str, &'a str)>>,
    body: Option<String>,
    read_timeout: Option<Duration>,
    headers: Option<header::Headers>,
}

impl<'a> Request<'a> {
    pub fn new(url: Url) -> Request<'a> {
        Request {
            url: url,
            params: None,
            body: None,
            read_timeout: None,
            headers: None,
        }
    }

    /// Sets one parameter. On a GET or DELETE request, this parameter will
    /// be stored in the URL. On a POST or PUT request, it is stored in the
    /// body of the request. Hence, if you call this method on a POST or
    /// PUT request, you cannot also call `body`.
    pub fn param(&mut self, key: &'a str, value: &'a str) -> &mut Request<'a> {
        if let Some(ref mut p) = self.params {
            p.push((key, value));
        } else {
            let mut v = Vec::new();
            v.push((key, value));
            self.params = Some(v);
        }
        self
    }

    /// Sets many parameters. On a GET or DELETE request, these parameters will
    /// be stored in the URL. On a POST or PUT request, they are stored in the
    /// body of the request. Hence, if you call this method on a POST or
    /// PUT request, you cannot also call `body`.
    pub fn params<T>(&mut self, values: T) -> &mut Request<'a>
        where T: IntoIterator<Item = (&'a str, &'a str)>
    {
        if let Some(ref mut p) = self.params {
            for value in values {
                p.push(value);
            }
        } else {
            let mut v = Vec::new();
            for value in values {
                v.push(value);
            }
            self.params = Some(v);
        }
        self
    }

    /// Writes a `String` to the body of the request. Don't call this
    /// method if you also call `param` on a PUT or POST request.
    pub fn body(&mut self, body: String) -> &mut Request<'a> {
        self.body = Some(body);
        self
    }

    /// Sets a header for the request.
    pub fn header<H: header::Header + header::HeaderFormat>(&mut self,
                                                            header: H)
                                                            -> &mut Request<'a> {
        if let Some(ref mut h) = self.headers {
            h.set(header);
        } else {
            let mut v = header::Headers::new();
            v.set(header);
            self.headers = Some(v);
        }
        self
    }

    /// Sets a read timeout for the response.
    pub fn read_timeout(&mut self, timeout: Duration) -> &mut Request<'a> {
        self.read_timeout = Some(timeout);
        self
    }

    fn send_request(&mut self, mut req: HyperRequest<Fresh>) -> Result<Response, Error> {
        if let Some(headers) = self.headers.as_ref() {
            req.headers_mut().extend(headers.iter());
        }

        let _ = req.set_read_timeout(self.read_timeout);

        let mut req = try!(req.start());

        if let Some(body) = self.body.as_ref() {
            try!(req.write_all(body.as_bytes()));
        }

        let resp = try!(req.send());
        let resp = try!(Response::from_hyper_response(resp));

        if resp.hyper_response.status.is_success() {
            Ok(resp)
        } else {
            Err(Error::UnsuccessfulResponse(resp))
        }
    }

    /// Sends a GET request and returns either an error
    /// or a `String` of the response.
    pub fn get(&mut self) -> Result<Response, Error> {
        let mut url = self.url.clone();

        if let Some(ref params) = self.params {
            url.query_pairs_mut().extend_pairs(params.into_iter().map(|&x| x));
        }

        let req = try!(HyperRequest::new(Method::Get, url));
        self.send_request(req)
    }

    /// Sends a DELETE request and returns either an error
    /// or a `String` of the response.
    pub fn delete(&mut self) -> Result<Response, Error> {
        let mut url = self.url.clone();

        if let Some(ref params) = self.params {
            url.query_pairs_mut().extend_pairs(params.into_iter().map(|&x| x));
        }

        let req = try!(HyperRequest::new(Method::Delete, url));
        self.send_request(req)
    }

    /// Sends a POST request and returns either an error
    /// or a `String` of the response.
    pub fn post(&mut self) -> Result<Response, Error> {
        let url = self.url.clone();

        if let Some(ref params) = self.params {
            let mut serializer = url::form_urlencoded::Serializer::new(String::new());
            serializer.extend_pairs(params);
            self.body = Some(serializer.finish());
        }

        let req = try!(HyperRequest::new(Method::Post, url));
        self.send_request(req)
    }

    /// Sends a PUT request and returns either an error
    /// or a `String` of the response.
    pub fn put(&mut self) -> Result<Response, Error> {
        let url = self.url.clone();

        if let Some(ref params) = self.params {
            let mut serializer = url::form_urlencoded::Serializer::new(String::new());
            serializer.extend_pairs(params);
            self.body = Some(serializer.finish());
        }

        let req = try!(HyperRequest::new(Method::Put, url));
        self.send_request(req)
    }
}
<commit_msg>Also support owned strings as request parameters<commit_after>extern crate hyper;
extern crate url;
extern crate serde;
extern crate serde_json;

use std::error::Error as StdError;
use std::fmt::{self, Display, Formatter};
use std::io::{Read, Write};
use std::io::Error as IoError;
use std::time::Duration;

use serde::de::Deserialize;
use hyper::client::Request as HyperRequest;
use hyper::client::Response as HyperResponse;
use hyper::method::Method;
use hyper::net::Fresh;

#[doc(no_inline)]
pub use hyper::header;
#[doc(no_inline)]
pub use url::Url;
#[doc(no_inline)]
pub use hyper::error::Error as HyperError;
#[doc(no_inline)]
pub use hyper::status::StatusCode;

#[derive(Debug)]
pub enum Error {
    UnsuccessfulResponse(Response),
    Json(serde_json::error::Error),
    Hyper(HyperError),
}

impl Display for Error {
    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
        match *self {
            Error::UnsuccessfulResponse(ref rsp) => write!(f, "server response failed: {:?}", rsp),
            Error::Json(ref error) => write!(f, "json parse error: {}", error),
            Error::Hyper(ref error) => write!(f, "HTTP communication error: {}", error),
        }
    }
}

impl StdError for Error {
    fn description(&self) -> &str {
        "ease error"
    }

    fn cause(&self) -> Option<&StdError> {
        match *self {
            Error::Json(ref error) => Some(error),
            Error::Hyper(ref error) => Some(error),
            _ => None,
        }
    }
}

impl From<HyperError> for Error {
    fn from(h: HyperError) -> Error {
        Error::Hyper(h)
    }
}

impl From<IoError> for Error {
    fn from(i: IoError) -> Error {
        Error::Hyper(HyperError::Io(i))
    }
}

#[derive(Debug)]
pub struct Response {
    pub hyper_response: HyperResponse,
    pub body: String,
}

impl Response {
    fn from_hyper_response(mut hyper_response: HyperResponse) -> Result<Response, IoError> {
        let mut body = String::new();
        hyper_response.read_to_string(&mut body).map(|_| {
            Response {
                hyper_response: hyper_response,
                body: body,
            }
        })
    }

    /// Deserializes the body of the response from JSON into
    /// a `T`.
    pub fn from_json<T: Deserialize>(&self) -> Result<T, Error> {
        serde_json::from_str(&*self.body).map_err(|e| Error::Json(e))
    }
}

#[derive(Clone)]
pub struct Request {
    url: Url,
    params: Option<Vec<(String, String)>>,
    body: Option<String>,
    read_timeout: Option<Duration>,
    headers: Option<header::Headers>,
}

impl Request {
    pub fn new(url: Url) -> Request {
        Request {
            url: url,
            params: None,
            body: None,
            read_timeout: None,
            headers: None,
        }
    }

    /// Sets one parameter. On a GET or DELETE request, this parameter will
    /// be stored in the URL. On a POST or PUT request, it is stored in the
    /// body of the request. Hence, if you call this method on a POST or
    /// PUT request, you cannot also call `body`.
    pub fn param<S>(&mut self, key: S, value: S) -> &mut Request where S: Into<String> {
        if let Some(ref mut p) = self.params {
            p.push((key.into(), value.into()));
        } else {
            let mut v = Vec::new();
            v.push((key.into(), value.into()));
            self.params = Some(v);
        }
        self
    }

    /// Sets many parameters. On a GET or DELETE request, these parameters will
    /// be stored in the URL. On a POST or PUT request, they are stored in the
    /// body of the request. Hence, if you call this method on a POST or
    /// PUT request, you cannot also call `body`.
    pub fn params<S, T>(&mut self, values: T) -> &mut Request
        where S: Into<String>,
              T: IntoIterator<Item = (S, S)>
    {
        if let Some(ref mut p) = self.params {
            for value in values {
                p.push((value.0.into(), value.1.into()));
            }
        } else {
            let mut v = Vec::new();
            for value in values {
                v.push((value.0.into(), value.1.into()));
            }
            self.params = Some(v);
        }
        self
    }

    /// Writes a `String` to the body of the request. Don't call this
    /// method if you also call `param` on a PUT or POST request.
    pub fn body(&mut self, body: String) -> &mut Request {
        self.body = Some(body);
        self
    }

    /// Sets a header for the request.
    pub fn header<H: header::Header + header::HeaderFormat>(&mut self,
                                                            header: H)
                                                            -> &mut Request {
        if let Some(ref mut h) = self.headers {
            h.set(header);
        } else {
            let mut v = header::Headers::new();
            v.set(header);
            self.headers = Some(v);
        }
        self
    }

    /// Sets a read timeout for the response.
    pub fn read_timeout(&mut self, timeout: Duration) -> &mut Request {
        self.read_timeout = Some(timeout);
        self
    }

    fn send_request(&mut self, mut req: HyperRequest<Fresh>) -> Result<Response, Error> {
        if let Some(headers) = self.headers.as_ref() {
            req.headers_mut().extend(headers.iter());
        }

        let _ = req.set_read_timeout(self.read_timeout);

        let mut req = try!(req.start());

        if let Some(body) = self.body.as_ref() {
            try!(req.write_all(body.as_bytes()));
        }

        let resp = try!(req.send());
        let resp = try!(Response::from_hyper_response(resp));

        if resp.hyper_response.status.is_success() {
            Ok(resp)
        } else {
            Err(Error::UnsuccessfulResponse(resp))
        }
    }

    /// Sends a GET request and returns either an error
    /// or a `String` of the response.
    pub fn get(&mut self) -> Result<Response, Error> {
        let mut url = self.url.clone();

        if let Some(ref params) = self.params {
            url.query_pairs_mut().extend_pairs(params);
        }

        let req = try!(HyperRequest::new(Method::Get, url));
        self.send_request(req)
    }

    /// Sends a DELETE request and returns either an error
    /// or a `String` of the response.
    pub fn delete(&mut self) -> Result<Response, Error> {
        let mut url = self.url.clone();

        if let Some(ref params) = self.params {
            url.query_pairs_mut().extend_pairs(params);
        }

        let req = try!(HyperRequest::new(Method::Delete, url));
        self.send_request(req)
    }

    /// Sends a POST request and returns either an error
    /// or a `String` of the response.
    pub fn post(&mut self) -> Result<Response, Error> {
        let url = self.url.clone();

        if let Some(ref params) = self.params {
            let mut serializer = url::form_urlencoded::Serializer::new(String::new());
            serializer.extend_pairs(params);
            self.body = Some(serializer.finish());
        }

        let req = try!(HyperRequest::new(Method::Post, url));
        self.send_request(req)
    }

    /// Sends a PUT request and returns either an error
    /// or a `String` of the response.
    pub fn put(&mut self) -> Result<Response, Error> {
        let url = self.url.clone();

        if let Some(ref params) = self.params {
            let mut serializer = url::form_urlencoded::Serializer::new(String::new());
            serializer.extend_pairs(params);
            self.body = Some(serializer.finish());
        }

        let req = try!(HyperRequest::new(Method::Put, url));
        self.send_request(req)
    }
}
<|endoftext|>


<commit_before>// Module for parsing ISO Base Media Format aka video/mp4 streams.

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

/// Basic ISO box structure.
pub struct Mp4BoxHeader {
    /// Four character box type
pub name: u32,
    /// Size of the box in bytes
pub size: u64,
    /// Offset to the start of the contained data (or header size).
pub offset: u64,
}

/// File type box 'ftyp'.
pub struct Mp4FileTypeBox {
    name: u32,
    size: u64,
    major_brand: u32,
    minor_version: u32,
    compatible_brands: Vec<u32>,
}

extern crate byteorder;
use byteorder::{BigEndian, ReadBytesExt};
use std::io::{Result, Seek, SeekFrom};

/// Parse a box out of a data buffer.
pub fn read_box_header<T: ReadBytesExt>(src: &mut T) -> Option<Mp4BoxHeader> {
    let tmp_size = src.read_u32::<BigEndian>().unwrap();
    let name = src.read_u32::<BigEndian>().unwrap();
    let size = match tmp_size {
        1 => src.read_u64::<BigEndian>().unwrap(),
        _ => tmp_size as u64,
    };
    assert!(size >= 8);
    if tmp_size == 1 {
        assert!(size >= 16);
    }
    let offset = match tmp_size {
        1 => 4 + 4 + 8,
        _ => 4 + 4,
    };
    assert!(offset <= size);
    Some(Mp4BoxHeader{
        name: name,
        size: size,
        offset: offset,
    })
}

/// Skip over the contents of a box.
pub fn skip_box_content<T: ReadBytesExt + std::io::Seek>
  (src: &mut T, header: &Mp4BoxHeader)
  -> std::io::Result<u64>
{
    src.seek(SeekFrom::Current((header.size - header.offset) as i64))
}

/// Parse an ftype box.
pub fn read_ftyp<T: ReadBytesExt>(src: &mut T) -> Option<Mp4FileTypeBox> {
    let head = read_box_header(src).unwrap();
    let major = src.read_u32::<BigEndian>().unwrap();
    let minor = src.read_u32::<BigEndian>().unwrap();
    let brand_count = (head.size - 8 - 8) /4;
    let mut brands = Vec::new();
    for _ in 0..brand_count {
        brands.push(src.read_u32::<BigEndian>().unwrap());
    }
    Some(Mp4FileTypeBox{
        name: head.name,
        size: head.size,
        major_brand: major,
        minor_version: minor,
        compatible_brands: brands,
    })
}

/// Split a u32 box type into a [u8].
fn u32_to_vec(x: u32) -> Vec<u8> {
    vec!((x >> 24 & 0xffu32) as u8,
         (x >> 16 & 0xffu32) as u8,
         (x >>  8 & 0xffu32) as u8,
         (x & 0xffu32) as u8)
}

use std::fmt;
impl fmt::Display for Mp4BoxHeader {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let name_bytes = u32_to_vec(self.name);
        let name = String::from_utf8_lossy(&name_bytes);
        write!(f, "'{}' {} bytes", name, self.size)
    }
}

impl fmt::Display for Mp4FileTypeBox {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let name_bytes = u32_to_vec(self.name);
        let name = String::from_utf8_lossy(&name_bytes);
        let brand_bytes = u32_to_vec(self.major_brand);
        let brand = String::from_utf8_lossy(&brand_bytes);
        write!(f, "'{}' {} bytes '{}' v{}", name, self.size,
            brand, self.minor_version)
    }
}

#[test]
fn test_read_box_header() {
    use std::io::Cursor;
    use std::io::Write;
    let mut test: Vec<u8> = vec![0, 0, 0, 8];  // minimal box length
    write!(&mut test, "test").unwrap(); // box type
    let mut stream = Cursor::new(test);
    let parsed = read_box_header(&mut stream).unwrap();
    assert_eq!(parsed.name, 1952805748);
    assert_eq!(parsed.size, 8);
    println!("box {}", parsed);
}

#[test]
fn test_read_box_header_long() {
    use std::io::Cursor;
    let mut test: Vec<u8> = vec![0, 0, 0, 1]; // long box extension code
    test.extend("long".to_string().into_bytes()); // box type
    test.extend(vec![0, 0, 0, 0, 0, 0, 16, 0]); // 64 bit size
    // Skip generating box content.
    let mut stream = Cursor::new(test);
    let parsed = read_box_header(&mut stream).unwrap();
    assert_eq!(parsed.name, 1819242087);
    assert_eq!(parsed.size, 4096);
    println!("box {}", parsed);
}

#[test]
fn test_read_ftyp() {
    use std::io::Cursor;
    use std::io::Write;
    let mut test: Vec<u8> = vec![0, 0, 0, 24]; // size
    write!(&mut test, "ftyp").unwrap(); // type
    write!(&mut test, "mp42").unwrap(); // major brand
    test.extend(vec![0, 0, 0, 0]);      // minor version
    write!(&mut test, "isom").unwrap(); // compatible brands...
    write!(&mut test, "mp42").unwrap();
    assert_eq!(test.len(), 24);

    let mut stream = Cursor::new(test);
    let parsed = read_ftyp(&mut stream).unwrap();
    assert_eq!(parsed.name, 1718909296);
    assert_eq!(parsed.size, 24);
    assert_eq!(parsed.major_brand, 1836069938);
    assert_eq!(parsed.minor_version, 0);
    assert_eq!(parsed.compatible_brands.len(), 2);
    println!("box {}", parsed);
}
<commit_msg>Convert u32 fourcc directly to strings.<commit_after>// Module for parsing ISO Base Media Format aka video/mp4 streams.

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

/// Basic ISO box structure.
pub struct Mp4BoxHeader {
    /// Four character box type
pub name: u32,
    /// Size of the box in bytes
pub size: u64,
    /// Offset to the start of the contained data (or header size).
pub offset: u64,
}

/// File type box 'ftyp'.
pub struct Mp4FileTypeBox {
    name: u32,
    size: u64,
    major_brand: u32,
    minor_version: u32,
    compatible_brands: Vec<u32>,
}

extern crate byteorder;
use byteorder::{BigEndian, ReadBytesExt};
use std::io::{Result, Seek, SeekFrom};

/// Parse a box out of a data buffer.
pub fn read_box_header<T: ReadBytesExt>(src: &mut T) -> Option<Mp4BoxHeader> {
    let tmp_size = src.read_u32::<BigEndian>().unwrap();
    let name = src.read_u32::<BigEndian>().unwrap();
    let size = match tmp_size {
        1 => src.read_u64::<BigEndian>().unwrap(),
        _ => tmp_size as u64,
    };
    assert!(size >= 8);
    if tmp_size == 1 {
        assert!(size >= 16);
    }
    let offset = match tmp_size {
        1 => 4 + 4 + 8,
        _ => 4 + 4,
    };
    assert!(offset <= size);
    Some(Mp4BoxHeader{
        name: name,
        size: size,
        offset: offset,
    })
}

/// Skip over the contents of a box.
pub fn skip_box_content<T: ReadBytesExt + std::io::Seek>
  (src: &mut T, header: &Mp4BoxHeader)
  -> std::io::Result<u64>
{
    src.seek(SeekFrom::Current((header.size - header.offset) as i64))
}

/// Parse an ftype box.
pub fn read_ftyp<T: ReadBytesExt>(src: &mut T) -> Option<Mp4FileTypeBox> {
    let head = read_box_header(src).unwrap();
    let major = src.read_u32::<BigEndian>().unwrap();
    let minor = src.read_u32::<BigEndian>().unwrap();
    let brand_count = (head.size - 8 - 8) /4;
    let mut brands = Vec::new();
    for _ in 0..brand_count {
        brands.push(src.read_u32::<BigEndian>().unwrap());
    }
    Some(Mp4FileTypeBox{
        name: head.name,
        size: head.size,
        major_brand: major,
        minor_version: minor,
        compatible_brands: brands,
    })
}

/// Convert the 4-character Mp4Box type to a string.
fn mp4_box_to_string(name: u32) -> String {
    let u32_to_vec = |u| {
        vec!((u >> 24 & 0xffu32) as u8,
             (u >> 16 & 0xffu32) as u8,
             (u >>  8 & 0xffu32) as u8,
             (u & 0xffu32) as u8)
    };
    let name_bytes = u32_to_vec(name);
    String::from_utf8_lossy(&name_bytes).into_owned()
}

use std::fmt;
impl fmt::Display for Mp4BoxHeader {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "'{}' {} bytes", mp4_box_to_string(self.name), self.size)
    }
}

impl fmt::Display for Mp4FileTypeBox {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let name = mp4_box_to_string(self.name);
        let brand = mp4_box_to_string(self.major_brand);
        write!(f, "'{}' {} bytes '{}' v{}", name, self.size,
            brand, self.minor_version)
    }
}

#[test]
fn test_read_box_header() {
    use std::io::Cursor;
    use std::io::Write;
    let mut test: Vec<u8> = vec![0, 0, 0, 8];  // minimal box length
    write!(&mut test, "test").unwrap(); // box type
    let mut stream = Cursor::new(test);
    let parsed = read_box_header(&mut stream).unwrap();
    assert_eq!(parsed.name, 1952805748);
    assert_eq!(parsed.size, 8);
    println!("box {}", parsed);
}

#[test]
fn test_read_box_header_long() {
    use std::io::Cursor;
    let mut test: Vec<u8> = vec![0, 0, 0, 1]; // long box extension code
    test.extend("long".to_string().into_bytes()); // box type
    test.extend(vec![0, 0, 0, 0, 0, 0, 16, 0]); // 64 bit size
    // Skip generating box content.
    let mut stream = Cursor::new(test);
    let parsed = read_box_header(&mut stream).unwrap();
    assert_eq!(parsed.name, 1819242087);
    assert_eq!(parsed.size, 4096);
    println!("box {}", parsed);
}

#[test]
fn test_read_ftyp() {
    use std::io::Cursor;
    use std::io::Write;
    let mut test: Vec<u8> = vec![0, 0, 0, 24]; // size
    write!(&mut test, "ftyp").unwrap(); // type
    write!(&mut test, "mp42").unwrap(); // major brand
    test.extend(vec![0, 0, 0, 0]);      // minor version
    write!(&mut test, "isom").unwrap(); // compatible brands...
    write!(&mut test, "mp42").unwrap();
    assert_eq!(test.len(), 24);

    let mut stream = Cursor::new(test);
    let parsed = read_ftyp(&mut stream).unwrap();
    assert_eq!(parsed.name, 1718909296);
    assert_eq!(parsed.size, 24);
    assert_eq!(parsed.major_brand, 1836069938);
    assert_eq!(parsed.minor_version, 0);
    assert_eq!(parsed.compatible_brands.len(), 2);
    println!("box {}", parsed);
}
<|endoftext|>


<commit_before>//! A memory-safer wrapper around system dynamic library primitives.
//!
//! With this library you can load [dynamic libraries](struct.Library.html) and retrieve
//! [symbols](struct.Symbol.html) from the loaded libraries.
//!
//! Less safe platform specific bindings are available in the [`os::platform`](os/index.html)
//! modules.
use std::ffi::OsStr;
use std::marker;

#[cfg(unix)]
#[macro_use]
extern crate lazy_static;

#[cfg(unix)]
use self::os::unix as imp;
#[cfg(windows)]
use self::os::windows as imp;

pub mod os;
mod util;

pub type Result<T> = ::std::io::Result<T>;

/// A dynamically loaded library.
pub struct Library(imp::Library);

impl Library {
    /// Find and load a shared library (module).
    ///
    /// Locations where library is searched for is platform specific and can’t be adjusted
    /// portably.
    ///
    /// # Examples
    ///
    /// ```ignore
    /// // on Unix
    /// let lib = Library::new("libm.so.6").unwrap();
    /// // on OS X
    /// let lib = Library::new("libm.dylib").unwrap();
    /// // on Windows
    /// let lib = Library::new("msvcrt.dll").unwrap();
    /// ```
    pub fn new<P: AsRef<OsStr>>(filename: P) -> Result<Library> {
        imp::Library::new(filename).map(Library)
    }

    /// Get a symbol by name.
    ///
    /// Mangling or symbol rustification is not done: trying to `get` something like `x::y`
    /// will not work.
    ///
    /// You may append a null byte at the end of the byte string to avoid string allocation in some
    /// cases. E.g. for symbol `sin` you may write `b"sin\0"` instead of `b"sin"`.
    ///
    /// # Unsafety
    ///
    /// Symbol of arbitrary requested type is returned. Using a symbol with wrong type is not
    /// memory safe.
    ///
    /// # Examples
    ///
    /// Simple function:
    ///
    /// ```ignore
    /// let sin: Symbol<extern fn(f64) -> f64> = unsafe {
    ///     lib.get(b"sin\0").unwrap()
    /// };
    /// ```
    ///
    /// A static or TLS variable:
    ///
    /// ```ignore
    /// let errno: Symbol<*mut u32> = unsafe {
    ///     lib.get(b"errno\0").unwrap()
    /// };
    /// ```
    pub unsafe fn get<'lib, T>(&'lib self, symbol: &[u8]) -> Result<Symbol<'lib, T>> {
        self.0.get(symbol).map(|from| {
            Symbol {
                inner: from,
                pd: marker::PhantomData
            }
        })
    }
}

/// Symbol from a library.
///
/// This type is a safeguard against using dynamically loaded symbols after a `Library` is
/// unloaded. Primary method to create an instance of a `Symbol` is via `Library::get`.
///
/// Due to implementation of the `Deref` trait, an instance of `Symbol` may be used as if it was a
/// function or variable directly, without taking care to “extract” function or variable manually
/// most of the time.
///
/// # Examples
///
/// ```ignore
/// let sin: Symbol<extern fn(f64) -> f64> = unsafe {
///     lib.get(b"sin\0").unwrap()
/// };
/// let sine0 = sin(0);
/// assert!(sine0 < 0.1E-10);
/// ```
pub struct Symbol<'lib, T: 'lib> {
    inner: imp::Symbol<T>,
    pd: marker::PhantomData<&'lib T>
}

impl<'lib, T> ::std::ops::Deref for Symbol<'lib, T> {
    type Target = T;
    fn deref(&self) -> &T {
        ::std::ops::Deref::deref(&self.inner)
    }
}

#[cfg(all(unix, not(any(target_os="macos", target_os="ios", target_os="android"))))]
#[test]
fn libm() {
    let lib = Library::new("libm.so.6").unwrap();
    let sin: Symbol<extern fn(f64) -> f64> = unsafe {
        lib.get(b"sin").unwrap()
    };
    assert!(sin(::std::f64::INFINITY).is_nan());
    let errno: Symbol<*mut u32> = unsafe {
        lib.get(b"errno").unwrap()
    };
    assert!(unsafe { **errno } != 0);
    unsafe { **errno = 0; }
}
<commit_msg>Add Debug impl for Library and Symbol<commit_after>//! A memory-safer wrapper around system dynamic library primitives.
//!
//! With this library you can load [dynamic libraries](struct.Library.html) and retrieve
//! [symbols](struct.Symbol.html) from the loaded libraries.
//!
//! Less safe platform specific bindings are available in the [`os::platform`](os/index.html)
//! modules.
use std::ffi::OsStr;
use std::fmt;
use std::marker;

#[cfg(unix)]
#[macro_use]
extern crate lazy_static;

#[cfg(unix)]
use self::os::unix as imp;
#[cfg(windows)]
use self::os::windows as imp;

pub mod os;
mod util;

pub type Result<T> = ::std::io::Result<T>;

/// A dynamically loaded library.
pub struct Library(imp::Library);

impl Library {
    /// Find and load a shared library (module).
    ///
    /// Locations where library is searched for is platform specific and can’t be adjusted
    /// portably.
    ///
    /// # Examples
    ///
    /// ```ignore
    /// // on Unix
    /// let lib = Library::new("libm.so.6").unwrap();
    /// // on OS X
    /// let lib = Library::new("libm.dylib").unwrap();
    /// // on Windows
    /// let lib = Library::new("msvcrt.dll").unwrap();
    /// ```
    pub fn new<P: AsRef<OsStr>>(filename: P) -> Result<Library> {
        imp::Library::new(filename).map(Library)
    }

    /// Get a symbol by name.
    ///
    /// Mangling or symbol rustification is not done: trying to `get` something like `x::y`
    /// will not work.
    ///
    /// You may append a null byte at the end of the byte string to avoid string allocation in some
    /// cases. E.g. for symbol `sin` you may write `b"sin\0"` instead of `b"sin"`.
    ///
    /// # Unsafety
    ///
    /// Symbol of arbitrary requested type is returned. Using a symbol with wrong type is not
    /// memory safe.
    ///
    /// # Examples
    ///
    /// Simple function:
    ///
    /// ```ignore
    /// let sin: Symbol<extern fn(f64) -> f64> = unsafe {
    ///     lib.get(b"sin\0").unwrap()
    /// };
    /// ```
    ///
    /// A static or TLS variable:
    ///
    /// ```ignore
    /// let errno: Symbol<*mut u32> = unsafe {
    ///     lib.get(b"errno\0").unwrap()
    /// };
    /// ```
    pub unsafe fn get<'lib, T>(&'lib self, symbol: &[u8]) -> Result<Symbol<'lib, T>> {
        self.0.get(symbol).map(|from| {
            Symbol {
                inner: from,
                pd: marker::PhantomData
            }
        })
    }
}

impl fmt::Debug for Library {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(f)
    }
}

/// Symbol from a library.
///
/// This type is a safeguard against using dynamically loaded symbols after a `Library` is
/// unloaded. Primary method to create an instance of a `Symbol` is via `Library::get`.
///
/// Due to implementation of the `Deref` trait, an instance of `Symbol` may be used as if it was a
/// function or variable directly, without taking care to “extract” function or variable manually
/// most of the time.
///
/// # Examples
///
/// ```ignore
/// let sin: Symbol<extern fn(f64) -> f64> = unsafe {
///     lib.get(b"sin\0").unwrap()
/// };
/// let sine0 = sin(0);
/// assert!(sine0 < 0.1E-10);
/// ```
pub struct Symbol<'lib, T: 'lib> {
    inner: imp::Symbol<T>,
    pd: marker::PhantomData<&'lib T>
}

impl<'lib, T> ::std::ops::Deref for Symbol<'lib, T> {
    type Target = T;
    fn deref(&self) -> &T {
        ::std::ops::Deref::deref(&self.inner)
    }
}

impl<'lib, T> fmt::Debug for Symbol<'lib, T> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.inner.fmt(f)
    }
}

#[cfg(all(unix, not(any(target_os="macos", target_os="ios", target_os="android"))))]
#[test]
fn libm() {
    let lib = Library::new("libm.so.6").unwrap();
    let sin: Symbol<extern fn(f64) -> f64> = unsafe {
        lib.get(b"sin").unwrap()
    };
    assert!(sin(::std::f64::INFINITY).is_nan());
    let errno: Symbol<*mut u32> = unsafe {
        lib.get(b"errno").unwrap()
    };
    assert!(unsafe { **errno } != 0);
    unsafe { **errno = 0; }
}
<|endoftext|>


<commit_before>#![feature(test)]
extern crate test;

use std::cell::UnsafeCell;
use std::ptr;
use std::sync::atomic;
use std::sync::atomic::Ordering;

const KEY_SIZE: usize = 16;
const INITIAL_SIZE: usize = 128;
const DJB2_START: usize = 5318;

type HashKey = [u8; KEY_SIZE];

#[derive(Debug)]
struct HashSlot {
    key: atomic::AtomicPtr<HashKey>,
    count: atomic::AtomicUsize,
}

/// The interior threadsafe counter struct. Everything here is atomic or
/// composed of atomics.
struct InnerCounter {
    size: usize,
    used: atomic::AtomicUsize,
    slots: atomic::AtomicPtr<Vec<HashSlot>>,
}

/// The public Counter structure which contains the inner mutable state. This
/// allows us to put this straight into an Arc and safely share between threads
pub struct Counter {
    inner: UnsafeCell<InnerCounter>
}

pub struct CounterIterator<'a> {
    counter: &'a Counter,
    index: usize,
    size: usize,
}

/// An implementation of the djb2 hash using rust iterators over the contents
/// of the `HashKey` array.
pub fn djb2_hash(key: &HashKey) -> usize {
    key.into_iter().take_while(|c| { **c != 0 }).fold(DJB2_START, |hash, c| { (hash * 33) ^ (*c as usize) })
}

/// A small inlined helper function that checks if a given character is ascii
fn is_ascii(c: &u8) -> bool {
    (*c > 47 && *c < 58) || (*c > 64 && *c < 91) || (*c > 96 && *c < 123)
}

/// Clean a given key string. This involves stripping non ascii characters out
/// and limiting the key to 16 characters. The resulting key is stack allocated
/// so be careful with it. If you need it long term put it in a Box.
pub fn clean_key(key: &str) -> HashKey {
    let mut cleaned: HashKey = [0; KEY_SIZE]; // Stack allocate a temporary key.
    for (index, v) in key.bytes().take(KEY_SIZE).filter(is_ascii).enumerate() {
        cleaned[index] = v;
    }
    cleaned
}

impl<'a> IntoIterator for &'a Counter {
    type Item = (String, usize);
    type IntoIter = CounterIterator<'a>;

    fn into_iter(self) -> Self::IntoIter {
        CounterIterator { counter: self, index: 0 , size: self.size() }
    }
}

impl<'a> Iterator for CounterIterator<'a> {
    type Item = (String, usize);
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            if self.index < self.size {
                match self.counter.get_index(self.index) {
                    Some((key, val)) => {
                        self.index += 1;
                        return Some((key, val))
                    },
                    None => self.index += 1,
                }
            } else {
                return None;
            }
        }
    }
}

impl HashSlot {
    /// Create a new HashSlot struct with the default values. The key is
    /// initially set to null so get/incr know this slot is empty.
    pub fn new() -> HashSlot {
        HashSlot {
            key: atomic::AtomicPtr::new(ptr::null_mut()),
            count: atomic::AtomicUsize::new(0),
        }
    }
}

impl InnerCounter {
    /// Create a new InnerCounter. Sets up all the top level atomic values for
    /// keeping track of global hashmap state.
    pub fn new(size: usize) -> InnerCounter {
        let mut slots: Vec<HashSlot> = Vec::with_capacity(size);
        for _ in 0..INITIAL_SIZE {
            slots.push(HashSlot::new());
        }
        let boxed = Box::new(slots);
        let counter: InnerCounter = InnerCounter {
            size: size,
            used: atomic::AtomicUsize::new(0),
            slots: atomic::AtomicPtr::new(Box::into_raw(boxed)),
        };
        counter
    }

    pub unsafe fn unsafe_get_index(&self, index: usize) -> Option<(HashKey, usize)> {
        if index > 0 && index < self.size {
            let slots = self.slots.load(Ordering::Relaxed);
            let slot = &((*slots)[index]);
            let key = slot.key.load(Ordering::Relaxed);
            if key.is_null() {
                return None;
            } else {
                let value = slot.count.load(Ordering::Relaxed);
                let key_copy: *mut HashKey = &mut [0; KEY_SIZE];
                ptr::copy(key, key_copy, 1);
                return Some((*key_copy, value));
            }
        }
        None
    }

    /// Unsafe method for returning the value of a counter.
    pub unsafe fn unsafe_get(&self, key: HashKey) -> usize {
        let size = self.size;
        let mut index = djb2_hash(&key) % size;
        let slots = self.slots.load(Ordering::Relaxed);
        loop {
            let slot = &((*slots)[index]);
            let other_key = slot.key.load(Ordering::Relaxed);
            if other_key.is_null() {
                return 0;
            }
            else if *other_key == key{
                return slot.count.load(Ordering::Relaxed);
            }
            index = (index + 1) % size;
        }
    }

    /// Unsafe method for incrementing a counter by the given amount. Returns
    /// the previous value of the counter.
    pub unsafe fn unsafe_incr(&mut self, key: HashKey, count: usize) -> usize {
        let size = self.size;
        let mut index = djb2_hash(&key) % size;
        let slots = self.slots.load(Ordering::Relaxed);
        let mut slot: &HashSlot;
        loop {
            slot = &((*slots)[index]);
            let other_key = slot.key.load(Ordering::Acquire);
            if other_key.is_null() {
                let boxed_key = Box::new(key);
                let res = slot.key.compare_exchange(other_key,
                                                    Box::into_raw(boxed_key),
                                                    Ordering::Release,
                                                    Ordering::Relaxed);
                let ptr = res.unwrap_or_else(|v| { v });
                if ptr.is_null() || *ptr == key {
                    self.used.fetch_add(1, Ordering::Relaxed);
                    break;
                }
            }
            else if *other_key == key {
                break;
            }
            index = (index + 1) % size
        }
        slot.count.fetch_add(count, Ordering::Relaxed)
    }
}

impl Drop for InnerCounter {
    /// Custom drop implementation to go through and explicitly re-box all the
    /// heap memory used to store keys within the Vec<HashSlot>. During the
    /// first call to unsafe_incr() the key is copied into a Box so it will
    /// persist.
    fn drop(&mut self) {
        unsafe {
            let slots = self.slots.load(Ordering::Relaxed);
            if !slots.is_null() {
                for hs in &*slots {
                    let key = hs.key.load(Ordering::Relaxed);
                    if !key.is_null() {
                        let _: Box<HashKey> = Box::from_raw(key);
                    }
                }
                let _: Box<Vec<HashSlot>> = Box::from_raw(slots);
            }
        }
    }
}

unsafe impl Sync for Counter {}
unsafe impl Send for Counter {}

impl Default for Counter {
    fn default() -> Self {
        Self::new()
    }
}
impl Counter {
    /// Create a new counter, which is has an UnsafeCell wrapping the actual
    /// counter implementation.
    pub fn new() -> Counter {
        Counter {
            inner: UnsafeCell::new(InnerCounter::new(INITIAL_SIZE)),
        }
    }

    /// Extract the mutable reference to the counter implementation and
    /// increment the given key.
    pub fn incr(&self, key: &str, count: usize) -> usize {
        unsafe {
            (*self.inner.get()).unsafe_incr(clean_key(key), count)
        }
    }

    /// Extract the mutable reference to the counter implementation and
    /// return the current counter value for the given key.
    pub fn get(&self, key: &str) -> usize {
        unsafe {
            (*self.inner.get()).unsafe_get(clean_key(key))
        }
    }

    /// Return the `HashKey` as a `String` and the count as a usize for the given
    /// index in the hashtable. NOTE: This breaks the abstraction of the hastable
    /// and is only used internally by IntoIterator
    pub fn get_index(&self, index: usize) -> Option<(String, usize)> {
        unsafe {
            (*self.inner.get())
                .unsafe_get_index(index)
                .map(|(hk, c)| (std::str::from_utf8(&hk).unwrap().to_owned(), c))
        }
    }

    pub fn size(&self) -> usize {
        unsafe {
            (*self.inner.get()).size
        }
    }
}

#[cfg(test)]
mod tests {
    use std::thread;
    use std::sync::Arc;
    use test::Bencher;
    use super::*;

    #[test]
    fn increment() {
        let counter = Counter::new();
        let res = counter.incr("foo", 1);
        assert_eq!(res, 0);
    }

    #[test]
    fn sequential_increment() {
        let counter = Counter::new();
        for _ in 0..100000 {
            counter.incr("foo", 1);
        }
        assert_eq!(counter.get("foo"), 100000);
    }

    #[test]
    fn sequential_iter() {
        let counter = Counter::new();
        for i in 0..20 {
            let key = format!("key_{0}", i);
            counter.incr(&key, 1);
        }
        for (key, val) in (&counter).into_iter() {
            println!("k: {}, v: {}", key, val);
            assert_eq!(val, 1)
        }
    }

    #[test]
    fn concurrent_iter() {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let nthreads = 8;
        let c = shared.clone();
        for i in 0..20 {
            let key = format!("key_{0}", i);
            c.incr(&key, 1);
        }
        let mut children = vec![];
        for _ in 0..nthreads {
            let c = shared.clone();
            children.push(thread::spawn(move|| {
                for (_, val) in (&c).into_iter() {
                    assert_eq!(val, 1);
                }
            }));
        }
        for t in children {
            let _ = t.join();
        }
    }

    #[test]
    fn concurrent_iter_and_incr() {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let mut children = vec![];
        let c1 = shared.clone();
        children.push(thread::spawn(move|| {
            for _ in 0..200 {
                for (key, _) in (&c1).into_iter() {
                    assert_eq!(key, "key_1")
                }
            }
        }));
        let c2 = shared.clone();
        children.push(thread::spawn(move|| {
            for _ in 0..10000 {
                c2.incr("key_1", 1);
            }
        }));

        for t in children {
            let _ = t.join();
        }
    }

    #[test]
    fn concurrent_increment() {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let nthreads = 8;
        let nincr = 10000;
        let mut children = vec![];
        for _ in 0..nthreads {
            let c = shared.clone();
            children.push(thread::spawn(move|| {
                for _ in 0..nincr {
                    c.incr("foo", 1);
                }
            }));
        }
        for t in children {
            let _ = t.join();
        }
        assert_eq!(shared.get("foo"), nincr * nthreads);
    }

    #[test]
    fn concurrent_multi_key_increment() {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let nthreads = 8;
        let nincr = 10000;
        let mut children = vec![];
        for i in 0..nthreads {
            let c = shared.clone();
            let key = format!("thread_{}", i);
            children.push(thread::spawn(move|| {
                for _ in 0..nincr {
                    c.incr(&key, 1);
                }
            }));
        }
        for t in children {
            let _ = t.join();
        }
        for i in 0..nthreads {
            let key = format!("thread_{}", i);
            assert_eq!(shared.get(&key), nincr);
        }
    }

    #[test]
    fn get() {
        let counter = Counter::new();
        let _ = counter.incr("foo", 1);
        let count = counter.get("foo");
        assert_eq!(count, 1);
    }

    #[bench]
    fn bench_clean_key(b: &mut Bencher) {
        b.iter(|| clean_key("foobar"));
    }

    #[bench]
    fn bench_djb2_hash(b: &mut Bencher) {
        let key = clean_key("foobar");
        b.iter(|| djb2_hash(&key));
    }

    #[bench]
    fn bench_incr(b: &mut Bencher) {
        let counter = Counter::new();
        b.iter(|| counter.incr("foo", 1))
    }

    #[bench]
    fn bench_existing_incr(b: &mut Bencher) {
        let counter = Counter::new();
        counter.incr("foo", 1);
        b.iter(|| counter.incr("foo", 1));
    }

    #[bench]
    fn bench_get(b: &mut Bencher) {
        let counter = Counter::new();
        counter.incr("foo", 100);
        b.iter(|| counter.get("foo"))
    }

    #[bench]
    fn bench_into_iter(b: &mut Bencher) {
        let counter = Counter::new();
        let key = format!("key_{0}", 1);
        counter.incr(&key, 1);
        b.iter(|| (&counter).into_iter())
    }

    #[bench]
    fn bench_iter(b: &mut Bencher) {
        let counter = Counter::new();
        let key = format!("key_{0}", 1);
        counter.incr(&key, 1);
        b.iter(|| {
            for (key, val) in counter.into_iter() {
                drop(key);
                drop(val);
            }
        })

    }

    #[bench]
    fn bench_arc_incr(b: &mut Bencher) {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let clone = shared.clone();
        b.iter(|| clone.incr("foo", 1))
    }

    #[bench]
    fn bench_arc_get(b: &mut Bencher) {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let clone = shared.clone();
        clone.incr("foo", 1);
        b.iter(|| clone.get("foo"))
    }
}
<commit_msg>Fix a comment<commit_after>#![feature(test)]
extern crate test;

use std::cell::UnsafeCell;
use std::ptr;
use std::sync::atomic;
use std::sync::atomic::Ordering;

const KEY_SIZE: usize = 16;
const INITIAL_SIZE: usize = 128;
const DJB2_START: usize = 5318;

type HashKey = [u8; KEY_SIZE];

#[derive(Debug)]
struct HashSlot {
    key: atomic::AtomicPtr<HashKey>,
    count: atomic::AtomicUsize,
}

/// The interior threadsafe counter struct. Everything here is atomic or
/// composed of atomics.
struct InnerCounter {
    size: usize,
    used: atomic::AtomicUsize,
    slots: atomic::AtomicPtr<Vec<HashSlot>>,
}

/// The public Counter structure which contains the inner mutable state. This
/// allows us to put this straight into an Arc and safely share between threads
pub struct Counter {
    inner: UnsafeCell<InnerCounter>
}

pub struct CounterIterator<'a> {
    counter: &'a Counter,
    index: usize,
    size: usize,
}

/// An implementation of the djb2 hash using rust iterators over the contents
/// of the `HashKey` array.
pub fn djb2_hash(key: &HashKey) -> usize {
    key.into_iter().take_while(|c| { **c != 0 }).fold(DJB2_START, |hash, c| { (hash * 33) ^ (*c as usize) })
}

/// A small inlinable helper function that checks if a given character is ascii
fn is_ascii(c: &u8) -> bool {
    (*c > 47 && *c < 58) || (*c > 64 && *c < 91) || (*c > 96 && *c < 123)
}

/// Clean a given key string. This involves stripping non ascii characters out
/// and limiting the key to 16 characters. The resulting key is stack allocated
/// so be careful with it. If you need it long term put it in a Box.
pub fn clean_key(key: &str) -> HashKey {
    let mut cleaned: HashKey = [0; KEY_SIZE]; // Stack allocate a temporary key.
    for (index, v) in key.bytes().take(KEY_SIZE).filter(is_ascii).enumerate() {
        cleaned[index] = v;
    }
    cleaned
}

impl<'a> IntoIterator for &'a Counter {
    type Item = (String, usize);
    type IntoIter = CounterIterator<'a>;

    fn into_iter(self) -> Self::IntoIter {
        CounterIterator { counter: self, index: 0 , size: self.size() }
    }
}

impl<'a> Iterator for CounterIterator<'a> {
    type Item = (String, usize);
    fn next(&mut self) -> Option<Self::Item> {
        loop {
            if self.index < self.size {
                match self.counter.get_index(self.index) {
                    Some((key, val)) => {
                        self.index += 1;
                        return Some((key, val))
                    },
                    None => self.index += 1,
                }
            } else {
                return None;
            }
        }
    }
}

impl HashSlot {
    /// Create a new HashSlot struct with the default values. The key is
    /// initially set to null so get/incr know this slot is empty.
    pub fn new() -> HashSlot {
        HashSlot {
            key: atomic::AtomicPtr::new(ptr::null_mut()),
            count: atomic::AtomicUsize::new(0),
        }
    }
}

impl InnerCounter {
    /// Create a new InnerCounter. Sets up all the top level atomic values for
    /// keeping track of global hashmap state.
    pub fn new(size: usize) -> InnerCounter {
        let mut slots: Vec<HashSlot> = Vec::with_capacity(size);
        for _ in 0..INITIAL_SIZE {
            slots.push(HashSlot::new());
        }
        let boxed = Box::new(slots);
        let counter: InnerCounter = InnerCounter {
            size: size,
            used: atomic::AtomicUsize::new(0),
            slots: atomic::AtomicPtr::new(Box::into_raw(boxed)),
        };
        counter
    }

    pub unsafe fn unsafe_get_index(&self, index: usize) -> Option<(HashKey, usize)> {
        if index > 0 && index < self.size {
            let slots = self.slots.load(Ordering::Relaxed);
            let slot = &((*slots)[index]);
            let key = slot.key.load(Ordering::Relaxed);
            if key.is_null() {
                return None;
            } else {
                let value = slot.count.load(Ordering::Relaxed);
                let key_copy: *mut HashKey = &mut [0; KEY_SIZE];
                ptr::copy(key, key_copy, 1);
                return Some((*key_copy, value));
            }
        }
        None
    }

    /// Unsafe method for returning the value of a counter.
    pub unsafe fn unsafe_get(&self, key: HashKey) -> usize {
        let size = self.size;
        let mut index = djb2_hash(&key) % size;
        let slots = self.slots.load(Ordering::Relaxed);
        loop {
            let slot = &((*slots)[index]);
            let other_key = slot.key.load(Ordering::Relaxed);
            if other_key.is_null() {
                return 0;
            }
            else if *other_key == key{
                return slot.count.load(Ordering::Relaxed);
            }
            index = (index + 1) % size;
        }
    }

    /// Unsafe method for incrementing a counter by the given amount. Returns
    /// the previous value of the counter.
    pub unsafe fn unsafe_incr(&mut self, key: HashKey, count: usize) -> usize {
        let size = self.size;
        let mut index = djb2_hash(&key) % size;
        let slots = self.slots.load(Ordering::Relaxed);
        let mut slot: &HashSlot;
        loop {
            slot = &((*slots)[index]);
            let other_key = slot.key.load(Ordering::Acquire);
            if other_key.is_null() {
                let boxed_key = Box::new(key);
                let res = slot.key.compare_exchange(other_key,
                                                    Box::into_raw(boxed_key),
                                                    Ordering::Release,
                                                    Ordering::Relaxed);
                let ptr = res.unwrap_or_else(|v| { v });
                if ptr.is_null() || *ptr == key {
                    self.used.fetch_add(1, Ordering::Relaxed);
                    break;
                }
            }
            else if *other_key == key {
                break;
            }
            index = (index + 1) % size
        }
        slot.count.fetch_add(count, Ordering::Relaxed)
    }
}

impl Drop for InnerCounter {
    /// Custom drop implementation to go through and explicitly re-box all the
    /// heap memory used to store keys within the Vec<HashSlot>. During the
    /// first call to unsafe_incr() the key is copied into a Box so it will
    /// persist.
    fn drop(&mut self) {
        unsafe {
            let slots = self.slots.load(Ordering::Relaxed);
            if !slots.is_null() {
                for hs in &*slots {
                    let key = hs.key.load(Ordering::Relaxed);
                    if !key.is_null() {
                        let _: Box<HashKey> = Box::from_raw(key);
                    }
                }
                let _: Box<Vec<HashSlot>> = Box::from_raw(slots);
            }
        }
    }
}

unsafe impl Sync for Counter {}
unsafe impl Send for Counter {}

impl Default for Counter {
    fn default() -> Self {
        Self::new()
    }
}
impl Counter {
    /// Create a new counter, which is has an UnsafeCell wrapping the actual
    /// counter implementation.
    pub fn new() -> Counter {
        Counter {
            inner: UnsafeCell::new(InnerCounter::new(INITIAL_SIZE)),
        }
    }

    /// Extract the mutable reference to the counter implementation and
    /// increment the given key.
    pub fn incr(&self, key: &str, count: usize) -> usize {
        unsafe {
            (*self.inner.get()).unsafe_incr(clean_key(key), count)
        }
    }

    /// Extract the mutable reference to the counter implementation and
    /// return the current counter value for the given key.
    pub fn get(&self, key: &str) -> usize {
        unsafe {
            (*self.inner.get()).unsafe_get(clean_key(key))
        }
    }

    /// Return the `HashKey` as a `String` and the count as a usize for the given
    /// index in the hashtable. NOTE: This breaks the abstraction of the hastable
    /// and is only used internally by IntoIterator
    pub fn get_index(&self, index: usize) -> Option<(String, usize)> {
        unsafe {
            (*self.inner.get())
                .unsafe_get_index(index)
                .map(|(hk, c)| (std::str::from_utf8(&hk).unwrap().to_owned(), c))
        }
    }

    pub fn size(&self) -> usize {
        unsafe {
            (*self.inner.get()).size
        }
    }
}

#[cfg(test)]
mod tests {
    use std::thread;
    use std::sync::Arc;
    use test::Bencher;
    use super::*;

    #[test]
    fn increment() {
        let counter = Counter::new();
        let res = counter.incr("foo", 1);
        assert_eq!(res, 0);
    }

    #[test]
    fn sequential_increment() {
        let counter = Counter::new();
        for _ in 0..100000 {
            counter.incr("foo", 1);
        }
        assert_eq!(counter.get("foo"), 100000);
    }

    #[test]
    fn sequential_iter() {
        let counter = Counter::new();
        for i in 0..20 {
            let key = format!("key_{0}", i);
            counter.incr(&key, 1);
        }
        for (key, val) in (&counter).into_iter() {
            println!("k: {}, v: {}", key, val);
            assert_eq!(val, 1)
        }
    }

    #[test]
    fn concurrent_iter() {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let nthreads = 8;
        let c = shared.clone();
        for i in 0..20 {
            let key = format!("key_{0}", i);
            c.incr(&key, 1);
        }
        let mut children = vec![];
        for _ in 0..nthreads {
            let c = shared.clone();
            children.push(thread::spawn(move|| {
                for (_, val) in (&c).into_iter() {
                    assert_eq!(val, 1);
                }
            }));
        }
        for t in children {
            let _ = t.join();
        }
    }

    #[test]
    fn concurrent_iter_and_incr() {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let mut children = vec![];
        let c1 = shared.clone();
        children.push(thread::spawn(move|| {
            for _ in 0..200 {
                for (key, _) in (&c1).into_iter() {
                    assert_eq!(key, "key_1")
                }
            }
        }));
        let c2 = shared.clone();
        children.push(thread::spawn(move|| {
            for _ in 0..10000 {
                c2.incr("key_1", 1);
            }
        }));

        for t in children {
            let _ = t.join();
        }
    }

    #[test]
    fn concurrent_increment() {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let nthreads = 8;
        let nincr = 10000;
        let mut children = vec![];
        for _ in 0..nthreads {
            let c = shared.clone();
            children.push(thread::spawn(move|| {
                for _ in 0..nincr {
                    c.incr("foo", 1);
                }
            }));
        }
        for t in children {
            let _ = t.join();
        }
        assert_eq!(shared.get("foo"), nincr * nthreads);
    }

    #[test]
    fn concurrent_multi_key_increment() {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let nthreads = 8;
        let nincr = 10000;
        let mut children = vec![];
        for i in 0..nthreads {
            let c = shared.clone();
            let key = format!("thread_{}", i);
            children.push(thread::spawn(move|| {
                for _ in 0..nincr {
                    c.incr(&key, 1);
                }
            }));
        }
        for t in children {
            let _ = t.join();
        }
        for i in 0..nthreads {
            let key = format!("thread_{}", i);
            assert_eq!(shared.get(&key), nincr);
        }
    }

    #[test]
    fn get() {
        let counter = Counter::new();
        let _ = counter.incr("foo", 1);
        let count = counter.get("foo");
        assert_eq!(count, 1);
    }

    #[bench]
    fn bench_clean_key(b: &mut Bencher) {
        b.iter(|| clean_key("foobar"));
    }

    #[bench]
    fn bench_djb2_hash(b: &mut Bencher) {
        let key = clean_key("foobar");
        b.iter(|| djb2_hash(&key));
    }

    #[bench]
    fn bench_incr(b: &mut Bencher) {
        let counter = Counter::new();
        b.iter(|| counter.incr("foo", 1))
    }

    #[bench]
    fn bench_existing_incr(b: &mut Bencher) {
        let counter = Counter::new();
        counter.incr("foo", 1);
        b.iter(|| counter.incr("foo", 1));
    }

    #[bench]
    fn bench_get(b: &mut Bencher) {
        let counter = Counter::new();
        counter.incr("foo", 100);
        b.iter(|| counter.get("foo"))
    }

    #[bench]
    fn bench_into_iter(b: &mut Bencher) {
        let counter = Counter::new();
        let key = format!("key_{0}", 1);
        counter.incr(&key, 1);
        b.iter(|| (&counter).into_iter())
    }

    #[bench]
    fn bench_iter(b: &mut Bencher) {
        let counter = Counter::new();
        let key = format!("key_{0}", 1);
        counter.incr(&key, 1);
        b.iter(|| {
            for (key, val) in counter.into_iter() {
                drop(key);
                drop(val);
            }
        })

    }

    #[bench]
    fn bench_arc_incr(b: &mut Bencher) {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let clone = shared.clone();
        b.iter(|| clone.incr("foo", 1))
    }

    #[bench]
    fn bench_arc_get(b: &mut Bencher) {
        let counter = Counter::new();
        let shared = Arc::new(counter);
        let clone = shared.clone();
        clone.incr("foo", 1);
        b.iter(|| clone.get("foo"))
    }
}
<|endoftext|>


<commit_before>//! `Small Box` optimization: store small item on the stack and fallback to heap for large item.
//!
//! # Usage
//!
//! First, add the following to your `Cargo.toml`:
//!
//! ```toml
//! [dependencies]
//! smallbox = "0.5"
//! ```
//!
//! Next, add this to your crate root:
//!
//! ```rust
//! extern crate smallbox;
//! ```
//!
//! If you want this crate to work with dynamic-sized type, you can request it via:
//!
//! ```toml
//! [dependencies]
//! smallbox = { version = "0.5", features = ["unsize"] }
//! ```
//!
//! Currently `smallbox` by default links to the standard library, but if you would
//! instead like to use this crate in a `#![no_std]` situation or crate, and want to
//! opt out heap dependency and `SmallBox<T>` type, you can request this via:
//!
//! ```toml
//! [dependencies.smallbox]
//! version = "0.5"
//! features = ["unsize"]
//! default-features = false
//! ```
//!
//! Enable `heap` feature for `#![no_std]` build to link to `alloc` crate
//! and bring `SmallBox<T>` back.
//!
//! ```toml
//! [dependencies.smallbox]
//! version = "0.5"
//! features = ["unsize", "heap"]
//! default-features = false
//! ```
//!
//!
//! # Feature Flags
//!
//! This crate has the following cargo feature flags:
//!
//! - `std`
//!   - Optional, enabled by default
//!   - Use libstd
//!
//!
//! - `heap`
//!   - Optional, enabled by default
//!   - Support heap fallback by including `SmallBox<T>`
//!   - If `std` feature flag is opted out, this will link
//!   `alloc` crate, and it need nightly rust for that.
//!
//! - `unsize`
//!   - Optional
//!   - Require nightly rust
//!   - Enable support for `DST` (dynamic-sized type).
//!
//!
//! # Stable Rust
//!
//! The only possible way to use this crate on stable rust is to use the default feature flag, which means you can't use it in `no_std`
//! environment or use it with `DST` (dynamic-sized type).
//!
//! # Unsized Type
//!
//! Once the feature `unsize` is enabled, the item type `T` of `SmallBox` and `StackBox` can
//! and should be a unsized type, such as trait object or owned array slice.
//!
//! # Overview
//! This crate delivers two core type:
//!
//! - `SmallBox<T, Space>`: Stores `T` on heap or stack depending on the size of `T`. It takes `StackBox<T, Space>` as an varience to store small item, and then fallback to heap allocated `Box<T>` when type `T` is larger then the capacity of `Space`.
//!
//! - `StackBox<T, Space>`: Represents as a fixed-capacity allocation, and  stores item on stack.
//!
//! # Example
//!
//! Eliminate heap alloction for small items by `SmallBox`:
//!
//! ```rust
//! # #[cfg(not(feature = "unsize"))]
//! # {
//! use smallbox::SmallBox;
//! use smallbox::space::S4;
//!
//! let small: SmallBox<_, S4> = SmallBox::new([0; 2]);
//! let large: SmallBox<_, S4> = SmallBox::new([0; 32]);
//!
//! assert_eq!(small.len(), 2);
//! assert_eq!(large.len(), 32);
//!
//! match small {
//!     SmallBox::Stack(val) => assert_eq!(*val, [0; 2]),
//!     _ => unreachable!()
//! }
//!
//! match large {
//!     SmallBox::Box(val) => assert_eq!(*val, [0; 32]),
//!     _ => unreachable!()
//! }
//! # }
//! ```
//!
//! ## DST
//!
//! The following examples requires `unsize` feature flag enabled.
//!
//! Trait object dynamic-dispatch:
//!
//! ```rust
//! # #[cfg(feature = "unsize")]
//! # {
//! use smallbox::StackBox;
//! use smallbox::space::S1;
//!
//! let val: StackBox<PartialEq<usize>, S1> = StackBox::new(5usize).unwrap();
//!
//! assert!(*val == 5)
//! # }
//! ```
//!
//! `Any` downcasting:
//!
//! ```rust
//! # #[cfg(feature = "unsize")]
//! # {
//! use std::any::Any;
//! use smallbox::SmallBox;
//! use smallbox::space::S2;
//!
//! let num: SmallBox<Any, S2> = SmallBox::new(1234u32);
//!
//! if let Some(num) = num.downcast_ref::<u32>() {
//!     assert_eq!(*num, 1234);
//! } else {
//!     unreachable!();
//! }
//! # }
//! ```
//!
//!
//! # Capacity
//! The capacity of `SmallBox<T, Space>` and `StackBox<T，Space>` is expressed by the size of type **`Space`**, regardless of what the `Space` actually is.
//!
//! This crate provides some spaces in module `smallbox::space`, from `S2`, 'S4' to `S64`, representing a `"n * usize"` space.
//!
//! Anyway, you can defind your own space type, such as a byte array `[u8;64]`.
//!
//! The `resize()` method on `StackBox<T, Space>` and `SmallBox<T, Space>` is used to transforms the capacity.
//!
//! ```rust
//! # #[cfg(not(feature = "unsize"))]
//! # {
//! use smallbox::SmallBox;
//! use smallbox::space::{S8, S16};
//!
//! let s = SmallBox::<[usize; 8], S8>::new([0usize; 8]);
//! let m = s.resize::<S16>();
//! # }
//! ```

#![cfg_attr(
    feature = "unsize",
    feature(unsize, box_syntax)
)]
#![cfg_attr(not(feature = "std"), no_std)]
#![cfg_attr(all(feature = "heap", not(feature = "std")), feature(alloc))]

#[cfg(all(feature = "heap", not(feature = "std")))]
extern crate alloc;
#[cfg(not(feature = "std"))]
extern crate core as std;

mod smallbox;
pub mod space;
mod stackbox;

pub use smallbox::SmallBox;
pub use stackbox::StackBox;

#[cfg(not(any(feature = "heap", feature = "unsize")))]
compile_error!("Either feature \"heap\" or \"unsize\" must be enabled for this crate.");
<commit_msg>fix mod<commit_after>//! `Small Box` optimization: store small item on the stack and fallback to heap for large item.
//!
//! # Usage
//!
//! First, add the following to your `Cargo.toml`:
//!
//! ```toml
//! [dependencies]
//! smallbox = "0.5"
//! ```
//!
//! Next, add this to your crate root:
//!
//! ```rust
//! extern crate smallbox;
//! ```
//!
//! If you want this crate to work with dynamic-sized type, you can request it via:
//!
//! ```toml
//! [dependencies]
//! smallbox = { version = "0.5", features = ["unsize"] }
//! ```
//!
//! Currently `smallbox` by default links to the standard library, but if you would
//! instead like to use this crate in a `#![no_std]` situation or crate, and want to
//! opt out heap dependency and `SmallBox<T>` type, you can request this via:
//!
//! ```toml
//! [dependencies.smallbox]
//! version = "0.5"
//! features = ["unsize"]
//! default-features = false
//! ```
//!
//! Enable `heap` feature for `#![no_std]` build to link to `alloc` crate
//! and bring `SmallBox<T>` back.
//!
//! ```toml
//! [dependencies.smallbox]
//! version = "0.5"
//! features = ["unsize", "heap"]
//! default-features = false
//! ```
//!
//!
//! # Feature Flags
//!
//! This crate has the following cargo feature flags:
//!
//! - `std`
//!   - Optional, enabled by default
//!   - Use libstd
//!
//!
//! - `heap`
//!   - Optional, enabled by default
//!   - Support heap fallback by including `SmallBox<T>`
//!   - If `std` feature flag is opted out, this will link
//!   `alloc` crate, and it need nightly rust for that.
//!
//! - `unsize`
//!   - Optional
//!   - Require nightly rust
//!   - Enable support for `DST` (dynamic-sized type).
//!
//!
//! # Stable Rust
//!
//! The only possible way to use this crate on stable rust is to use the default feature flag, which means you can't use it in `no_std`
//! environment or use it with `DST` (dynamic-sized type).
//!
//! # Unsized Type
//!
//! Once the feature `unsize` is enabled, the item type `T` of `SmallBox` and `StackBox` can
//! and should be a unsized type, such as trait object or owned array slice.
//!
//! # Overview
//! This crate delivers two core type:
//!
//! - `SmallBox<T, Space>`: Stores `T` on heap or stack depending on the size of `T`. It takes `StackBox<T, Space>` as an varience to store small item, and then fallback to heap allocated `Box<T>` when type `T` is larger then the capacity of `Space`.
//!
//! - `StackBox<T, Space>`: Represents as a fixed-capacity allocation, and  stores item on stack.
//!
//! # Example
//!
//! Eliminate heap alloction for small items by `SmallBox`:
//!
//! ```rust
//! # #[cfg(not(feature = "unsize"))]
//! # {
//! use smallbox::SmallBox;
//! use smallbox::space::S4;
//!
//! let small: SmallBox<_, S4> = SmallBox::new([0; 2]);
//! let large: SmallBox<_, S4> = SmallBox::new([0; 32]);
//!
//! assert_eq!(small.len(), 2);
//! assert_eq!(large.len(), 32);
//!
//! match small {
//!     SmallBox::Stack(val) => assert_eq!(*val, [0; 2]),
//!     _ => unreachable!()
//! }
//!
//! match large {
//!     SmallBox::Box(val) => assert_eq!(*val, [0; 32]),
//!     _ => unreachable!()
//! }
//! # }
//! ```
//!
//! ## DST
//!
//! The following examples requires `unsize` feature flag enabled.
//!
//! Trait object dynamic-dispatch:
//!
//! ```rust
//! # #[cfg(feature = "unsize")]
//! # {
//! use smallbox::StackBox;
//! use smallbox::space::S1;
//!
//! let val: StackBox<PartialEq<usize>, S1> = StackBox::new(5usize).unwrap();
//!
//! assert!(*val == 5)
//! # }
//! ```
//!
//! `Any` downcasting:
//!
//! ```rust
//! # #[cfg(feature = "unsize")]
//! # {
//! use std::any::Any;
//! use smallbox::SmallBox;
//! use smallbox::space::S2;
//!
//! let num: SmallBox<Any, S2> = SmallBox::new(1234u32);
//!
//! if let Some(num) = num.downcast_ref::<u32>() {
//!     assert_eq!(*num, 1234);
//! } else {
//!     unreachable!();
//! }
//! # }
//! ```
//!
//!
//! # Capacity
//! The capacity of `SmallBox<T, Space>` and `StackBox<T，Space>` is expressed by the size of type **`Space`**, regardless of what the `Space` actually is.
//!
//! This crate provides some spaces in module `smallbox::space`, from `S2`, 'S4' to `S64`, representing a `"n * usize"` space.
//!
//! Anyway, you can defind your own space type, such as a byte array `[u8;64]`.
//!
//! The `resize()` method on `StackBox<T, Space>` and `SmallBox<T, Space>` is used to transforms the capacity.
//!
//! ```rust
//! # #[cfg(not(feature = "unsize"))]
//! # {
//! use smallbox::SmallBox;
//! use smallbox::space::{S8, S16};
//!
//! let s = SmallBox::<[usize; 8], S8>::new([0usize; 8]);
//! let m = s.resize::<S16>();
//! # }
//! ```

#![cfg_attr(feature = "unsize", feature(unsize, box_syntax))]
#![cfg_attr(not(feature = "std"), no_std)]
#![cfg_attr(all(feature = "heap", not(feature = "std")), feature(alloc))]

#[cfg(all(feature = "heap", not(feature = "std")))]
extern crate alloc;
#[cfg(not(feature = "std"))]
extern crate core as std;

#[cfg(feature = "heap")]
mod smallbox;
pub mod space;
mod stackbox;

#[cfg(feature = "heap")]
pub use smallbox::SmallBox;
pub use stackbox::StackBox;

#[cfg(not(any(feature = "heap", feature = "unsize")))]
compile_error!("Either feature \"heap\" or \"unsize\" must be enabled for this crate.");
<|endoftext|>