<commit_before>
use std::f32;
use std::vec;
use std::collections::HashMap;
use std::collections::hash_set::HashSet;

extern crate ocl;
use ocl::{ProQue, Buffer};

static KERNEL_SOURCE: &'static str = include_str!("kernel_source.cl");

type Dimension = (usize, usize);
type NodeId = usize;

//		ActivationFunction::Sigmoid => 1.0f32/(1.0 + (-value).exp()),

#[derive(Hash, Eq, PartialEq, Debug, Copy, Clone)]
enum Operation<F> {
	Input,
	MatrixMultiply(NodeId, NodeId),
	MatrixBinaryOp(NodeId, NodeId, F),
	MatrixUnaryOp(NodeId, F),
//	Transpose(NodeId),
}

struct Node<F> {
	id : NodeId,
	shape : Dimension,
	operation : Operation<F>,
	buffer : Vec<f32>,
}

impl<F> Node<F> {
	fn get_value(&self, x : usize, y : usize) -> f32 {
		self.buffer[x + y*self.shape.1]
	}

	fn set_value(&mut self, x : usize, y : usize, value : f32) {
		self.buffer[x + y*self.shape.1] = value;
	}

	fn get_rows(&self) -> usize {
		self.shape.0
	}

	fn get_columns(&self) -> usize {
		self.shape.1
	}

	fn get_width(&self) -> usize {
		self.shape.1
	}

	fn get_height(&self) -> usize {
		self.shape.0
	}
}

struct Graph<F> {
//	proque : ProQue,
	nodes : Vec<Node<F>>,
}

impl<F> Graph<F> where F: Fn(f32, f32)->f32 {
	fn new() -> Graph<F> {
		Graph {
//			proque : ProQue::builder().src(KERNEL_SOURCE).dims([3]).build().unwrap(),
			nodes : vec![],
		}
	}

	// Graph methods
	fn get_output(&mut self, node_id : NodeId, input_map : &HashMap<NodeId, Vec<f32>>) -> Vec<f32> {
		//(node.operation)(&self, input_map)
		match self.nodes[node_id].operation {
			Operation::Input => input_map.get(&node_id).unwrap().clone(),
			Operation::MatrixMultiply(n1, n2) => {
				// Verify shapes
				let a_width = self.nodes[n1].shape.1; // Columns (j)
				let a_height = self.nodes[n1].shape.0; // Rows (i)
				let b_width = self.nodes[n2].shape.1; // Columns (k)
				let b_height = self.nodes[n2].shape.0; // Rows (j)
				let c_width = b_width;
				let c_height = a_height;
				assert_eq!(a_width, b_height);
				assert_eq!(a_height*b_width, self.nodes[node_id].buffer.len());

				// Set up the buffer for this node.
				let a : Vec<f32> = self.get_output(n1, &input_map);
				let b : Vec<f32> = self.get_output(n2, &input_map);

				// Multiply
				for i in 0..a_height { // Column [Iterating over row]
					for k in 0..b_width { // Row/Width [Iterating over column]
						let mut accumulator = 0.0;
						for j in 0..a_width { // Column
							accumulator += a[j + i*a_width]*b[k + j*b_width];
						}
						self.nodes[node_id].buffer[k + i*c_width] = accumulator;
					}
				}

				self.nodes[node_id].buffer.clone()
			},
			Operation::MatrixBinaryOp(n1, n2, f) => {
				let a : Vec<f32> = self.get_output(n1, &input_map);
				let b : Vec<f32> = self.get_output(n2, &input_map);

				let vec_len = self.nodes[node_id].buffer.len();

				for i in 0..vec_len {
					self.nodes[node_id].buffer[i] = f(a[i], b[i]);
				}

				self.nodes[node_id].buffer.clone()
			},
			Operation::MatrixUnaryOp(n1, f) => {
				let a : Vec<f32> = self.get_output(n1, &input_map);
				let vec_len = self.nodes[node_id].buffer.len();

				for i in 0..vec_len {
					self.nodes[node_id].buffer[i] = f(a[i], i as f32);
				}

				self.nodes[node_id].buffer.clone()
			}
		}
	}

	// Node creation
	fn new_input(&mut self, shape : Dimension) -> NodeId {
		let mut n = Node {
			id : 0,
			shape : shape,
			operation : Operation::Input,
			buffer : vec![0.0; shape.0*shape.1], //self.proque.create_buffer::<f32>().unwrap()
		};
		self.nodes.push(n);
		let id = self.nodes.len()-1;
		n.id = id;
		id
	}

	fn new_matmul(&mut self, node_a_id : NodeId, node_b_id : NodeId) -> NodeId {
		let mut n = Node {
			id : 0,
			shape : (self.nodes[node_a_id].shape.0, self.nodes[node_b_id].shape.1),
			operation : Operation::MatrixMultiply(node_a_id, node_b_id),
			buffer : vec![0.0; self.nodes[node_a_id].shape.0*self.nodes[node_b_id].shape.1],
		};
		self.nodes.push(n);
		let id = self.nodes.len()-1;
		n.id = id;
		id
	}
}

#[cfg(test)]
mod tests {
	use super::{Graph, Dimension, Node, NodeId};
	use std::collections::HashMap;

	#[test]
	fn test_identity() {
		// Build graph
		let mut input = HashMap::new();
		let mut g = Graph::new();
		let m : NodeId = g.new_input((3, 3));
		let n : NodeId = g.new_input((1, 3));
		let o = g.new_matmul(n, m);
		let j : NodeId = g.new_input((3, 5));
		let k = g.new_matmul(m, j);

		// Shape check
		assert_eq!(g.nodes[m].shape, (3, 3));
		assert_eq!(g.nodes[n].shape, (1, 3));
		assert_eq!(g.nodes[o].shape, (1, 3));

		// Configure data
		input.insert(m, vec![
			1.0, 0.0, 0.0,
			0.0, 1.0, 0.0,
			0.0, 0.0, 1.0
		]);
		input.insert(n, vec![
			0.0, 1.0, 2.0
		]);
		input.insert(j, vec![
			1.0, 2.0, 3.0,
			4.0, 5.0, 6.0,
			7.0, 8.0, 9.0,
			10.0, 11.0, 12.0,
			13.0, 14.0, 15.0
		]);

		// Verify outputs
		let res = g.get_output(o, &input);
		assert_eq!(res[0], 0.0);
		assert_eq!(res[1], 1.0);
		assert_eq!(res[2], 2.0);
		let res2 = g.get_output(k, &input);
		for i in 0..15 {
			assert_eq!(res2[i], (i as f32+1.0));
		}
	}

    #[test]
    fn integration_full_test() {
		let g = Graph::new();
    }
}
<commit_msg>Replace operation with close.  Comment out a lot.<commit_after>
use std::f32;
use std::vec;
use std::collections::HashMap;
use std::collections::hash_set::HashSet;

extern crate ocl;
use ocl::{ProQue, Buffer};

static KERNEL_SOURCE: &'static str = include_str!("kernel_source.cl");

type Dimension = (usize, usize);
type NodeId = usize;

//		ActivationFunction::Sigmoid => 1.0f32/(1.0 + (-value).exp()),

struct Node<F> {
	id : NodeId,
	shape : Dimension,
	inputs : Vec<NodeId>,
	operation : F,
	buffer : Vec<f32>,
}

impl<F> Node<F> {
	fn get_value(&self, x : usize, y : usize) -> f32 {
		self.buffer[x + y*self.shape.1]
	}

	fn set_value(&mut self, x : usize, y : usize, value : f32) {
		self.buffer[x + y*self.shape.1] = value;
	}

	fn get_rows(&self) -> usize {
		self.shape.0
	}

	fn get_columns(&self) -> usize {
		self.shape.1
	}

	fn get_width(&self) -> usize {
		self.shape.1
	}

	fn get_height(&self) -> usize {
		self.shape.0
	}
}

struct Graph<F> {
//	proque : ProQue,
	nodes : Vec<Node<F>>,
}

impl<F> Graph<F> where F: Fn(f32, f32)->f32 {
	fn new() -> Graph<F> {
		Graph {
//			proque : ProQue::builder().src(KERNEL_SOURCE).dims([3]).build().unwrap(),
			nodes : vec![],
		}
	}

	// Graph methods
	fn get_output(&mut self, node_id : NodeId, input_map : &HashMap<NodeId, Vec<f32>>) -> Vec<f32> {
		//(node.operation)(&self, input_map)
		vec![]
/*
		match self.nodes[node_id].operation {
			Operation::Input => input_map.get(&node_id).unwrap().clone(),
			Operation::MatrixMultiply(n1, n2) => {
				// Verify shapes
				let a_width = self.nodes[n1].shape.1; // Columns (j)
				let a_height = self.nodes[n1].shape.0; // Rows (i)
				let b_width = self.nodes[n2].shape.1; // Columns (k)
				let b_height = self.nodes[n2].shape.0; // Rows (j)
				let c_width = b_width;
				let c_height = a_height;
				assert_eq!(a_width, b_height);
				assert_eq!(a_height*b_width, self.nodes[node_id].buffer.len());

				// Set up the buffer for this node.
				let a : Vec<f32> = self.get_output(n1, &input_map);
				let b : Vec<f32> = self.get_output(n2, &input_map);

				// Multiply
				for i in 0..a_height { // Column [Iterating over row]
					for k in 0..b_width { // Row/Width [Iterating over column]
						let mut accumulator = 0.0;
						for j in 0..a_width { // Column
							accumulator += a[j + i*a_width]*b[k + j*b_width];
						}
						self.nodes[node_id].buffer[k + i*c_width] = accumulator;
					}
				}

				self.nodes[node_id].buffer.clone()
			},
			Operation::MatrixBinaryOp(n1, n2, f) => {
				let a : Vec<f32> = self.get_output(n1, &input_map);
				let b : Vec<f32> = self.get_output(n2, &input_map);

				let vec_len = self.nodes[node_id].buffer.len();

				for i in 0..vec_len {
					self.nodes[node_id].buffer[i] = f(a[i], b[i]);
				}

				self.nodes[node_id].buffer.clone()
			},
			Operation::MatrixUnaryOp(n1, f) => {
				let a : Vec<f32> = self.get_output(n1, &input_map);
				let vec_len = self.nodes[node_id].buffer.len();

				for i in 0..vec_len {
					self.nodes[node_id].buffer[i] = f(a[i], i as f32);
				}

				self.nodes[node_id].buffer.clone()
			}
		}
*/
	}

	// Node creation
	fn new_input(&mut self, shape : Dimension) -> NodeId {
		let mut n = Node {
			id : 0,
			inputs: vec![],
			shape : shape,
			operation : |a, b| { 0.0 },
			buffer : vec![0.0; shape.0*shape.1], //self.proque.create_buffer::<f32>().unwrap()
		};
		self.nodes.push(n);
		let id = self.nodes.len()-1;
		n.id = id;
		id
	}

	fn new_matmul(&mut self, node_a_id : NodeId, node_b_id : NodeId) -> NodeId {
		let mut n = Node {
			id : 0,
			inputs: vec![],
			shape : (self.nodes[node_a_id].shape.0, self.nodes[node_b_id].shape.1),
			operation : |a, b| { 0.0 },
			buffer : vec![0.0; self.nodes[node_a_id].shape.0*self.nodes[node_b_id].shape.1],
		};
		self.nodes.push(n);
		let id = self.nodes.len()-1;
		n.id = id;
		id
	}
}

#[cfg(test)]
mod tests {
	use super::{Graph, Dimension, Node, NodeId};
	use std::collections::HashMap;

	#[test]
	fn test_identity() {
		// Build graph
		let mut input = HashMap::new();
		let mut g = Graph::new();
		let m : NodeId = g.new_input((3, 3));
		let n : NodeId = g.new_input((1, 3));
		let o = g.new_matmul(n, m);
		let j : NodeId = g.new_input((3, 5));
		let k = g.new_matmul(m, j);

		// Shape check
		assert_eq!(g.nodes[m].shape, (3, 3));
		assert_eq!(g.nodes[n].shape, (1, 3));
		assert_eq!(g.nodes[o].shape, (1, 3));

		// Configure data
		input.insert(m, vec![
			1.0, 0.0, 0.0,
			0.0, 1.0, 0.0,
			0.0, 0.0, 1.0
		]);
		input.insert(n, vec![
			0.0, 1.0, 2.0
		]);
		input.insert(j, vec![
			1.0, 2.0, 3.0,
			4.0, 5.0, 6.0,
			7.0, 8.0, 9.0,
			10.0, 11.0, 12.0,
			13.0, 14.0, 15.0
		]);

		// Verify outputs
		let res = g.get_output(o, &input);
		assert_eq!(res[0], 0.0);
		assert_eq!(res[1], 1.0);
		assert_eq!(res[2], 2.0);
		let res2 = g.get_output(k, &input);
		for i in 0..15 {
			assert_eq!(res2[i], (i as f32+1.0));
		}
	}

    #[test]
    fn integration_full_test() {
		let g = Graph::new();
    }
}
<|endoftext|>


<commit_before>extern crate json;
extern crate tiled;

extern crate cgmath;
extern crate genmesh;
#[macro_use]
extern crate gfx;
extern crate gfx_device_gl;
extern crate gfx_window_glutin;
extern crate glutin;
extern crate image;
extern crate rand;
extern crate rodio;
extern crate specs;

mod bullet;
mod gfx_app;
mod game;
mod data;
mod critter;
pub mod graphics;
mod terrain_object;
mod terrain;
mod character;
mod shaders;
mod zombie;

pub fn main() {
  let mut window = gfx_app::GlutinWindow::new();
  #[cfg_attr(feature = "cargo-clippy", allow(never_loop))]
    loop {
    match gfx_app::init::run(&mut window) {
      gfx_app::GameStatus::Quit => break,
    }
  }
}
<commit_msg>Reformat imports in lib.rs<commit_after>extern crate cgmath;
extern crate genmesh;
#[macro_use]
extern crate gfx;
extern crate gfx_device_gl;
extern crate gfx_window_glutin;
extern crate glutin;
extern crate image;
extern crate json;
extern crate rand;
extern crate rodio;
extern crate specs;
extern crate tiled;

mod bullet;
mod gfx_app;
mod game;
mod data;
mod critter;
pub mod graphics;
mod terrain_object;
mod terrain;
mod character;
mod shaders;
mod zombie;

pub fn main() {
  let mut window = gfx_app::GlutinWindow::new();
  #[cfg_attr(feature = "cargo-clippy", allow(never_loop))]
    loop {
    match gfx_app::init::run(&mut window) {
      gfx_app::GameStatus::Quit => break,
    }
  }
}
<|endoftext|>


<commit_before>#![feature(io, std_misc)]
extern crate "readline-sys" as ffi;
use std::io::Result;

use std::ffi::{CString, c_str_to_bytes};
use std::str;

pub fn readline(prompt: &str) -> Result<String> {
    use std::io::{Error, ErrorKind};
    unsafe {
        // It doesn't matter if there is an interior null
        // It just won't prompt all the way
        let line_ptr =
            ffi::readline(CString::from_slice(prompt.as_bytes()).as_ptr());

        if line_ptr.is_null() {
            return Err(Error::new(ErrorKind::Other, "end of file", None));
        }

        let line = str::from_utf8(c_str_to_bytes(&(line_ptr as *const i8))).unwrap().to_string();

        Ok(line)
    }
}

pub fn add_history(line: &str) {
    let l = CString::from_slice(line.as_bytes()).as_ptr();
    unsafe { ffi::add_history(l as *const i8) };
}
<commit_msg>essentially a rewrite<commit_after>#![feature(alloc,libc,std_misc,core)]
extern crate "readline-sys" as readline_ffi;
extern crate libc;
extern crate alloc;

use std::ffi::c_str_to_bytes;
use libc::c_char;

/* copies the contents of given str to a new heap-allocated buffer
 * note: the returned buffer needs to be freed manually */
fn str_to_cstr(line: &str) -> *const c_char {
    let l = line.as_bytes();
    unsafe {
        //alignment, whats that?
        let b = alloc::heap::allocate(line.len()+1, 8);
        let s = std::slice::from_raw_parts_mut(b, line.len()+1);
        std::slice::bytes::copy_memory(s, l);
        s[line.len()] = 0;
        return b as *const c_char;
    }
}

pub enum ReadlineError {
    EndOfFile,
    InvalidUtf8(std::string::FromUtf8Error)
}

impl std::error::FromError<std::string::FromUtf8Error> for ReadlineError {
    fn from_error(err: std::string::FromUtf8Error) -> ReadlineError {
        ReadlineError::InvalidUtf8(err)
    }
}

pub fn readline(prompt: &str) -> Result<String, ReadlineError> {
    unsafe {
        // It doesn't matter if there is an interior null
        // It just won't prompt all the way
        let line_ptr: *const c_char=
            readline_ffi::readline(prompt.as_ptr() as *const c_char);

        if line_ptr.is_null() {
            return Err(ReadlineError::EndOfFile);
        }

        return Ok(try!(String::from_utf8(c_str_to_bytes(&line_ptr).to_vec())));
    }
}

pub fn add_history(line: &str) {
    unsafe {
        readline_ffi::add_history(str_to_cstr(line));
    }
}
<|endoftext|>


<commit_before>#[macro_use]
extern crate log;
#[macro_use]
extern crate lazy_static;
mod ansi;
mod engine;
mod event;
mod field;
mod header;
mod input;
mod item;
mod matcher;
mod model;
mod options;
mod orderedvec;
mod output;
mod previewer;
mod query;
mod reader;
mod score;
mod selection;
mod spinlock;
mod theme;
mod util;

use crate::event::Event::*;
use crate::event::{EventReceiver, EventSender};
use crate::model::Model;
pub use crate::options::{SkimOptions, SkimOptionsBuilder};
pub use crate::output::SkimOutput;
use crate::reader::Reader;
use nix::unistd::isatty;
use std::env;
use std::io::BufRead;
use std::io::BufReader;
use std::os::unix::io::AsRawFd;
use std::sync::mpsc::channel;
use std::sync::Arc;
use std::thread;
use tuikit::term::{Term, TermHeight, TermOptions};

pub struct Skim {}

impl Skim {
    pub fn run_with(options: &SkimOptions, source: Option<Box<BufRead + Send>>) -> Option<SkimOutput> {
        let min_height = options
            .min_height
            .map(Skim::parse_height_string)
            .expect("min_height should have default values");
        let height = options
            .height
            .map(Skim::parse_height_string)
            .expect("height should have default values");

        let (tx, rx): (EventSender, EventReceiver) = channel();
        let term = Arc::new(Term::with_options(TermOptions::default().min_height(min_height).height(height)).unwrap());

        //------------------------------------------------------------------------------
        // input
        let mut input = input::Input::new();
        input.parse_keymaps(&options.bind);
        input.parse_expect_keys(options.expect.as_ref().map(|x| &**x));
        let tx_clone = tx.clone();
        let term_clone = term.clone();
        thread::spawn(move || 'outer: loop {
            if let Ok(key) = term_clone.poll_event() {
                for (ev, arg) in input.translate_event(key).into_iter() {
                    let _ = tx_clone.send((ev, arg));
                    if ev == EvActAccept || ev == EvActAbort {
                        break 'outer;
                    }
                }
            }
        });

        //------------------------------------------------------------------------------
        // reader

        // in piped situation(e.g. `echo "a" | sk`) set source to the pipe
        let source = source.or_else(|| {
            let stdin = std::io::stdin();
            if !isatty(stdin.as_raw_fd()).unwrap_or(true) {
                Some(Box::new(BufReader::new(stdin)))
            } else {
                None
            }
        });

        let reader = Reader::with_options(&options).source(source);

        //------------------------------------------------------------------------------
        // start a timer for notifying refresh
        let _ = tx.send((EvHeartBeat, Box::new(true)));

        //------------------------------------------------------------------------------
        // model + previewer
        let mut model = Model::new(rx, tx, reader, term, &options);
        model.start()
    }

    pub fn filter(options: &SkimOptions, source: Option<Box<BufRead + Send>>) -> i32 {
        use crate::engine::{EngineFactory, MatcherMode};

        let output_ending = if options.print0 { "\0" } else { "\n" };
        let query = options.filter;
        let default_command = match env::var("SKIM_DEFAULT_COMMAND").as_ref().map(String::as_ref) {
            Ok("") | Err(_) => "find .".to_owned(),
            Ok(val) => val.to_owned(),
        };

        let cmd = options.cmd.unwrap_or(&default_command);

        // output query
        if options.print_query {
            print!("{}{}", query, output_ending);
        }

        if options.print_cmd {
            print!("{}{}", cmd, output_ending);
        }

        //------------------------------------------------------------------------------
        // reader

        // in piped situation(e.g. `echo "a" | sk`) set source to the pipe
        let source = source.or_else(|| {
            let stdin = std::io::stdin();
            if !isatty(stdin.as_raw_fd()).unwrap_or(true) {
                Some(Box::new(BufReader::new(stdin)))
            } else {
                None
            }
        });

        let mut reader = Reader::with_options(&options).source(source);

        //------------------------------------------------------------------------------
        // matcher
        let matcher_mode = if options.regex {
            MatcherMode::Regex
        } else if options.exact {
            MatcherMode::Exact
        } else {
            MatcherMode::Fuzzy
        };

        let engine = EngineFactory::build(query, matcher_mode);

        //------------------------------------------------------------------------------
        // start
        let reader_control = reader.run(cmd);

        let mut match_count = 0;
        while !reader_control.is_done() {
            for item in reader_control.take().into_iter() {
                if let Some(matched) = engine.match_item(item) {
                    println!("{}\t{}", -matched.rank.score, matched.item.get_output_text());
                    match_count += 1;
                }
            }
        }

        if match_count == 0 {
            return 1;
        } else {
            return 0;
        }
    }

    // 10 -> TermHeight::Fixed(10)
    // 10% -> TermHeight::Percent(10)
    fn parse_height_string(string: &str) -> TermHeight {
        if string.ends_with('%') {
            TermHeight::Percent(string[0..string.len() - 1].parse().unwrap_or(100))
        } else {
            TermHeight::Fixed(string.parse().unwrap_or(0))
        }
    }
}
<commit_msg>Add some cleanup to allow multiple calls from the same process<commit_after>#[macro_use]
extern crate log;
#[macro_use]
extern crate lazy_static;
mod ansi;
mod engine;
mod event;
mod field;
mod header;
mod input;
mod item;
mod matcher;
mod model;
mod options;
mod orderedvec;
mod output;
mod previewer;
mod query;
mod reader;
mod score;
mod selection;
mod spinlock;
mod theme;
mod util;

use crate::event::Event::*;
use crate::event::{EventReceiver, EventSender};
use crate::model::Model;
pub use crate::options::{SkimOptions, SkimOptionsBuilder};
pub use crate::output::SkimOutput;
use crate::reader::Reader;
use nix::unistd::isatty;
use std::env;
use std::io::BufRead;
use std::io::BufReader;
use std::os::unix::io::AsRawFd;
use std::sync::mpsc::channel;
use std::sync::Arc;
use std::thread;
use tuikit::term::{Term, TermHeight, TermOptions};

pub struct Skim {}

impl Skim {
    pub fn run_with(options: &SkimOptions, source: Option<Box<BufRead + Send>>) -> Option<SkimOutput> {
        let min_height = options
            .min_height
            .map(Skim::parse_height_string)
            .expect("min_height should have default values");
        let height = options
            .height
            .map(Skim::parse_height_string)
            .expect("height should have default values");

        let (tx, rx): (EventSender, EventReceiver) = channel();
        let term = Arc::new(Term::with_options(TermOptions::default().min_height(min_height).height(height)).unwrap());

        //------------------------------------------------------------------------------
        // input
        let mut input = input::Input::new();
        input.parse_keymaps(&options.bind);
        input.parse_expect_keys(options.expect.as_ref().map(|x| &**x));
        let tx_clone = tx.clone();
        let term_clone = term.clone();
        let input_thread = thread::spawn(move || 'outer: loop {
            if let Ok(key) = term_clone.poll_event() {
                for (ev, arg) in input.translate_event(key).into_iter() {
                    let _ = tx_clone.send((ev, arg));
                    if ev == EvActAccept || ev == EvActAbort {
                        break 'outer;
                    }
                }
            }
        });

        //------------------------------------------------------------------------------
        // reader

        // in piped situation(e.g. `echo "a" | sk`) set source to the pipe
        let source = source.or_else(|| {
            let stdin = std::io::stdin();
            if !isatty(stdin.as_raw_fd()).unwrap_or(true) {
                Some(Box::new(BufReader::new(stdin)))
            } else {
                None
            }
        });

        let reader = Reader::with_options(&options).source(source);

        //------------------------------------------------------------------------------
        // start a timer for notifying refresh
        let _ = tx.send((EvHeartBeat, Box::new(true)));

        //------------------------------------------------------------------------------
        // model + previewer
        let mut model = Model::new(rx, tx, reader, term.clone(), &options);
        let ret = model.start();
        let _ = input_thread.join();
        let _ = term.pause();
        ret
    }

    pub fn filter(options: &SkimOptions, source: Option<Box<BufRead + Send>>) -> i32 {
        use crate::engine::{EngineFactory, MatcherMode};

        let output_ending = if options.print0 { "\0" } else { "\n" };
        let query = options.filter;
        let default_command = match env::var("SKIM_DEFAULT_COMMAND").as_ref().map(String::as_ref) {
            Ok("") | Err(_) => "find .".to_owned(),
            Ok(val) => val.to_owned(),
        };

        let cmd = options.cmd.unwrap_or(&default_command);

        // output query
        if options.print_query {
            print!("{}{}", query, output_ending);
        }

        if options.print_cmd {
            print!("{}{}", cmd, output_ending);
        }

        //------------------------------------------------------------------------------
        // reader

        // in piped situation(e.g. `echo "a" | sk`) set source to the pipe
        let source = source.or_else(|| {
            let stdin = std::io::stdin();
            if !isatty(stdin.as_raw_fd()).unwrap_or(true) {
                Some(Box::new(BufReader::new(stdin)))
            } else {
                None
            }
        });

        let mut reader = Reader::with_options(&options).source(source);

        //------------------------------------------------------------------------------
        // matcher
        let matcher_mode = if options.regex {
            MatcherMode::Regex
        } else if options.exact {
            MatcherMode::Exact
        } else {
            MatcherMode::Fuzzy
        };

        let engine = EngineFactory::build(query, matcher_mode);

        //------------------------------------------------------------------------------
        // start
        let reader_control = reader.run(cmd);

        let mut match_count = 0;
        while !reader_control.is_done() {
            for item in reader_control.take().into_iter() {
                if let Some(matched) = engine.match_item(item) {
                    println!("{}\t{}", -matched.rank.score, matched.item.get_output_text());
                    match_count += 1;
                }
            }
        }

        if match_count == 0 {
            return 1;
        } else {
            return 0;
        }
    }

    // 10 -> TermHeight::Fixed(10)
    // 10% -> TermHeight::Percent(10)
    fn parse_height_string(string: &str) -> TermHeight {
        if string.ends_with('%') {
            TermHeight::Percent(string[0..string.len() - 1].parse().unwrap_or(100))
        } else {
            TermHeight::Fixed(string.parse().unwrap_or(0))
        }
    }
}
<|endoftext|>


<commit_before>//! An implementation of [FLAC](https://xiph.org/flac), free lossless audio
//! codec, written in Rust.
//!
//! The code is available on [GitHub](https://github.com/sourrust/flac).
//!
//! # Examples
//!
//! Basic decoding from a file.
//!
//! ```
//! use flac::StreamReader;
//! use std::fs::File;
//!
//! match StreamReader::<File>::from_file("path/to/file.flac") {
//!   Ok(mut stream) => {
//!     // Copy of `StreamInfo` to help convert to a different audio format.
//!     let info = stream.info();
//!
//!     // The explicit type needs to be one bit size up from the goal
//!     // size. (ie `i8` -> `i16`, `i16` -> `i32`, `i32` -> `i64`)
//!     for sample in stream.iter::<i32>() {
//!       // Iterate over each decoded sample
//!     }
//!   }
//!   Err(error)     => println!("{:?}", error),
//! }
//! ```

#[macro_use]
extern crate nom;

#[macro_use]
mod utility;
mod frame;
mod subframe;
pub mod metadata;
pub mod stream;

pub use metadata::Metadata;
pub use stream::{Stream, StreamBuffer, StreamReader};
pub use utility::{
  Sample, SampleSize,
  StreamProducer, ReadStream, ByteStream,
  ErrorKind
};
<commit_msg>Remove the not about extended sample size<commit_after>//! An implementation of [FLAC](https://xiph.org/flac), free lossless audio
//! codec, written in Rust.
//!
//! The code is available on [GitHub](https://github.com/sourrust/flac).
//!
//! # Examples
//!
//! Basic decoding from a file.
//!
//! ```
//! use flac::StreamReader;
//! use std::fs::File;
//!
//! match StreamReader::<File>::from_file("path/to/file.flac") {
//!   Ok(mut stream) => {
//!     // Copy of `StreamInfo` to help convert to a different audio format.
//!     let info = stream.info();
//!
//!     for sample in stream.iter::<i16>() {
//!       // Iterate over each decoded sample
//!     }
//!   }
//!   Err(error)     => println!("{:?}", error),
//! }
//! ```

#[macro_use]
extern crate nom;

#[macro_use]
mod utility;
mod frame;
mod subframe;
pub mod metadata;
pub mod stream;

pub use metadata::Metadata;
pub use stream::{Stream, StreamBuffer, StreamReader};
pub use utility::{
  Sample, SampleSize,
  StreamProducer, ReadStream, ByteStream,
  ErrorKind
};
<|endoftext|>


<commit_before>/*!
rust-memcache is a [memcached](https://memcached.org/) client written in pure rust.

# Install:

The crate is called `memcache` and you can depend on it via cargo:

```ini
[dependencies]
memcache = "*"
```

# Features:

- <input type="checkbox"  disabled checked /> All memcached supported protocols
  - <input type="checkbox"  disabled checked /> Binary protocol
  - <input type="checkbox"  disabled checked /> ASCII protocol
- <input type="checkbox"  disabled checked /> All memcached supported connections
  - <input type="checkbox"  disabled checked /> TCP connection
  - <input type="checkbox"  disabled checked /> UDP connection
  - <input type="checkbox"  disabled checked/> UNIX Domain socket connection
  - <input type="checkbox"  disabled checked/> TLS connection
- <input type="checkbox"  disabled /> Encodings
  - <input type="checkbox"  disabled checked /> Typed interface
  - <input type="checkbox"  disabled /> Automatically compress
  - <input type="checkbox"  disabled /> Automatically serialize to JSON / msgpack etc
- <input type="checkbox"  disabled checked /> Mutiple server support with custom key hash algorithm
- <input type="checkbox"  disabled checked /> Authority
  - <input type="checkbox"  disabled checked /> Binary protocol (plain SASL authority)
  - <input type="checkbox"  disabled checked /> ASCII protocol

# Basic usage:

```rust
// create connection with to memcached server node:
let client = memcache::connect("memcache://127.0.0.1:12345?timeout=10&tcp_nodelay=true").unwrap();

// flush the database:
client.flush().unwrap();

// set a string value:
client.set("foo", "bar", 0).unwrap();

// retrieve from memcached:
let value: Option<String> = client.get("foo").unwrap();
assert_eq!(value, Some(String::from("bar")));
assert_eq!(value.unwrap(), "bar");

// prepend, append:
client.prepend("foo", "foo").unwrap();
client.append("foo", "baz").unwrap();
let value: String = client.get("foo").unwrap().unwrap();
assert_eq!(value, "foobarbaz");

// delete value:
client.delete("foo").unwrap();

// using counter:
client.set("counter", 40, 0).unwrap();
client.increment("counter", 2).unwrap();
let answer: i32 = client.get("counter").unwrap().unwrap();
assert_eq!(answer, 42);
```
!*/

#![cfg_attr(feature = "cargo-clippy", allow(clippy::needless_return))]

extern crate byteorder;
extern crate enum_dispatch;
#[cfg(feature = "tls")]
extern crate openssl;
extern crate r2d2;
extern crate rand;
extern crate url;

mod client;
mod connection;
mod error;
mod protocol;
mod stream;
mod value;

pub use crate::client::{Client, Connectable};
pub use crate::error::{ClientError, CommandError, MemcacheError, ServerError};
pub use crate::value::{FromMemcacheValue, FromMemcacheValueExt, ToMemcacheValue};
pub use r2d2::Error;

/// Create a memcached client instance and connect to memcached server.
///
/// Example:
///
/// ```rust
/// let client = memcache::connect("memcache://localhost:12345").unwrap();
/// ```
pub fn connect<C: Connectable>(target: C) -> Result<Client, MemcacheError> {
    Client::connect(target)
}
<commit_msg>:bug: Export Stream as public (#126)<commit_after>/*!
rust-memcache is a [memcached](https://memcached.org/) client written in pure rust.

# Install:

The crate is called `memcache` and you can depend on it via cargo:

```ini
[dependencies]
memcache = "*"
```

# Features:

- <input type="checkbox"  disabled checked /> All memcached supported protocols
  - <input type="checkbox"  disabled checked /> Binary protocol
  - <input type="checkbox"  disabled checked /> ASCII protocol
- <input type="checkbox"  disabled checked /> All memcached supported connections
  - <input type="checkbox"  disabled checked /> TCP connection
  - <input type="checkbox"  disabled checked /> UDP connection
  - <input type="checkbox"  disabled checked/> UNIX Domain socket connection
  - <input type="checkbox"  disabled checked/> TLS connection
- <input type="checkbox"  disabled /> Encodings
  - <input type="checkbox"  disabled checked /> Typed interface
  - <input type="checkbox"  disabled /> Automatically compress
  - <input type="checkbox"  disabled /> Automatically serialize to JSON / msgpack etc
- <input type="checkbox"  disabled checked /> Mutiple server support with custom key hash algorithm
- <input type="checkbox"  disabled checked /> Authority
  - <input type="checkbox"  disabled checked /> Binary protocol (plain SASL authority)
  - <input type="checkbox"  disabled checked /> ASCII protocol

# Basic usage:

```rust
// create connection with to memcached server node:
let client = memcache::connect("memcache://127.0.0.1:12345?timeout=10&tcp_nodelay=true").unwrap();

// flush the database:
client.flush().unwrap();

// set a string value:
client.set("foo", "bar", 0).unwrap();

// retrieve from memcached:
let value: Option<String> = client.get("foo").unwrap();
assert_eq!(value, Some(String::from("bar")));
assert_eq!(value.unwrap(), "bar");

// prepend, append:
client.prepend("foo", "foo").unwrap();
client.append("foo", "baz").unwrap();
let value: String = client.get("foo").unwrap().unwrap();
assert_eq!(value, "foobarbaz");

// delete value:
client.delete("foo").unwrap();

// using counter:
client.set("counter", 40, 0).unwrap();
client.increment("counter", 2).unwrap();
let answer: i32 = client.get("counter").unwrap().unwrap();
assert_eq!(answer, 42);
```
!*/

#![cfg_attr(feature = "cargo-clippy", allow(clippy::needless_return))]

extern crate byteorder;
extern crate enum_dispatch;
#[cfg(feature = "tls")]
extern crate openssl;
extern crate r2d2;
extern crate rand;
extern crate url;

mod client;
mod connection;
mod error;
mod protocol;
mod stream;
mod value;

pub use crate::client::{Client, Connectable};
pub use crate::error::{ClientError, CommandError, MemcacheError, ServerError};
pub use crate::stream::Stream;
pub use crate::value::{FromMemcacheValue, FromMemcacheValueExt, ToMemcacheValue};
pub use r2d2::Error;

/// Create a memcached client instance and connect to memcached server.
///
/// Example:
///
/// ```rust
/// let client = memcache::connect("memcache://localhost:12345").unwrap();
/// ```
pub fn connect<C: Connectable>(target: C) -> Result<Client, MemcacheError> {
    Client::connect(target)
}
<|endoftext|>


<commit_before>// Copyright (C) 2013-2015  Mikhail Zabaluev <mikhail.zabaluev@gmail.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#![crate_name = "gobject-2_0-sys"]
#![crate_type = "lib"]

#![allow(missing_copy_implementations)]

extern crate "glib-2_0-sys" as glib;

use glib::types::{gboolean, gchar, gdouble, gfloat, gint, glong};
use glib::types::{gsize, guchar, guint, gulong};
use glib::types::{gpointer, gconstpointer};

pub type GType = gsize;

#[repr(C)]
pub struct GTypeInstance {
    g_class: gpointer
}

#[repr(C)]
pub struct GObject {
    g_type_instance: GTypeInstance,
    ref_count: guint,
    data: gpointer
}

#[repr(C)]
pub struct GValue {
    pub g_type: GType,
    // If the data enum layout turns out to be different on some arch,
    // we'd need arch-specific redefinitions
    data: [u64; 2]
}

pub type GBoxedCopyFunc = extern "C" fn (gpointer) -> gpointer;
pub type GBoxedFreeFunc = extern "C" fn (gpointer);

extern {
    pub fn g_boxed_type_register_static(name: *const gchar, boxed_copy: GBoxedCopyFunc, boxed_free: GBoxedFreeFunc) -> GType;
    pub fn g_error_get_type() -> GType;
    pub fn g_main_context_get_type() -> GType;
    pub fn g_main_loop_get_type() -> GType;
    pub fn g_object_get_type() -> GType;
    pub fn g_object_ref(obj: gpointer) -> gpointer;
    pub fn g_object_unref(obj: gpointer);
    pub fn g_strdup_value_contents(value: *const GValue) -> *mut gchar;
    pub fn g_type_check_instance_is_a(instance: *const GTypeInstance, iface_type: GType) -> gboolean;
    pub fn g_type_from_name(name: *const gchar) -> GType;
    pub fn g_type_name(t: GType) -> *const gchar;
    pub fn g_value_copy(src: *const GValue, dst: *mut GValue);
    pub fn g_value_get_boolean(value: *const GValue) -> gboolean;
    pub fn g_value_get_boxed(value: *const GValue) -> gpointer;
    pub fn g_value_get_char(value: *const GValue) -> gchar;
    pub fn g_value_get_double(value: *const GValue) -> gdouble;
    pub fn g_value_get_enum(value: *const GValue) -> gint;
    pub fn g_value_get_float(value: *const GValue) -> gfloat;
    pub fn g_value_get_int(value: *const GValue) -> gint;
    pub fn g_value_get_int64(value: *const GValue) -> i64;
    pub fn g_value_get_long(value: *const GValue) -> glong;
    pub fn g_value_get_object(value: *const GValue) -> gpointer;
    pub fn g_value_get_schar(value: *const GValue) -> i8;
    pub fn g_value_get_string(value: *const GValue) -> *const gchar;
    pub fn g_value_get_uchar(value: *const GValue) -> guchar;
    pub fn g_value_get_uint(value: *const GValue) -> guint;
    pub fn g_value_get_uint64(value: *const GValue) -> u64;
    pub fn g_value_get_ulong(value: *const GValue) -> gulong;
    pub fn g_value_dup_boxed(value: *const GValue) -> gpointer;
    pub fn g_value_init(value: *mut GValue, type_id: GType) -> *mut GValue;
    pub fn g_value_reset(value: *mut GValue) -> *mut GValue;
    pub fn g_value_set_boolean(value: *mut GValue, v_boolean: gboolean);
    pub fn g_value_set_char(value: *mut GValue, v_char: gchar);
    pub fn g_value_set_double(value: *mut GValue, v_double: gdouble);
    pub fn g_value_set_enum(value: *mut GValue, v_enum: gint);
    pub fn g_value_set_float(value: *mut GValue, v_float: gfloat);
    pub fn g_value_set_int(value: *mut GValue, v_int: gint);
    pub fn g_value_set_int64(value: *mut GValue, v_int: i64);
    pub fn g_value_set_long(value: *mut GValue, v_long: glong);
    pub fn g_value_set_object(value: *mut GValue, v_object: gpointer);
    pub fn g_value_set_schar(value: *mut GValue, v_char: i8);
    pub fn g_value_set_string(value: *mut GValue, v_string: *const gchar);
    pub fn g_value_set_static_string(value: *mut GValue, v_string: *const gchar);
    pub fn g_value_set_uchar(value: *mut GValue, v_char: guchar);
    pub fn g_value_set_uint(value: *mut GValue, v_uint: guint);
    pub fn g_value_set_uint64(value: *mut GValue, v_uint: u64);
    pub fn g_value_set_ulong(value: *mut GValue, v_ulong: gulong);
    pub fn g_value_take_boxed(value: *mut GValue, v_boxed: gconstpointer);
    pub fn g_value_take_string(value: *mut GValue, v_string: *mut gchar);
    pub fn g_value_unset(value: *mut GValue);
}
<commit_msg>Added g_value_{get,set}_flags<commit_after>// Copyright (C) 2013-2015  Mikhail Zabaluev <mikhail.zabaluev@gmail.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#![crate_name = "gobject-2_0-sys"]
#![crate_type = "lib"]

#![allow(missing_copy_implementations)]

extern crate "glib-2_0-sys" as glib;

use glib::types::{gboolean, gchar, gdouble, gfloat, gint, glong};
use glib::types::{gsize, guchar, guint, gulong};
use glib::types::{gpointer, gconstpointer};

pub type GType = gsize;

#[repr(C)]
pub struct GTypeInstance {
    g_class: gpointer
}

#[repr(C)]
pub struct GObject {
    g_type_instance: GTypeInstance,
    ref_count: guint,
    data: gpointer
}

#[repr(C)]
pub struct GValue {
    pub g_type: GType,
    // If the data enum layout turns out to be different on some arch,
    // we'd need arch-specific redefinitions
    data: [u64; 2]
}

pub type GBoxedCopyFunc = extern "C" fn (gpointer) -> gpointer;
pub type GBoxedFreeFunc = extern "C" fn (gpointer);

extern {
    pub fn g_boxed_type_register_static(name: *const gchar, boxed_copy: GBoxedCopyFunc, boxed_free: GBoxedFreeFunc) -> GType;
    pub fn g_error_get_type() -> GType;
    pub fn g_main_context_get_type() -> GType;
    pub fn g_main_loop_get_type() -> GType;
    pub fn g_object_get_type() -> GType;
    pub fn g_object_ref(obj: gpointer) -> gpointer;
    pub fn g_object_unref(obj: gpointer);
    pub fn g_strdup_value_contents(value: *const GValue) -> *mut gchar;
    pub fn g_type_check_instance_is_a(instance: *const GTypeInstance, iface_type: GType) -> gboolean;
    pub fn g_type_from_name(name: *const gchar) -> GType;
    pub fn g_type_name(t: GType) -> *const gchar;
    pub fn g_value_copy(src: *const GValue, dst: *mut GValue);
    pub fn g_value_get_boolean(value: *const GValue) -> gboolean;
    pub fn g_value_get_boxed(value: *const GValue) -> gpointer;
    pub fn g_value_get_char(value: *const GValue) -> gchar;
    pub fn g_value_get_double(value: *const GValue) -> gdouble;
    pub fn g_value_get_enum(value: *const GValue) -> gint;
    pub fn g_value_get_flags(value: *const GValue) -> guint;
    pub fn g_value_get_float(value: *const GValue) -> gfloat;
    pub fn g_value_get_int(value: *const GValue) -> gint;
    pub fn g_value_get_int64(value: *const GValue) -> i64;
    pub fn g_value_get_long(value: *const GValue) -> glong;
    pub fn g_value_get_object(value: *const GValue) -> gpointer;
    pub fn g_value_get_schar(value: *const GValue) -> i8;
    pub fn g_value_get_string(value: *const GValue) -> *const gchar;
    pub fn g_value_get_uchar(value: *const GValue) -> guchar;
    pub fn g_value_get_uint(value: *const GValue) -> guint;
    pub fn g_value_get_uint64(value: *const GValue) -> u64;
    pub fn g_value_get_ulong(value: *const GValue) -> gulong;
    pub fn g_value_dup_boxed(value: *const GValue) -> gpointer;
    pub fn g_value_init(value: *mut GValue, type_id: GType) -> *mut GValue;
    pub fn g_value_reset(value: *mut GValue) -> *mut GValue;
    pub fn g_value_set_boolean(value: *mut GValue, v_boolean: gboolean);
    pub fn g_value_set_char(value: *mut GValue, v_char: gchar);
    pub fn g_value_set_double(value: *mut GValue, v_double: gdouble);
    pub fn g_value_set_enum(value: *mut GValue, v_enum: gint);
    pub fn g_value_set_flags(value: *mut GValue, v_enum: guint);
    pub fn g_value_set_float(value: *mut GValue, v_float: gfloat);
    pub fn g_value_set_int(value: *mut GValue, v_int: gint);
    pub fn g_value_set_int64(value: *mut GValue, v_int: i64);
    pub fn g_value_set_long(value: *mut GValue, v_long: glong);
    pub fn g_value_set_object(value: *mut GValue, v_object: gpointer);
    pub fn g_value_set_schar(value: *mut GValue, v_char: i8);
    pub fn g_value_set_string(value: *mut GValue, v_string: *const gchar);
    pub fn g_value_set_static_string(value: *mut GValue, v_string: *const gchar);
    pub fn g_value_set_uchar(value: *mut GValue, v_char: guchar);
    pub fn g_value_set_uint(value: *mut GValue, v_uint: guint);
    pub fn g_value_set_uint64(value: *mut GValue, v_uint: u64);
    pub fn g_value_set_ulong(value: *mut GValue, v_ulong: gulong);
    pub fn g_value_take_boxed(value: *mut GValue, v_boxed: gconstpointer);
    pub fn g_value_take_string(value: *mut GValue, v_string: *mut gchar);
    pub fn g_value_unset(value: *mut GValue);
}
<|endoftext|>


<commit_before>//! Variable length records are used to store additional metadata not defined in the header.
//!
//! Variable length records (VLRs) can be "regular" or "extended". "Regular" vlrs are stored right
//! after the header, before the point records. "Extended" vlrs (EVLRs) are stored at the end of
//! the file, after the point records.
//!
//! Vlrs contain arbitrary data:
//!
//! ```
//! use las::Vlr;
//! let vlr = Vlr {
//!     user_id: "gadomski".to_string(),
//!     record_id: 42,
//!     description: "Some really important data".to_string(),
//!     data: vec![1, 2, 3],
//!     is_extended: false,
//! };
//! ```
//!
//! Use the `is_extended` flag to indicate that this vlr should be stored as an extended variable
//! length record. Note that a header may choose to ignore this flag if the version doesn't support
//! evlrs and the evlr can be safely converted to a vlr.
//!
//! ```
//! use las::{Vlr, Header};
//! let mut header = Header::default();
//! header.version = (1, 4).into();
//!
//! let evlr = Vlr { is_extended: true, ..Default::default() };
//! header.push_vlr(evlr);
//! assert_eq!(1, header.evlrs().count());
//!
//! header.version = (1, 2).into(); // las 1.2 doesn't support evlrs
//! assert_eq!(0, header.evlrs().count());
//! assert_eq!(1, header.vlrs().count());
//! ```

use {Result, raw};

const HEADER_SIZE: usize = 54;

quick_error! {
    /// Vlr-specific errors.
    #[derive(Debug, Clone, Copy)]
    pub enum Error {
        /// The vlr data is too long.
        TooLong(len: usize) {
            description("The vlr is too long")
            display("the vlr is too long: {}", len)
        }
    }
}

/// A variable length record.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Vlr {
    /// The user that created this record.
    ///
    /// This value is often an official, "registered" user_id, such as "LASF_Spec" or
    /// "LASF_Projection".
    pub user_id: String,
    /// This value specifies the type of record, and depends on the user id.
    pub record_id: u16,
    /// Textual description of these data.
    pub description: String,
    /// The data themselves.
    pub data: Vec<u8>,
    /// Should this vlr be written "extended", i.e. at the end of the file.
    ///
    /// Note that not all las versions support extended vlrs, and extra-long vlrs might be written
    /// as extended even if this flag is not set.
    pub is_extended: bool,
}

impl Vlr {
    /// Creates a vlr from a raw vlr.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::{Vlr, raw};
    /// let raw_vlr = raw::Vlr::default();
    /// let vlr = Vlr::new(raw_vlr).unwrap();
    /// ```
    pub fn new(raw_vlr: raw::Vlr) -> Result<Vlr> {
        use utils::AsLasStr;
        Ok(Vlr {
            user_id: raw_vlr.user_id.as_ref().as_las_str()?.to_string(),
            record_id: raw_vlr.record_id,
            description: raw_vlr.description.as_ref().as_las_str()?.to_string(),
            is_extended: raw_vlr.is_extended(),
            data: raw_vlr.data,
        })
    }

    /// Converts this vlr to a raw vlr.
    ///
    /// The second argument works like this:
    ///
    /// - If `Some(false)`, create a normal vlr.
    /// - If `Some(true)`, create an extended vlr.
    /// - If `None`, fall back to the default as defined by the `extended` flag on this vlr.
    ///
    /// Note that you can just pass `true` or `false` and it will be converted to the option type.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::Vlr;
    /// let raw_vlr =  Vlr::default().into_raw(false).unwrap();
    /// let raw_evlr =  Vlr::default().into_raw(true).unwrap();
    /// let raw_vlr2 =  Vlr::default().into_raw(None).unwrap();
    /// ```
    pub fn into_raw<T>(self, force_extended: T) -> Result<raw::Vlr>
    where
        T: Into<Option<bool>>,
    {
        use utils::FromLasStr;

        let extended = force_extended.into().unwrap_or(self.is_extended);
        let mut user_id = [0; 16];
        user_id.as_mut().from_las_str(&self.user_id)?;
        let mut description = [0; 32];
        description.as_mut().from_las_str(&self.description)?;
        Ok(raw::Vlr {
            reserved: 0,
            user_id: user_id,
            record_id: self.record_id,
            record_length_after_header: self.record_length_after_header(extended)?,
            description: description,
            data: self.data,
        })
    }

    /// Returns the total length of this vlr, header and data.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::Vlr;
    /// let vlr = Vlr::default();
    /// assert_eq!(54, vlr.len());
    /// ```
    pub fn len(&self) -> usize {
        self.data.len() + HEADER_SIZE
    }

    /// Returns true if this vlr is extended.
    ///
    /// True either if the flag is set, or the data is too long for a normal vlr.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::u16;
    /// use las::Vlr;
    /// let mut vlr = Vlr::default();
    /// assert!(!vlr.is_extended());
    /// vlr.is_extended = true;
    /// assert!(vlr.is_extended());
    ///
    /// vlr.is_extended = false;
    /// vlr.data = vec![0; u16::MAX as usize + 1];
    /// assert!(vlr.is_extended());
    /// ```
    pub fn is_extended(&self) -> bool {
        use std::u16;
        self.is_extended || self.data.len() > u16::MAX as usize
    }

    fn record_length_after_header(&self, extended: bool) -> Result<raw::vlr::RecordLength> {
        if extended {
            Ok(raw::vlr::RecordLength::Evlr(self.data.len() as u64))
        } else {
            use std::u16;
            if self.data.len() > u16::MAX as usize {
                Err(Error::TooLong(self.data.len()).into())
            } else {
                Ok(raw::vlr::RecordLength::Vlr(self.data.len() as u16))
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn len() {
        let data = vec![0; 1];
        let vlr = Vlr {
            data: data,
            ..Default::default()
        };
        assert_eq!(55, vlr.len());
    }

    #[test]
    fn too_long() {
        use std::u16;
        let data = vec![0; u16::MAX as usize + 1];
        let vlr = Vlr {
            data: data,
            ..Default::default()
        };
        assert!(vlr.into_raw(false).is_err());
    }
}
<commit_msg>Space out the vlr struct<commit_after>//! Variable length records are used to store additional metadata not defined in the header.
//!
//! Variable length records (VLRs) can be "regular" or "extended". "Regular" vlrs are stored right
//! after the header, before the point records. "Extended" vlrs (EVLRs) are stored at the end of
//! the file, after the point records.
//!
//! Vlrs contain arbitrary data:
//!
//! ```
//! use las::Vlr;
//! let vlr = Vlr {
//!     user_id: "gadomski".to_string(),
//!     record_id: 42,
//!     description: "Some really important data".to_string(),
//!     data: vec![1, 2, 3],
//!     is_extended: false,
//! };
//! ```
//!
//! Use the `is_extended` flag to indicate that this vlr should be stored as an extended variable
//! length record. Note that a header may choose to ignore this flag if the version doesn't support
//! evlrs and the evlr can be safely converted to a vlr.
//!
//! ```
//! use las::{Vlr, Header};
//! let mut header = Header::default();
//! header.version = (1, 4).into();
//!
//! let evlr = Vlr { is_extended: true, ..Default::default() };
//! header.push_vlr(evlr);
//! assert_eq!(1, header.evlrs().count());
//!
//! header.version = (1, 2).into(); // las 1.2 doesn't support evlrs
//! assert_eq!(0, header.evlrs().count());
//! assert_eq!(1, header.vlrs().count());
//! ```

use {Result, raw};

const HEADER_SIZE: usize = 54;

quick_error! {
    /// Vlr-specific errors.
    #[derive(Debug, Clone, Copy)]
    pub enum Error {
        /// The vlr data is too long.
        TooLong(len: usize) {
            description("The vlr is too long")
            display("the vlr is too long: {}", len)
        }
    }
}

/// A variable length record.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Vlr {
    /// The user that created this record.
    ///
    /// This value is often an official, "registered" user_id, such as "LASF_Spec" or
    /// "LASF_Projection".
    pub user_id: String,

    /// This value specifies the type of record, and depends on the user id.
    pub record_id: u16,

    /// Textual description of these data.
    pub description: String,

    /// The data themselves.
    pub data: Vec<u8>,

    /// Should this vlr be written "extended", i.e. at the end of the file.
    ///
    /// Note that not all las versions support extended vlrs, and extra-long vlrs might be written
    /// as extended even if this flag is not set.
    pub is_extended: bool,
}

impl Vlr {
    /// Creates a vlr from a raw vlr.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::{Vlr, raw};
    /// let raw_vlr = raw::Vlr::default();
    /// let vlr = Vlr::new(raw_vlr).unwrap();
    /// ```
    pub fn new(raw_vlr: raw::Vlr) -> Result<Vlr> {
        use utils::AsLasStr;
        Ok(Vlr {
            user_id: raw_vlr.user_id.as_ref().as_las_str()?.to_string(),
            record_id: raw_vlr.record_id,
            description: raw_vlr.description.as_ref().as_las_str()?.to_string(),
            is_extended: raw_vlr.is_extended(),
            data: raw_vlr.data,
        })
    }

    /// Converts this vlr to a raw vlr.
    ///
    /// The second argument works like this:
    ///
    /// - If `Some(false)`, create a normal vlr.
    /// - If `Some(true)`, create an extended vlr.
    /// - If `None`, fall back to the default as defined by the `extended` flag on this vlr.
    ///
    /// Note that you can just pass `true` or `false` and it will be converted to the option type.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::Vlr;
    /// let raw_vlr =  Vlr::default().into_raw(false).unwrap();
    /// let raw_evlr =  Vlr::default().into_raw(true).unwrap();
    /// let raw_vlr2 =  Vlr::default().into_raw(None).unwrap();
    /// ```
    pub fn into_raw<T>(self, force_extended: T) -> Result<raw::Vlr>
    where
        T: Into<Option<bool>>,
    {
        use utils::FromLasStr;

        let extended = force_extended.into().unwrap_or(self.is_extended);
        let mut user_id = [0; 16];
        user_id.as_mut().from_las_str(&self.user_id)?;
        let mut description = [0; 32];
        description.as_mut().from_las_str(&self.description)?;
        Ok(raw::Vlr {
            reserved: 0,
            user_id: user_id,
            record_id: self.record_id,
            record_length_after_header: self.record_length_after_header(extended)?,
            description: description,
            data: self.data,
        })
    }

    /// Returns the total length of this vlr, header and data.
    ///
    /// # Examples
    ///
    /// ```
    /// use las::Vlr;
    /// let vlr = Vlr::default();
    /// assert_eq!(54, vlr.len());
    /// ```
    pub fn len(&self) -> usize {
        self.data.len() + HEADER_SIZE
    }

    /// Returns true if this vlr is extended.
    ///
    /// True either if the flag is set, or the data is too long for a normal vlr.
    ///
    /// # Examples
    ///
    /// ```
    /// use std::u16;
    /// use las::Vlr;
    /// let mut vlr = Vlr::default();
    /// assert!(!vlr.is_extended());
    /// vlr.is_extended = true;
    /// assert!(vlr.is_extended());
    ///
    /// vlr.is_extended = false;
    /// vlr.data = vec![0; u16::MAX as usize + 1];
    /// assert!(vlr.is_extended());
    /// ```
    pub fn is_extended(&self) -> bool {
        use std::u16;
        self.is_extended || self.data.len() > u16::MAX as usize
    }

    fn record_length_after_header(&self, extended: bool) -> Result<raw::vlr::RecordLength> {
        if extended {
            Ok(raw::vlr::RecordLength::Evlr(self.data.len() as u64))
        } else {
            use std::u16;
            if self.data.len() > u16::MAX as usize {
                Err(Error::TooLong(self.data.len()).into())
            } else {
                Ok(raw::vlr::RecordLength::Vlr(self.data.len() as u16))
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn len() {
        let data = vec![0; 1];
        let vlr = Vlr {
            data: data,
            ..Default::default()
        };
        assert_eq!(55, vlr.len());
    }

    #[test]
    fn too_long() {
        use std::u16;
        let data = vec![0; u16::MAX as usize + 1];
        let vlr = Vlr {
            data: data,
            ..Default::default()
        };
        assert!(vlr.into_raw(false).is_err());
    }
}
<|endoftext|>


<commit_before>use crate::command_prelude::*;

use cargo::core::{GitReference, SourceId};
use cargo::ops;
use cargo::util::ToUrl;

pub fn cli() -> App {
    subcommand("install")
        .about("Install a Rust binary. Default location is $HOME/.cargo/bin")
        .arg(Arg::with_name("crate").empty_values(false).multiple(true))
        .arg(
            opt("version", "Specify a version to install from crates.io")
                .alias("vers")
                .value_name("VERSION"),
        )
        .arg(opt("git", "Git URL to install the specified crate from").value_name("URL"))
        .arg(opt("branch", "Branch to use when installing from git").value_name("BRANCH"))
        .arg(opt("tag", "Tag to use when installing from git").value_name("TAG"))
        .arg(opt("rev", "Specific commit to use when installing from git").value_name("SHA"))
        .arg(opt("path", "Filesystem path to local crate to install").value_name("PATH"))
        .arg(opt(
            "list",
            "list all installed packages and their versions",
        ))
        .arg_jobs()
        .arg(opt("force", "Force overwriting existing crates or binaries").short("f"))
        .arg_features()
        .arg(opt("debug", "Build in debug mode instead of release mode"))
        .arg_targets_bins_examples(
            "Install only the specified binary",
            "Install all binaries",
            "Install only the specified example",
            "Install all examples",
        )
        .arg_target_triple("Build for the target triple")
        .arg(opt("root", "Directory to install packages into").value_name("DIR"))
        .arg(opt("registry", "Registry to use").value_name("REGISTRY"))
        .after_help(
            "\
This command manages Cargo's local set of installed binary crates. Only packages
which have [[bin]] targets can be installed, and all binaries are installed into
the installation root's `bin` folder. The installation root is determined, in
order of precedence, by `--root`, `$CARGO_INSTALL_ROOT`, the `install.root`
configuration key, and finally the home directory (which is either
`$CARGO_HOME` if set or `$HOME/.cargo` by default).

There are multiple sources from which a crate can be installed. The default
location is crates.io but the `--git` and `--path` flags can change this source.
If the source contains more than one package (such as crates.io or a git
repository with multiple crates) the `<crate>` argument is required to indicate
which crate should be installed.

Crates from crates.io can optionally specify the version they wish to install
via the `--vers` flags, and similarly packages from git repositories can
optionally specify the branch, tag, or revision that should be installed. If a
crate has multiple binaries, the `--bin` argument can selectively install only
one of them, and if you'd rather install examples the `--example` argument can
be used as well.

By default cargo will refuse to overwrite existing binaries. The `--force` flag
enables overwriting existing binaries. Thus you can reinstall a crate with
`cargo install --force <crate>`.

Omitting the <crate> specification entirely will
install the crate in the current directory. That is, `install` is equivalent to
the more explicit `install --path .`.  This behaviour is deprecated, and no
longer supported as of the Rust 2018 edition.

If the source is crates.io or `--git` then by default the crate will be built
in a temporary target directory.  To avoid this, the target directory can be
specified by setting the `CARGO_TARGET_DIR` environment variable to a relative
path.  In particular, this can be useful for caching build artifacts on
continuous integration systems.",
        )
}

pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult {
    let registry = args.registry(config)?;

    config.reload_rooted_at_cargo_home()?;
    let mut compile_opts = args.compile_options(config, CompileMode::Build)?;

    compile_opts.build_config.release = !args.is_present("debug");

    let krates = args
        .values_of("crate")
        .unwrap_or_default()
        .collect::<Vec<_>>();

    let mut from_cwd = false;

    let source = if let Some(url) = args.value_of("git") {
        let url = url.to_url()?;
        let gitref = if let Some(branch) = args.value_of("branch") {
            GitReference::Branch(branch.to_string())
        } else if let Some(tag) = args.value_of("tag") {
            GitReference::Tag(tag.to_string())
        } else if let Some(rev) = args.value_of("rev") {
            GitReference::Rev(rev.to_string())
        } else {
            GitReference::Branch("master".to_string())
        };
        SourceId::for_git(&url, gitref)?
    } else if let Some(path) = args.value_of_path("path", config) {
        SourceId::for_path(&path)?
    } else if krates.is_empty() {
        from_cwd = true;
        SourceId::for_path(config.cwd())?
    } else if let Some(registry) = registry {
        SourceId::alt_registry(config, &registry)?
    } else {
        SourceId::crates_io(config)?
    };

    let version = args.value_of("version");
    let root = args.value_of("root");

    if args.is_present("list") {
        ops::install_list(root, config)?;
    } else {
        ops::install(
            root,
            krates,
            source,
            from_cwd,
            version,
            &compile_opts,
            args.is_present("force"),
        )?;
    }
    Ok(())
}
<commit_msg>Auto merge of #6495 - dwijnand:fixup-cargo-install-help, r=ehuss<commit_after>use crate::command_prelude::*;

use cargo::core::{GitReference, SourceId};
use cargo::ops;
use cargo::util::ToUrl;

pub fn cli() -> App {
    subcommand("install")
        .about("Install a Rust binary. Default location is $HOME/.cargo/bin")
        .arg(Arg::with_name("crate").empty_values(false).multiple(true))
        .arg(
            opt("version", "Specify a version to install from crates.io")
                .alias("vers")
                .value_name("VERSION"),
        )
        .arg(opt("git", "Git URL to install the specified crate from").value_name("URL"))
        .arg(opt("branch", "Branch to use when installing from git").value_name("BRANCH"))
        .arg(opt("tag", "Tag to use when installing from git").value_name("TAG"))
        .arg(opt("rev", "Specific commit to use when installing from git").value_name("SHA"))
        .arg(opt("path", "Filesystem path to local crate to install").value_name("PATH"))
        .arg(opt(
            "list",
            "list all installed packages and their versions",
        ))
        .arg_jobs()
        .arg(opt("force", "Force overwriting existing crates or binaries").short("f"))
        .arg_features()
        .arg(opt("debug", "Build in debug mode instead of release mode"))
        .arg_targets_bins_examples(
            "Install only the specified binary",
            "Install all binaries",
            "Install only the specified example",
            "Install all examples",
        )
        .arg_target_triple("Build for the target triple")
        .arg(opt("root", "Directory to install packages into").value_name("DIR"))
        .arg(opt("registry", "Registry to use").value_name("REGISTRY"))
        .after_help(
            "\
This command manages Cargo's local set of installed binary crates. Only packages
which have [[bin]] targets can be installed, and all binaries are installed into
the installation root's `bin` folder. The installation root is determined, in
order of precedence, by `--root`, `$CARGO_INSTALL_ROOT`, the `install.root`
configuration key, and finally the home directory (which is either
`$CARGO_HOME` if set or `$HOME/.cargo` by default).

There are multiple sources from which a crate can be installed. The default
location is crates.io but the `--git` and `--path` flags can change this source.
If the source contains more than one package (such as crates.io or a git
repository with multiple crates) the `<crate>` argument is required to indicate
which crate should be installed.

Crates from crates.io can optionally specify the version they wish to install
via the `--version` flags, and similarly packages from git repositories can
optionally specify the branch, tag, or revision that should be installed. If a
crate has multiple binaries, the `--bin` argument can selectively install only
one of them, and if you'd rather install examples the `--example` argument can
be used as well.

By default cargo will refuse to overwrite existing binaries. The `--force` flag
enables overwriting existing binaries. Thus you can reinstall a crate with
`cargo install --force <crate>`.

Omitting the <crate> specification entirely will
install the crate in the current directory. That is, `install` is equivalent to
the more explicit `install --path .`.  This behaviour is deprecated, and no
longer supported as of the Rust 2018 edition.

If the source is crates.io or `--git` then by default the crate will be built
in a temporary target directory.  To avoid this, the target directory can be
specified by setting the `CARGO_TARGET_DIR` environment variable to a relative
path.  In particular, this can be useful for caching build artifacts on
continuous integration systems.",
        )
}

pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult {
    let registry = args.registry(config)?;

    config.reload_rooted_at_cargo_home()?;
    let mut compile_opts = args.compile_options(config, CompileMode::Build)?;

    compile_opts.build_config.release = !args.is_present("debug");

    let krates = args
        .values_of("crate")
        .unwrap_or_default()
        .collect::<Vec<_>>();

    let mut from_cwd = false;

    let source = if let Some(url) = args.value_of("git") {
        let url = url.to_url()?;
        let gitref = if let Some(branch) = args.value_of("branch") {
            GitReference::Branch(branch.to_string())
        } else if let Some(tag) = args.value_of("tag") {
            GitReference::Tag(tag.to_string())
        } else if let Some(rev) = args.value_of("rev") {
            GitReference::Rev(rev.to_string())
        } else {
            GitReference::Branch("master".to_string())
        };
        SourceId::for_git(&url, gitref)?
    } else if let Some(path) = args.value_of_path("path", config) {
        SourceId::for_path(&path)?
    } else if krates.is_empty() {
        from_cwd = true;
        SourceId::for_path(config.cwd())?
    } else if let Some(registry) = registry {
        SourceId::alt_registry(config, &registry)?
    } else {
        SourceId::crates_io(config)?
    };

    let version = args.value_of("version");
    let root = args.value_of("root");

    if args.is_present("list") {
        ops::install_list(root, config)?;
    } else {
        ops::install(
            root,
            krates,
            source,
            from_cwd,
            version,
            &compile_opts,
            args.is_present("force"),
        )?;
    }
    Ok(())
}
<|endoftext|>


<commit_before>use regex::Regex;
use std::collections::HashSet;
use std::fmt;
use std::fmt::Write as _;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::path::MAIN_SEPARATOR;
use std::str;
use tempfile::NamedTempFile;

use protobuf::descriptor::field_descriptor_proto;
use protobuf::descriptor::DescriptorProto;
use protobuf::descriptor::EnumDescriptorProto;
use protobuf::descriptor::FieldDescriptorProto;
use protobuf::descriptor::FileDescriptorProto;
use protobuf::descriptor::FileDescriptorSet;
use protobuf::descriptor::OneofDescriptorProto;
use protobuf::text_format::lexer::float::parse_protobuf_float;
use protobuf::Message;
use protobuf_test_common::build::copy_tests_v2_v3;
use protobuf_test_common::build::glob_simple;
use std::process::Command;
use std::process::Stdio;

fn to_paths(iter: impl IntoIterator<Item = impl Into<String>>) -> Vec<PathBuf> {
    iter.into_iter()
        .map(|item| item.into().replace(MAIN_SEPARATOR, "/").into())
        .collect()
}

#[derive(Default, Debug)]
struct TestStats {
    passed: u32,
    passed_marked_skip: u32,
    skipped: u32,
    failed: u32,
}

fn normalize_generated_file(contents: &str) -> String {
    let parsed_by = Regex::new("^// \\.proto file is parsed by.*").unwrap();

    let mut r = String::new();
    let mut inside_descriptor_data = false;
    for line in contents.lines() {
        let line = if inside_descriptor_data {
            if line == "\";" {
                inside_descriptor_data = false;
                line.to_owned()
            } else {
                continue;
            }
        } else if line.starts_with("static file_descriptor_proto_data") {
            inside_descriptor_data = true;
            line.to_owned()
        } else {
            parsed_by.replace(line, "").into_owned()
        };

        writeln!(&mut r, "{}", line).unwrap();
    }

    // sanity check
    assert!(!inside_descriptor_data);

    r
}

fn normalize_generated_file_in_place(path: &Path) {
    let contents = fs::read_to_string(path).unwrap();
    let contents = normalize_generated_file(&contents);
    fs::write(path, &contents).unwrap();
}

fn print_diff(dir: &Path, a: &Path, b: &Path) {
    if cfg!(windows) {
        // likely we don't have `diff` command on Windows
        return;
    }

    // sanity check
    let mut a_full = dir.to_path_buf();
    a_full.push(a);
    assert!(a_full.exists());
    let mut b_full = dir.to_path_buf();
    b_full.push(b);
    assert!(b_full.exists());

    // Not using difference crate because it's slow for large files
    let output = Command::new("diff")
        .current_dir(dir)
        .arg("-u")
        .arg(a)
        .arg(b)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .output()
        .unwrap();

    print!("{}", str::from_utf8(&output.stdout).unwrap());
    print!("{}", str::from_utf8(&output.stderr).unwrap());
}

fn protoc_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut temp_file = NamedTempFile::new().unwrap();
    protoc::Protoc::from_path(protoc_bin_vendored::protoc_bin_path().unwrap())
        .descriptor_set_out_args()
        .out(temp_file.path())
        .includes(includes)
        .inputs(inputs)
        .write_descriptor_set()
        .unwrap();
    FileDescriptorSet::parse_from_reader(&mut temp_file).unwrap()
}

// TODO: expose this utility from protobuf-codegen-pure crate.
fn pure_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut codegen = protobuf_codegen_pure::parse_and_typecheck(includes, inputs).unwrap();
    let relative_paths: HashSet<_> = codegen
        .relative_paths
        .iter()
        .map(|path| path.to_string_lossy())
        .collect();
    codegen
        .file_descriptors
        .retain(|fd| relative_paths.contains(fd.get_name()));
    let mut fds = FileDescriptorSet::new();
    fds.file = codegen.file_descriptors;
    fds
}

fn normalize_descriptor_set(fds: &mut FileDescriptorSet) {
    for desc in &mut fds.file {
        normalize_file_descriptor(desc)
    }
}

fn normalize_file_descriptor(desc: &mut FileDescriptorProto) {
    if !desc.has_syntax() {
        desc.set_syntax("proto2".into())
    }
    for desc in &mut *desc.message_type {
        normalize_descriptor(desc)
    }
    for desc in &mut desc.enum_type {
        normalize_enum_descriptor(desc)
    }

    for desc in &mut desc.extension {
        desc.options.mut_or_default();
    }
    desc.options.mut_or_default();

    // TODO: don't clear services.
    desc.service.clear();

    // for unittest_custom_options.proto where a custom option
    // is an extension. Probably nobody outside of Google uses it.
    desc.options
        .mut_or_default()
        .unknown_fields
        .remove(15478479);
}

fn normalize_enum_descriptor(desc: &mut EnumDescriptorProto) {
    desc.options.mut_or_default();

    for value in &mut desc.value {
        value.options.mut_or_default();
    }
}

fn normalize_oneof_descriptor(desc: &mut OneofDescriptorProto) {
    desc.options.mut_or_default();
}

fn normalize_descriptor(desc: &mut DescriptorProto) {
    for desc in &mut desc.nested_type {
        normalize_descriptor(desc);
    }
    for desc in &mut desc.enum_type {
        normalize_enum_descriptor(desc);
    }
    for desc in &mut desc.oneof_decl {
        normalize_oneof_descriptor(desc);
    }

    // TODO: don't clear options.
    desc.options.clear();

    for field in &mut desc.field {
        normalize_field(field);
    }

    for ext in &mut desc.extension {
        ext.options.mut_or_default();
    }

    for ext in &mut desc.extension_range {
        // If ext range end exceeds max field number,
        // the actual upper limit does not matter.
        // protoc is not consistent in behavior thus flush
        // the value to some arbitrary compatible value.
        if ext.has_end() && ext.get_end() >= 0x20000000 {
            ext.set_end(0x20000000);
        }
    }
}

fn normalize_field(field: &mut FieldDescriptorProto) {
    field.options.mut_or_default();

    if field.has_default_value() {
        if field.get_field_type() == field_descriptor_proto::Type::TYPE_FLOAT {
            field.set_default_value(format!(
                "{}",
                parse_protobuf_float(field.get_default_value()).unwrap()
            ));
        }
    }
}

fn pretty_message<M: protobuf::Message>(message: &M) -> String {
    // TODO: pull this handy utility into the protobuf crate.
    struct FormatMessage<'a, M>(&'a M);

    impl<M: protobuf::Message> fmt::Display for FormatMessage<'_, M> {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            protobuf::text_format::fmt(self.0, f)
        }
    }

    format!("{:#}", FormatMessage(message))
}

fn descriptor_for_file<'a>(fds: &'a FileDescriptorSet, file_name: &str) -> &'a FileDescriptorProto {
    for file in &fds.file {
        if Path::new(file.get_name()).file_name().unwrap()
            == Path::new(file_name).file_name().unwrap()
        {
            return file;
        }
    }
    panic!(
        "file not found: {}; all files: {}",
        file_name,
        fds.file
            .iter()
            .map(|f| f.get_name())
            .collect::<Vec<_>>()
            .join(", ")
    );
}

fn test_diff_in<F>(root: &str, sources_dir: &str, include: &str, should_skip: F)
where
    F: Fn(&str) -> bool,
{
    let mut stats = TestStats {
        passed: 0,
        passed_marked_skip: 0,
        skipped: 0,
        failed: 0,
    };

    let mut include_root = Path::new(root).to_path_buf();
    include_root.push(include);

    let include_full = format!("{}/{}", root, include);
    let s_full = format!("{}/{}", root, sources_dir);

    let inputs_glob = format!("{}/*.proto", s_full);
    let inputs = to_paths(glob_simple(&inputs_glob));
    assert!(!inputs.is_empty(), "glob is empty: {}", inputs_glob);
    let includes = to_paths(vec![include_full.as_str(), "../proto"]);

    let temp_dir = tempfile::Builder::new()
        .prefix("protobuf-codegen-identical-test")
        .tempdir()
        .unwrap();

    let protoc_dir = format!("{}/protoc", temp_dir.path().to_str().unwrap());
    let pure_dir = format!("{}/pure", temp_dir.path().to_str().unwrap());

    fs::create_dir(&protoc_dir).unwrap();
    fs::create_dir(&pure_dir).unwrap();

    protoc_rust::Codegen::new()
        .protoc_path(protoc_bin_vendored::protoc_bin_path().unwrap())
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&protoc_dir)
        .run()
        .unwrap();

    protobuf_codegen_pure::Codegen::new()
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&pure_dir)
        .run()
        .unwrap();

    let mut protoc_descriptors = protoc_descriptor_set(&includes, &inputs);
    let mut pure_descriptors = pure_descriptor_set(&includes, &inputs);
    normalize_descriptor_set(&mut protoc_descriptors);
    normalize_descriptor_set(&mut pure_descriptors);

    for input in &inputs {
        let label = input.strip_prefix(root).unwrap().to_str().unwrap();
        let proto_file_name = input.strip_prefix(&include_root).unwrap().to_str().unwrap();
        let proto_name = input.file_name().unwrap().to_str().unwrap();
        let rs_name = protobuf_codegen::proto_name_to_rs(proto_name);
        let protoc_rs = format!("{}/{}", protoc_dir, rs_name);
        let pure_rs = format!("{}/{}", pure_dir, rs_name);

        normalize_generated_file_in_place(Path::new(&protoc_rs));
        normalize_generated_file_in_place(Path::new(&pure_rs));

        let protoc_rs_contents =
            fs::read_to_string(&protoc_rs).expect(&format!("while reading {}", protoc_rs));
        let pure_rs_contents =
            fs::read_to_string(&pure_rs).expect(&format!("while reading {}", pure_rs));

        let protoc_descriptor_for_file = descriptor_for_file(&protoc_descriptors, &proto_file_name);
        let pure_descriptor_for_file = descriptor_for_file(&pure_descriptors, &proto_file_name);

        let skip = should_skip(input.to_str().unwrap());
        if protoc_rs_contents == pure_rs_contents
            && protoc_descriptor_for_file == pure_descriptor_for_file
        {
            if !skip {
                stats.passed += 1;
                println!("{}: PASSED", label);
            } else {
                stats.passed_marked_skip += 1;
                println!("{}: PASSED BUT MARKED SKIP", label);
            }
        } else {
            if skip {
                stats.skipped += 1;
                println!("{} SKIPPED", label);
            } else {
                stats.failed += 1;
                println!("{} FAILED", label);
            }

            print_diff(
                temp_dir.path(),
                Path::new(&protoc_rs).strip_prefix(temp_dir.path()).unwrap(),
                Path::new(&pure_rs).strip_prefix(temp_dir.path()).unwrap(),
            );

            fs::write(
                format!("{}/{}.descriptors", protoc_dir, proto_name),
                pretty_message(protoc_descriptor_for_file),
            )
            .unwrap();
            fs::write(
                format!("{}/{}.descriptors", pure_dir, proto_name),
                pretty_message(pure_descriptor_for_file),
            )
            .unwrap();
            print_diff(
                temp_dir.path(),
                Path::new(&format!("protoc/{}.descriptors", proto_name)),
                Path::new(&format!("pure/{}.descriptors", proto_name)),
            );
        }
    }

    println!("{:?}", stats);
    assert!(stats.failed == 0, "at least one test failed");
}

fn should_skip_with_marker(path: &str) -> bool {
    fs::read_to_string(path)
        .unwrap()
        .contains("@skip-codegen-identical-test")
}

#[test]
fn common_v2() {
    test_diff_in(
        "../protobuf-test",
        "src/common/v2",
        "src/common/v2",
        should_skip_with_marker,
    );
}

#[test]
fn common_v3() {
    let common_v3_root = tempfile::Builder::new()
        .prefix("common-v3")
        .tempdir()
        .unwrap();
    fs::create_dir_all(format!(
        "{}/src/common/v3",
        common_v3_root.path().to_str().unwrap()
    ))
    .unwrap();

    copy_tests_v2_v3(
        "../protobuf-test/src/common/v2",
        &format!("{}/src/common/v3", common_v3_root.path().to_str().unwrap()),
    );

    test_diff_in(
        common_v3_root.path().to_str().unwrap(),
        "src/common/v3",
        "src/common/v3",
        should_skip_with_marker,
    );
}

#[test]
fn v2() {
    test_diff_in(
        "../protobuf-test",
        "src/v2",
        "src/v2",
        should_skip_with_marker,
    );
}

#[test]
fn v3() {
    test_diff_in(
        "../protobuf-test",
        "src/v3",
        "src/v3",
        should_skip_with_marker,
    );
}

#[test]
fn interop() {
    test_diff_in(
        "../protobuf-test",
        "../interop/cxx",
        "../interop/cxx",
        should_skip_with_marker,
    );
}

#[test]
fn google() {
    test_diff_in("../protobuf-test", "src/google/protobuf", "src", |_| false);
}
<commit_msg>Include *.proto3 files in identical output tests<commit_after>use regex::Regex;
use std::collections::HashSet;
use std::fmt;
use std::fmt::Write as _;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::path::MAIN_SEPARATOR;
use std::str;
use tempfile::NamedTempFile;

use protobuf::descriptor::field_descriptor_proto;
use protobuf::descriptor::DescriptorProto;
use protobuf::descriptor::EnumDescriptorProto;
use protobuf::descriptor::FieldDescriptorProto;
use protobuf::descriptor::FileDescriptorProto;
use protobuf::descriptor::FileDescriptorSet;
use protobuf::descriptor::OneofDescriptorProto;
use protobuf::text_format::lexer::float::parse_protobuf_float;
use protobuf::Message;
use protobuf_test_common::build::copy_tests_v2_v3;
use protobuf_test_common::build::glob_simple;
use std::process::Command;
use std::process::Stdio;

fn to_paths(iter: impl IntoIterator<Item = impl Into<String>>) -> Vec<PathBuf> {
    iter.into_iter()
        .map(|item| item.into().replace(MAIN_SEPARATOR, "/").into())
        .collect()
}

#[derive(Default, Debug)]
struct TestStats {
    passed: u32,
    passed_marked_skip: u32,
    skipped: u32,
    failed: u32,
}

fn normalize_generated_file(contents: &str) -> String {
    let parsed_by = Regex::new("^// \\.proto file is parsed by.*").unwrap();

    let mut r = String::new();
    let mut inside_descriptor_data = false;
    for line in contents.lines() {
        let line = if inside_descriptor_data {
            if line == "\";" {
                inside_descriptor_data = false;
                line.to_owned()
            } else {
                continue;
            }
        } else if line.starts_with("static file_descriptor_proto_data") {
            inside_descriptor_data = true;
            line.to_owned()
        } else {
            parsed_by.replace(line, "").into_owned()
        };

        writeln!(&mut r, "{}", line).unwrap();
    }

    // sanity check
    assert!(!inside_descriptor_data);

    r
}

fn normalize_generated_file_in_place(path: &Path) {
    let contents = fs::read_to_string(path).unwrap();
    let contents = normalize_generated_file(&contents);
    fs::write(path, &contents).unwrap();
}

fn print_diff(dir: &Path, a: &Path, b: &Path) {
    if cfg!(windows) {
        // likely we don't have `diff` command on Windows
        return;
    }

    // sanity check
    let mut a_full = dir.to_path_buf();
    a_full.push(a);
    assert!(a_full.exists());
    let mut b_full = dir.to_path_buf();
    b_full.push(b);
    assert!(b_full.exists());

    // Not using difference crate because it's slow for large files
    let output = Command::new("diff")
        .current_dir(dir)
        .arg("-u")
        .arg(a)
        .arg(b)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .output()
        .unwrap();

    print!("{}", str::from_utf8(&output.stdout).unwrap());
    print!("{}", str::from_utf8(&output.stderr).unwrap());
}

fn protoc_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut temp_file = NamedTempFile::new().unwrap();
    protoc::Protoc::from_path(protoc_bin_vendored::protoc_bin_path().unwrap())
        .descriptor_set_out_args()
        .out(temp_file.path())
        .includes(includes)
        .inputs(inputs)
        .write_descriptor_set()
        .unwrap();
    FileDescriptorSet::parse_from_reader(&mut temp_file).unwrap()
}

// TODO: expose this utility from protobuf-codegen-pure crate.
fn pure_descriptor_set(includes: &[PathBuf], inputs: &[PathBuf]) -> FileDescriptorSet {
    let mut codegen = protobuf_codegen_pure::parse_and_typecheck(includes, inputs).unwrap();
    let relative_paths: HashSet<_> = codegen
        .relative_paths
        .iter()
        .map(|path| path.to_string_lossy())
        .collect();
    codegen
        .file_descriptors
        .retain(|fd| relative_paths.contains(fd.get_name()));
    let mut fds = FileDescriptorSet::new();
    fds.file = codegen.file_descriptors;
    fds
}

fn normalize_descriptor_set(fds: &mut FileDescriptorSet) {
    for desc in &mut fds.file {
        normalize_file_descriptor(desc)
    }
}

fn normalize_file_descriptor(desc: &mut FileDescriptorProto) {
    if !desc.has_syntax() {
        desc.set_syntax("proto2".into())
    }
    for desc in &mut *desc.message_type {
        normalize_descriptor(desc)
    }
    for desc in &mut desc.enum_type {
        normalize_enum_descriptor(desc)
    }

    for desc in &mut desc.extension {
        desc.options.mut_or_default();
    }
    desc.options.mut_or_default();

    // TODO: don't clear services.
    desc.service.clear();

    // for unittest_custom_options.proto where a custom option
    // is an extension. Probably nobody outside of Google uses it.
    desc.options
        .mut_or_default()
        .unknown_fields
        .remove(15478479);
}

fn normalize_enum_descriptor(desc: &mut EnumDescriptorProto) {
    desc.options.mut_or_default();

    for value in &mut desc.value {
        value.options.mut_or_default();
    }
}

fn normalize_oneof_descriptor(desc: &mut OneofDescriptorProto) {
    desc.options.mut_or_default();
}

fn normalize_descriptor(desc: &mut DescriptorProto) {
    for desc in &mut desc.nested_type {
        normalize_descriptor(desc);
    }
    for desc in &mut desc.enum_type {
        normalize_enum_descriptor(desc);
    }
    for desc in &mut desc.oneof_decl {
        normalize_oneof_descriptor(desc);
    }

    // TODO: don't clear options.
    desc.options.clear();

    for field in &mut desc.field {
        normalize_field(field);
    }

    for ext in &mut desc.extension {
        ext.options.mut_or_default();
    }

    for ext in &mut desc.extension_range {
        // If ext range end exceeds max field number,
        // the actual upper limit does not matter.
        // protoc is not consistent in behavior thus flush
        // the value to some arbitrary compatible value.
        if ext.has_end() && ext.get_end() >= 0x20000000 {
            ext.set_end(0x20000000);
        }
    }
}

fn normalize_field(field: &mut FieldDescriptorProto) {
    field.options.mut_or_default();

    if field.has_default_value() {
        if field.get_field_type() == field_descriptor_proto::Type::TYPE_FLOAT {
            field.set_default_value(format!(
                "{}",
                parse_protobuf_float(field.get_default_value()).unwrap()
            ));
        }
    }
}

fn pretty_message<M: protobuf::Message>(message: &M) -> String {
    // TODO: pull this handy utility into the protobuf crate.
    struct FormatMessage<'a, M>(&'a M);

    impl<M: protobuf::Message> fmt::Display for FormatMessage<'_, M> {
        fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
            protobuf::text_format::fmt(self.0, f)
        }
    }

    format!("{:#}", FormatMessage(message))
}

fn descriptor_for_file<'a>(fds: &'a FileDescriptorSet, file_name: &str) -> &'a FileDescriptorProto {
    for file in &fds.file {
        if Path::new(file.get_name()).file_name().unwrap()
            == Path::new(file_name).file_name().unwrap()
        {
            return file;
        }
    }
    panic!(
        "file not found: {}; all files: {}",
        file_name,
        fds.file
            .iter()
            .map(|f| f.get_name())
            .collect::<Vec<_>>()
            .join(", ")
    );
}

fn test_diff_in<F>(root: &str, sources_dir: &str, include: &str, should_skip: F)
where
    F: Fn(&str) -> bool,
{
    let mut stats = TestStats {
        passed: 0,
        passed_marked_skip: 0,
        skipped: 0,
        failed: 0,
    };

    let mut include_root = Path::new(root).to_path_buf();
    include_root.push(include);

    let include_full = format!("{}/{}", root, include);
    let s_full = format!("{}/{}", root, sources_dir);

    let inputs_glob = format!("{}/*.proto*", s_full);
    let inputs = to_paths(glob_simple(&inputs_glob));
    assert!(!inputs.is_empty(), "glob is empty: {}", inputs_glob);
    let includes = to_paths(vec![include_full.as_str(), "../proto"]);

    let temp_dir = tempfile::Builder::new()
        .prefix("protobuf-codegen-identical-test")
        .tempdir()
        .unwrap();

    let protoc_dir = format!("{}/protoc", temp_dir.path().to_str().unwrap());
    let pure_dir = format!("{}/pure", temp_dir.path().to_str().unwrap());

    fs::create_dir(&protoc_dir).unwrap();
    fs::create_dir(&pure_dir).unwrap();

    protoc_rust::Codegen::new()
        .protoc_path(protoc_bin_vendored::protoc_bin_path().unwrap())
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&protoc_dir)
        .run()
        .unwrap();

    protobuf_codegen_pure::Codegen::new()
        .inputs(&inputs)
        .includes(&includes)
        .out_dir(&pure_dir)
        .run()
        .unwrap();

    let mut protoc_descriptors = protoc_descriptor_set(&includes, &inputs);
    let mut pure_descriptors = pure_descriptor_set(&includes, &inputs);
    normalize_descriptor_set(&mut protoc_descriptors);
    normalize_descriptor_set(&mut pure_descriptors);

    for input in &inputs {
        let label = input.strip_prefix(root).unwrap().to_str().unwrap();
        let proto_file_name = input.strip_prefix(&include_root).unwrap().to_str().unwrap();
        let proto_name = input.file_name().unwrap().to_str().unwrap();
        let rs_name = protobuf_codegen::proto_name_to_rs(proto_name);
        let protoc_rs = format!("{}/{}", protoc_dir, rs_name);
        let pure_rs = format!("{}/{}", pure_dir, rs_name);

        normalize_generated_file_in_place(Path::new(&protoc_rs));
        normalize_generated_file_in_place(Path::new(&pure_rs));

        let protoc_rs_contents =
            fs::read_to_string(&protoc_rs).expect(&format!("while reading {}", protoc_rs));
        let pure_rs_contents =
            fs::read_to_string(&pure_rs).expect(&format!("while reading {}", pure_rs));

        let protoc_descriptor_for_file = descriptor_for_file(&protoc_descriptors, &proto_file_name);
        let pure_descriptor_for_file = descriptor_for_file(&pure_descriptors, &proto_file_name);

        let skip = should_skip(input.to_str().unwrap());
        if protoc_rs_contents == pure_rs_contents
            && protoc_descriptor_for_file == pure_descriptor_for_file
        {
            if !skip {
                stats.passed += 1;
                println!("{}: PASSED", label);
            } else {
                stats.passed_marked_skip += 1;
                println!("{}: PASSED BUT MARKED SKIP", label);
            }
        } else {
            if skip {
                stats.skipped += 1;
                println!("{} SKIPPED", label);
            } else {
                stats.failed += 1;
                println!("{} FAILED", label);
            }

            print_diff(
                temp_dir.path(),
                Path::new(&protoc_rs).strip_prefix(temp_dir.path()).unwrap(),
                Path::new(&pure_rs).strip_prefix(temp_dir.path()).unwrap(),
            );

            fs::write(
                format!("{}/{}.descriptors", protoc_dir, proto_name),
                pretty_message(protoc_descriptor_for_file),
            )
            .unwrap();
            fs::write(
                format!("{}/{}.descriptors", pure_dir, proto_name),
                pretty_message(pure_descriptor_for_file),
            )
            .unwrap();
            print_diff(
                temp_dir.path(),
                Path::new(&format!("protoc/{}.descriptors", proto_name)),
                Path::new(&format!("pure/{}.descriptors", proto_name)),
            );
        }
    }

    println!("{:?}", stats);
    assert!(stats.failed == 0, "at least one test failed");
}

fn should_skip_with_marker(path: &str) -> bool {
    fs::read_to_string(path)
        .unwrap()
        .contains("@skip-codegen-identical-test")
}

#[test]
fn common_v2() {
    test_diff_in(
        "../protobuf-test",
        "src/common/v2",
        "src/common/v2",
        should_skip_with_marker,
    );
}

#[test]
fn common_v3() {
    let common_v3_root = tempfile::Builder::new()
        .prefix("common-v3")
        .tempdir()
        .unwrap();
    fs::create_dir_all(format!(
        "{}/src/common/v3",
        common_v3_root.path().to_str().unwrap()
    ))
    .unwrap();

    copy_tests_v2_v3(
        "../protobuf-test/src/common/v2",
        &format!("{}/src/common/v3", common_v3_root.path().to_str().unwrap()),
    );

    test_diff_in(
        common_v3_root.path().to_str().unwrap(),
        "src/common/v3",
        "src/common/v3",
        should_skip_with_marker,
    );
}

#[test]
fn v2() {
    test_diff_in(
        "../protobuf-test",
        "src/v2",
        "src/v2",
        should_skip_with_marker,
    );
}

#[test]
fn v3() {
    test_diff_in(
        "../protobuf-test",
        "src/v3",
        "src/v3",
        should_skip_with_marker,
    );
}

#[test]
fn interop() {
    test_diff_in(
        "../protobuf-test",
        "../interop/cxx",
        "../interop/cxx",
        should_skip_with_marker,
    );
}

#[test]
fn google() {
    test_diff_in("../protobuf-test", "src/google/protobuf", "src", |_| false);
}
<|endoftext|>