<commit_before>use redox::ops::DerefMut;
use redox::string::*;
use redox::vec::Vec;
use redox::boxed::Box;
use redox::fs::*;
use redox::io::*;
use redox::console::*;
use redox::env::*;
use redox::time::Duration;
use redox::to_num::*;

/* Magic Macros { */
static mut application: *mut Application<'static> = 0 as *mut Application;

/// Execute a command
macro_rules! exec {
    ($cmd:expr) => ({
        unsafe {
            (*application).on_command(&$cmd.to_string());
        }
    })
}
/* } Magic Macros */

/// A command
pub struct Command<'a> {
    pub name: &'a str,
    pub main: Box<Fn(&Vec<String>)>,
}

impl<'a> Command<'a> {
    /// Return the vector of the commands
    // TODO: Use a more efficient collection instead
    pub fn vec() -> Vec<Self> {
        let mut commands: Vec<Self> = Vec::new();
        commands.push(Command {
            name: "echo",
            main: box |args: &Vec<String>| {
                let echo = args.iter()
                    .skip(1)
                    .fold(String::new(), |string, arg| string + " " + arg);
                println!("{}", echo.trim());
            },
        });

        commands.push(Command {
            name: "open",
            main: box |args: &Vec<String>| {
                if let Some(arg) = args.get(1) {
                    File::exec(arg);
                }
            },
        });

        commands.push(Command {
            name: "run",
            main: box |args: &Vec<String>| {
                if let Some(path) = args.get(1) {

                    let mut commands = String::new();
                    if let Some(mut file) = File::open(path) {
                        println!("URL: {:?}", file.path());

                        file.read_to_string(&mut commands);
                    }

                    for command in commands.split('\n') {
                        exec!(command);
                    }
                }
            },
        });

        commands.push(Command {
            name: "send",
            main: box |args: &Vec<String>| {
                if args.len() < 3 {
                    println!("Error: incorrect arguments");
                    println!("Usage: send [url] [data]");
                    return;
                }

                let path = {
                    match args.get(1) {
                        Some(arg) => arg.clone(),
                        None => String::new(),
                    }
                };

                if let Some(mut file) = File::open(&path) {
                    println!("URL: {:?}", file.path());

                    let string: String = args.iter()
                        .skip(2)
                        .fold(String::new(), |s, arg| s + " " + arg)
                        + "\r\n\r\n";

                    match file.write(string.trim_left().as_bytes()) {
                        Some(size) => println!("Wrote {} bytes", size),
                        None => println!("Failed to write"),
                    }

                    let mut string = String::new();
                    match file.read_to_string(&mut string) {
                        Some(_) => println!("{}", string),
                        None => println!("Failed to read"),
                    }
                }
            },
        });

        commands.push(Command {
            name: "sleep",
            main: box |args: &Vec<String>| {
                let secs = {
                    match args.get(1) {
                        Some(arg) => arg.to_num() as i64,
                        None => 0,
                    }
                };

                let nanos = {
                    match args.get(2) {
                        Some(arg) => arg.to_num() as i32,
                        None => 0,
                    }
                };

                println!("Sleep: {} {}", secs, nanos);
                let remaining = Duration::new(secs, nanos).sleep();
                println!("Remaining: {} {}", remaining.secs, remaining.nanos);
            },
        });

        commands.push(Command {
            name: "test_ht",
            main: box |args: &Vec<String>| {
                ::redox::hashmap::test();
            },
        });

        commands.push(Command {
            name: "url",
            main: box |args: &Vec<String>| {
                let path = {
                    match args.get(1) {
                        Some(arg) => arg.clone(),
                        None => String::new(),
                    }
                };

                if let Some(mut file) = File::open(&path) {
                    println!("URL: {:?}", file.path());

                    let mut string = String::new();
                    match file.read_to_string(&mut string) {
                        Some(_) => println!("{}", string),
                        None => println!("Failed to read"),
                    }
                }
            },
        });

        commands.push(Command {
            name: "url_hex",
            main: box |args: &Vec<String>| {
                let path = {
                    match args.get(1) {
                        Some(arg) => arg.clone(),
                        None => String::new(),
                    }
                };

                if let Some(mut file) = File::open(&path) {
                    println!("URL: {:?}", file.path());

                    let mut vec: Vec<u8> = Vec::new();
                    match file.read_to_end(&mut vec) {
                        Some(_) => {
                            let mut line = "HEX:".to_string();
                            for byte in vec.iter() {
                                line = line + " " + &format!("{:X}", *byte);
                            }
                            println!("{}", line);
                        }
                        None => println!("Failed to read"),
                    }
                }
            },
        });

        commands.push(Command {
            name: "wget",
            main: box |args: &Vec<String>| {
                if let Some(host) = args.get(1) {
                    if let Some(req) = args.get(2) {
                        if let Some(mut con) = File::open(&("tcp://".to_string() + host)) {
                            con.write(("GET ".to_string() + req + " HTTP/1.1").as_bytes());

                            let mut res = Vec::new();
                            con.read_to_end(&mut res);

                            if let Some(mut file) = File::open(&req) {
                                file.write(&res);
                            }
                        }
                    } else {
                        println!("No request given");
                    }
                } else {
                    println!("No url given");
                }
            },
        });

        commands.push(Command {
            name: "pwd",
            main: box |args: &Vec<String>| {
                let mut err = false;
                if let Some(file) = File::open("") {
                    if let Some(path) = file.path() {
                        println!("{}", path);
                    } else {
                        err = true;
                    }
                } else {
                    err = true;
                }
                if err {
                    println!("Could not get the path");
                }
            },
        });

        commands.push(Command {
            name: "cd",
            main: box |args: &Vec<String>| {
                match args.get(1) {
                    Some(path) => {
                        if !change_cwd(&path) {
                            println!("Bad path: {}", path);
                        }
                    }
                    None => println!("No path given")
                }
            },
        });

        let command_list = commands.iter().fold(String::new(), |l , c| l + " " + c.name) + " exit";

        commands.push(Command {
            name: "help",
            main: box move |args: &Vec<String>| {
                println!("Commands:{}", command_list);
            },
         });

        commands
    }
}

/// A (env) variable
pub struct Variable {
    pub name: String,
    pub value: String,
}

pub struct Mode {
    value: bool,
}

/// An application
pub struct Application<'a> {
    commands: Vec<Command<'a>>,
    variables: Vec<Variable>,
    modes: Vec<Mode>,
}

impl<'a> Application<'a> {
    /// Create a new empty application
    pub fn new() -> Self {
        return Application {
            commands: Command::vec(),
            variables: Vec::new(),
            modes: Vec::new(),
        };
    }

    fn on_command(&mut self, command_string: &str) {
        //Comment
        if command_string.starts_with('#') {
            return;
        }

        //Show variables
        if command_string == "$" {
            let variables = self.variables.iter()
                .fold(String::new(),
                      |string, variable| string + "\n" + &variable.name + "=" + &variable.value);
            println!("{}", variables);
            return;
        }

        //Explode into arguments, replace variables
        let mut args: Vec<String> = Vec::<String>::new();
        for arg in command_string.split(' ') {
            if !arg.is_empty() {
                if arg.starts_with('$') {
                    let name = arg[1 .. arg.len()].to_string();
                    for variable in self.variables.iter() {
                        if variable.name == name {
                            args.push(variable.value.clone());
                            break;
                        }
                    }
                } else {
                    args.push(arg.to_string());
                }
            }
        }

        //Execute commands
        if let Some(cmd) = args.get(0) {
            if cmd == "if" {
                let mut value = false;

                if let Some(left) = args.get(1) {
                    if let Some(cmp) = args.get(2) {
                        if let Some(right) = args.get(3) {
                            if cmp == "==" {
                                value = *left == *right;
                            } else if cmp == "!=" {
                                value = *left != *right;
                            } else if cmp == ">" {
                                value = left.to_num_signed() > right.to_num_signed();
                            } else if cmp == ">=" {
                                value = left.to_num_signed() >= right.to_num_signed();
                            } else if cmp == "<" {
                                value = left.to_num_signed() < right.to_num_signed();
                            } else if cmp == "<=" {
                                value = left.to_num_signed() <= right.to_num_signed();
                            } else {
                                println!("Unknown comparison: {}", cmp);
                            }
                        }
                    }
                }

                self.modes.insert(0, Mode { value: value });
                return;
            }

            if cmd == "else" {
                let mut syntax_error = false;
                match self.modes.get_mut(0) {
                    Some(mode) => mode.value = !mode.value,
                    None => syntax_error = true,
                }
                if syntax_error {
                    println!("Syntax error: else found with no previous if");
                }
                return;
            }

            if cmd == "fi" {
                let mut syntax_error = false;
                if !self.modes.is_empty() {
                    self.modes.remove(0);
                } else {
                    syntax_error = true;
                }
                if syntax_error {
                    println!("Syntax error: fi found with no previous if");
                }
                return;
            }

            for mode in self.modes.iter() {
                if !mode.value {
                    return;
                }
            }

            //Set variables
            if let Some(i) = cmd.find('=') {
                let name = cmd[0 .. i].to_string();
                let mut value = cmd[i + 1 .. cmd.len()].to_string();

                if name.is_empty() {
                    return;
                }

                for i in 1..args.len() {
                    if let Some(arg) = args.get(i) {
                        value = value + " " + &arg;
                    }
                }

                if value.is_empty() {
                    let mut remove = -1;
                    for i in 0..self.variables.len() {
                        match self.variables.get(i) {
                            Some(variable) => if variable.name == name {
                                remove = i as isize;
                                break;
                            },
                            None => break,
                        }
                    }

                    if remove >= 0 {
                        self.variables.remove(remove as usize);
                    }
                } else {
                    for variable in self.variables.iter_mut() {
                        if variable.name == name {
                            variable.value = value;
                            return;
                        }
                    }

                    self.variables.push(Variable {
                        name: name,
                        value: value,
                    });
                }
                return;
            }

            //Commands
            for command in self.commands.iter() {
                if &command.name == cmd {
                    (*command.main)(&args);
                    return;
                }
            }

            println!("Unknown command: '{}'", cmd);
        }
    }

    /// Run the application
    pub fn main(&mut self) {
        console_title("Terminal");

        println!("Type help for a command list");
        if let Some(arg) = args().get(1) {
            let command = "run ".to_string() + arg;
            println!("# {}", command);
            self.on_command(&command);
        }

        while let Some(command) = readln!() {
            println!("# {}", command);
            if command == "exit" {
                break;
            } else if !command.is_empty() {
                self.on_command(&command);
            }
        }
    }
}

pub fn main() {
    unsafe {
        let mut app = box Application::new();
        application = app.deref_mut();
        app.main();
    }
}
<commit_msg>Single command to create a file, with your terminal<commit_after>use redox::ops::DerefMut;
use redox::string::*;
use redox::vec::Vec;
use redox::boxed::Box;
use redox::fs::*;
use redox::io::*;
use redox::console::*;
use redox::env::*;
use redox::time::Duration;
use redox::to_num::*;

/* Magic Macros { */
static mut application: *mut Application<'static> = 0 as *mut Application;

/// Execute a command
macro_rules! exec {
    ($cmd:expr) => ({
        unsafe {
            (*application).on_command(&$cmd.to_string());
        }
    })
}
/* } Magic Macros */

/// A command
pub struct Command<'a> {
    pub name: &'a str,
    pub main: Box<Fn(&Vec<String>)>,
}

impl<'a> Command<'a> {
    /// Return the vector of the commands
    // TODO: Use a more efficient collection instead
    pub fn vec() -> Vec<Self> {
        let mut commands: Vec<Self> = Vec::new();
        commands.push(Command {
            name: "echo",
            main: box |args: &Vec<String>| {
                let echo = args.iter()
                    .skip(1)
                    .fold(String::new(), |string, arg| string + " " + arg);
                println!("{}", echo.trim());
            },
        });

        commands.push(Command {
            name: "open",
            main: box |args: &Vec<String>| {
                if let Some(arg) = args.get(1) {
                    File::exec(arg);
                }
            },
        });

        commands.push(Command {
            name: "run",
            main: box |args: &Vec<String>| {
                if let Some(path) = args.get(1) {

                    let mut commands = String::new();
                    if let Some(mut file) = File::open(path) {
                        println!("URL: {:?}", file.path());

                        file.read_to_string(&mut commands);
                    }

                    for command in commands.split('\n') {
                        exec!(command);
                    }
                }
            },
        });

        commands.push(Command {
            name: "send",
            main: box |args: &Vec<String>| {
                if args.len() < 3 {
                    println!("Error: incorrect arguments");
                    println!("Usage: send [url] [data]");
                    return;
                }

                let path = {
                    match args.get(1) {
                        Some(arg) => arg.clone(),
                        None => String::new(),
                    }
                };

                if let Some(mut file) = File::open(&path) {
                    println!("URL: {:?}", file.path());

                    let string: String = args.iter()
                        .skip(2)
                        .fold(String::new(), |s, arg| s + " " + arg)
                        + "\r\n\r\n";

                    match file.write(string.trim_left().as_bytes()) {
                        Some(size) => println!("Wrote {} bytes", size),
                        None => println!("Failed to write"),
                    }

                    let mut string = String::new();
                    match file.read_to_string(&mut string) {
                        Some(_) => println!("{}", string),
                        None => println!("Failed to read"),
                    }
                }
            },
        });

        commands.push(Command {
            name: "sleep",
            main: box |args: &Vec<String>| {
                let secs = {
                    match args.get(1) {
                        Some(arg) => arg.to_num() as i64,
                        None => 0,
                    }
                };

                let nanos = {
                    match args.get(2) {
                        Some(arg) => arg.to_num() as i32,
                        None => 0,
                    }
                };

                println!("Sleep: {} {}", secs, nanos);
                let remaining = Duration::new(secs, nanos).sleep();
                println!("Remaining: {} {}", remaining.secs, remaining.nanos);
            },
        });

        commands.push(Command {
            name: "test_ht",
            main: box |args: &Vec<String>| {
                ::redox::hashmap::test();
            },
        });

        commands.push(Command {
            name: "url",
            main: box |args: &Vec<String>| {
                let path = {
                    match args.get(1) {
                        Some(arg) => arg.clone(),
                        None => String::new(),
                    }
                };

                if let Some(mut file) = File::open(&path) {
                    println!("URL: {:?}", file.path());

                    let mut string = String::new();
                    match file.read_to_string(&mut string) {
                        Some(_) => println!("{}", string),
                        None => println!("Failed to read"),
                    }
                }
            },
        });

        commands.push(Command {
            name: "url_hex",
            main: box |args: &Vec<String>| {
                let path = {
                    match args.get(1) {
                        Some(arg) => arg.clone(),
                        None => String::new(),
                    }
                };

                if let Some(mut file) = File::open(&path) {
                    println!("URL: {:?}", file.path());

                    let mut vec: Vec<u8> = Vec::new();
                    match file.read_to_end(&mut vec) {
                        Some(_) => {
                            let mut line = "HEX:".to_string();
                            for byte in vec.iter() {
                                line = line + " " + &format!("{:X}", *byte);
                            }
                            println!("{}", line);
                        }
                        None => println!("Failed to read"),
                    }
                }
            },
        });

        commands.push(Command {
            name: "wget",
            main: box |args: &Vec<String>| {
                if let Some(host) = args.get(1) {
                    if let Some(req) = args.get(2) {
                        if let Some(mut con) = File::open(&("tcp://".to_string() + host)) {
                            con.write(("GET ".to_string() + req + " HTTP/1.1").as_bytes());

                            let mut res = Vec::new();
                            con.read_to_end(&mut res);

                            if let Some(mut file) = File::open(&req) {
                                file.write(&res);
                            }
                        }
                    } else {
                        println!("No request given");
                    }
                } else {
                    println!("No url given");
                }
            },
        });

        /// Simple command to create a file, in the current directory
        /// The file has got the name given as the first argument of the command
        /// If the command have no arguments, the command don't create the file
        commands.push(Command {
            name: "cfile",
            main: box |args: &Vec<String>| {
                match args.get(1) {
                    Some(file_name) => {
                        File::create(file_name);
                    }
                    None => {
                        println!("Could not create a file without a name");
                    }
                }
            }
        });

        commands.push(Command {
            name: "pwd",
            main: box |args: &Vec<String>| {
                let mut err = false;
                if let Some(file) = File::open("") {
                    if let Some(path) = file.path() {
                        println!("{}", path);
                    } else {
                        err = true;
                    }
                } else {
                    err = true;
                }
                if err {
                    println!("Could not get the path");
                }
            },
        });

        commands.push(Command {
            name: "cd",
            main: box |args: &Vec<String>| {
                match args.get(1) {
                    Some(path) => {
                        if !change_cwd(&path) {
                            println!("Bad path: {}", path);
                        }
                    }
                    None => println!("No path given")
                }
            },
        });

        let command_list = commands.iter().fold(String::new(), |l , c| l + " " + c.name) + " exit";

        commands.push(Command {
            name: "help",
            main: box move |args: &Vec<String>| {
                println!("Commands:{}", command_list);
            },
         });

        commands
    }
}

/// A (env) variable
pub struct Variable {
    pub name: String,
    pub value: String,
}

pub struct Mode {
    value: bool,
}

/// An application
pub struct Application<'a> {
    commands: Vec<Command<'a>>,
    variables: Vec<Variable>,
    modes: Vec<Mode>,
}

impl<'a> Application<'a> {
    /// Create a new empty application
    pub fn new() -> Self {
        return Application {
            commands: Command::vec(),
            variables: Vec::new(),
            modes: Vec::new(),
        };
    }

    fn on_command(&mut self, command_string: &str) {
        //Comment
        if command_string.starts_with('#') {
            return;
        }

        //Show variables
        if command_string == "$" {
            let variables = self.variables.iter()
                .fold(String::new(),
                      |string, variable| string + "\n" + &variable.name + "=" + &variable.value);
            println!("{}", variables);
            return;
        }

        //Explode into arguments, replace variables
        let mut args: Vec<String> = Vec::<String>::new();
        for arg in command_string.split(' ') {
            if !arg.is_empty() {
                if arg.starts_with('$') {
                    let name = arg[1 .. arg.len()].to_string();
                    for variable in self.variables.iter() {
                        if variable.name == name {
                            args.push(variable.value.clone());
                            break;
                        }
                    }
                } else {
                    args.push(arg.to_string());
                }
            }
        }

        //Execute commands
        if let Some(cmd) = args.get(0) {
            if cmd == "if" {
                let mut value = false;

                if let Some(left) = args.get(1) {
                    if let Some(cmp) = args.get(2) {
                        if let Some(right) = args.get(3) {
                            if cmp == "==" {
                                value = *left == *right;
                            } else if cmp == "!=" {
                                value = *left != *right;
                            } else if cmp == ">" {
                                value = left.to_num_signed() > right.to_num_signed();
                            } else if cmp == ">=" {
                                value = left.to_num_signed() >= right.to_num_signed();
                            } else if cmp == "<" {
                                value = left.to_num_signed() < right.to_num_signed();
                            } else if cmp == "<=" {
                                value = left.to_num_signed() <= right.to_num_signed();
                            } else {
                                println!("Unknown comparison: {}", cmp);
                            }
                        }
                    }
                }

                self.modes.insert(0, Mode { value: value });
                return;
            }

            if cmd == "else" {
                let mut syntax_error = false;
                match self.modes.get_mut(0) {
                    Some(mode) => mode.value = !mode.value,
                    None => syntax_error = true,
                }
                if syntax_error {
                    println!("Syntax error: else found with no previous if");
                }
                return;
            }

            if cmd == "fi" {
                let mut syntax_error = false;
                if !self.modes.is_empty() {
                    self.modes.remove(0);
                } else {
                    syntax_error = true;
                }
                if syntax_error {
                    println!("Syntax error: fi found with no previous if");
                }
                return;
            }

            for mode in self.modes.iter() {
                if !mode.value {
                    return;
                }
            }

            //Set variables
            if let Some(i) = cmd.find('=') {
                let name = cmd[0 .. i].to_string();
                let mut value = cmd[i + 1 .. cmd.len()].to_string();

                if name.is_empty() {
                    return;
                }

                for i in 1..args.len() {
                    if let Some(arg) = args.get(i) {
                        value = value + " " + &arg;
                    }
                }

                if value.is_empty() {
                    let mut remove = -1;
                    for i in 0..self.variables.len() {
                        match self.variables.get(i) {
                            Some(variable) => if variable.name == name {
                                remove = i as isize;
                                break;
                            },
                            None => break,
                        }
                    }

                    if remove >= 0 {
                        self.variables.remove(remove as usize);
                    }
                } else {
                    for variable in self.variables.iter_mut() {
                        if variable.name == name {
                            variable.value = value;
                            return;
                        }
                    }

                    self.variables.push(Variable {
                        name: name,
                        value: value,
                    });
                }
                return;
            }

            //Commands
            for command in self.commands.iter() {
                if &command.name == cmd {
                    (*command.main)(&args);
                    return;
                }
            }

            println!("Unknown command: '{}'", cmd);
        }
    }

    /// Run the application
    pub fn main(&mut self) {
        console_title("Terminal");

        println!("Type help for a command list");
        if let Some(arg) = args().get(1) {
            let command = "run ".to_string() + arg;
            println!("# {}", command);
            self.on_command(&command);
        }

        while let Some(command) = readln!() {
            println!("# {}", command);
            if command == "exit" {
                break;
            } else if !command.is_empty() {
                self.on_command(&command);
            }
        }
    }
}

pub fn main() {
    unsafe {
        let mut app = box Application::new();
        application = app.deref_mut();
        app.main();
    }
}
<|endoftext|>


<commit_before>use libc::{int32_t,size_t};
use super::graph::Graph;
use ffi::{Context,Job,PixelFormat, EdgeKind, NodeType, BitmapBgra};
use daggy::{Dag,EdgeIndex,NodeIndex};
use petgraph::{EdgeDirection};
mod rotate_flip_transpose;
mod clone_crop_fill_expand;
mod scale_render;
mod create_canvas;
mod codecs_and_pointer;

extern crate imageflow_serde as s;
use super::definitions::*;

//TODO: Implement Scale1D
//TODO: Implement Decoder + APPLY_ORIENTATION, Encoder
//TODO: implement TRANSPOSE, APPLY_ORIENTATION
//TODO: Implement EXPAND_CANVAS, CROP

pub use self::codecs_and_pointer::PRIMITIVE_DECODER;
pub use self::codecs_and_pointer::DECODER;
pub use self::codecs_and_pointer::BITMAP_BGRA_POINTER;
pub use self::create_canvas::CREATE_CANVAS;
pub use self::clone_crop_fill_expand::CLONE;
pub use self::clone_crop_fill_expand::COPY_RECT;
pub use self::clone_crop_fill_expand::FILL_RECT;
pub use self::clone_crop_fill_expand::EXPAND_CANVAS;
pub use self::scale_render::SCALE;
pub use self::rotate_flip_transpose::FLIP_V;
pub use self::rotate_flip_transpose::FLIP_V_PRIMITIVE;
pub use self::rotate_flip_transpose::FLIP_H;
pub use self::rotate_flip_transpose::FLIP_H_PRIMITIVE;
pub use self::rotate_flip_transpose::ROTATE_90;
pub use self::rotate_flip_transpose::ROTATE_180;
pub use self::rotate_flip_transpose::ROTATE_270;
pub use self::rotate_flip_transpose::TRANSPOSE;
pub use self::rotate_flip_transpose::APPLY_ORIENTATION;
pub use self::rotate_flip_transpose::NO_OP;

struct NodeDefHelpers {}
impl NodeDefHelpers {
    fn copy_frame_est_from_first_input(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.copy_frame_est_from_first_input(ix);
    }
    fn copy_frame_est_from_first_canvas(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.copy_frame_est_from_first_canvas(ix);
    }
    fn rotate_frame_info(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.rotate_frame_est_from_first_input(ix);
    }
    fn flatten_flip_v(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        // ctx.graph.node_weight_mut(ix).unwrap()
    }

    fn delete_node_and_snap_together(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.delete_node_and_snap_together(ix);
    }
}

impl<'c> OpCtxMut<'c> {
    pub fn first_parent_of_kind<'a>(&'a self, of_node: NodeIndex<u32>, filter_by_kind: EdgeKind) -> Option<NodeIndex<u32>> {
        self.graph
            .graph()
            .edges_directed(of_node, EdgeDirection::Incoming)
            .filter(|&(node, &kind)| kind == filter_by_kind)
            .map(|(node, kind)| node)
            .nth(0)
    }

    pub fn first_parent_input<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<NodeIndex<u32>> {
        self.first_parent_of_kind(of_node, EdgeKind::Input)
    }
    pub fn first_parent_canvas<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<NodeIndex<u32>> {
        self.first_parent_of_kind(of_node, EdgeKind::Input)
    }

    pub fn first_parent_input_weight<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<Node> {
        self.first_parent_input(of_node).map(|ix| self.graph.node_weight(ix).unwrap().clone())
    }

    pub fn first_parent_frame_info_some<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<FrameInfo> {
        self.first_parent_input(of_node).and_then(|ix| {
            self.graph.node_weight(ix).and_then(|w| {
                match w.frame_est{
                    FrameEstimate::Some(ref frame_info) => Some(*frame_info),
                    _ => None
                }
            })
        })
    }

    pub fn get_json_params<'a>(&'a self, ix: NodeIndex<u32>) -> Option<s::Node> {
        self.graph.node_weight(ix).and_then(|w| {
            match w.params {
                NodeParams::Json(ref node) => Some(node.clone()),
                _ => None
            }
        })
    }

    pub fn first_parent_canvas_weight<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<&Node> {
        self.first_parent_canvas(of_node).map(|ix| self.graph.node_weight(ix).unwrap())
    }

    pub fn first_parent_result_frame<'a, 'b>(&'a self, of_node: NodeIndex<u32>, kind: EdgeKind) -> Option<*mut BitmapBgra> {
        self.first_parent_of_kind(of_node, kind)
            .and_then(|ix| self.graph.node_weight(ix))
            .and_then(|w|
            match w.result {
                NodeResult::Frame(ptr) => Some(ptr),
                _ => None
            }
        )
    }

    pub fn first_parent_input_weight_mut<'a>(&'a mut self,
                                             of_node: NodeIndex<u32>)
                                             -> Option<&mut Node> {
        self.first_parent_input(of_node).map(move |ix| self.graph.node_weight_mut(ix).unwrap())
    }

    pub fn has_other_children<'a>(&'a self,
                                  of_node: NodeIndex<u32>,
                                  except_child: NodeIndex<u32>)
                                  -> bool {
        self.graph
            .graph()
            .neighbors_directed(of_node, EdgeDirection::Incoming)
            .any(|n| n != except_child)
    }

    pub fn weight<'a>(&'a mut self, node_to_update: NodeIndex<u32>) -> &'a Node {
        self.graph.node_weight(node_to_update).unwrap()
    }

    pub fn weight_mut<'a>(&'a mut self, node_to_update: NodeIndex<u32>) -> &'a mut Node {
        self.graph.node_weight_mut(node_to_update).unwrap()
    }
    pub fn copy_frame_est_from_first_input<'a>(&'a mut self, node_to_update: NodeIndex<u32>) {
        match self.first_parent_input(node_to_update) {
            Some(input_ix) => {
                if self.graph.node_weight(input_ix).unwrap().frame_est == FrameEstimate::None{
                    panic!("Parent frame {} is not estimated", input_ix.index());
                }
                self.graph.node_weight_mut(node_to_update).unwrap().frame_est =
                    self.graph.node_weight(input_ix).unwrap().frame_est.clone();
            }
            None => {}
        }
    }
    pub fn copy_frame_est_from_first_canvas<'a>(&'a mut self, node_to_update: NodeIndex<u32>) {
        match self.first_parent_canvas(node_to_update) {
            Some(input_ix) => {
                self.graph.node_weight_mut(node_to_update).unwrap().frame_est =
                    self.graph.node_weight(input_ix).unwrap().frame_est.clone();
            }
            None => {}
        }
    }

    pub fn assert_ok(&self){
        unsafe {
            ::ContextPtr::from_ptr(self.c).assert_ok(None);
        }
    }

    pub fn rotate_frame_est_from_first_input<'a, 'b>(&'a mut self,
                                                     node_to_update: NodeIndex<u32>) {
        // TODO: select by EdgeKind=Input
        let input = self.graph
            .graph()
            .neighbors_directed(node_to_update, EdgeDirection::Incoming)
            .nth(0);
        match input {
            Some(input_ix) => {
                let input_est = self.graph.node_weight(input_ix).unwrap().frame_est.clone();
                let mut w = self.graph.node_weight_mut(node_to_update).unwrap();
                w.frame_est = match input_est {
                    FrameEstimate::Some(info) => {
                        FrameEstimate::Some(FrameInfo {
                            w: info.h,
                            h: info.w,
                            ..info
                        })
                    }
                    FrameEstimate::UpperBound(info) => {
                        FrameEstimate::UpperBound(FrameInfo {
                            w: info.h,
                            h: info.w,
                            ..info
                        })
                    }
                    other => other,
                };
            }
            None => {}
        }
    }

    pub fn copy_edges_to<'a>(&'a mut self,
                             from_node: NodeIndex<u32>,
                             to_node: NodeIndex<u32>,
                             direction: EdgeDirection) {
        let edges = self.graph
            .graph()
            .edges_directed(from_node, direction)
            .map(|(a, b)| (a, b.clone()))
            .collect::<Vec<_>>();

        for (other_node, weight) in edges {
            match direction {
                EdgeDirection::Incoming => {
                    self.graph.add_edge(other_node, to_node, weight.clone()).unwrap()
                }
                EdgeDirection::Outgoing => {
                    self.graph.add_edge(to_node, other_node, weight.clone()).unwrap()
                }
            };
        }
    }
    pub fn delete_node_and_snap_together<'a>(&'a mut self, node_to_delete: NodeIndex<u32>) {
        // Prefer EdgeKind=Input
        let input = self.graph
            .graph()
            .neighbors_directed(node_to_delete, EdgeDirection::Incoming)
            .nth(0);
        match input {
            None => {}
            Some(from_node) => {
                self.copy_edges_to(node_to_delete, from_node, EdgeDirection::Outgoing);
                self.graph.remove_node(node_to_delete).unwrap();
            }
        };
    }

    pub fn replace_node<'a>(&'a mut self, index: NodeIndex<u32>, with_list: Vec<Node>) {
        let mut with = with_list.clone();
        match with.len() {
            0 => self.delete_node_and_snap_together(index),
            n => {
                with.reverse();
                let mut last_ix = self.graph.add_node(with.pop().unwrap());
                self.copy_edges_to(index, last_ix, EdgeDirection::Incoming);
                while with.len() > 0 {
                    last_ix = self.graph.add_node(with.pop().unwrap());
                }
                self.copy_edges_to(index, last_ix, EdgeDirection::Outgoing);
                self.graph.remove_node(index).unwrap();
            }
        }
    }

    pub fn replace_node_with_existing<'a>(&'a mut self,
                                          index: NodeIndex<u32>,
                                          with_index: NodeIndex<u32>) {
        self.copy_edges_to(index, with_index, EdgeDirection::Incoming);
        self.copy_edges_to(index, with_index, EdgeDirection::Outgoing);
        self.graph.remove_node(index).unwrap();
    }
}<commit_msg>Fix has_other_children<commit_after>use libc::{int32_t,size_t};
use super::graph::Graph;
use ffi::{Context,Job,PixelFormat, EdgeKind, NodeType, BitmapBgra};
use daggy::{Dag,EdgeIndex,NodeIndex};
use petgraph::{EdgeDirection};
mod rotate_flip_transpose;
mod clone_crop_fill_expand;
mod scale_render;
mod create_canvas;
mod codecs_and_pointer;

extern crate imageflow_serde as s;
use super::definitions::*;

//TODO: Implement Scale1D
//TODO: Implement Decoder + APPLY_ORIENTATION, Encoder
//TODO: implement TRANSPOSE, APPLY_ORIENTATION
//TODO: Implement EXPAND_CANVAS, CROP

pub use self::codecs_and_pointer::PRIMITIVE_DECODER;
pub use self::codecs_and_pointer::DECODER;
pub use self::codecs_and_pointer::BITMAP_BGRA_POINTER;
pub use self::create_canvas::CREATE_CANVAS;
pub use self::clone_crop_fill_expand::CLONE;
pub use self::clone_crop_fill_expand::COPY_RECT;
pub use self::clone_crop_fill_expand::FILL_RECT;
pub use self::clone_crop_fill_expand::EXPAND_CANVAS;
pub use self::scale_render::SCALE;
pub use self::rotate_flip_transpose::FLIP_V;
pub use self::rotate_flip_transpose::FLIP_V_PRIMITIVE;
pub use self::rotate_flip_transpose::FLIP_H;
pub use self::rotate_flip_transpose::FLIP_H_PRIMITIVE;
pub use self::rotate_flip_transpose::ROTATE_90;
pub use self::rotate_flip_transpose::ROTATE_180;
pub use self::rotate_flip_transpose::ROTATE_270;
pub use self::rotate_flip_transpose::TRANSPOSE;
pub use self::rotate_flip_transpose::APPLY_ORIENTATION;
pub use self::rotate_flip_transpose::NO_OP;

struct NodeDefHelpers {}
impl NodeDefHelpers {
    fn copy_frame_est_from_first_input(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.copy_frame_est_from_first_input(ix);
    }
    fn copy_frame_est_from_first_canvas(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.copy_frame_est_from_first_canvas(ix);
    }
    fn rotate_frame_info(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.rotate_frame_est_from_first_input(ix);
    }
    fn flatten_flip_v(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        // ctx.graph.node_weight_mut(ix).unwrap()
    }

    fn delete_node_and_snap_together(ctx: &mut OpCtxMut, ix: NodeIndex<u32>) {
        ctx.delete_node_and_snap_together(ix);
    }
}

impl<'c> OpCtxMut<'c> {
    pub fn first_parent_of_kind<'a>(&'a self, of_node: NodeIndex<u32>, filter_by_kind: EdgeKind) -> Option<NodeIndex<u32>> {
        self.graph
            .graph()
            .edges_directed(of_node, EdgeDirection::Incoming)
            .filter(|&(node, &kind)| kind == filter_by_kind)
            .map(|(node, kind)| node)
            .nth(0)
    }

    pub fn first_parent_input<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<NodeIndex<u32>> {
        self.first_parent_of_kind(of_node, EdgeKind::Input)
    }
    pub fn first_parent_canvas<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<NodeIndex<u32>> {
        self.first_parent_of_kind(of_node, EdgeKind::Input)
    }

    pub fn first_parent_input_weight<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<Node> {
        self.first_parent_input(of_node).map(|ix| self.graph.node_weight(ix).unwrap().clone())
    }

    pub fn first_parent_frame_info_some<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<FrameInfo> {
        self.first_parent_input(of_node).and_then(|ix| {
            self.graph.node_weight(ix).and_then(|w| {
                match w.frame_est{
                    FrameEstimate::Some(ref frame_info) => Some(*frame_info),
                    _ => None
                }
            })
        })
    }

    pub fn get_json_params<'a>(&'a self, ix: NodeIndex<u32>) -> Option<s::Node> {
        self.graph.node_weight(ix).and_then(|w| {
            match w.params {
                NodeParams::Json(ref node) => Some(node.clone()),
                _ => None
            }
        })
    }

    pub fn first_parent_canvas_weight<'a>(&'a self, of_node: NodeIndex<u32>) -> Option<&Node> {
        self.first_parent_canvas(of_node).map(|ix| self.graph.node_weight(ix).unwrap())
    }

    pub fn first_parent_result_frame<'a, 'b>(&'a self, of_node: NodeIndex<u32>, kind: EdgeKind) -> Option<*mut BitmapBgra> {
        self.first_parent_of_kind(of_node, kind)
            .and_then(|ix| self.graph.node_weight(ix))
            .and_then(|w|
            match w.result {
                NodeResult::Frame(ptr) => Some(ptr),
                _ => None
            }
        )
    }

    pub fn first_parent_input_weight_mut<'a>(&'a mut self,
                                             of_node: NodeIndex<u32>)
                                             -> Option<&mut Node> {
        self.first_parent_input(of_node).map(move |ix| self.graph.node_weight_mut(ix).unwrap())
    }

    pub fn has_other_children<'a>(&'a self,
                                  of_node: NodeIndex<u32>,
                                  except_child: NodeIndex<u32>)
                                  -> bool {
        self.graph
            .graph()
            .neighbors_directed(of_node, EdgeDirection::Outgoing)
            .any(|n| n != except_child)
    }

    pub fn weight<'a>(&'a mut self, node_to_update: NodeIndex<u32>) -> &'a Node {
        self.graph.node_weight(node_to_update).unwrap()
    }

    pub fn weight_mut<'a>(&'a mut self, node_to_update: NodeIndex<u32>) -> &'a mut Node {
        self.graph.node_weight_mut(node_to_update).unwrap()
    }
    pub fn copy_frame_est_from_first_input<'a>(&'a mut self, node_to_update: NodeIndex<u32>) {
        match self.first_parent_input(node_to_update) {
            Some(input_ix) => {
                if self.graph.node_weight(input_ix).unwrap().frame_est == FrameEstimate::None{
                    panic!("Parent frame {} is not estimated", input_ix.index());
                }
                self.graph.node_weight_mut(node_to_update).unwrap().frame_est =
                    self.graph.node_weight(input_ix).unwrap().frame_est.clone();
            }
            None => {}
        }
    }
    pub fn copy_frame_est_from_first_canvas<'a>(&'a mut self, node_to_update: NodeIndex<u32>) {
        match self.first_parent_canvas(node_to_update) {
            Some(input_ix) => {
                self.graph.node_weight_mut(node_to_update).unwrap().frame_est =
                    self.graph.node_weight(input_ix).unwrap().frame_est.clone();
            }
            None => {}
        }
    }

    pub fn assert_ok(&self){
        unsafe {
            ::ContextPtr::from_ptr(self.c).assert_ok(None);
        }
    }

    pub fn rotate_frame_est_from_first_input<'a, 'b>(&'a mut self,
                                                     node_to_update: NodeIndex<u32>) {
        // TODO: select by EdgeKind=Input
        let input = self.graph
            .graph()
            .neighbors_directed(node_to_update, EdgeDirection::Incoming)
            .nth(0);
        match input {
            Some(input_ix) => {
                let input_est = self.graph.node_weight(input_ix).unwrap().frame_est.clone();
                let mut w = self.graph.node_weight_mut(node_to_update).unwrap();
                w.frame_est = match input_est {
                    FrameEstimate::Some(info) => {
                        FrameEstimate::Some(FrameInfo {
                            w: info.h,
                            h: info.w,
                            ..info
                        })
                    }
                    FrameEstimate::UpperBound(info) => {
                        FrameEstimate::UpperBound(FrameInfo {
                            w: info.h,
                            h: info.w,
                            ..info
                        })
                    }
                    other => other,
                };
            }
            None => {}
        }
    }

    pub fn copy_edges_to<'a>(&'a mut self,
                             from_node: NodeIndex<u32>,
                             to_node: NodeIndex<u32>,
                             direction: EdgeDirection) {
        let edges = self.graph
            .graph()
            .edges_directed(from_node, direction)
            .map(|(a, b)| (a, b.clone()))
            .collect::<Vec<_>>();

        for (other_node, weight) in edges {
            match direction {
                EdgeDirection::Incoming => {
                    self.graph.add_edge(other_node, to_node, weight.clone()).unwrap()
                }
                EdgeDirection::Outgoing => {
                    self.graph.add_edge(to_node, other_node, weight.clone()).unwrap()
                }
            };
        }
    }
    pub fn delete_node_and_snap_together<'a>(&'a mut self, node_to_delete: NodeIndex<u32>) {
        // Prefer EdgeKind=Input
        let input = self.graph
            .graph()
            .neighbors_directed(node_to_delete, EdgeDirection::Incoming)
            .nth(0);
        match input {
            None => {}
            Some(from_node) => {
                self.copy_edges_to(node_to_delete, from_node, EdgeDirection::Outgoing);
                self.graph.remove_node(node_to_delete).unwrap();
            }
        };
    }

    pub fn replace_node<'a>(&'a mut self, index: NodeIndex<u32>, with_list: Vec<Node>) {
        let mut with = with_list.clone();
        match with.len() {
            0 => self.delete_node_and_snap_together(index),
            n => {
                with.reverse();
                let mut last_ix = self.graph.add_node(with.pop().unwrap());
                self.copy_edges_to(index, last_ix, EdgeDirection::Incoming);
                while with.len() > 0 {
                    last_ix = self.graph.add_node(with.pop().unwrap());
                }
                self.copy_edges_to(index, last_ix, EdgeDirection::Outgoing);
                self.graph.remove_node(index).unwrap();
            }
        }
    }

    pub fn replace_node_with_existing<'a>(&'a mut self,
                                          index: NodeIndex<u32>,
                                          with_index: NodeIndex<u32>) {
        self.copy_edges_to(index, with_index, EdgeDirection::Incoming);
        self.copy_edges_to(index, with_index, EdgeDirection::Outgoing);
        self.graph.remove_node(index).unwrap();
    }
}<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use apache_avro::{
    schema::Schema,
    types::{Record, Value},
    Reader, Writer,
};
use std::{
    io::{BufReader, BufWriter},
    time::{Duration, Instant},
};

fn nanos(duration: Duration) -> u64 {
    duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64
}

fn seconds(nanos: u64) -> f64 {
    (nanos as f64) / 1_000_000_000f64
}

/*
fn duration(nanos: u64) -> Duration {
    Duration::new(nanos / 1_000_000_000, (nanos % 1_000_000_000) as u32)
}
*/

fn benchmark(
    schema: &Schema,
    record: &Value,
    big_or_small: &str,
    count: usize,
    runs: usize,
) -> anyhow::Result<()> {
    let mut records = Vec::new();
    for __ in 0..count {
        records.push(record.clone());
    }

    let mut durations = Vec::with_capacity(runs);

    let mut bytes = None;
    for _ in 0..runs {
        let records = records.clone();

        let start = Instant::now();
        let mut writer = Writer::new(schema, BufWriter::new(Vec::new()));
        writer.extend(records.into_iter())?;

        let duration = Instant::now().duration_since(start);
        durations.push(duration);

        bytes = Some(writer.into_inner()?.into_inner()?);
    }

    let total_duration_write = durations.into_iter().fold(0u64, |a, b| a + nanos(b));

    // println!("Write: {} {} {:?}", count, runs, seconds(total_duration));

    let bytes = bytes.unwrap();

    let mut durations = Vec::with_capacity(runs);

    for _ in 0..runs {
        let start = Instant::now();
        let reader = Reader::with_schema(schema, BufReader::new(&bytes[..]))?;

        let mut read_records = Vec::with_capacity(count);
        for record in reader {
            read_records.push(record);
        }

        let duration = Instant::now().duration_since(start);
        durations.push(duration);

        assert_eq!(count, read_records.len());
    }

    let total_duration_read = durations.into_iter().fold(0u64, |a, b| a + nanos(b));

    // println!("Read: {} {} {:?}", count, runs, seconds(total_duration));
    let (total_write_secs, total_read_secs) =
        (seconds(total_duration_write), seconds(total_duration_read));

    println!(
        "{},{},{},{},{}",
        count, runs, big_or_small, total_write_secs, total_read_secs
    );
    Ok(())
}

fn main() -> anyhow::Result<()> {
    let raw_small_schema = r#"
        {"namespace": "test", "type": "record", "name": "Test", "fields": [{"type": {"type": "string"}, "name": "field"}]}
    "#;

    let raw_big_schema = r#"
        {"namespace": "my.example", "type": "record", "name": "userInfo", "fields": [{"default": "NONE", "type": "string", "name": "username"}, {"default": -1, "type": "int", "name": "age"}, {"default": "NONE", "type": "string", "name": "phone"}, {"default": "NONE", "type": "string", "name": "housenum"}, {"default": {}, "type": {"fields": [{"default": "NONE", "type": "string", "name": "street"}, {"default": "NONE", "type": "string", "name": "city"}, {"default": "NONE", "type": "string", "name": "state_prov"}, {"default": "NONE", "type": "string", "name": "country"}, {"default": "NONE", "type": "string", "name": "zip"}], "type": "record", "name": "mailing_address"}, "name": "address"}]}
    "#;

    let small_schema = Schema::parse_str(raw_small_schema)?;
    let big_schema = Schema::parse_str(raw_big_schema)?;

    println!("{:?}", small_schema);
    println!("{:?}", big_schema);

    let mut small_record = Record::new(&small_schema).unwrap();
    small_record.put("field", "foo");
    let small_record = small_record.into();

    let raw_address_schema = r#"{"fields": [{"default": "NONE", "type": "string", "name": "street"}, {"default": "NONE", "type": "string", "name": "city"}, {"default": "NONE", "type": "string", "name": "state_prov"}, {"default": "NONE", "type": "string", "name": "country"}, {"default": "NONE", "type": "string", "name": "zip"}], "type": "record", "name": "mailing_address"}"#;
    let address_schema = Schema::parse_str(raw_address_schema).unwrap();
    let mut address = Record::new(&address_schema).unwrap();
    address.put("street", "street");
    address.put("city", "city");
    address.put("state_prov", "state_prov");
    address.put("country", "country");
    address.put("zip", "zip");

    let mut big_record = Record::new(&big_schema).unwrap();
    big_record.put("username", "username");
    big_record.put("age", 10i32);
    big_record.put("phone", "000000000");
    big_record.put("housenum", "0000");
    big_record.put("address", address);
    let big_record = big_record.into();

    println!();

    benchmark(&small_schema, &small_record, "Small", 10_000, 1)?;
    benchmark(&big_schema, &big_record, "Big", 10_000, 1)?;

    benchmark(&small_schema, &small_record, "Small", 1, 100_000)?;
    benchmark(&small_schema, &small_record, "Small", 100, 1000)?;
    benchmark(&small_schema, &small_record, "Small", 10_000, 10)?;

    benchmark(&big_schema, &big_record, "Big", 1, 100_000)?;
    benchmark(&big_schema, &big_record, "Big", 100, 1000)?;
    benchmark(&big_schema, &big_record, "Big", 10_000, 10)?;

    Ok(())
}
<commit_msg>AVRO-3464: Print a header and use tabs as separators (#1614)<commit_after>// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

use apache_avro::{
    schema::Schema,
    types::{Record, Value},
    Reader, Writer,
};
use std::{
    io::{BufReader, BufWriter},
    time::{Duration, Instant},
};

fn nanos(duration: Duration) -> u64 {
    duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64
}

fn seconds(nanos: u64) -> f64 {
    (nanos as f64) / 1_000_000_000f64
}

/*
fn duration(nanos: u64) -> Duration {
    Duration::new(nanos / 1_000_000_000, (nanos % 1_000_000_000) as u32)
}
*/

fn benchmark(
    schema: &Schema,
    record: &Value,
    big_or_small: &str,
    count: usize,
    runs: usize,
) -> anyhow::Result<()> {
    let mut records = Vec::new();
    for __ in 0..count {
        records.push(record.clone());
    }

    let mut durations = Vec::with_capacity(runs);

    let mut bytes = None;
    for _ in 0..runs {
        let records = records.clone();

        let start = Instant::now();
        let mut writer = Writer::new(schema, BufWriter::new(Vec::new()));
        writer.extend(records.into_iter())?;

        let duration = Instant::now().duration_since(start);
        durations.push(duration);

        bytes = Some(writer.into_inner()?.into_inner()?);
    }

    let total_duration_write = durations.into_iter().fold(0u64, |a, b| a + nanos(b));

    // println!("Write: {} {} {:?}", count, runs, seconds(total_duration));

    let bytes = bytes.unwrap();

    let mut durations = Vec::with_capacity(runs);

    for _ in 0..runs {
        let start = Instant::now();
        let reader = Reader::with_schema(schema, BufReader::new(&bytes[..]))?;

        let mut read_records = Vec::with_capacity(count);
        for record in reader {
            read_records.push(record);
        }

        let duration = Instant::now().duration_since(start);
        durations.push(duration);

        assert_eq!(count, read_records.len());
    }

    let total_duration_read = durations.into_iter().fold(0u64, |a, b| a + nanos(b));

    // println!("Read: {} {} {:?}", count, runs, seconds(total_duration));
    let (total_write_secs, total_read_secs) =
        (seconds(total_duration_write), seconds(total_duration_read));

    println!(
        "{}\t\t{}\t\t{}\t\t{}\t\t{}",
        count, runs, big_or_small, total_write_secs, total_read_secs
    );
    Ok(())
}

fn main() -> anyhow::Result<()> {
    let raw_small_schema = r#"
        {"namespace": "test", "type": "record", "name": "Test", "fields": [{"type": {"type": "string"}, "name": "field"}]}
    "#;

    let raw_big_schema = r#"
        {"namespace": "my.example", "type": "record", "name": "userInfo", "fields": [{"default": "NONE", "type": "string", "name": "username"}, {"default": -1, "type": "int", "name": "age"}, {"default": "NONE", "type": "string", "name": "phone"}, {"default": "NONE", "type": "string", "name": "housenum"}, {"default": {}, "type": {"fields": [{"default": "NONE", "type": "string", "name": "street"}, {"default": "NONE", "type": "string", "name": "city"}, {"default": "NONE", "type": "string", "name": "state_prov"}, {"default": "NONE", "type": "string", "name": "country"}, {"default": "NONE", "type": "string", "name": "zip"}], "type": "record", "name": "mailing_address"}, "name": "address"}]}
    "#;

    let small_schema = Schema::parse_str(raw_small_schema)?;
    let big_schema = Schema::parse_str(raw_big_schema)?;

    println!("{:?}", small_schema);
    println!("{:?}", big_schema);

    let mut small_record = Record::new(&small_schema).unwrap();
    small_record.put("field", "foo");
    let small_record = small_record.into();

    let raw_address_schema = r#"{"fields": [{"default": "NONE", "type": "string", "name": "street"}, {"default": "NONE", "type": "string", "name": "city"}, {"default": "NONE", "type": "string", "name": "state_prov"}, {"default": "NONE", "type": "string", "name": "country"}, {"default": "NONE", "type": "string", "name": "zip"}], "type": "record", "name": "mailing_address"}"#;
    let address_schema = Schema::parse_str(raw_address_schema).unwrap();
    let mut address = Record::new(&address_schema).unwrap();
    address.put("street", "street");
    address.put("city", "city");
    address.put("state_prov", "state_prov");
    address.put("country", "country");
    address.put("zip", "zip");

    let mut big_record = Record::new(&big_schema).unwrap();
    big_record.put("username", "username");
    big_record.put("age", 10i32);
    big_record.put("phone", "000000000");
    big_record.put("housenum", "0000");
    big_record.put("address", address);
    let big_record = big_record.into();

    println!();
    println!("Count\t\tRuns\t\tBig/Small\tTotal write secs\tTotal read secs");

    benchmark(&small_schema, &small_record, "Small", 10_000, 1)?;
    benchmark(&big_schema, &big_record, "Big", 10_000, 1)?;

    benchmark(&small_schema, &small_record, "Small", 1, 100_000)?;
    benchmark(&small_schema, &small_record, "Small", 100, 1000)?;
    benchmark(&small_schema, &small_record, "Small", 10_000, 10)?;

    benchmark(&big_schema, &big_record, "Big", 1, 100_000)?;
    benchmark(&big_schema, &big_record, "Big", 100, 1000)?;
    benchmark(&big_schema, &big_record, "Big", 10_000, 10)?;

    Ok(())
}
<|endoftext|>


<commit_before>// pest. The Elegant Parser
// Copyright (C) 2017  Dragoș Tiselice
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::cell::Cell;
use std::{mem, ptr};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering};

use futures::{Async, Poll};
use futures::stream::Stream;
use futures::task::{park, Task};

#[cfg(target_pointer_width = "32")]
const PAD: usize = 16;
#[cfg(target_pointer_width = "64")]
const PAD: usize = 8;

#[repr(C)]
struct RingBuffer<T, E> {
    read:          AtomicUsize,
    last_write:    Cell<usize>,
    _padding1:     [usize; PAD - 2],
    write:         AtomicUsize,
    last_read:     Cell<usize>,
    _padding2:     [usize; PAD - 2],
    buffer:        *mut T,
    capacity:      usize,
    size:          usize,
    pub error:     AtomicPtr<E>,
    pub is_closed: AtomicBool,
    pub task:      AtomicPtr<Task>
}

impl<T, E> RingBuffer<T, E> {
    #[inline]
    pub fn new(capacity: usize) -> RingBuffer<T, E> {
        let size = capacity.next_power_of_two();
        let mut vec = Vec::with_capacity(size);
        let ptr = vec.as_mut_ptr();

        mem::forget(vec);

        RingBuffer {
            read:       AtomicUsize::new(0),
            last_write: Cell::new(0),
            _padding1:  [0; PAD - 2],
            write:      AtomicUsize::new(0),
            last_read:  Cell::new(0),
            _padding2:  [0; PAD - 2],
            buffer:     ptr,
            capacity:   capacity,
            size:       size,
            error:      AtomicPtr::new(ptr::null_mut()),
            is_closed:  AtomicBool::new(false),
            task:       AtomicPtr::new(ptr::null_mut())
        }
    }

    #[inline]
    pub fn try_push(&self, value: T) -> Option<T> {
        let read = self.last_read.get();
        let write = self.write.load(Ordering::Relaxed);

        if read + self.capacity <= write {
            let read = self.read.load(Ordering::Relaxed);
            self.last_read.set(read);

            if read + self.capacity <= write {
                return Some(value);
            }
        }

        self.push(write, value);

        self.write.store(write.wrapping_add(1), Ordering::Relaxed);

        self.try_unpark();

        None
    }

    #[inline]
    pub fn try_pop(&self) -> Option<T> {
        let read = self.read.load(Ordering::Relaxed);
        let write = self.last_write.get();

        if read == write {
            let write = self.write.load(Ordering::Relaxed);
            self.last_write.set(write);

            if read == write {
                return None;
            }
        }

        let result = self.pop(read);

        self.read.store(read.wrapping_add(1), Ordering::Relaxed);

        Some(result)
    }

    #[inline]
    pub fn try_unpark(&self) {
        let task = self.task.load(Ordering::Acquire);

        if !task.is_null() {
            unsafe { Box::from_raw(task).unpark(); }
            self.task.store(ptr::null_mut(), Ordering::Release);
        }
    }

    #[inline]
    fn push(&self, index: usize, value: T) {
        unsafe {
            ptr::write(&mut *self.buffer.offset(self.index(index)), value);
        }
    }

    #[inline]
    fn pop(&self, index: usize) -> T {
        unsafe {
            ptr::read(self.buffer.offset(self.index(index)))
        }
    }

    #[inline]
    fn index(&self, index: usize) -> isize {
        (index & (self.size - 1)) as isize
    }
}

impl<T, E> Drop for RingBuffer<T, E> {
    fn drop(&mut self) {
        unsafe  {
            Vec::from_raw_parts(self.buffer, 0, self.size);
        }
    }
}

unsafe impl<T: Send, E: Send> Send for RingBuffer<T, E> {}
unsafe impl<T: Sync, E: Sync> Sync for RingBuffer<T, E> {}

pub struct BufferedStream<T, E> {
    buffer: Arc<RingBuffer<T, E>>
}

impl<T, E> Stream for BufferedStream<T, E> {
    type Item  = T;
    type Error = E;

    fn poll(&mut self) -> Poll<Option<T>, E> {
        let error = self.buffer.error.load(Ordering::Acquire);

        if !error.is_null() {
            let error = unsafe { Err(*Box::from_raw(error)) };
            self.buffer.error.store(ptr::null_mut(), Ordering::Release);

            return error;
        }

        match self.buffer.try_pop() {
            Some(value) => Ok(Async::Ready(Some(value))),
            None => {
                if self.buffer.is_closed.load(Ordering::Relaxed) {
                    Ok(Async::Ready(None))
                } else {
                    let boxed = Box::new(park());
                    self.buffer.task.store(Box::into_raw(boxed), Ordering::Release);

                    Ok(Async::NotReady)
                }
            }
        }
    }
}

pub struct BufferedSender<T, E> {
    buffer: Arc<RingBuffer<T, E>>
}

impl<T, E> BufferedSender<T, E> {
    #[inline]
    pub fn send(&self, value: T) {
        let mut value = value;

        loop {
            value = match self.buffer.try_push(value) {
                Some(value) => value,
                None        => break
            };
        }
    }

    #[inline]
    pub fn fail(self, error: E) {
        let boxed = Box::new(error);
        self.buffer.error.store(Box::into_raw(boxed), Ordering::Release);
    }
}

impl<T, E> Drop for BufferedSender<T, E> {
    fn drop(&mut self) {
        self.buffer.try_unpark();

        self.buffer.is_closed.store(true, Ordering::Relaxed);
    }
}

pub fn buffered<T, E>(capacity: usize) -> (BufferedSender<T, E>, BufferedStream<T, E>) {
    let buffer = Arc::new(RingBuffer::new(capacity));

    (BufferedSender { buffer: buffer.clone() }, BufferedStream { buffer: buffer })
}

#[cfg(test)]
mod tests {
    use std::sync::Arc;
    use std::thread;
    use std::time::Duration;

    use futures::future::Future;
    use futures::stream::Stream;

    use super::*;

    #[test]
    fn ring_push_pop() {
        let ring: RingBuffer<i32, ()> = RingBuffer::new(3);

        assert_eq!(ring.try_push(1), None);
        assert_eq!(ring.try_push(2), None);
        assert_eq!(ring.try_push(3), None);
        assert_eq!(ring.try_push(4), Some(4));

        assert_eq!(ring.try_pop(), Some(1));
        assert_eq!(ring.try_pop(), Some(2));
        assert_eq!(ring.try_pop(), Some(3));
        assert_eq!(ring.try_pop(), None);

        assert_eq!(ring.try_push(1), None);
        assert_eq!(ring.try_push(2), None);
        assert_eq!(ring.try_push(3), None);
        assert_eq!(ring.try_push(4), Some(4));

        assert_eq!(ring.try_pop(), Some(1));
        assert_eq!(ring.try_pop(), Some(2));
        assert_eq!(ring.try_pop(), Some(3));
        assert_eq!(ring.try_pop(), None);
    }

    #[test]
    fn buffer_fail() {
        let (sender, stream) = buffered::<(), String>(1);

        sender.fail("error".to_owned());

        assert_eq!(stream.collect().wait(), Err("error".to_owned()))
    }

    #[test]
    fn ring_async() {
        let ring: Arc<RingBuffer<i32, ()>> = Arc::new(RingBuffer::new(3));
        let clone = ring.clone();

        thread::spawn(move || {
            for _ in 0..10 {
                thread::sleep(Duration::from_millis(1));

                assert_eq!(clone.try_push(1), None);
            }
        });

        let mut count = 0;

        loop {
            if ring.try_pop().is_some() {
                count += 1;
            }

            if count == 10 {
                break;
            }
        }
    }

    #[test]
    fn sender_stream() {
        let (sender, stream) = buffered::<i32, ()>(3);

        thread::spawn(move || {
            thread::sleep(Duration::from_millis(10));

            for _ in 0..100 {
                sender.send(1);
            }
        });

        let sum = stream.fold(0, |sum, value| {
            Ok(sum + value)
        }).wait().unwrap();

        assert_eq!(sum, 100);
    }
}
<commit_msg>Fixed concurrency bug.<commit_after>// pest. The Elegant Parser
// Copyright (C) 2017  Dragoș Tiselice
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

use std::cell::{Cell, UnsafeCell};
use std::{mem, ptr};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering};

use futures::{Async, Poll};
use futures::stream::Stream;
use futures::task::{self, Task};

#[cfg(target_pointer_width = "32")]
const PAD: usize = 16;
#[cfg(target_pointer_width = "64")]
const PAD: usize = 8;

#[repr(C)]
struct RingBuffer<T, E> {
    read:          AtomicUsize,
    last_write:    Cell<usize>,
    _padding1:     [usize; PAD - 2],
    write:         AtomicUsize,
    last_read:     Cell<usize>,
    _padding2:     [usize; PAD - 2],
    buffer:        *mut T,
    capacity:      usize,
    size:          usize,
    pub error:     AtomicPtr<E>,
    pub is_closed: AtomicBool,
    task:          UnsafeCell<Option<Task>>,
    pub task_lock: AtomicBool
}

impl<T, E> RingBuffer<T, E> {
    #[inline]
    pub fn new(capacity: usize) -> RingBuffer<T, E> {
        let size = capacity.next_power_of_two();
        let mut vec = Vec::with_capacity(size);
        let ptr = vec.as_mut_ptr();

        mem::forget(vec);

        RingBuffer {
            read:       AtomicUsize::new(0),
            last_write: Cell::new(0),
            _padding1:  [0; PAD - 2],
            write:      AtomicUsize::new(0),
            last_read:  Cell::new(0),
            _padding2:  [0; PAD - 2],
            buffer:     ptr,
            capacity:   capacity,
            size:       size,
            error:      AtomicPtr::new(ptr::null_mut()),
            is_closed:  AtomicBool::new(false),
            task:       UnsafeCell::new(None),
            task_lock:  AtomicBool::new(false)
        }
    }

    #[inline]
    pub fn try_push(&self, value: T) -> Option<T> {
        let read = self.last_read.get();
        let write = self.write.load(Ordering::Relaxed);

        if read + self.capacity <= write {
            let read = self.read.load(Ordering::Relaxed);
            self.last_read.set(read);

            if read + self.capacity <= write {
                return Some(value);
            }
        }

        self.push(write, value);

        self.write.store(write.wrapping_add(1), Ordering::Relaxed);

        self.try_unpark();

        None
    }

    #[inline]
    pub fn try_pop(&self) -> Option<T> {
        let read = self.read.load(Ordering::Relaxed);
        let write = self.last_write.get();

        if read == write {
            let write = self.write.load(Ordering::Relaxed);
            self.last_write.set(write);

            if read == write {
                return None;
            }
        }

        let result = self.pop(read);

        self.read.store(read.wrapping_add(1), Ordering::Relaxed);

        Some(result)
    }

    #[inline]
    pub fn try_park(&self) -> bool {
        let locked = self.task_lock.swap(true, Ordering::Relaxed);

        if !locked {
            self.swap(Some(task::park()));

            self.task_lock.store(false, Ordering::Relaxed);
        }

        !locked
    }

    #[inline]
    pub fn try_unpark(&self) -> bool {
        let locked = self.task_lock.swap(true, Ordering::Relaxed);

        if !locked {
            let task = self.swap(None);

            if let Some(task) = task {
                task.unpark();
            }

            self.task_lock.store(false, Ordering::Relaxed);
        }

        !locked
    }

    #[inline]
    pub fn swap(&self, option: Option<Task>) -> Option<Task> {
        let mut task = unsafe { &mut *self.task.get()};
        mem::replace(task, option)
    }

    #[inline]
    fn push(&self, index: usize, value: T) {
        unsafe {
            ptr::write(&mut *self.buffer.offset(self.index(index)), value);
        }
    }

    #[inline]
    fn pop(&self, index: usize) -> T {
        unsafe {
            ptr::read(self.buffer.offset(self.index(index)))
        }
    }

    #[inline]
    fn index(&self, index: usize) -> isize {
        (index & (self.size - 1)) as isize
    }
}

impl<T, E> Drop for RingBuffer<T, E> {
    fn drop(&mut self) {
        unsafe  {
            Vec::from_raw_parts(self.buffer, 0, self.size);
        }
    }
}

unsafe impl<T: Send, E: Send> Send for RingBuffer<T, E> {}
unsafe impl<T: Sync, E: Sync> Sync for RingBuffer<T, E> {}

pub struct BufferedStream<T, E> {
    buffer: Arc<RingBuffer<T, E>>
}

impl<T, E> Stream for BufferedStream<T, E> {
    type Item  = T;
    type Error = E;

    fn poll(&mut self) -> Poll<Option<T>, E> {
        let error = self.buffer.error.load(Ordering::Acquire);

        if !error.is_null() {
            let error = unsafe { Err(*Box::from_raw(error)) };
            self.buffer.error.store(ptr::null_mut(), Ordering::Release);

            return error;
        }

        match self.buffer.try_pop() {
            Some(value) => Ok(Async::Ready(Some(value))),
            None => {
                if self.buffer.is_closed.load(Ordering::Relaxed) {
                    Ok(Async::Ready(None))
                } else {
                    loop {
                        if self.buffer.try_park() {
                            break;
                        }
                    }

                    Ok(Async::NotReady)
                }
            }
        }
    }
}

pub struct BufferedSender<T, E> {
    buffer: Arc<RingBuffer<T, E>>
}

impl<T, E> BufferedSender<T, E> {
    #[inline]
    pub fn send(&self, value: T) {
        let mut value = value;

        loop {
            value = match self.buffer.try_push(value) {
                Some(value) => value,
                None        => break
            };
        }
    }

    #[inline]
    pub fn fail(self, error: E) {
        let boxed = Box::new(error);
        self.buffer.error.store(Box::into_raw(boxed), Ordering::Release);
    }
}

impl<T, E> Drop for BufferedSender<T, E> {
    fn drop(&mut self) {
        self.buffer.is_closed.store(true, Ordering::Relaxed);

        loop {
            if self.buffer.try_unpark() {
                break;
            }
        }
    }
}

pub fn buffered<T, E>(capacity: usize) -> (BufferedSender<T, E>, BufferedStream<T, E>) {
    let buffer = Arc::new(RingBuffer::new(capacity));

    (BufferedSender { buffer: buffer.clone() }, BufferedStream { buffer: buffer })
}

#[cfg(test)]
mod tests {
    use std::sync::Arc;
    use std::thread;
    use std::time::Duration;

    use futures::future::Future;
    use futures::stream::Stream;

    use super::*;

    #[test]
    fn ring_push_pop() {
        let ring: RingBuffer<i32, ()> = RingBuffer::new(3);

        assert_eq!(ring.try_push(1), None);
        assert_eq!(ring.try_push(2), None);
        assert_eq!(ring.try_push(3), None);
        assert_eq!(ring.try_push(4), Some(4));

        assert_eq!(ring.try_pop(), Some(1));
        assert_eq!(ring.try_pop(), Some(2));
        assert_eq!(ring.try_pop(), Some(3));
        assert_eq!(ring.try_pop(), None);

        assert_eq!(ring.try_push(1), None);
        assert_eq!(ring.try_push(2), None);
        assert_eq!(ring.try_push(3), None);
        assert_eq!(ring.try_push(4), Some(4));

        assert_eq!(ring.try_pop(), Some(1));
        assert_eq!(ring.try_pop(), Some(2));
        assert_eq!(ring.try_pop(), Some(3));
        assert_eq!(ring.try_pop(), None);
    }

    #[test]
    fn buffer_fail() {
        let (sender, stream) = buffered::<(), String>(1);

        sender.fail("error".to_owned());

        assert_eq!(stream.collect().wait(), Err("error".to_owned()))
    }

    #[test]
    fn ring_async() {
        let ring: Arc<RingBuffer<i32, ()>> = Arc::new(RingBuffer::new(3));
        let clone = ring.clone();

        thread::spawn(move || {
            for _ in 0..10 {
                thread::sleep(Duration::from_millis(1));

                assert_eq!(clone.try_push(1), None);
            }
        });

        let mut count = 0;

        loop {
            if ring.try_pop().is_some() {
                count += 1;
            }

            if count == 10 {
                break;
            }
        }
    }

    #[test]
    fn sender_stream() {
        let (sender, stream) = buffered::<i32, ()>(3);

        thread::spawn(move || {
            thread::sleep(Duration::from_millis(10));

            for _ in 0..100 {
                sender.send(1);
            }
        });

        let sum = stream.fold(0, |sum, value| {
            Ok(sum + value)
        }).wait().unwrap();

        assert_eq!(sum, 100);
    }
}
<|endoftext|>


<commit_before>//! Broker fuzzer.  See README.md for a full description.

use std::collections::HashMap;
use std::thread;
use std::time::Duration;

use libc::c_void;
use rand::{self, Rng};

use futures::{Future, Stream, Sink, Complete};
use futures::sync::mpsc::{unbounded, Sender};

use tickgrinder_util::strategies::Strategy;
use tickgrinder_util::trading::broker::{Broker, BrokerResult};
use tickgrinder_util::trading::objects::BrokerAction;
use tickgrinder_util::trading::tick::Tick;
use tickgrinder_util::transport::command_server::CommandServer;
use tickgrinder_util::transport::query_server::QueryServer;
use tickgrinder_util::transport::textlog::get_logger_handle;
use tickgrinder_util::conf::CONF;

// link with the libboost_random wrapper
#[link(name="rand_bindings")]
extern {
    fn init_rng(seed: u32) -> *mut c_void;
    fn rand_int_range(void_rng: *mut c_void, min: i32, max: i32) -> u32;
}

pub struct Fuzzer {
    pub gen: *mut c_void,
    pub logger: EventLogger,
    pub pairs: Vec<String>,
}

impl Strategy for Fuzzer {
    fn new(cs: CommandServer, qs: QueryServer, conf: HashMap<String, String>) -> Fuzzer {
        // convert the seed string into an integer we can use to seen the PNRG if deterministic fuzzing is enabled
        let seed: u32 = if CONF.fuzzer_deterministic_rng {
            let mut sum = 0;
            // convert the seed string into an integer for seeding the fuzzer
            for c in CONF.fuzzer_seed.chars() {
                sum += c as u32;
            }
            sum
        } else {
            let mut rng = rand::thread_rng();
            rng.gen()
        };

        // parse the settings HashMap to get the list of pairs to subscribe to
        let pairs_list = conf.get("pairs")
            .expect("This needs a list of pairs to subscribe to from the connected broker, else we can't do anything!");
        let pairs: Vec<String> = (&pairs_list).split(", ").map(|x| String::from(x)).collect();

        Fuzzer {
            gen: unsafe { init_rng(seed)},
            logger: EventLogger::new(),
            pairs: pairs,
        }
    }

    fn init(&mut self, mut broker: Box<Broker>) {
        // subscribe to all the tickstreams as supplied in the configuration and combine the streams
        let (streams_tx, streams_rx) = unbounded();
        let mut symbol_enumeration = Vec::new(); // way to match symbols with their id
        for (i, symbol) in (&self.pairs).iter().enumerate() {
            let streams_tx = &streams_tx;
            symbol_enumeration.push((i, symbol,));
            let rx = broker.sub_ticks(symbol.clone())
                .expect(&format!("Unable to sub ticks for symbol {}", symbol))
                .map(move |t| (i, t));
            streams_tx.send(rx).unwrap();
        }
        let master_rx = streams_rx.flatten();
        self.logger.log_misc(format!("Subscribed to {} tickstreams", symbol_enumeration.len()));

        let pushstream_rx = broker.get_stream().unwrap();
        thread::spawn(move || {
            for msg in pushstream_rx.wait() {
                println!("PUSHTREAM: {:?}", msg.unwrap());
            }
        });

        // start responding to ticks from all the streams.
        self.logger.log_misc(String::from("Initializing fuzzer loop..."));
        for msg in master_rx.wait() {
            let (i, t) = msg.unwrap();
            self.logger.log_tick(t, i);

            match get_action(t, self.gen) {
                Some(action) => {
                    let id = self.logger.log_request(&action);
                    let fut = broker.execute(action);
                    let res = fut.wait().unwrap();
                    self.logger.log_response(&res, id);
                },
                None => (),
            }
        }
    }

    fn exit_now(&mut self, ready: Complete<()>) {
        unimplemented!();
    }
}

/// Called during each iteration of the fuzzer loop.  Picks a random action to take based on the
/// internally held PRNG and executes it.
pub fn get_action(t: Tick, gen: *mut c_void) -> Option<BrokerAction> {
    let rand = unsafe { rand_int_range(gen, 0, 5) };
    let action_opt: Option<BrokerAction> = match rand {
        0 => Some(BrokerAction::Ping),
        1 => None, // TODO
        // sleep for a few milliseconds, then do either do nothing or perform an action
        2 => {
            let sleep_time = unsafe { rand_int_range(gen, 0, 25) };
            thread::sleep(Duration::from_millis(sleep_time as u64));
            let action_or_no = unsafe { rand_int_range(gen, 0, 5) };
            if action_or_no > 3 {
                get_action(t, gen)
            } else {
                None
            }
        },
        // do nothing at all in response to the tick
        _ => None,
    };

    action_opt
}

pub struct EventLogger {
    tx: Option<Sender<String>>,
    i: usize, // incremented every time an event is logged in order to record order
}

impl EventLogger {
    /// Initializes a new logger thread and returns handle to it
    /// TODO: write header info into the log file about symbol/symbol_id pairing etc.
    pub fn new() -> EventLogger {
        let tx = get_logger_handle(String::from("fuzzer"), 50);

        EventLogger {
            tx: Some(tx),
            i: 0,
        }
    }

    /// Logs an event taking place during the fuzzing process.  Returns a number to be used to match
    /// the request to a response.
    pub fn log_request(&mut self, action: &BrokerAction) -> usize {
        println!("Sending request to broker: {:?}", action);
        self.i += 1;
        let tx = self.tx.take().unwrap();
        let new_tx = tx.send(format!("{} -  REQUEST: {:?}", self.i, action))
            .wait().expect("Unable to log request!");
        self.tx = Some(new_tx);
        self.i
    }

    /// Logs a response received from the broker
    pub fn log_response(&mut self, res: &BrokerResult, id: usize) {
        println!("Got response from broker: {:?}", res);
        let tx = self.tx.take().unwrap();
        let new_tx = tx.send(format!("{} - RESPONSE: {:?}", id, res))
            .wait().expect("Unable to log response!");
        self.tx = Some(new_tx);
    }

    /// Logs the fuzzer receiving a tick from the broker.  `i` is the index of that symbol.
    pub fn log_tick(&mut self, t: Tick, i: usize) {
        println!("Received new tick from broker: {:?}", t);
        let tx = self.tx.take().unwrap();
        let new_tx = tx.send(format!("Received tick from symbol with index {}: {:?}", i, t))
            .wait().expect("Unable to log tick!");
        self.tx = Some(new_tx);
    }

    /// Logs a plain old text message
    pub fn log_misc(&mut self, msg: String) {
        println!("Message: {}", msg);
        let tx = self.tx.take().unwrap();
        let new_tx = tx.send(msg).wait().expect("Logging tick failed");
        self.tx = Some(new_tx);
    }
}

// Make sure that the values we pull out of the seeded random number generator really are deterministic.
#[test]
fn deterministic_rng() {
    unsafe {
        let gen1 = init_rng(12345u32);
        let gen2 = init_rng(12345u32);

        let rand1 = rand_int_range(gen1, 1i32, 1000000i32);
        let rand2 = rand_int_range(gen2, 1i32, 1000000i32);

        assert_eq!(rand1, rand2);
    }
}
<commit_msg>Implemented beginnings of the callback handler for Fuzzer  - All oneshot receivers returned by broker actions are pushed into a queue where they are processed in the order they are fulfilled  - Looks like we got some order debugging to do still but things look pretty ok atm.  - Integrated the events stream into the fuzzer loop and tested  - Deadlocks are gone!  Testing works as expected. (HURRAY)<commit_after>//! Broker fuzzer.  See README.md for a full description.

use std::collections::HashMap;
use std::thread;
use std::time::Duration;
use std::sync::mpsc;

use libc::c_void;
use rand::{self, Rng};

use futures::{Future, Stream, Sink, Complete, Canceled, BoxFuture};
use futures::stream::{BufferUnordered, MapErr, BoxStream};
use futures::sync::mpsc::{unbounded, channel, Sender, Receiver};
use futures::sync::oneshot;

use tickgrinder_util::strategies::Strategy;
use tickgrinder_util::trading::broker::{Broker, BrokerResult};
use tickgrinder_util::trading::objects::BrokerAction;
use tickgrinder_util::trading::tick::Tick;
use tickgrinder_util::transport::command_server::CommandServer;
use tickgrinder_util::transport::query_server::QueryServer;
use tickgrinder_util::transport::textlog::get_logger_handle;
use tickgrinder_util::conf::CONF;

// link with the libboost_random wrapper
#[link(name="rand_bindings")]
extern {
    fn init_rng(seed: u32) -> *mut c_void;
    fn rand_int_range(void_rng: *mut c_void, min: i32, max: i32) -> u32;
}

pub struct Fuzzer {
    pub gen: *mut c_void,
    pub logger: EventLogger,
    pub pairs: Vec<String>,
    pub events_tx: Option<Sender<oneshot::Receiver<BrokerResult>>>,
    pub events_rx: mpsc::Receiver<BrokerResult>,
}

impl Strategy for Fuzzer {
    fn new(cs: CommandServer, qs: QueryServer, conf: HashMap<String, String>) -> Fuzzer {
        // convert the seed string into an integer we can use to seen the PNRG if deterministic fuzzing is enabled
        let seed: u32 = if CONF.fuzzer_deterministic_rng {
            let mut sum = 0;
            // convert the seed string into an integer for seeding the fuzzer
            for c in CONF.fuzzer_seed.chars() {
                sum += c as u32;
            }
            sum
        } else {
            let mut rng = rand::thread_rng();
            rng.gen()
        };

        // parse the settings HashMap to get the list of pairs to subscribe to
        let pairs_list = conf.get("pairs")
            .expect("This needs a list of pairs to subscribe to from the connected broker, else we can't do anything!");
        let pairs: Vec<String> = (&pairs_list).split(", ").map(|x| String::from(x)).collect();

        // create the stream over which we receive callbacks from the broker
        let buffer_size = 8;//pairs.len() * 2;
        let (tx, rx) = channel(buffer_size);

        // map the output of the events buffer strea into a stdlib mpsc channel so we can try_get it
        let (mpsc_tx, mpsc_rx) = mpsc::sync_channel::<BrokerResult>(0);
        thread::spawn(move || {
            let mod_rx: BoxStream<oneshot::Receiver<BrokerResult>, Canceled> = rx.map_err(|()| Canceled).boxed();
            let buf_rx = mod_rx.buffer_unordered(buffer_size);
            for msg in buf_rx.wait() {
                println!("Mapping message through mpsc...");
                mpsc_tx.send(msg.unwrap()).unwrap();
            }
        });

        Fuzzer {
            gen: unsafe { init_rng(seed)},
            logger: EventLogger::new(),
            pairs: pairs,
            events_tx: Some(tx),
            events_rx: mpsc_rx,
        }
    }

    fn init(&mut self, mut broker: Box<Broker>) {
        // subscribe to all the tickstreams as supplied in the configuration and combine the streams
        let (streams_tx, streams_rx) = unbounded();
        let mut symbol_enumeration = Vec::new(); // way to match symbols with their id
        for (i, symbol) in (&self.pairs).iter().enumerate() {
            let streams_tx = &streams_tx;
            symbol_enumeration.push((i, symbol,));
            let rx = broker.sub_ticks(symbol.clone())
                .expect(&format!("Unable to sub ticks for symbol {}", symbol))
                .map(move |t| (i, t));
            streams_tx.send(rx).unwrap();
        }
        let master_rx = streams_rx.flatten();
        self.logger.log_misc(format!("Subscribed to {} tickstreams", symbol_enumeration.len()));

        let pushstream_rx = broker.get_stream().unwrap();
        thread::spawn(move || {
            for msg in pushstream_rx.wait() {
                println!("PUSHTREAM: {:?}", msg.unwrap());
                // TODO: log/handle
            }
        });

        // start responding to ticks from all the streams.
        self.logger.log_misc(String::from("Initializing fuzzer loop..."));
        for msg in master_rx.wait() {
            while let Ok(msg) = self.events_rx.try_recv() {
                self.logger.log_misc(format!("EVENT: {:?}", msg));
            }

            let (i, t) = msg.unwrap();
            self.logger.log_tick(t, i);

            match get_action(t, self.gen) {
                Some(action) => {
                    let id = self.logger.log_request(&action);
                    let fut = broker.execute(action);
                    // store the pending future into the buffered queue
                    let mut tx = self.events_tx.take().unwrap();
                    tx = tx.send(fut).wait().unwrap();
                    self.events_tx = Some(tx);
                    // self.logger.log_response(&res, id);
                },
                None => (),
            }
        }
    }

    fn exit_now(&mut self, ready: Complete<()>) {
        unimplemented!();
    }
}

/// Called during each iteration of the fuzzer loop.  Picks a random action to take based on the
/// internally held PRNG and executes it.
pub fn get_action(t: Tick, gen: *mut c_void) -> Option<BrokerAction> {
    let rand = unsafe { rand_int_range(gen, 0, 5) };
    let action_opt: Option<BrokerAction> = match rand {
        0 => Some(BrokerAction::Ping),
        1 => None, // TODO
        // sleep for a few milliseconds, then do either do nothing or perform an action
        2 => {
            let action_or_no = unsafe { rand_int_range(gen, 0, 5) };
            if action_or_no > 3 {
                get_action(t, gen)
            } else {
                None
            }
        },
        // do nothing at all in response to the tick
        _ => None,
    };

    action_opt
}

pub struct EventLogger {
    tx: Option<Sender<String>>,
    i: usize, // incremented every time an event is logged in order to record order
}

impl EventLogger {
    /// Initializes a new logger thread and returns handle to it
    /// TODO: write header info into the log file about symbol/symbol_id pairing etc.
    pub fn new() -> EventLogger {
        let tx = get_logger_handle(String::from("fuzzer"), 50);

        EventLogger {
            tx: Some(tx),
            i: 0,
        }
    }

    /// Logs an event taking place during the fuzzing process.  Returns a number to be used to match
    /// the request to a response.
    pub fn log_request(&mut self, action: &BrokerAction) -> usize {
        println!("Sending request to broker: {:?}", action);
        self.i += 1;
        let tx = self.tx.take().unwrap();
        let new_tx = tx.send(format!("{} -  REQUEST: {:?}", self.i, action))
            .wait().expect("Unable to log request!");
        self.tx = Some(new_tx);
        self.i
    }

    /// Logs a response received from the broker
    pub fn log_response(&mut self, res: &BrokerResult, id: usize) {
        println!("Got response from broker: {:?}", res);
        let tx = self.tx.take().unwrap();
        let new_tx = tx.send(format!("{} - RESPONSE: {:?}", id, res))
            .wait().expect("Unable to log response!");
        self.tx = Some(new_tx);
    }

    /// Logs the fuzzer receiving a tick from the broker.  `i` is the index of that symbol.
    pub fn log_tick(&mut self, t: Tick, i: usize) {
        println!("Received new tick from broker: {:?}", t);
        let tx = self.tx.take().unwrap();
        let new_tx = tx.send(format!("Received tick from symbol with index {}: {:?}", i, t))
            .wait().expect("Unable to log tick!");
        self.tx = Some(new_tx);
    }

    /// Logs a plain old text message
    pub fn log_misc(&mut self, msg: String) {
        println!("Message: {}", msg);
        let tx = self.tx.take().unwrap();
        let new_tx = tx.send(msg).wait().expect("Logging tick failed");
        self.tx = Some(new_tx);
    }
}

// Make sure that the values we pull out of the seeded random number generator really are deterministic.
#[test]
fn deterministic_rng() {
    unsafe {
        let gen1 = init_rng(12345u32);
        let gen2 = init_rng(12345u32);

        let rand1 = rand_int_range(gen1, 1i32, 1000000i32);
        let rand2 = rand_int_range(gen2, 1i32, 1000000i32);

        assert_eq!(rand1, rand2);
    }
}
<|endoftext|>


<commit_before>use arch::x86_64::pti;
use syscall;

#[naked]
pub unsafe extern fn syscall() {
    #[inline(never)]
    unsafe fn inner(stack: &mut SyscallStack) {
        let mut a;
        let b;
        let rbp;
        asm!("" : "={rax}"(a), "={rbx}"(b), "={rbp}"(rbp)
                : : : "intel", "volatile");

        // Map kernel
        pti::map();

        a = syscall::syscall(a, b, stack.rcx, stack.rdx, stack.rsi, stack.rdi, rbp, stack);

        // Unmap kernel
        pti::unmap();

        asm!("" : : "{rax}"(a) : : "intel", "volatile");
    }

    // Push scratch registers, minus rax for the return value
    asm!("push rcx
        push rdx
        push rdi
        push rsi
        push r8
        push r9
        push r10
        push r11
        push fs
        mov r11, 0x18
        mov fs, r11"
        : : : : "intel", "volatile");

    // Get reference to stack variables
    let rsp: usize;
    asm!("" : "={rsp}"(rsp) : : : "intel", "volatile");

    inner(&mut *(rsp as *mut SyscallStack));

    // Interrupt return
    asm!("pop fs
        pop r11
        pop r10
        pop r9
        pop r8
        pop rsi
        pop rdi
        pop rdx
        pop rcx
        iretq"
        : : : : "intel", "volatile");
}

#[allow(dead_code)]
#[repr(packed)]
pub struct SyscallStack {
    pub fs: usize,
    pub r11: usize,
    pub r10: usize,
    pub r9: usize,
    pub r8: usize,
    pub rsi: usize,
    pub rdi: usize,
    pub rdx: usize,
    pub rcx: usize,
    pub rip: usize,
    pub cs: usize,
    pub rflags: usize,
}

#[naked]
pub unsafe extern fn clone_ret() -> usize {
    asm!("pop rbp"
        : : : : "intel", "volatile");
        0
}
<commit_msg>Add rbx to saved registers in syscall stack<commit_after>use arch::x86_64::pti;
use syscall;

#[naked]
pub unsafe extern fn syscall() {
    #[inline(never)]
    unsafe fn inner(stack: &mut SyscallStack) {
        let mut a;
        let rbp;
        asm!("" : "={rax}"(a), "={rbp}"(rbp)
                : : : "intel", "volatile");

        // Map kernel
        pti::map();

        a = syscall::syscall(a, stack.rbx, stack.rcx, stack.rdx, stack.rsi, stack.rdi, rbp, stack);

        // Unmap kernel
        pti::unmap();

        asm!("" : : "{rax}"(a) : : "intel", "volatile");
    }

    // Push scratch registers, minus rax for the return value
    asm!("push rcx
        push rdx
        push rdi
        push rsi
        push r8
        push r9
        push r10
        push r11
        push rbx
        push fs
        mov r11, 0x18
        mov fs, r11"
        : : : : "intel", "volatile");

    // Get reference to stack variables
    let rsp: usize;
    asm!("" : "={rsp}"(rsp) : : : "intel", "volatile");

    inner(&mut *(rsp as *mut SyscallStack));

    // Interrupt return
    asm!("pop fs
        pop rbx
        pop r11
        pop r10
        pop r9
        pop r8
        pop rsi
        pop rdi
        pop rdx
        pop rcx
        iretq"
        : : : : "intel", "volatile");
}

#[allow(dead_code)]
#[repr(packed)]
pub struct SyscallStack {
    pub fs: usize,
    pub rbx: usize,
    pub r11: usize,
    pub r10: usize,
    pub r9: usize,
    pub r8: usize,
    pub rsi: usize,
    pub rdi: usize,
    pub rdx: usize,
    pub rcx: usize,
    pub rip: usize,
    pub cs: usize,
    pub rflags: usize,
}

#[naked]
pub unsafe extern fn clone_ret() {
    asm!("pop rbp" : : : : "intel", "volatile");
    asm!("" : : "{rax}"(0) : : "intel", "volatile");
}
<|endoftext|>