<commit_before>//! This module serves as the entry point into Xtensis's main binary.

// This file is part of Xtensis.

// This is the Xtensis text editor; it edits text.
// Copyright (C) 2016-2017  The Xtensis Developers

// This program is free software: you can redistribute it and/or
// modify it under the terms of the GNU General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.

// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program.  If not, see
// <http://www.gnu.org/licenses/>.

fn main() {
    println!("Hello, XTENSIS!");
}
<commit_msg>Add arguments and logging initialisation to binary<commit_after>//! This module serves as the entry point into Xtensis's main binary.

// This file is part of Xtensis.

// This is the Xtensis text editor; it edits text.
// Copyright (C) 2016-2017  The Xtensis Developers

// This program is free software: you can redistribute it and/or
// modify it under the terms of the GNU General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.

// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with this program.  If not, see
// <http://www.gnu.org/licenses/>.

extern crate clap;
extern crate xtensis;

#[macro_use]
extern crate slog;
extern crate slog_term;
extern crate slog_scope;

use slog_scope::logger;
use clap::{App, Arg, ArgMatches, SubCommand};
use xtensis::logging::init_logger;
use xtensis::utils::get_version;

fn retrieve_arguments() -> ArgMatches<'static> {
    App::new("xtensis-core")
        .version(get_version())
        .author("Dom Rodriguez <shymega@shymega.org.uk>")
        .about("Extensible editing: screw limits.")
        .arg(Arg::with_name("verbose")
            .short("v")
            .multiple(true)
            .required(false)
            .help("Set the level of logging verbosity"))
        .subcommand(SubCommand::with_name("spawn")
            .about("Spawn a new instance of xtensis-core.")
            .version(get_version())
            .author("Dom Rodriguez <shymega@shymega.org.uk>"))
        .get_matches()
}

fn main() {
    let cargs = retrieve_arguments();

    init_logger(cargs.clone());

    println!("Hello, XTENSIS!");
}
<|endoftext|>


<commit_before>extern crate sdl2;
extern crate minifb;
use minifb::*;
pub mod cpu;
pub mod opcodes;
//mod clock;
mod memory;
mod io;
mod cia;
mod vic;

pub const SCREEN_WIDTH:  usize = 384; // extend 20 pixels left and right for the borders
pub const SCREEN_HEIGHT: usize = 272; // extend 36 pixels top and down for the borders

pub struct C64
{
    pub window: minifb::Window,
    memory: memory::MemShared,
    keyboard: io::Keyboard,
    //clock: clock::Clock,
    cpu: cpu::CPUShared,
    cia1: cia::CIAShared,
    cia2: cia::CIAShared,
    vic: vic::VICShared,

    cycle_count: u32,
}

impl C64
{
    pub fn new() -> C64
    {
        let memory = memory::Memory::new_shared();
        let vic    = vic::VIC::new_shared();
        let cia1   = cia::CIA::new_shared(true);
        let cia2   = cia::CIA::new_shared(false);
        let cpu    = cpu::CPU::new_shared();

        let c64 = C64
        {
            window: Window::new("Rust64", SCREEN_WIDTH, SCREEN_HEIGHT, Scale::X1, Vsync::No).unwrap(),
            memory: memory.clone(), // shared system memory (RAM, ROM, IO registers)
            keyboard: io::Keyboard::new(),
            //clock: clock::Clock::new(),
            cpu: cpu.clone(),
            cia1: cia1.clone(),
            cia2: cia2.clone(),
            vic: vic.clone(),
            cycle_count: 0,
        };

        // cyclic dependencies are not possible in Rust (yet?), so we have
        // to resort to setting references manually
        c64.cia1.borrow_mut().set_references(memory.clone(), cpu.clone(), vic.clone());
        c64.cia2.borrow_mut().set_references(memory.clone(), cpu.clone(), vic.clone());
        c64.vic.borrow_mut().set_references(memory.clone(), cpu.clone());
        c64.cpu.borrow_mut().set_references(memory.clone(), vic.clone(), cia1.clone(), cia2.clone());

        drop(memory);
        drop(cia1);
        drop(cia2);
        drop(vic);
        drop(cpu);

        c64
    }

    pub fn reset(&mut self)
    {
        self.memory.borrow_mut().reset();
        self.cpu.borrow_mut().reset();
        self.cia1.borrow_mut().reset();
        self.cia2.borrow_mut().reset();
    }

    pub fn update(&mut self)
    {
        let mut should_trigger_vblank = false;
        //if self.clock.tick() { println!("Clock tick"); }

        self.keyboard.update_keystates(&self.window);

        self.vic.borrow_mut().update(self.cycle_count, &mut should_trigger_vblank);
        // update sid here when it's done

        self.cia1.borrow_mut().process_irq();
        self.cia2.borrow_mut().process_irq();
        self.cia1.borrow_mut().update();
        self.cia2.borrow_mut().update();

        self.cpu.borrow_mut().update();

        if should_trigger_vblank
        {
            //self.keyboard.update_keystates(&self.window);
            //let w = self.vic.borrow_mut().window_buffer[0];
            //println!("w: {}", w);
            self.window.update(&self.vic.borrow_mut().window_buffer);
            self.cia1.borrow_mut().count_tod();
            self.cia2.borrow_mut().count_tod();
        }

        self.cycle_count += 1;
    }

    // debug
    pub fn render(&mut self)
    {
        //self.window.update(&self.vic.borrow_mut().window_buffer);
    }
}
<commit_msg>fetch keystates in VBlank (experimenting with delays)<commit_after>extern crate sdl2;
extern crate minifb;
use minifb::*;
pub mod cpu;
pub mod opcodes;
//mod clock;
mod memory;
mod io;
mod cia;
mod vic;

pub const SCREEN_WIDTH:  usize = 384; // extend 20 pixels left and right for the borders
pub const SCREEN_HEIGHT: usize = 272; // extend 36 pixels top and down for the borders

pub struct C64
{
    pub window: minifb::Window,
    memory: memory::MemShared,
    keyboard: io::Keyboard,
    //clock: clock::Clock,
    cpu: cpu::CPUShared,
    cia1: cia::CIAShared,
    cia2: cia::CIAShared,
    vic: vic::VICShared,

    cycle_count: u32,
}

impl C64
{
    pub fn new() -> C64
    {
        let memory = memory::Memory::new_shared();
        let vic    = vic::VIC::new_shared();
        let cia1   = cia::CIA::new_shared(true);
        let cia2   = cia::CIA::new_shared(false);
        let cpu    = cpu::CPU::new_shared();

        let c64 = C64
        {
            window: Window::new("Rust64", SCREEN_WIDTH, SCREEN_HEIGHT, Scale::X1, Vsync::No).unwrap(),
            memory: memory.clone(), // shared system memory (RAM, ROM, IO registers)
            keyboard: io::Keyboard::new(),
            //clock: clock::Clock::new(),
            cpu: cpu.clone(),
            cia1: cia1.clone(),
            cia2: cia2.clone(),
            vic: vic.clone(),
            cycle_count: 0,
        };

        // cyclic dependencies are not possible in Rust (yet?), so we have
        // to resort to setting references manually
        c64.cia1.borrow_mut().set_references(memory.clone(), cpu.clone(), vic.clone());
        c64.cia2.borrow_mut().set_references(memory.clone(), cpu.clone(), vic.clone());
        c64.vic.borrow_mut().set_references(memory.clone(), cpu.clone());
        c64.cpu.borrow_mut().set_references(memory.clone(), vic.clone(), cia1.clone(), cia2.clone());

        drop(memory);
        drop(cia1);
        drop(cia2);
        drop(vic);
        drop(cpu);

        c64
    }

    pub fn reset(&mut self)
    {
        self.memory.borrow_mut().reset();
        self.cpu.borrow_mut().reset();
        self.cia1.borrow_mut().reset();
        self.cia2.borrow_mut().reset();
    }

    pub fn update(&mut self)
    {
        let mut should_trigger_vblank = false;
        //if self.clock.tick() { println!("Clock tick"); }

        self.vic.borrow_mut().update(self.cycle_count, &mut should_trigger_vblank);
        // update sid here when it's done

        self.cia1.borrow_mut().process_irq();
        self.cia2.borrow_mut().process_irq();
        self.cia1.borrow_mut().update();
        self.cia2.borrow_mut().update();

        self.cpu.borrow_mut().update();

        if should_trigger_vblank
        {
            //let w = self.vic.borrow_mut().window_buffer[0];
            //println!("w: {}", w);
            self.window.update(&self.vic.borrow_mut().window_buffer);
            self.keyboard.update_keystates(&self.window);
            self.cia1.borrow_mut().count_tod();
            self.cia2.borrow_mut().count_tod();
        }

        self.cycle_count += 1;
    }

    // debug
    pub fn render(&mut self)
    {
        //self.window.update(&self.vic.borrow_mut().window_buffer);
    }
}
<|endoftext|>


<commit_before>use std::cmp;
use std::error::Error;
use std::ffi::OsStr;
use std::fs::{self, OpenOptions};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::result;
use std::str::FromStr;
use std::time::Duration;

use config::{ControllerPath, Limits, SpaceUsage};
use errors::CGroupError;
use ffi;
use run_info::RunUsage;

type Result<T> = result::Result<T, CGroupError>;

fn cgroup_write<T1: AsRef<Path>, T2: AsRef<str>>(
    controller_path: &Path,
    file: T1,
    line: T2,
) -> Result<()> {
    let path = controller_path.join(file.as_ref());
    let mut cgroup_file = OpenOptions::new().write(true).open(&path).map_err(|err| {
        CGroupError::OpenCGroupFileError {
            controller_path: controller_path.to_path_buf(),
            file: file.as_ref().to_path_buf(),
            error: err.description().into(),
        }
    })?;

    cgroup_file
        .write_all(line.as_ref().as_bytes())
        .map_err(|err| CGroupError::WriteCGroupFileError {
            controller_path: controller_path.to_path_buf(),
            file: file.as_ref().to_path_buf(),
            error: err.description().into(),
        })
}

fn cgroup_read<T1: AsRef<Path>, T2: FromStr>(controller_path: &Path, file: T1) -> Result<T2>
where
    <T2 as FromStr>::Err: Error,
{
    let path = controller_path.join(file.as_ref());
    let mut cgroup_file = OpenOptions::new().read(true).open(&path).map_err(|err| {
        CGroupError::OpenCGroupFileError {
            controller_path: controller_path.to_path_buf(),
            file: file.as_ref().to_path_buf(),
            error: err.description().into(),
        }
    })?;

    let mut buffer = String::new();
    let _ = cgroup_file.read_to_string(&mut buffer).map_err(|err| {
        CGroupError::ReadCGroupFileError {
            controller_path: controller_path.to_path_buf(),
            file: file.as_ref().to_path_buf(),
            error: err.description().into(),
        }
    })?;

    buffer
        .trim()
        .parse::<T2>()
        .map_err(|err| CGroupError::ParseCGroupFileError {
            controller_path: controller_path.to_path_buf(),
            file: file.as_ref().to_path_buf(),
            buffer,
            error: err.description().into(),
        })
}

pub fn enter_cgroup(controller_path: &Path) -> Result<()> {
    cgroup_write(controller_path, "tasks", format!("{}\n", ffi::getpid()))
}

const DEFAULT_INSTANCE_NAME: &'static str = "default";
fn get_instance_path(controller_path: &Path, instance_name: Option<&OsStr>) -> Result<PathBuf> {
    let instance = instance_name.unwrap_or(OsStr::new(DEFAULT_INSTANCE_NAME));
    if !controller_path.exists() {
        return Err(CGroupError::ControllerMissing(
            controller_path.to_path_buf(),
        ));
    }

    let instance_path = controller_path.join(instance);
    if !instance_path.exists() {
        fs::create_dir(&instance_path).map_err(|err| CGroupError::InstanceControllerCreateError {
            controller_path: controller_path.to_path_buf(),
            instance_name: instance.to_os_string(),
            error: err.description().into(),
        })?;
    }
    Ok(instance_path)
}

const CPUACCT_DEFAULT_CONTROLLER_PATH: &'static str = "/sys/fs/cgroup/cpuacct/ia-sandbox";
pub fn enter_cpuacct_cgroup(
    controller_path: Option<&Path>,
    instance_name: Option<&OsStr>,
) -> Result<()> {
    let instance_path = get_instance_path(
        controller_path.unwrap_or(Path::new(CPUACCT_DEFAULT_CONTROLLER_PATH)),
        instance_name,
    )?;
    cgroup_write(&instance_path, "cpuacct.usage", "0\n")?;
    enter_cgroup(&instance_path)
}

const MEMORY_DEFAULT_CONTROLLER_PATH: &'static str = "/sys/fs/cgroup/memory/ia-sandbox";
const EXTRA_MEMORY_GIVEN: u64 = 16 * 1_024;
pub fn enter_memory_cgroup(
    controller_path: Option<&Path>,
    instance_name: Option<&OsStr>,
    memory_limit: Option<SpaceUsage>,
) -> Result<()> {
    let instance_path = get_instance_path(
        controller_path.unwrap_or(Path::new(MEMORY_DEFAULT_CONTROLLER_PATH)),
        instance_name,
    )?;
    cgroup_write(&instance_path, "memory.max_usage_in_bytes", "0\n")?;
    cgroup_write(&instance_path, "memory.memsw.max_usage_in_bytes", "0\n")?;

    // Reset limits to infinite in case there is no memory limit but also because we need at all
    // times for limit_in_bytes < memsw.limit_in_bytes
    cgroup_write(&instance_path, "memory.memsw.limit_in_bytes", "-1\n")?;
    cgroup_write(&instance_path, "memory.limit_in_bytes", "-1\n")?;
    if let Some(memory_limit) = memory_limit {
        // Assign some extra memory so that we can tell when a killed by signal 9 is actually a
        // memory limit exceeded
        cgroup_write(
            &instance_path,
            "memory.limit_in_bytes",
            format!("{}\n", memory_limit.as_bytes() + EXTRA_MEMORY_GIVEN),
        )?;
        cgroup_write(
            &instance_path,
            "memory.memsw.limit_in_bytes",
            format!("{}\n", memory_limit.as_bytes() + EXTRA_MEMORY_GIVEN),
        )?;
    }

    enter_cgroup(&instance_path)
}

const PIDS_DEFAULT_CONTROLLER_PATH: &'static str = "/sys/fs/cgroup/pids/ia-sandbox";
pub fn enter_pids_cgroup(
    controller_path: Option<&Path>,
    instance_name: Option<&OsStr>,
    pids_limit: Option<usize>,
) -> Result<()> {
    let instance_path = get_instance_path(
        controller_path.unwrap_or(Path::new(PIDS_DEFAULT_CONTROLLER_PATH)),
        instance_name,
    )?;

    if let Some(pids_limit) = pids_limit {
        cgroup_write(&instance_path, "pids.max", format!("{}\n", pids_limit))?;
    } else {
        cgroup_write(&instance_path, "pids.max", "max\n")?;
    }

    enter_cgroup(&instance_path)
}

pub fn enter_all_cgroups(
    controller_path: &ControllerPath,
    instance_name: Option<&OsStr>,
    limits: Limits,
) -> Result<()> {
    enter_cpuacct_cgroup(controller_path.cpuacct(), instance_name)?;
    enter_memory_cgroup(controller_path.memory(), instance_name, limits.memory())?;
    enter_pids_cgroup(controller_path.pids(), instance_name, limits.pids())
}

pub fn get_usage(
    controller_path: &ControllerPath,
    instance_name: Option<&OsStr>,
    wall_time: Duration,
) -> Result<RunUsage> {
    let cpuacct_controller_path = controller_path
        .cpuacct()
        .unwrap_or(Path::new(CPUACCT_DEFAULT_CONTROLLER_PATH));
    let memory_controller_path = controller_path
        .memory()
        .unwrap_or(Path::new(MEMORY_DEFAULT_CONTROLLER_PATH));
    let instance = instance_name.unwrap_or(OsStr::new(DEFAULT_INSTANCE_NAME));

    let cpuacct_instance_path = cpuacct_controller_path.join(instance);
    let user_time = Duration::from_nanos(cgroup_read(&cpuacct_instance_path, "cpuacct.usage")?);

    let memory_instance_path = memory_controller_path.join(instance);
    let memory = SpaceUsage::from_bytes(cmp::max(
        cgroup_read(&memory_instance_path, "memory.max_usage_in_bytes")?,
        cgroup_read(&memory_instance_path, "memory.memsw.max_usage_in_bytes")?,
    ));
    Ok(RunUsage::new(user_time, wall_time, memory))
}
<commit_msg>Fixed memory.memsw.* files read/write since they might be missing on<commit_after>use std::cmp;
use std::error::Error;
use std::ffi::OsStr;
use std::fs::{self, OpenOptions};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::result;
use std::str::FromStr;
use std::time::Duration;

use config::{ControllerPath, Limits, SpaceUsage};
use errors::CGroupError;
use ffi;
use run_info::RunUsage;

type Result<T> = result::Result<T, CGroupError>;

fn cgroup_write<T1: AsRef<Path>, T2: AsRef<str>>(
    controller_path: &Path,
    file: T1,
    line: T2,
) -> Result<()> {
    let path = controller_path.join(file.as_ref());
    let mut cgroup_file = OpenOptions::new().write(true).open(&path).map_err(|err| {
        CGroupError::OpenCGroupFileError {
            controller_path: controller_path.to_path_buf(),
            file: file.as_ref().to_path_buf(),
            error: err.description().into(),
        }
    })?;

    cgroup_file
        .write_all(line.as_ref().as_bytes())
        .map_err(|err| CGroupError::WriteCGroupFileError {
            controller_path: controller_path.to_path_buf(),
            file: file.as_ref().to_path_buf(),
            error: err.description().into(),
        })
}

fn cgroup_read<T1: AsRef<Path>, T2: FromStr>(controller_path: &Path, file: T1) -> Result<T2>
where
    <T2 as FromStr>::Err: Error,
{
    let path = controller_path.join(file.as_ref());
    let mut cgroup_file = OpenOptions::new().read(true).open(&path).map_err(|err| {
        CGroupError::OpenCGroupFileError {
            controller_path: controller_path.to_path_buf(),
            file: file.as_ref().to_path_buf(),
            error: err.description().into(),
        }
    })?;

    let mut buffer = String::new();
    let _ = cgroup_file.read_to_string(&mut buffer).map_err(|err| {
        CGroupError::ReadCGroupFileError {
            controller_path: controller_path.to_path_buf(),
            file: file.as_ref().to_path_buf(),
            error: err.description().into(),
        }
    })?;

    buffer
        .trim()
        .parse::<T2>()
        .map_err(|err| CGroupError::ParseCGroupFileError {
            controller_path: controller_path.to_path_buf(),
            file: file.as_ref().to_path_buf(),
            buffer,
            error: err.description().into(),
        })
}

pub fn enter_cgroup(controller_path: &Path) -> Result<()> {
    cgroup_write(controller_path, "tasks", format!("{}\n", ffi::getpid()))
}

const DEFAULT_INSTANCE_NAME: &'static str = "default";
fn get_instance_path(controller_path: &Path, instance_name: Option<&OsStr>) -> Result<PathBuf> {
    let instance = instance_name.unwrap_or(OsStr::new(DEFAULT_INSTANCE_NAME));
    if !controller_path.exists() {
        return Err(CGroupError::ControllerMissing(
            controller_path.to_path_buf(),
        ));
    }

    let instance_path = controller_path.join(instance);
    if !instance_path.exists() {
        fs::create_dir(&instance_path).map_err(|err| CGroupError::InstanceControllerCreateError {
            controller_path: controller_path.to_path_buf(),
            instance_name: instance.to_os_string(),
            error: err.description().into(),
        })?;
    }
    Ok(instance_path)
}

const CPUACCT_DEFAULT_CONTROLLER_PATH: &'static str = "/sys/fs/cgroup/cpuacct/ia-sandbox";
pub fn enter_cpuacct_cgroup(
    controller_path: Option<&Path>,
    instance_name: Option<&OsStr>,
) -> Result<()> {
    let instance_path = get_instance_path(
        controller_path.unwrap_or(Path::new(CPUACCT_DEFAULT_CONTROLLER_PATH)),
        instance_name,
    )?;
    cgroup_write(&instance_path, "cpuacct.usage", "0\n")?;
    enter_cgroup(&instance_path)
}

const MEMORY_DEFAULT_CONTROLLER_PATH: &'static str = "/sys/fs/cgroup/memory/ia-sandbox";
const EXTRA_MEMORY_GIVEN: u64 = 16 * 1_024;
pub fn enter_memory_cgroup(
    controller_path: Option<&Path>,
    instance_name: Option<&OsStr>,
    memory_limit: Option<SpaceUsage>,
) -> Result<()> {
    let instance_path = get_instance_path(
        controller_path.unwrap_or(Path::new(MEMORY_DEFAULT_CONTROLLER_PATH)),
        instance_name,
    )?;
    cgroup_write(&instance_path, "memory.max_usage_in_bytes", "0\n")?;
    cgroup_write(&instance_path, "memory.memsw.max_usage_in_bytes", "0\n").unwrap_or(());

    // Reset limits to infinite in case there is no memory limit but also because we need at all
    // times for limit_in_bytes < memsw.limit_in_bytes
    cgroup_write(&instance_path, "memory.memsw.limit_in_bytes", "-1\n").unwrap_or(());
    cgroup_write(&instance_path, "memory.limit_in_bytes", "-1\n")?;
    if let Some(memory_limit) = memory_limit {
        // Assign some extra memory so that we can tell when a killed by signal 9 is actually a
        // memory limit exceeded
        cgroup_write(
            &instance_path,
            "memory.limit_in_bytes",
            format!("{}\n", memory_limit.as_bytes() + EXTRA_MEMORY_GIVEN),
        )?;
        cgroup_write(
            &instance_path,
            "memory.memsw.limit_in_bytes",
            format!("{}\n", memory_limit.as_bytes() + EXTRA_MEMORY_GIVEN),
        ).unwrap_or(());
    }

    enter_cgroup(&instance_path)
}

const PIDS_DEFAULT_CONTROLLER_PATH: &'static str = "/sys/fs/cgroup/pids/ia-sandbox";
pub fn enter_pids_cgroup(
    controller_path: Option<&Path>,
    instance_name: Option<&OsStr>,
    pids_limit: Option<usize>,
) -> Result<()> {
    let instance_path = get_instance_path(
        controller_path.unwrap_or(Path::new(PIDS_DEFAULT_CONTROLLER_PATH)),
        instance_name,
    )?;

    if let Some(pids_limit) = pids_limit {
        cgroup_write(&instance_path, "pids.max", format!("{}\n", pids_limit))?;
    } else {
        cgroup_write(&instance_path, "pids.max", "max\n")?;
    }

    enter_cgroup(&instance_path)
}

pub fn enter_all_cgroups(
    controller_path: &ControllerPath,
    instance_name: Option<&OsStr>,
    limits: Limits,
) -> Result<()> {
    enter_cpuacct_cgroup(controller_path.cpuacct(), instance_name)?;
    enter_memory_cgroup(controller_path.memory(), instance_name, limits.memory())?;
    enter_pids_cgroup(controller_path.pids(), instance_name, limits.pids())
}

pub fn get_usage(
    controller_path: &ControllerPath,
    instance_name: Option<&OsStr>,
    wall_time: Duration,
) -> Result<RunUsage> {
    let cpuacct_controller_path = controller_path
        .cpuacct()
        .unwrap_or(Path::new(CPUACCT_DEFAULT_CONTROLLER_PATH));
    let memory_controller_path = controller_path
        .memory()
        .unwrap_or(Path::new(MEMORY_DEFAULT_CONTROLLER_PATH));
    let instance = instance_name.unwrap_or(OsStr::new(DEFAULT_INSTANCE_NAME));

    let cpuacct_instance_path = cpuacct_controller_path.join(instance);
    let user_time = Duration::from_nanos(cgroup_read(&cpuacct_instance_path, "cpuacct.usage")?);

    let memory_instance_path = memory_controller_path.join(instance);
    let memory = SpaceUsage::from_bytes(cmp::max(
        cgroup_read(&memory_instance_path, "memory.max_usage_in_bytes")?,
        cgroup_read(&memory_instance_path, "memory.memsw.max_usage_in_bytes").unwrap_or(0),
    ));
    Ok(RunUsage::new(user_time, wall_time, memory))
}
<|endoftext|>


<commit_before>use structs::Data;
use nn::Network;
use nalgebra::{DVector, DMatrix};

pub fn sgd(mut nn: &mut Network,
           mut training_data: Vec<Data>,
           epochs: u8, mini_batch_size: u8,
           eta: f32) {
    use rand::{ self, Rng};
    let mut rng = rand::thread_rng();

    for j in 0..epochs {
        rng.shuffle(&mut training_data);
        for mut mini_batch in training_data.chunks_mut(mini_batch_size as usize) {
            update_mini_batch(&mut nn, &mut mini_batch, eta);
        }
        println!("Epoch {} complete!", j);
    }
}

fn update_mini_batch(mut nn: &mut Network, mut mini_batch: &mut [Data], eta: f32) {
    // holds all biases of the network
    let mut nabla_b: Vec<DVector<f32>> = Vec::with_capacity(nn.get_biases().len());
    for biases in nn.get_biases() {
        nabla_b.push(DVector::from_element(biases.len(), 0.0f32));
    }
    // holds all weights of the network
    let mut nabla_w: Vec<DMatrix<f32>> = Vec::with_capacity(nn.get_weights().len());
    for weights in nn.get_weights() {
        nabla_w.push(DMatrix::from_element(weights.nrows(), weights.ncols(), 0.0f32));
    }

    let mini_batch_len = mini_batch.len();

    // for each dataset in mini_batch: calculate gradients, add to nablas
    for data in mini_batch {
        let (delta_nabla_b, delta_nabla_w) = backprop(&mut nn,
                                                      data.get_input(),
                                                      data.get_class_vector());
        for (mut nb, dnb) in nabla_b.iter_mut().zip(delta_nabla_b.iter()) {
            // TODO: This is really not good. Someone needs to fix this.
            *nb += dnb.clone();
        }
        for (mut nw, dnw) in nabla_w.iter_mut().zip(delta_nabla_w.iter()) {
            // TODO: Same as above.
            *nw += dnw.clone();
        }
    }

    for (mut w, nw) in nn.get_weights_mut().iter_mut().zip( nabla_w.iter() ) {
        *w -= eta / (mini_batch_len as f32) * nw.clone();
    }

    for (mut b, nb) in nn.get_biases_mut().iter_mut().zip( nabla_b.iter() ) {
        *b -= eta / (mini_batch_len as f32) * nb.clone();
    }
}

fn backprop(nn: &mut Network, data: &DVector<f32>, desired_output: &DVector<f32>)
            -> (Vec<DVector<f32>>, Vec<DMatrix<f32>>) {
    use nalgebra::{self, Dot};
    //use nalgebra::Dot;
    use nn;

    // holds all biases of the network
    let mut nabla_b: Vec<DVector<f32>> = Vec::with_capacity(nn.get_biases().len());
    for biases in nn.get_biases() {
        nabla_b.push(DVector::from_element(biases.len(), 0.0f32));
    }
    // holds all weights of the network
    let mut nabla_w: Vec<DMatrix<f32>> = Vec::with_capacity(nn.get_weights().len());
    for weights in nn.get_weights() {
        nabla_w.push(DMatrix::from_element(weights.nrows(), weights.ncols(), 0.0f32));
    }

    // feedforward
    // current activation layer, at the beginning this is the input
    // Note: Clone here because in later iterations activation will actually hold the
    // ownership of the Vector (or rather the activations vector will). However this might
    // still be a performance issue since we call this once per training data.
    let mut activation = data.clone();

    // hold all activation layers (including output)
    let mut activations: Vec<DVector<f32>> = Vec::with_capacity(nn.get_layers().len());
    // hold z where z is the input of the sigmoid function for each layer
    let mut zs: Vec<DVector<f32>> = Vec::with_capacity(nn.get_layers().len());

    activations.push(activation);
    // execute feedforward
    for (biases, weights) in nn.get_biases().iter().zip(nn.get_weights().iter()) {
        // TODO: Remove Clone
        let z = weights * &activations[activations.len()-1] + biases.clone();
        zs.push(z);
        activation = nn::sigmoid(&zs[zs.len()-1]);
        activations.push(activation)
    }

    // backward pass
    let delta = cost_derivative(&activations[activations.len()], desired_output)
        * sigmoid_prime(&zs[zs.len()-1]);
    let nabla_b_len = nabla_b.len() -1;
    let nabla_w_len = nabla_w.len() -1;
    nabla_b[nabla_b_len] = delta;
    //let mut act_transp = DMatrix::from_column_iter(
    nabla_w[nabla_w_len] = (&nabla_b[nabla_b_len]).clone().dot(&activations[activations.len()-2]).clone();

    unimplemented!();
}
/*

        # backward pass
        delta = self.cost_derivative(activations[-1], y) * \
            sigmoid_prime(zs[-1])
        nabla_b[-1] = delta
        nabla_w[-1] = np.dot(delta, activations[-2].transpose())
        # Note that the variable l in the loop below is used a little
        # differently to the notation in Chapter 2 of the book.  Here,
        # l = 1 means the last layer of neurons, l = 2 is the
        # second-last layer, and so on.  It's a renumbering of the
        # scheme in the book, used here to take advantage of the fact
        # that Python can use negative indices in lists.
        for l in xrange(2, self.num_layers):
            z = zs[-l]
            sp = sigmoid_prime(z)
            delta = np.dot(self.weights[-l+1].transpose(), delta) * sp
            nabla_b[-l] = delta
            nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())
        return (nabla_b, nabla_w)

   def backprop(self, x, y):
        """Return a tuple "(nabla_b, nabla_w)" representing the
        gradient for the cost function C_x.  "nabla_b" and
        "nabla_w" are layer-by-layer lists of numpy arrays, similar
        to "self.biases" and "self.weights"."""
        nabla_b = [np.zeros(b.shape) for b in self.biases]
        nabla_w = [np.zeros(w.shape) for w in self.weights]
        # feedforward
        activation = x
        activations = [x] # list to store all the activations, layer by layer
        zs = [] # list to store all the z vectors, layer by layer
        for b, w in zip(self.biases, self.weights):
            z = np.dot(w, activation)+b
            zs.append(z)
            activation = sigmoid(z)
            activations.append(activation)
*/

fn transpose_vec(vec: &Vec<DVector<f32>>) -> DMatrix<&f32> {
    // this eventually has to go. If we need this for any vector of DVectors we should probably
    // think of storing it all in a DMatrix instead.
    use nalgebra::Iterable;
    let iter = vec.iter().flat_map(|x| x.iter());

    let mut m = DMatrix::from_column_iter(vec[0].len(), vec.len(), iter);
    m = m.transpose()
}

/// Derivative of the cost function
fn cost_derivative(output_activations: &DVector<f32>, desired_output: &DVector<f32>)
                   -> DVector<f32> {
    // easy, derivative of quadratic cost function is:
    //TODO: Get rid of clone
    output_activations.clone()-desired_output.clone()
}

/// Derivative of the sigmoid function
fn sigmoid_prime(z: &DVector<f32>) -> DVector<f32>{
    use nn;
    // Derivative of sigmoid function, ask wolfram alpha if you don't believe me
    nn::sigmoid(z) * (1.0f32 - nn::sigmoid(z))
}
<commit_msg>Finished adapting the python code, still needs some work<commit_after>use structs::Data;
use nn::Network;
use nalgebra::{DVector, DMatrix};

pub fn sgd(mut nn: &mut Network,
           mut training_data: Vec<Data>,
           epochs: u8, mini_batch_size: u8,
           eta: f32) {
    use rand::{ self, Rng};
    let mut rng = rand::thread_rng();

    for j in 0..epochs {
        rng.shuffle(&mut training_data);
        for mut mini_batch in training_data.chunks_mut(mini_batch_size as usize) {
            update_mini_batch(&mut nn, &mut mini_batch, eta);
        }
        println!("Epoch {} complete!", j);
    }
}

fn update_mini_batch(mut nn: &mut Network, mut mini_batch: &mut [Data], eta: f32) {
    // holds all biases of the network
    let mut nabla_b: Vec<DVector<f32>> = Vec::with_capacity(nn.get_biases().len());
    for biases in nn.get_biases() {
        nabla_b.push(DVector::from_element(biases.len(), 0.0f32));
    }
    // holds all weights of the network
    let mut nabla_w: Vec<DMatrix<f32>> = Vec::with_capacity(nn.get_weights().len());
    for weights in nn.get_weights() {
        nabla_w.push(DMatrix::from_element(weights.nrows(), weights.ncols(), 0.0f32));
    }

    let mini_batch_len = mini_batch.len();

    // for each dataset in mini_batch: calculate gradients, add to nablas
    for data in mini_batch {
        let (delta_nabla_b, delta_nabla_w) = backprop(&mut nn,
                                                      data.get_input(),
                                                      data.get_class_vector());
        for (mut nb, dnb) in nabla_b.iter_mut().zip(delta_nabla_b.iter()) {
            // TODO: This is really not good. Someone needs to fix this.
            *nb += dnb.clone();
        }
        for (mut nw, dnw) in nabla_w.iter_mut().zip(delta_nabla_w.iter()) {
            // TODO: Same as above.
            *nw += dnw.clone();
        }
    }

    for (mut w, nw) in nn.get_weights_mut().iter_mut().zip( nabla_w.iter() ) {
        *w -= eta / (mini_batch_len as f32) * nw.clone();
    }

    for (mut b, nb) in nn.get_biases_mut().iter_mut().zip( nabla_b.iter() ) {
        *b -= eta / (mini_batch_len as f32) * nb.clone();
    }
}

/// Gets the desired changes in weights and biases for one training example
fn backprop(nn: &mut Network, data: &DVector<f32>, desired_output: &DVector<f32>)
            -> (Vec<DVector<f32>>, Vec<DMatrix<f32>>) {
    use nalgebra::{self, Dot, Outer};
    //use nalgebra::Dot;
    use nn;

    // holds all biases of the network
    let mut nabla_b: Vec<DVector<f32>> = Vec::with_capacity(nn.get_biases().len());
    for biases in nn.get_biases() {
        nabla_b.push(DVector::new_zeros(biases.len()));
    }
    // holds all weights of the network
    let mut nabla_w: Vec<DMatrix<f32>> = Vec::with_capacity(nn.get_weights().len());
    for weights in nn.get_weights() {
        nabla_w.push(DMatrix::new_zeros(weights.nrows(), weights.ncols()));
    }

    // feedforward
    // current activation layer, at the beginning this is the input
    // Note: Clone here because in later iterations activation will actually hold the
    // ownership of the Vector (or rather the activations vector will). However this might
    // still be a performance issue since we call this once per training data.
    let mut activation = data.clone();

    // hold all activation layers (including output)
    let mut activations: Vec<DVector<f32>> = Vec::with_capacity(nn.get_layers().len());
    // hold z where z is the input of the sigmoid function for each layer
    let mut zs: Vec<DVector<f32>> = Vec::with_capacity(nn.get_layers().len());

    activations.push(activation);
    // execute feedforward
    for (biases, weights) in nn.get_biases().iter().zip(nn.get_weights().iter()) {
        // TODO: Remove Clone
        let z = weights * &activations[activations.len()-1] + biases.clone();
        zs.push(z);
        activation = nn::sigmoid(&zs[zs.len()-1]);
        activations.push(activation)
    }

    // backward pass
    let mut delta = cost_derivative(&activations[activations.len()], desired_output)
        * sigmoid_prime(&zs[zs.len()-1]);
    let nabla_b_len = nabla_b.len() -1;
    let nabla_w_len = nabla_w.len() -1;
    // TODO: Remove clone
    nabla_b[nabla_b_len] = delta.clone();
    nabla_w[nabla_w_len] = (&nabla_b[nabla_b_len]).outer(&activations[activations.len()-2]);

    //TODO: Verify if we need to iterate only to ...len()-1 because of input layer
    for l in 2..nn.get_layers().len() {
        let z = &zs[zs.len()-l];
        let sp = sigmoid_prime(&z);
        //TODO: Verify that this line does what it's supposed to
        delta = &nn.get_weights()[nn.get_weights().len()-l+1] * (&delta) * sp;
        nabla_b[nabla_b_len-l] = delta.clone();
        nabla_w[nabla_w_len-l] = (&delta).outer(&activations[activations.len()-l-1]);
    }
    (nabla_b, nabla_w)
}

/// Derivative of the cost function
fn cost_derivative(output_activations: &DVector<f32>, desired_output: &DVector<f32>)
                   -> DVector<f32> {
    // easy, derivative of quadratic cost function is:
    //TODO: Get rid of clone
    output_activations.clone()-desired_output.clone()
}

/// Derivative of the sigmoid function
fn sigmoid_prime(z: &DVector<f32>) -> DVector<f32>{
    use nn;
    // Derivative of sigmoid function, ask wolfram alpha if you don't believe me
    nn::sigmoid(z) * (1.0f32 - nn::sigmoid(z))
}
<|endoftext|>


<commit_before>//! Support for patterns of arbitrary element types.
use std;
use std::marker::MarkerTrait;

mod walk;
use self::walk::{action,Action,Walkable,Walker};

/// An iterator that produces its input item exactly once.
struct Once<'a,T: 'a>(Option<&'a T>);

impl<'a,T> Once<'a,T> {
    fn new(obj: &'a T) -> Once<'a,T> {
        Once(Some(obj))
    }
}

impl<'a,T> Iterator for Once<'a,T> {
    type Item = &'a T;
    fn next(&mut self) -> Option<&'a T> {
        let out = self.0;
        self.0 = None;
        out
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        match self.0 {
            Some(..) => (1, Some(1)),
            None => (0, Some(0))
        }
    }
}

/// Interface for descriptions of syntactic patterns.
pub trait Pattern<T>: Walkable<FirstSet,Element<T>> + Walkable<AtomicFirstSet,T> + Nullable {}

/// A pattern's first set is the set of elements that may begin a sequence
/// matching that pattern.
trait FirstSet: MarkerTrait {}
/// The atomic first set is like the first set above, but yields only the
/// atomic values within the set.
trait AtomicFirstSet: MarkerTrait {}

/// Provides a method to determine whether a pattern is "nullable", i.e., can
/// match an empty sequence.
pub trait Nullable {
    /// Check if the token is nullable.
    fn is_nullable(&self) -> bool { false }
}

/// A component in a pattern.  Each element is either an atomic value, or
/// a Pattern trait object.
pub enum Element<T> {
    Atom(T),
    Pattern(Box<Pattern<T>>)
}

impl<'a,T> Nullable for Element<T> {
    fn is_nullable(&self) -> bool {
        match *self {
            Element::Atom(..) => false,
            Element::Pattern(ref pat) => pat.is_nullable(),
        }
    }
}

/// A sequence of atoms or subpatterns.
struct Sequence<T> {
    elements: Vec<Element<T>>
}

impl<'a,T: 'a> Walkable<AtomicFirstSet,&'a T> for Sequence<T> {
    type BaseIterator = std::slice::Iter<'a,Element<T>>;

    #[inline(always)]
    fn iter(&self) -> <Self as Walkable<AtomicFirstSet,&T>>::BaseIterator { self.elements.iter() }

    fn action<'b>(&self, element: &'b Element<T>) -> (Option<&T>, u8) {
        match *element {
            Element::Atom(..) =>
                Action{yield_value: Some(element),
                       recurse_on: None,
                       flags: action::TERMINATE},
            Element::Pattern(ref pat) =>
                Action{yield_value: None,
                       recurse_on: Some(element),
                       flags: if pat.is_nullable() { 0 } else { action::TERMINATE }}
        }
    }
}

impl<'a,T: 'a> Walkable<FirstSet,&'a Element<T>> for Sequence<T> {
    type BaseIterator = std::slice::Iter<'a,Element<T>>;

    #[inline(always)]
    fn iter(&self) -> <Self as Walkable<FirstSet,&Element<T>>>::BaseIterator { self.elements.iter() }

    #[inline(always)]
    fn action<'b>(&self, element: &'b Element<T>) -> (Option<&Element<T>>, u8) {
        match *element {
            Element::Atom(..) =>
                Action{yield_value: Some(element),
                       recurse_on: None,
                       flags: action::TERMINATE},
            Element::Pattern(ref pat) =>
                Action{yield_value: Some(element),
                       recurse_on: Some(element),
                       flags: if pat.is_nullable() { 0 } else { action::TERMINATE }}
        }
    }
}

impl<T> Nullable for Sequence<T> {
    fn is_nullable(&self) -> bool {
        if self.elements.is_empty() { true }
        else {
            for elt in self.elements.iter() {
                match *elt {
                    Element::Atom(..) => return false,
                    Element::Pattern(pat) => if ! pat.is_nullable() { return false; }
                }
            }
            true
        }
    }
}
<commit_msg>pattern: let's try using struct types as tags... nope, still doesn't work!<commit_after>//! Support for patterns of arbitrary element types.
use std;
use std::marker::{MarkerTrait,PhantomData};

mod walk;
use self::walk::{action,Action,Walkable,Walker};

/// An iterator that produces its input item exactly once.
struct Once<'a,T: 'a>(Option<&'a T>);

impl<'a,T> Once<'a,T> {
    fn new(obj: &'a T) -> Once<'a,T> {
        Once(Some(obj))
    }
}

impl<'a,T> Iterator for Once<'a,T> {
    type Item = &'a T;
    fn next(&mut self) -> Option<&'a T> {
        let out = self.0;
        self.0 = None;
        out
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        match self.0 {
            Some(..) => (1, Some(1)),
            None => (0, Some(0))
        }
    }
}

/// Interface for descriptions of syntactic patterns.
pub trait Pattern<T>: Walkable<FirstSet,Element<T>> + Walkable<AtomicFirstSet,T> + Nullable {}

/// A pattern's first set is the set of elements that may begin a sequence
/// matching that pattern.
pub struct FirstSet { _marker: PhantomData<u8> }
/// The atomic first set is like the first set above, but yields only the
/// atomic values within the set.
pub struct AtomicFirstSet { _marker: PhantomData<u8>}

/// Provides a method to determine whether a pattern is "nullable", i.e., can
/// match an empty sequence.
pub trait Nullable {
    /// Check if the token is nullable.
    fn is_nullable(&self) -> bool { false }
}

/// A component in a pattern.  Each element is either an atomic value, or
/// a Pattern trait object.
pub enum Element<T> {
    Atom(T),
    Pattern(Box<Pattern<T>>)
}

impl<'a,T> Nullable for Element<T> {
    fn is_nullable(&self) -> bool {
        match *self {
            Element::Atom(..) => false,
            Element::Pattern(ref pat) => pat.is_nullable(),
        }
    }
}

/// A sequence of atoms or subpatterns.
struct Sequence<T> {
    elements: Vec<Element<T>>
}

impl<'a,T: 'a> Walkable<AtomicFirstSet,&'a T> for Sequence<T> {
    type BaseIterator = std::slice::Iter<'a,Element<T>>;

    #[inline(always)]
    fn iter(&self) -> <Self as Walkable<AtomicFirstSet,&T>>::BaseIterator { self.elements.iter() }

    fn action<'b>(&self, element: &'b Element<T>) -> (Option<&T>, u8) {
        match *element {
            Element::Atom(..) =>
                Action{yield_value: Some(element),
                       recurse_on: None,
                       flags: action::TERMINATE},
            Element::Pattern(ref pat) =>
                Action{yield_value: None,
                       recurse_on: Some(element),
                       flags: if pat.is_nullable() { 0 } else { action::TERMINATE }}
        }
    }
}

impl<'a,T: 'a> Walkable<FirstSet,&'a Element<T>> for Sequence<T> {
    type BaseIterator = std::slice::Iter<'a,Element<T>>;

    #[inline(always)]
    fn iter(&self) -> <Self as Walkable<FirstSet,&Element<T>>>::BaseIterator { self.elements.iter() }

    #[inline(always)]
    fn action<'b>(&self, element: &'b Element<T>) -> (Option<&Element<T>>, u8) {
        match *element {
            Element::Atom(..) =>
                Action{yield_value: Some(element),
                       recurse_on: None,
                       flags: action::TERMINATE},
            Element::Pattern(ref pat) =>
                Action{yield_value: Some(element),
                       recurse_on: Some(element),
                       flags: if pat.is_nullable() { 0 } else { action::TERMINATE }}
        }
    }
}

impl<T> Nullable for Sequence<T> {
    fn is_nullable(&self) -> bool {
        if self.elements.is_empty() { true }
        else {
            for elt in self.elements.iter() {
                match *elt {
                    Element::Atom(..) => return false,
                    Element::Pattern(pat) => if ! pat.is_nullable() { return false; }
                }
            }
            true
        }
    }
}
<|endoftext|>


<commit_before>use super::*;
use std::collections::{BTreeMap, VecDeque};

#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub struct TermId {
    address: Vec<usize>,
    compos_var_pos: usize,
}

impl From<(Vec<usize>, usize)> for TermId {
    fn from((address, compos_var_pos): (Vec<usize>, usize)) -> TermId {
        TermId { address, compos_var_pos }
    }
}

pub fn identify_terminals<A>(tree_map: &GornTree<Composition<A>>)
        -> (GornTree<Composition<TermId>>, BTreeMap<TermId, A>)
    where A: Clone,
{
    let mut identified_tree_map = GornTree::new();
    let mut terminal_map = BTreeMap::new();

    for (address, composition) in tree_map {
        let vec_compos = &composition.composition;
        let mut identified_compos = Vec::new();
        let mut compos_var_pos = 0;

        for component in vec_compos {
            let mut identified_compon = Vec::new();

            for variable in component {
                match variable {
                    &VarT::Var(x, y) => {
                        identified_compon.push(VarT::Var(x, y));
                    },
                    &VarT::T(ref terminal) => {
                        let terminal_id = TermId { address: address.clone(), compos_var_pos };
                        identified_compon.push(VarT::T(terminal_id.clone()));
                        terminal_map.insert(terminal_id, terminal.clone());
                    },
                };

                compos_var_pos += 1;
            }

            identified_compos.push(identified_compon);
        }

        identified_tree_map.insert(address.clone(), Composition::from(identified_compos));
    }

    (identified_tree_map, terminal_map)
}

pub fn to_negra<H, T, W>(tree_map: &GornTree<PMCFGRule<H, T, W>>, sentence_num: usize)
        -> String
    where H: Clone + ToString,
          T: Clone + ToString,
{
    let negra_vector = to_negra_vector(&tree_map);
    let mut output = format!("#BOS {}\n", sentence_num);

    for (symbol1, symbol2, number) in negra_vector {
        output.push_str(&format!("{}\t{}\t{}\n", symbol1, symbol2, number));
    }

    output.push_str(&format!("#EOS {}", sentence_num));
    output
}

pub fn to_negra_vector<H, T, W>(tree_map: &GornTree<PMCFGRule<H, T, W>>)
        -> Vec<(String, String, usize)>
    where H: Clone + ToString,
          T: Clone + ToString,
{
    let (term_map, nonterminal_map) = to_term(&tree_map);
    // TODO: Enforce negra grammar restrictions
    let (identified_tree_map, terminal_map) = identify_terminals(&term_map);
    let evaluated_compos = evaluate(&identified_tree_map);

    let mut negra_vector = Vec::new();
    let mut rule_queue = VecDeque::new();
    let mut rule_number_map = GornTree::new();
    let mut rule_counter = 1;

    for component in evaluated_compos.composition {
        for variable in component {
            match variable {
                VarT::Var(_, _) => {
                    panic!("Nonterminals must not appear in a fully evaluated configuration!");
                },
                VarT::T(terminal_id) => {
                    let terminal_symbol = terminal_map.get(&terminal_id).unwrap();
                    let address = terminal_id.address;
                    let rule_label = nonterminal_map.get(&address).unwrap();

                    let mut parent_address = address;
                    let parent_number = if let None = parent_address.pop() {
                        panic!("Terminals must have a nonterminal-only rule as their parent!");
                    } else {
                        get_rule_number(&parent_address, &mut rule_queue, &mut rule_number_map,
                                        &mut rule_counter)
                    };
                    negra_vector.push((terminal_symbol.to_string(), rule_label.to_string(), parent_number));
                }
            }
        }
    }

    while let Some((address, rule_number)) = rule_queue.pop_front() {
        let mut parent_address = address.clone();

        let parent_number = if let None = parent_address.pop() {
            0
        } else {
            get_rule_number(&parent_address, &mut rule_queue, &mut rule_number_map, &mut rule_counter)
        };

        let rule_label = nonterminal_map.get(&address).unwrap();
        negra_vector.push((format!("#{}", rule_number), rule_label.to_string(), parent_number));
    }

    negra_vector
}

fn get_rule_number(address: &Vec<usize>, rule_queue: &mut VecDeque<(Vec<usize>, usize)>, rule_number_map: &mut GornTree<usize>, rule_counter: &mut usize)
        -> usize
{
    if let Some(rule_number) = rule_number_map.get(address) {
        return *rule_number
    }

    let rule_number = *rule_counter;
    *rule_counter = *rule_counter + 1;

    rule_number_map.insert(address.clone(), rule_number.clone());
    rule_queue.push_back((address.clone(), rule_number));
    rule_number
}

#[cfg(test)]
mod tests {
    use super::*;
    use self::VarT::{Var, T};

    #[test]
    fn test_identify_terminals() {
        let mut tree_map = GornTree::new();
        tree_map.insert(vec![], Composition::from(vec![
            vec![Var(0, 0), T("a"), Var(0, 1), T("b")]
        ]));
        tree_map.insert(vec![0], Composition::from(vec![
            vec![Var(1, 0)],
            vec![T("c")]
        ]));
        tree_map.insert(vec![0, 1], Composition::from(vec![
            vec![T("d")]
        ]));

        let mut identified_tree_map = GornTree::new();
        identified_tree_map.insert(vec![], Composition::from(vec![
            vec![Var(0, 0), T(TermId::from((vec![], 1))), Var(0, 1), T(TermId::from((vec![], 3)))]
        ]));
        identified_tree_map.insert(vec![0], Composition::from(vec![
            vec![Var(1, 0)],
            vec![T(TermId::from((vec![0], 1)))]
        ]));
        identified_tree_map.insert(vec![0, 1], Composition::from(vec![
            vec![T(TermId::from((vec![0, 1], 0)))]
        ]));

        let mut terminal_map = BTreeMap::new();
        terminal_map.insert(TermId::from((vec![], 1)), "a");
        terminal_map.insert(TermId::from((vec![], 3)), "b");
        terminal_map.insert(TermId::from((vec![0], 1)), "c");
        terminal_map.insert(TermId::from((vec![0, 1], 0)), "d");

        assert_eq!((identified_tree_map, terminal_map), identify_terminals(&tree_map));
    }
}
<commit_msg>src/pmcfg/negra.rs: added test_to_negra_vector()<commit_after>use super::*;
use std::collections::{BTreeMap, VecDeque};

#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
pub struct TermId {
    address: Vec<usize>,
    compos_var_pos: usize,
}

impl From<(Vec<usize>, usize)> for TermId {
    fn from((address, compos_var_pos): (Vec<usize>, usize)) -> TermId {
        TermId { address, compos_var_pos }
    }
}

pub fn identify_terminals<A>(tree_map: &GornTree<Composition<A>>)
        -> (GornTree<Composition<TermId>>, BTreeMap<TermId, A>)
    where A: Clone,
{
    let mut identified_tree_map = GornTree::new();
    let mut terminal_map = BTreeMap::new();

    for (address, composition) in tree_map {
        let vec_compos = &composition.composition;
        let mut identified_compos = Vec::new();
        let mut compos_var_pos = 0;

        for component in vec_compos {
            let mut identified_compon = Vec::new();

            for variable in component {
                match variable {
                    &VarT::Var(x, y) => {
                        identified_compon.push(VarT::Var(x, y));
                    },
                    &VarT::T(ref terminal) => {
                        let terminal_id = TermId { address: address.clone(), compos_var_pos };
                        identified_compon.push(VarT::T(terminal_id.clone()));
                        terminal_map.insert(terminal_id, terminal.clone());
                    },
                };

                compos_var_pos += 1;
            }

            identified_compos.push(identified_compon);
        }

        identified_tree_map.insert(address.clone(), Composition::from(identified_compos));
    }

    (identified_tree_map, terminal_map)
}

pub fn to_negra<H, T, W>(tree_map: &GornTree<PMCFGRule<H, T, W>>, sentence_num: usize)
        -> String
    where H: Clone + ToString,
          T: Clone + ToString,
{
    let negra_vector = to_negra_vector(&tree_map);
    let mut output = format!("#BOS {}\n", sentence_num);

    for (symbol1, symbol2, number) in negra_vector {
        output.push_str(&format!("{}\t{}\t{}\n", symbol1, symbol2, number));
    }

    output.push_str(&format!("#EOS {}", sentence_num));
    output
}

pub fn to_negra_vector<H, T, W>(tree_map: &GornTree<PMCFGRule<H, T, W>>)
        -> Vec<(String, String, usize)>
    where H: Clone + ToString,
          T: Clone + ToString,
{
    let (term_map, nonterminal_map) = to_term(&tree_map);
    // TODO: Enforce negra grammar restrictions
    let (identified_tree_map, terminal_map) = identify_terminals(&term_map);
    let evaluated_compos = evaluate(&identified_tree_map);

    let mut negra_vector = Vec::new();
    let mut rule_queue = VecDeque::new();
    let mut rule_number_map = GornTree::new();
    let mut rule_counter = 1;

    for component in evaluated_compos.composition {
        for variable in component {
            match variable {
                VarT::Var(_, _) => {
                    panic!("Nonterminals must not appear in a fully evaluated configuration!");
                },
                VarT::T(terminal_id) => {
                    let terminal_symbol = terminal_map.get(&terminal_id).unwrap();
                    let address = terminal_id.address;
                    let rule_label = nonterminal_map.get(&address).unwrap();

                    let mut parent_address = address;
                    let parent_number = if let None = parent_address.pop() {
                        panic!("Terminals must have a nonterminal-only rule as their parent!");
                    } else {
                        get_rule_number(&parent_address, &mut rule_queue, &mut rule_number_map,
                                        &mut rule_counter)
                    };
                    negra_vector.push((terminal_symbol.to_string(), rule_label.to_string(), parent_number));
                }
            }
        }
    }

    while let Some((address, rule_number)) = rule_queue.pop_front() {
        let mut parent_address = address.clone();

        let parent_number = if let None = parent_address.pop() {
            0
        } else {
            get_rule_number(&parent_address, &mut rule_queue, &mut rule_number_map, &mut rule_counter)
        };

        let rule_label = nonterminal_map.get(&address).unwrap();
        negra_vector.push((format!("#{}", rule_number), rule_label.to_string(), parent_number));
    }

    negra_vector
}

fn get_rule_number(address: &Vec<usize>, rule_queue: &mut VecDeque<(Vec<usize>, usize)>, rule_number_map: &mut GornTree<usize>, rule_counter: &mut usize)
        -> usize
{
    if let Some(rule_number) = rule_number_map.get(address) {
        return *rule_number
    }

    let rule_number = *rule_counter;
    *rule_counter = *rule_counter + 1;

    rule_number_map.insert(address.clone(), rule_number.clone());
    rule_queue.push_back((address.clone(), rule_number));
    rule_number
}

#[cfg(test)]
mod tests {
    use super::*;
    use super::super::tests::*;
    use self::VarT::{Var, T};

    #[test]
    fn test_identify_terminals() {
        let mut tree_map = GornTree::new();
        tree_map.insert(vec![], Composition::from(vec![
            vec![Var(0, 0), T("a"), Var(0, 1), T("b")]
        ]));
        tree_map.insert(vec![0], Composition::from(vec![
            vec![Var(1, 0)],
            vec![T("c")]
        ]));
        tree_map.insert(vec![0, 1], Composition::from(vec![
            vec![T("d")]
        ]));

        let mut identified_tree_map = GornTree::new();
        identified_tree_map.insert(vec![], Composition::from(vec![
            vec![Var(0, 0), T(TermId::from((vec![], 1))), Var(0, 1), T(TermId::from((vec![], 3)))]
        ]));
        identified_tree_map.insert(vec![0], Composition::from(vec![
            vec![Var(1, 0)],
            vec![T(TermId::from((vec![0], 1)))]
        ]));
        identified_tree_map.insert(vec![0, 1], Composition::from(vec![
            vec![T(TermId::from((vec![0, 1], 0)))]
        ]));

        let mut terminal_map = BTreeMap::new();
        terminal_map.insert(TermId::from((vec![], 1)), "a");
        terminal_map.insert(TermId::from((vec![], 3)), "b");
        terminal_map.insert(TermId::from((vec![0], 1)), "c");
        terminal_map.insert(TermId::from((vec![0, 1], 0)), "d");

        assert_eq!((identified_tree_map, terminal_map), identify_terminals(&tree_map));
    }

    #[test]
    fn test_to_negra_vector() {
        let tree_map = example_tree_map();
        let negra_vector = vec![
            (String::from("a"), String::from("a"), 1),
            (String::from("a"), String::from("a"), 2),
            (String::from("b"), String::from("b"), 3),
            (String::from("c"), String::from("c"), 1),
            (String::from("c"), String::from("c"), 2),
            (String::from("d"), String::from("d"), 3),
            (String::from("#1"), String::from("A"), 4),
            (String::from("#2"), String::from("A"), 1),
            (String::from("#3"), String::from("B"), 4),
            (String::from("#4"), String::from("S"), 0)
        ];

        assert_eq!(negra_vector, to_negra_vector(&tree_map));
    }
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//! Unix implementation of `tutil::screen`, tested on Linux, FreeBSD and macOS.

use super::{Width, Height};

use std::os::raw::c_ushort;
use libc::{ioctl, isatty, STDOUT_FILENO, TIOCGWINSZ};

/// The struct required by the `TIOCGWINSZ` syscall; specified in the following
/// [man page](http://www.delorie.com/djgpp/doc/libc/libc_495.html).
#[derive(Debug)]
struct WinSize {
    /// Rows, in characters.
    ws_row: c_ushort,
    /// Columns, in characters.
    ws_col: c_ushort,
    /// Horizontal size, in pixels.
    ws_xpixel: c_ushort,
    /// Vertical size, in pixels.
    ws_ypixel: c_ushort,
}

/// Returns the terminal screen size (in columns and rows).
///
/// Returns `None` if the screen size is `(0, 0)` or is not able to be
/// determined.
pub fn size() -> Option<(Width, Height)> {
    let is_tty = unsafe { isatty(STDOUT_FILENO) == 1 };

    if !is_tty { return None; }

    let mut winsize = WinSize { ws_row: 0, ws_col: 0, ws_xpixel: 0, ws_ypixel: 0 };

    let success: bool = unsafe {
        ioctl(STDOUT_FILENO, TIOCGWINSZ, &mut winsize) == 0
    };

    if success {
        Some((Width(winsize.ws_col), Height(winsize.ws_row)))
    } else {
        None
    }
}

/// Returns the terminal screen width (in columns).
///
/// Returns `None` if the terminal width is detected as being <= 0 columns or is
/// not able to be determined at all.
pub fn width() -> Option<Width> {
    let size = size();

    if let Some((Width(width), Height(_))) = size {
        Some(Width(width))
    } else {
        None
    }
}

/// Returns the terminal screen height (in rows).
///
/// Returns `None` if the terminal height is detected as being <= 0 rows or is
/// not able to be determined at all.
pub fn height() -> Option<Height> {
    let size = size();

    if let Some((Width(_), Height(height))) = size {
        Some(Height(height))
    } else {
        None
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use super::super::{Width, Height};

    use std::process::{Command, Stdio};

    #[cfg(target_os = "linux")]
    fn create_command() -> Command {
        let mut cmd = Command::new("stty");
        cmd.arg("--file");
        cmd.arg("/dev/stderr");
        cmd.arg("size");
        cmd.stderr(Stdio::inherit());
        cmd
    }

    #[cfg(any(target_os = "freebsd", target_os = "macos"))]
    fn create_command() -> Command {
        let mut cmd = Command::new("stty");
        cmd.arg("-f");
        cmd.arg("/dev/stderr");
        cmd.arg("size");
        cmd.stderr(Stdio::inherit());
        cmd
    }

    #[test]
    fn correct_size() {
        let output = create_command().output().unwrap();
        let stdout = String::from_utf8(output.stdout).unwrap();
        assert!(output.status.success());

        let cols = u16::from_str_radix(stdout.split_whitespace().last().unwrap(), 10).unwrap();
        let rows = u16::from_str_radix(stdout.split_whitespace().next().unwrap(), 10).unwrap();

        if let Some((Width(width), Height(height))) = size() {
            assert_eq!(width, cols);
            assert_eq!(height, rows);
        }
    }

    #[test]
    fn correct_width() {
        let output = create_command().output().unwrap();
        let stdout = String::from_utf8(output.stdout).unwrap();
        assert!(output.status.success());

        let cols = u16::from_str_radix(stdout.split_whitespace().last().unwrap(), 10).unwrap();

        if let Some((Width(width), Height(_))) = size() {
            assert_eq!(width, cols);
        }
    }

    #[test]
    fn correct_height() {
        let output = create_command().output().unwrap();
        let stdout = String::from_utf8(output.stdout).unwrap();
        assert!(output.status.success());

        let rows = u16::from_str_radix(stdout.split_whitespace().next().unwrap(), 10).unwrap();

        if let Some((Width(_), Height(height))) = size() {
            assert_eq!(height, rows);
        }
    }
}
<commit_msg>Improve the tests for tutil::screen::unix<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//! Unix implementation of `tutil::screen`, tested on Linux, FreeBSD and macOS.

use super::{Width, Height};

use std::os::raw::c_ushort;
use libc::{ioctl, isatty, STDOUT_FILENO, TIOCGWINSZ};

/// The struct required by the `TIOCGWINSZ` syscall; specified in the following
/// [man page](http://www.delorie.com/djgpp/doc/libc/libc_495.html).
#[derive(Debug)]
struct WinSize {
    /// Rows, in characters.
    ws_row: c_ushort,
    /// Columns, in characters.
    ws_col: c_ushort,
    /// Horizontal size, in pixels.
    ws_xpixel: c_ushort,
    /// Vertical size, in pixels.
    ws_ypixel: c_ushort,
}

/// Returns the terminal screen size (in columns and rows).
///
/// Returns `None` if the screen size is `(0, 0)` or is not able to be
/// determined.
pub fn size() -> Option<(Width, Height)> {
    let is_tty = unsafe { isatty(STDOUT_FILENO) == 1 };

    if !is_tty { return None; }

    let mut winsize = WinSize { ws_row: 0, ws_col: 0, ws_xpixel: 0, ws_ypixel: 0 };

    let success: bool = unsafe {
        ioctl(STDOUT_FILENO, TIOCGWINSZ, &mut winsize) == 0
    };

    if success {
        Some((Width(winsize.ws_col), Height(winsize.ws_row)))
    } else {
        None
    }
}

/// Returns the terminal screen width (in columns).
///
/// Returns `None` if the terminal width is detected as being <= 0 columns or is
/// not able to be determined at all.
pub fn width() -> Option<Width> {
    let size = size();

    if let Some((Width(width), Height(_))) = size {
        Some(Width(width))
    } else {
        None
    }
}

/// Returns the terminal screen height (in rows).
///
/// Returns `None` if the terminal height is detected as being <= 0 rows or is
/// not able to be determined at all.
pub fn height() -> Option<Height> {
    let size = size();

    if let Some((Width(_), Height(height))) = size {
        Some(Height(height))
    } else {
        None
    }
}

#[cfg(test)]
mod test {
    use super::*;
    use super::super::{Width, Height};

    use std::process::{Command, Stdio};

    #[cfg(target_os = "linux")]
    fn create_command() -> Command {
        let mut cmd = Command::new("stty");
        cmd.arg("--file");
        cmd.arg("/dev/stderr");
        cmd.arg("size");
        cmd.stderr(Stdio::inherit());
        cmd
    }

    #[cfg(any(target_os = "freebsd", target_os = "macos"))]
    fn create_command() -> Command {
        let mut cmd = Command::new("stty");
        cmd.arg("-f");
        cmd.arg("/dev/stderr");
        cmd.arg("size");
        cmd.stderr(Stdio::inherit());
        cmd
    }

    #[test]
    fn correct_size() {
        let output = create_command().output().unwrap();
        let stdout = String::from_utf8(output.stdout).unwrap();
        assert!(output.status.success());

        let cols = u16::from_str_radix(stdout.split_whitespace().last().unwrap(), 10).unwrap();
        let rows = u16::from_str_radix(stdout.split_whitespace().next().unwrap(), 10).unwrap();

        if let Some((Width(width), Height(height))) = size() {
            assert_eq!(width, cols);
            assert_eq!(height, rows);
        } else {
            // If the terminal size cannot be found, than stty must not be able to find it either.
            assert_eq!(cols, 0);
            assert_eq!(rows, 0);
        }
    }

    #[test]
    fn correct_width() {
        let output = create_command().output().unwrap();
        let stdout = String::from_utf8(output.stdout).unwrap();
        assert!(output.status.success());

        let cols = u16::from_str_radix(stdout.split_whitespace().last().unwrap(), 10).unwrap();

        if let Some(Width(width)) = width() {
            assert_eq!(width, cols);
        } else {
            // If the terminal size cannot be found, than stty must not be able to find it either.
            assert_eq!(cols, 0);
        };
    }

    #[test]
    fn correct_height() {
        let output = create_command().output().unwrap();
        let stdout = String::from_utf8(output.stdout).unwrap();
        assert!(output.status.success());

        let rows = u16::from_str_radix(stdout.split_whitespace().next().unwrap(), 10).unwrap();

        if let Some(Height(height)) = height() {
            assert_eq!(height, rows);
        } else {
            // If the terminal size cannot be found, than stty must not be able to find it either.
            assert_eq!(rows, 0);
        };
    }
}
<|endoftext|>


<commit_before>//! Indirect system call
//!
use libc::c_int;

pub use self::arch::*;

#[cfg(target_arch = "x86_64")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 155;
    pub static MEMFD_CREATE: Syscall = 319;
}

#[cfg(target_arch = "x86")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 217;
    pub static MEMFD_CREATE: Syscall = 356;
}

#[cfg(target_arch = "aarch64")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 41;
    pub static MEMFD_CREATE: Syscall = 279;
}

#[cfg(target_arch = "arm")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 218;
    pub static MEMFD_CREATE: Syscall = 385;
}

#[cfg(target_arch = "mips")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 216;
    pub static MEMFD_CREATE: Syscall = 354;
}

#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 203;
    pub static MEMFD_CREATE: Syscall = 360;
}

extern {
    pub fn syscall(num: Syscall, ...) -> c_int;
}
<commit_msg>Enable syscalls on s390x<commit_after>//! Indirect system call
//!
use libc::c_int;

pub use self::arch::*;

#[cfg(target_arch = "x86_64")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 155;
    pub static MEMFD_CREATE: Syscall = 319;
}

#[cfg(target_arch = "x86")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 217;
    pub static MEMFD_CREATE: Syscall = 356;
}

#[cfg(target_arch = "aarch64")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 41;
    pub static MEMFD_CREATE: Syscall = 279;
}

#[cfg(target_arch = "arm")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 218;
    pub static MEMFD_CREATE: Syscall = 385;
}

#[cfg(target_arch = "mips")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 216;
    pub static MEMFD_CREATE: Syscall = 354;
}

#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 203;
    pub static MEMFD_CREATE: Syscall = 360;
}

#[cfg(target_arch = "s390x")]
mod arch {
    use libc::c_long;

    pub type Syscall = c_long;

    pub static SYSPIVOTROOT: Syscall = 217;
    pub static MEMFD_CREATE: Syscall = 350;
}

extern {
    pub fn syscall(num: Syscall, ...) -> c_int;
}
<|endoftext|>


<commit_before>pub mod syscall_table;

use libc;
use std::ffi::CStr;
use std::{ptr, slice};
use std::mem::size_of;
use std::collections::HashMap;
use chrono::{DateTime, UTC};
use value::Value;

#[repr(C, packed)]
pub struct SyscallHdr {
    /// timestamp in nanoseconds from epoch
    pub ts: u64,
    /// the thread id that generated the syscall
    pub tid: u64,
    /// length of the syscall + header
    pub len: u32,
    /// the identifier of the syscall
    pub id: u16
}

pub struct SyscallMeta {
    /// name of the system call
    pub name: &'static str,
    /// syscall category
    pub category: Category,
    /// flags for this syscall
    pub flags: Vec<Flags>,
    /// the number of parameters
    pub nparams: usize,
    /// the list of syscall's parameters
    pub params: Vec<SyscallParam>
}

pub struct SyscallParam {
    /// parameter name
    pub name: &'static str,
    /// parameter type
    pub kind: ParamType,
    /// parameter rendering format
    pub fmt: ParamFormat
}

#[derive(Serialize, Debug)]
pub struct Syscall {
    /// timestamp expressed as UTC date/time structure
    pub ts: DateTime<UTC>,
    /// name of the system call
    pub name: String,
    /// syscall's parameter map
    pub params: HashMap<String, Value>

}

/// determines the syscall category
pub enum Category {
    Unknown,
    Other,
    File,
    Net,
    IPC,
    Memory,
    Process,
    Sleep,
    System,
    Signal,
    User,
    Time,
    Processing,
    IOBase,
    IORead,
    IOWrite,
    IOOther,
    Wait,
    Scheduler,
    Internal

}

pub enum Flags {
    None,
    CreatesFd,
    DestroysFd,
    UsesFd,
    ReadsFromFd,
    WritesToFd,
    ModifiesState,
    Unused,
    Waits,
    SkipParser,
    OldVesion

}

pub enum ParamType {
    None,
    Int8,
    Int16,
    Int32,
    Int64,
    UInt8,
    UInt16,
    UInt32,
    UInt64,
    /// a NULL terminated printable buffer of bytes
    CharBuffer,
    ByteBuffer,
    ErrNo,
    SockAddr,
    SockTuple,
    Fd,
    Pid,
    FdList,
    FsPath,
    SyscallId,
    SigType,
    /// a relative time
    RelTime,
    /// an absolute time interval
    AbsTime,
    Port,
    L4Proto,
    SockFamily,
    Bool,
    Ipv4Addr,
    Dyn,
    Flags8,
    Flags16,
    Flags32,
    Uid,
    Gid,
    Double,
    Sigset,
    CharBufferArray,
    CharBufferPairArray,
    Ipv4Net
}

pub enum ParamFormat {
    Na,
    Dec,
    Hex,
    PaddedDec,
    Id,
    Dir
}

pub enum Direction {
    Enter,
    Exit
}

impl SyscallMeta {
    /// Populates the syscall parameter map by applying pointer arithmetic
    /// operations on the system call header structure. The parameter's buffer is extracted
    /// from the slice which is constructed from syscall header structure pointer
    /// and the number of parameters.
    /// For each parameter found in the slice, the parsing stage is delegated to
    /// the `SyscallParam::parse` method. The return value of the previous method is being put into
    /// the hash map and indexed by parameter name.
    ///
    pub fn build_params(&self, syscall_hdr: *mut SyscallHdr) -> HashMap<String, Value> {
        let mut params = HashMap::<String, Value>::default();
        unsafe {
            let lens = syscall_hdr.offset(1) as *const u16;
            let mut buf = lens.offset(self.nparams as isize) as *const u8;
            let mut i = 0;

            for len in slice::from_raw_parts(lens, self.nparams) {
                let ref param = self.params[i];
                params.insert(param.name.to_string(), param.parse(buf));
                buf = buf.offset(*len as isize);
                i += 1;
            }
        }
        params
    }
}

impl SyscallParam {
    /// Transforms the raw buffer which contains the parameter value to a native
    /// data type suitable for serialization.
    pub fn parse(&self, buf: *const u8) -> Value {
        match self.kind {
            ParamType::Int8 => {
                unsafe { Value::Int8(*(buf as *const i8)) }
            },
            ParamType::Int16 => {
                unsafe { Value::Int16(*(buf as *const i16)) }
            },
            ParamType::Int32 => {
                unsafe { Value::Int32(*(buf as *const i32)) }
            },
            ParamType::Int64 => {
                unsafe { Value::Int64(*(buf as *const i64)) }
            },
            ParamType::UInt8 => {
                unsafe { Value::UInt8(*(buf as *const u8)) }
            },
            ParamType::UInt16 => {
                unsafe { Value::UInt16(*(buf as *const u16)) }
            },
            ParamType::UInt32 => {
                unsafe { Value::UInt32(*(buf as *const u32)) }
            },
            ParamType::UInt64 => {
                unsafe { Value::UInt64(*(buf as *const u64)) }
            },
            ParamType::FsPath => {
                Value::String(self.to_string(buf))
            },
            ParamType::ErrNo => {
                unsafe { Value::Int64(*(buf as *const i64)) }
            },
            ParamType::Fd => {
                unsafe { Value::Int64(*(buf as *const i64)) }
            },
            ParamType::Pid => {
                unsafe { Value::Int64(*(buf as *const i64)) }
            },
            ParamType::Uid | ParamType::Gid  => {
                unsafe { Value::UInt32(*(buf as *const u32)) }
            },
            ParamType::SyscallId => {
                unsafe { Value::UInt16(*(buf as *const u16)) }
            },
            ParamType::CharBuffer => {
                Value::String(self.to_string(buf))
            },
            ParamType::ByteBuffer => {
                Value::String(self.to_string(buf))
            }
            _ => Value::None
        }
    }

    fn to_string(&self, buf: *const u8) -> String {
        unsafe {
            CStr::from_ptr(buf as *const libc::c_char).to_string_lossy().into_owned()
        }
    }
}<commit_msg>`Syscall` to `SyscallInfo` refactoring. using enumerate on`from_raw_parts` to avoid having an extra counter binding.<commit_after>pub mod syscall_table;

use libc;
use std::ffi::CStr;
use std::{ptr, slice};
use std::mem::size_of;
use std::collections::HashMap;
use chrono::{DateTime, UTC};
use value::Value;

#[repr(C, packed)]
pub struct Syscall {
    /// timestamp in nanoseconds from epoch
    pub ts: u64,
    /// the thread id that generated the syscall
    pub tid: u64,
    /// length of the syscall + header
    pub len: u32,
    /// the identifier of the syscall
    pub id: u16
}

pub struct SyscallMeta {
    /// name of the system call
    pub name: &'static str,
    /// syscall category
    pub category: Category,
    /// flags for this syscall
    pub flags: Vec<Flags>,
    /// the number of parameters
    pub nparams: usize,
    /// the list of syscall's parameters
    pub params: Vec<SyscallParam>
}

pub struct SyscallParam {
    /// parameter name
    pub name: &'static str,
    /// parameter type
    pub kind: ParamType,
    /// parameter rendering format
    pub fmt: ParamFormat
}

#[derive(Serialize, Debug)]
pub struct SyscallInfo {
    /// timestamp expressed as UTC date/time structure
    pub ts: DateTime<UTC>,
    /// name of the system call
    pub name: String,
    /// syscall's parameter map
    pub params: HashMap<String, Value>

}

/// determines the syscall category
pub enum Category {
    Unknown,
    Other,
    File,
    Net,
    IPC,
    Memory,
    Process,
    Sleep,
    System,
    Signal,
    User,
    Time,
    Processing,
    IOBase,
    IORead,
    IOWrite,
    IOOther,
    Wait,
    Scheduler,
    Internal

}

pub enum Flags {
    None,
    CreatesFd,
    DestroysFd,
    UsesFd,
    ReadsFromFd,
    WritesToFd,
    ModifiesState,
    Unused,
    Waits,
    SkipParser,
    OldVesion

}

pub enum ParamType {
    None,
    Int8,
    Int16,
    Int32,
    Int64,
    UInt8,
    UInt16,
    UInt32,
    UInt64,
    /// a NULL terminated printable buffer of bytes
    CharBuffer,
    ByteBuffer,
    ErrNo,
    SockAddr,
    SockTuple,
    Fd,
    Pid,
    FdList,
    FsPath,
    SyscallId,
    SigType,
    /// a relative time
    RelTime,
    /// an absolute time interval
    AbsTime,
    Port,
    L4Proto,
    SockFamily,
    Bool,
    Ipv4Addr,
    Dyn,
    Flags8,
    Flags16,
    Flags32,
    Uid,
    Gid,
    Double,
    Sigset,
    CharBufferArray,
    CharBufferPairArray,
    Ipv4Net
}

pub enum ParamFormat {
    Na,
    Dec,
    Hex,
    PaddedDec,
    Id,
    Dir
}

pub enum Direction {
    Enter,
    Exit
}

impl SyscallMeta {
    /// Populates the syscall parameter map by applying pointer arithmetic
    /// operations on the system call structure. The parameter's buffer is extracted
    /// from the slice which is constructed from syscall structure pointer
    /// and the number of parameters.
    /// For each parameter found in the slice, the parsing stage is delegated to
    /// the `SyscallParam::parse` method. The return value of the previous method is put into
    /// the hash map and indexed by parameter name.
    ///
    pub fn build_params(&self, syscall: *mut Syscall) -> HashMap<String, Value> {
        let mut params = HashMap::<String, Value>::default();
        unsafe {
            let lens = syscall.offset(1) as *const u16;
            let mut buf = lens.offset(self.nparams as isize) as *const u8;
            for (i, len) in slice::from_raw_parts(lens, self.nparams)
                        .iter()
                        .enumerate() {
                let ref param = self.params[i];
                params.insert(param.name.to_string(), param.parse(buf));
                buf = buf.offset(*len as isize);
            }
        }
        params
    }
}

impl SyscallParam {
    /// Transforms the raw buffer which contains the parameter value to a native
    /// data type suitable for serialization.
    pub fn parse(&self, buf: *const u8) -> Value {
        match self.kind {
            ParamType::Int8 => {
                unsafe { Value::Int8(*(buf as *const i8)) }
            },
            ParamType::Int16 => {
                unsafe { Value::Int16(*(buf as *const i16)) }
            },
            ParamType::Int32 => {
                unsafe { Value::Int32(*(buf as *const i32)) }
            },
            ParamType::Int64 => {
                unsafe { Value::Int64(*(buf as *const i64)) }
            },
            ParamType::UInt8 => {
                unsafe { Value::UInt8(*(buf as *const u8)) }
            },
            ParamType::UInt16 => {
                unsafe { Value::UInt16(*(buf as *const u16)) }
            },
            ParamType::UInt32 => {
                unsafe { Value::UInt32(*(buf as *const u32)) }
            },
            ParamType::UInt64 => {
                unsafe { Value::UInt64(*(buf as *const u64)) }
            },
            ParamType::FsPath => {
                Value::String(self.to_string(buf))
            },
            ParamType::ErrNo => {
                unsafe { Value::Int64(*(buf as *const i64)) }
            },
            ParamType::Fd => {
                unsafe { Value::Int64(*(buf as *const i64)) }
            },
            ParamType::Pid => {
                unsafe { Value::Int64(*(buf as *const i64)) }
            },
            ParamType::Uid | ParamType::Gid  => {
                unsafe { Value::UInt32(*(buf as *const u32)) }
            },
            ParamType::SyscallId => {
                unsafe { Value::UInt16(*(buf as *const u16)) }
            },
            ParamType::CharBuffer => {
                Value::String(self.to_string(buf))
            },
            ParamType::ByteBuffer => {
                Value::String(self.to_string(buf))
            }
            _ => Value::None
        }
    }

    fn to_string(&self, buf: *const u8) -> String {
        unsafe {
            CStr::from_ptr(buf as *const libc::c_char).to_string_lossy().into_owned()
        }
    }
}<|endoftext|>


<commit_before>
use super::Fixed;
use Error;
use Result;
use std::io::Cursor;
use byteorder::{BigEndian, ReadBytesExt};

#[derive(Debug, Default)]
pub struct HEAD {
    version: Fixed,
    font_revision: Fixed,
    check_sum_adjustment: u32,
    magic_number: u32,
    flags: u16,
    units_per_em: u16,
    created: i64,
    modified: i64,
    x_min: i16,
    y_min: i16,
    x_max: i16,
    y_max: i16,
    mac_style: u16,
    lowest_rec_ppem: u16,
    font_direction_hint: i16,
    index_to_loc_format: i16,
    glyph_data_format: i16,
}

impl HEAD {
    pub fn from_data(data: &[u8]) -> Result<HEAD> {
        let mut cursor = Cursor::new(data);
        let version = Fixed(try!(cursor.read_i32::<BigEndian>()));
        if version != Fixed(0x00010000) {
            return Err(Error::HHEAVersionIsNotSupported);
        }

        let mut head = HEAD::default();
        head.version = version;
        head.font_revision = Fixed(try!(cursor.read_i32::<BigEndian>()));
        head.check_sum_adjustment = try!(cursor.read_u32::<BigEndian>());
        head.magic_number = try!(cursor.read_u32::<BigEndian>());
        head.flags = try!(cursor.read_u16::<BigEndian>());
        head.units_per_em = try!(cursor.read_u16::<BigEndian>());
        head.created = try!(cursor.read_i64::<BigEndian>());
        head.modified = try!(cursor.read_i64::<BigEndian>());
        head.x_min = try!(cursor.read_i16::<BigEndian>());
        head.y_min = try!(cursor.read_i16::<BigEndian>());
        head.x_max = try!(cursor.read_i16::<BigEndian>());
        head.y_max = try!(cursor.read_i16::<BigEndian>());
        head.mac_style = try!(cursor.read_u16::<BigEndian>());
        head.lowest_rec_ppem = try!(cursor.read_u16::<BigEndian>());
        head.font_direction_hint = try!(cursor.read_i16::<BigEndian>());
        head.index_to_loc_format = try!(cursor.read_i16::<BigEndian>());
        head.glyph_data_format = try!(cursor.read_i16::<BigEndian>());

        Ok(head)
    }

    #[cfg(test)]
    fn bytes(&self) -> Vec<u8> {
        use byteorder::WriteBytesExt;

        let mut data = vec![];
        data.write_i32::<BigEndian>(self.version.0).unwrap();
        data.write_i32::<BigEndian>(self.font_revision.0).unwrap();
        data.write_u32::<BigEndian>(self.check_sum_adjustment).unwrap();
        data.write_u32::<BigEndian>(self.magic_number).unwrap();
        data.write_u16::<BigEndian>(self.flags).unwrap();
        data.write_u16::<BigEndian>(self.units_per_em).unwrap();
        data.write_i64::<BigEndian>(self.created).unwrap();
        data.write_i64::<BigEndian>(self.modified).unwrap();
        data.write_i16::<BigEndian>(self.x_min).unwrap();
        data.write_i16::<BigEndian>(self.y_min).unwrap();
        data.write_i16::<BigEndian>(self.x_max).unwrap();
        data.write_i16::<BigEndian>(self.y_max).unwrap();
        data.write_u16::<BigEndian>(self.mac_style).unwrap();
        data.write_u16::<BigEndian>(self.lowest_rec_ppem).unwrap();
        data.write_i16::<BigEndian>(self.font_direction_hint).unwrap();
        data.write_i16::<BigEndian>(self.index_to_loc_format).unwrap();
        data.write_i16::<BigEndian>(self.glyph_data_format).unwrap();
        data
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    const OFFSET: usize = 284;
    const SIZE: usize = 4 * 4 + 2 * 2 + 8 * 2 + 2 * 9;

    #[test]
    fn runner() {
        let data = super::super::read_file("tests/Tuffy_Bold.ttf");
        test_read_write(&data);
    }

    fn test_read_write(data: &[u8]) {
        let data = &data[OFFSET..OFFSET + SIZE];
        let head = HEAD::from_data(data).unwrap();
        assert_eq!(head.bytes(), data);
    }
}
<commit_msg>Add more unit tests for the head table. Fixes.<commit_after>
use super::Fixed;
use Error;
use Result;
use std::io::Cursor;
use byteorder::{BigEndian, ReadBytesExt};

#[derive(Debug, Default)]
pub struct HEAD {
    version: Fixed,
    font_revision: Fixed,
    check_sum_adjustment: u32,
    magic_number: u32,
    flags: u16,
    units_per_em: u16,
    created: i64,
    modified: i64,
    x_min: i16,
    y_min: i16,
    x_max: i16,
    y_max: i16,
    mac_style: u16,
    lowest_rec_ppem: u16,
    font_direction_hint: i16,
    index_to_loc_format: i16,
    glyph_data_format: i16,
}

impl HEAD {
    pub fn from_data(data: &[u8], offset: usize) -> Result<HEAD> {
        if offset >= data.len() {
            return Err(Error::Malformed);
        }

        let mut cursor = Cursor::new(&data[offset..]);
        let version = Fixed(try!(cursor.read_i32::<BigEndian>()));
        if version != Fixed(0x00010000) {
            return Err(Error::HEADVersionIsNotSupported);
        }

        let mut head = HEAD::default();
        head.version = version;
        head.font_revision = Fixed(try!(cursor.read_i32::<BigEndian>()));
        head.check_sum_adjustment = try!(cursor.read_u32::<BigEndian>());
        head.magic_number = try!(cursor.read_u32::<BigEndian>());
        head.flags = try!(cursor.read_u16::<BigEndian>());
        head.units_per_em = try!(cursor.read_u16::<BigEndian>());
        head.created = try!(cursor.read_i64::<BigEndian>());
        head.modified = try!(cursor.read_i64::<BigEndian>());
        head.x_min = try!(cursor.read_i16::<BigEndian>());
        head.y_min = try!(cursor.read_i16::<BigEndian>());
        head.x_max = try!(cursor.read_i16::<BigEndian>());
        head.y_max = try!(cursor.read_i16::<BigEndian>());
        head.mac_style = try!(cursor.read_u16::<BigEndian>());
        head.lowest_rec_ppem = try!(cursor.read_u16::<BigEndian>());
        head.font_direction_hint = try!(cursor.read_i16::<BigEndian>());
        head.index_to_loc_format = try!(cursor.read_i16::<BigEndian>());
        head.glyph_data_format = try!(cursor.read_i16::<BigEndian>());

        Ok(head)
    }

    #[cfg(test)]
    fn bytes(&self) -> Vec<u8> {
        use byteorder::WriteBytesExt;

        let mut data = vec![];
        data.write_i32::<BigEndian>(self.version.0).unwrap();
        data.write_i32::<BigEndian>(self.font_revision.0).unwrap();
        data.write_u32::<BigEndian>(self.check_sum_adjustment).unwrap();
        data.write_u32::<BigEndian>(self.magic_number).unwrap();
        data.write_u16::<BigEndian>(self.flags).unwrap();
        data.write_u16::<BigEndian>(self.units_per_em).unwrap();
        data.write_i64::<BigEndian>(self.created).unwrap();
        data.write_i64::<BigEndian>(self.modified).unwrap();
        data.write_i16::<BigEndian>(self.x_min).unwrap();
        data.write_i16::<BigEndian>(self.y_min).unwrap();
        data.write_i16::<BigEndian>(self.x_max).unwrap();
        data.write_i16::<BigEndian>(self.y_max).unwrap();
        data.write_u16::<BigEndian>(self.mac_style).unwrap();
        data.write_u16::<BigEndian>(self.lowest_rec_ppem).unwrap();
        data.write_i16::<BigEndian>(self.font_direction_hint).unwrap();
        data.write_i16::<BigEndian>(self.index_to_loc_format).unwrap();
        data.write_i16::<BigEndian>(self.glyph_data_format).unwrap();
        data
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    const OFFSET: usize = 284;
    const SIZE: usize = 4 * 4 + 2 * 2 + 8 * 2 + 2 * 9;

    #[test]
    fn runner() {
        let data = super::super::read_file("tests/Tuffy_Bold.ttf");
        test_read_write(&data);
        test_version_mismatch(&data);
        test_read_malformed(&data);
    }

    fn test_read_write(data: &[u8]) {
        let head = HEAD::from_data(data, OFFSET).unwrap();
        assert_eq!(head.bytes(), &data[OFFSET..OFFSET + SIZE]);
    }

    fn test_version_mismatch(data: &[u8]) {
        let mut head = HEAD::from_data(data, OFFSET).unwrap();
        head.version = super::super::Fixed(0);
        match HEAD::from_data(&head.bytes(), 0) {
            Err(::Error::HEADVersionIsNotSupported) => (),
            _ => panic!("should return error on version mismatch"),
        }
    }

    fn test_read_malformed(data: &[u8]) {
        match HEAD::from_data(data, data.len()) {
            Err(::Error::Malformed) => (),
            _ => panic!("should return error on malformed data"),
        }
    }
}
<|endoftext|>


<commit_before>//! Transformation context

use draw_state::DrawState;
use default_draw_state;
use vecmath::{
    abs_transform,
    identity,
    Matrix2d,
    Scalar
};

/// Transform property
#[derive(Copy, Clone)]
pub struct Transform(pub Matrix2d);

/// View transform property
#[derive(Copy, Clone)]
pub struct ViewTransform(pub Matrix2d);

/// Drawing 2d context.
#[derive(Copy, Clone)]
pub struct Context {
    /// View transformation.
    pub view: Matrix2d,
    /// Current transformation.
    pub transform: Matrix2d,
    /// Current draw state settings.
    pub draw_state: DrawState,
}

/*
quack! {
    c: Context[]
    get:
        fn () -> Transform [] { Transform(c.transform) }
        fn () -> ViewTransform [] { ViewTransform(c.view) }
    set:
        fn (val: Transform) [] { c.transform = val.0 }
        fn (val: ViewTransform) [] { c.view = val.0 }
    action:
}
*/

impl Context {
    /// Creates a new drawing context.
    #[inline(always)]
    pub fn new() -> Context {
        Context {
            view: identity(),
            transform: identity(),
            draw_state: *default_draw_state(),
        }
    }

    /// Creates a new drawing context in absolute coordinates.
    ///
    /// This function assumes the default coordinate system
    /// being centered with x axis pointing to the right
    /// and y axis pointing up.
    ///
    /// Returns a drawing context
    /// with origin in the upper left corner
    /// and x axis pointing to the right
    /// and y axis pointing down.
    #[inline(always)]
    pub fn abs(w: Scalar, h: Scalar) -> Context {
        let mat = abs_transform(w, h);
        Context {
            view: mat,
            transform: mat,
            draw_state: *default_draw_state(),
        }
    }
}

#[cfg(test)]
mod test {
    use super::Context;
    use std::num::Float;

    #[test]
    fn test_context() {
        use RelativeTransform;

        let c = Context::new();
        {
            let d = c.trans(20.0, 40.0);
            let d = d.trans(10.0, 10.0);
            let transform = d.transform;
            assert_eq!(transform[0][2], 30.0);
            assert_eq!(transform[1][2], 50.0);
        }

        let transform = c.transform;
        assert_eq!(transform[0][2], 0.0);
        assert_eq!(transform[1][2], 0.0);

        let c = c.rot_deg(90.0);
        let transform = c.transform;
        assert!((transform[0][0] - 0.0).abs() < 0.00001);
        assert!((transform[0][1] + 1.0).abs() < 0.00001);
    }

    #[test]
    fn test_scale() {
        use RelativeTransform;

        let c = Context::new();
        let c = c.scale(2.0, 3.0);
        let transform = c.transform;
        assert!((transform[0][0] - 2.0).abs() < 0.00001);
        assert!((transform[1][1] - 3.0).abs() < 0.00001);
    }
}
<commit_msg>Removed property methods on `Context`<commit_after>//! Transformation context

use draw_state::DrawState;
use default_draw_state;
use vecmath::{
    abs_transform,
    identity,
    Matrix2d,
    Scalar
};

/// Transform property
#[derive(Copy, Clone)]
pub struct Transform(pub Matrix2d);

/// View transform property
#[derive(Copy, Clone)]
pub struct ViewTransform(pub Matrix2d);

/// Drawing 2d context.
#[derive(Copy, Clone)]
pub struct Context {
    /// View transformation.
    pub view: Matrix2d,
    /// Current transformation.
    pub transform: Matrix2d,
    /// Current draw state settings.
    pub draw_state: DrawState,
}

impl Context {
    /// Creates a new drawing context.
    #[inline(always)]
    pub fn new() -> Context {
        Context {
            view: identity(),
            transform: identity(),
            draw_state: *default_draw_state(),
        }
    }

    /// Creates a new drawing context in absolute coordinates.
    ///
    /// This function assumes the default coordinate system
    /// being centered with x axis pointing to the right
    /// and y axis pointing up.
    ///
    /// Returns a drawing context
    /// with origin in the upper left corner
    /// and x axis pointing to the right
    /// and y axis pointing down.
    #[inline(always)]
    pub fn abs(w: Scalar, h: Scalar) -> Context {
        let mat = abs_transform(w, h);
        Context {
            view: mat,
            transform: mat,
            draw_state: *default_draw_state(),
        }
    }
}

#[cfg(test)]
mod test {
    use super::Context;
    use std::num::Float;

    #[test]
    fn test_context() {
        use RelativeTransform;

        let c = Context::new();
        {
            let d = c.trans(20.0, 40.0);
            let d = d.trans(10.0, 10.0);
            let transform = d.transform;
            assert_eq!(transform[0][2], 30.0);
            assert_eq!(transform[1][2], 50.0);
        }

        let transform = c.transform;
        assert_eq!(transform[0][2], 0.0);
        assert_eq!(transform[1][2], 0.0);

        let c = c.rot_deg(90.0);
        let transform = c.transform;
        assert!((transform[0][0] - 0.0).abs() < 0.00001);
        assert!((transform[0][1] + 1.0).abs() < 0.00001);
    }

    #[test]
    fn test_scale() {
        use RelativeTransform;

        let c = Context::new();
        let c = c.scale(2.0, 3.0);
        let transform = c.transform;
        assert!((transform[0][0] - 2.0).abs() < 0.00001);
        assert!((transform[1][1] - 3.0).abs() < 0.00001);
    }
}
<|endoftext|>