<commit_before>// Derived from uBPF <https://github.com/iovisor/ubpf>
// Copyright 2015 Big Switch Networks, Inc
//      (uBPF: safety checks, originally in C)
// Copyright 2016 6WIND S.A. <quentin.monnet@6wind.com>
//      (Translation to Rust)
//
// Licensed under the Apache License, Version 2.0 <http://www.apache.org/licenses/LICENSE-2.0> or
// the MIT license <http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

// This “verifier” performs simple checks when the eBPF program is loaded into the VM (before it is
// interpreted or JIT-compiled). It has nothing to do with the much more elaborated verifier inside
// Linux kernel. There is no verification regarding the program flow control (should be a Direct
// Acyclic Graph) or the consistency for registers usage (the verifier of the kernel assigns types
// to the registers and is much stricter).
//
// On the other hand, rbpf is not expected to run in kernel space.
//
// Improving the verifier would be nice, but this is not trivial (and Linux kernel is under GPL
// license, so we cannot copy it).
//
// Contrary to the verifier of the Linux kernel, this one does not modify the bytecode at all.

use ebpf;
use std::io::{Error, ErrorKind};

fn reject<S: AsRef<str>>(msg: S) -> Result<(), Error> {
    let full_msg = format!("[Verifier] Error: {}", msg.as_ref());
    Err(Error::new(ErrorKind::Other, full_msg))
}

fn check_prog_len(prog: &[u8]) -> Result<(), Error> {
    if prog.len() % ebpf::INSN_SIZE != 0 {
        reject(format!("eBPF program length must be a multiple of {:?} octets",
                       ebpf::INSN_SIZE))?;
    }
    if prog.len() > ebpf::PROG_MAX_SIZE {
        reject(format!("eBPF program length limited to {:?}, here {:?}",
                       ebpf::PROG_MAX_INSNS, prog.len() / ebpf::INSN_SIZE))?;
    }

    if prog.is_empty() {
        reject("program does not end with “EXIT” instruction".to_string())?;
    }
    let last_insn = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1);
    if last_insn.opc != ebpf::EXIT {
        reject("program does not end with “EXIT” instruction".to_string())?;
    }

    Ok(())
}

fn check_imm_nonzero(insn: &ebpf::Insn, insn_ptr: usize) -> Result<(), Error> {
    if insn.imm == 0 {
        reject(format!("division by 0 (insn #{:?})", insn_ptr))?;
    }

    Ok(())
}

fn check_imm_endian(insn: &ebpf::Insn, insn_ptr: usize) -> Result<(), Error> {
    match insn.imm {
        16 | 32 | 64 => Ok(()),
        _ => reject(format!("unsupported argument for LE/BE (insn #{:?})", insn_ptr))
    }
}

fn check_load_dw(prog: &[u8], insn_ptr: usize) -> Result<(), Error> {
    // We know we can reach next insn since we enforce an EXIT insn at the end of program, while
    // this function should be called only for LD_DW insn, that cannot be last in program.
    let next_insn = ebpf::get_insn(prog, insn_ptr + 1);
    if next_insn.opc != 0 {
        reject(format!("incomplete LD_DW instruction (insn #{:?})", insn_ptr))?;
    }

    Ok(())
}

fn check_jmp_offset(prog: &[u8], insn_ptr: usize) -> Result<(), Error> {
    let insn = ebpf::get_insn(prog, insn_ptr);
    if insn.off == -1 {
        reject(format!("infinite loop (insn #{:?})", insn_ptr))?;
    }

    let dst_insn_ptr = insn_ptr as isize + 1 + insn.off as isize;
    if dst_insn_ptr < 0 || dst_insn_ptr as usize >= (prog.len() / ebpf::INSN_SIZE) {
        reject(format!("jump out of code to #{:?} (insn #{:?})", dst_insn_ptr, insn_ptr))?;
    }

    let dst_insn = ebpf::get_insn(prog, dst_insn_ptr as usize);
    if dst_insn.opc == 0 {
        reject(format!("jump to middle of LD_DW at #{:?} (insn #{:?})", dst_insn_ptr, insn_ptr))?;
    }

    Ok(())
}

fn check_registers(insn: &ebpf::Insn, store: bool, insn_ptr: usize) -> Result<(), Error> {
    if insn.src > 10 {
        reject(format!("invalid source register (insn #{:?})", insn_ptr))?;
    }

    match (insn.dst, store) {
        (0 ... 9, _) | (10, true) => Ok(()),
        (10, false) => reject(format!("cannot write into register r10 (insn #{:?})", insn_ptr)),
        (_, _)      => reject(format!("invalid destination register (insn #{:?})", insn_ptr))
    }
}

pub fn check(prog: &[u8]) -> Result<(), Error> {
    check_prog_len(prog)?;

    let mut insn_ptr:usize = 0;
    while insn_ptr * ebpf::INSN_SIZE < prog.len() {
        let insn = ebpf::get_insn(prog, insn_ptr);
        let mut store = false;

        match insn.opc {

            // BPF_LD class
            ebpf::LD_ABS_B   => {},
            ebpf::LD_ABS_H   => {},
            ebpf::LD_ABS_W   => {},
            ebpf::LD_ABS_DW  => {},
            ebpf::LD_IND_B   => {},
            ebpf::LD_IND_H   => {},
            ebpf::LD_IND_W   => {},
            ebpf::LD_IND_DW  => {},

            ebpf::LD_DW_IMM  => {
                store = true;
                check_load_dw(prog, insn_ptr)?;
                insn_ptr += 1;
            },

            // BPF_LDX class
            ebpf::LD_B_REG   => {},
            ebpf::LD_H_REG   => {},
            ebpf::LD_W_REG   => {},
            ebpf::LD_DW_REG  => {},

            // BPF_ST class
            ebpf::ST_B_IMM   => store = true,
            ebpf::ST_H_IMM   => store = true,
            ebpf::ST_W_IMM   => store = true,
            ebpf::ST_DW_IMM  => store = true,

            // BPF_STX class
            ebpf::ST_B_REG   => store = true,
            ebpf::ST_H_REG   => store = true,
            ebpf::ST_W_REG   => store = true,
            ebpf::ST_DW_REG  => store = true,
            ebpf::ST_W_XADD  => { unimplemented!(); },
            ebpf::ST_DW_XADD => { unimplemented!(); },

            // BPF_ALU class
            ebpf::ADD32_IMM  => {},
            ebpf::ADD32_REG  => {},
            ebpf::SUB32_IMM  => {},
            ebpf::SUB32_REG  => {},
            ebpf::MUL32_IMM  => {},
            ebpf::MUL32_REG  => {},
            ebpf::DIV32_IMM  => { check_imm_nonzero(&insn, insn_ptr)?; },
            ebpf::DIV32_REG  => {},
            ebpf::OR32_IMM   => {},
            ebpf::OR32_REG   => {},
            ebpf::AND32_IMM  => {},
            ebpf::AND32_REG  => {},
            ebpf::LSH32_IMM  => {},
            ebpf::LSH32_REG  => {},
            ebpf::RSH32_IMM  => {},
            ebpf::RSH32_REG  => {},
            ebpf::NEG32      => {},
            ebpf::MOD32_IMM  => { check_imm_nonzero(&insn, insn_ptr)?; },
            ebpf::MOD32_REG  => {},
            ebpf::XOR32_IMM  => {},
            ebpf::XOR32_REG  => {},
            ebpf::MOV32_IMM  => {},
            ebpf::MOV32_REG  => {},
            ebpf::ARSH32_IMM => {},
            ebpf::ARSH32_REG => {},
            ebpf::LE         => { check_imm_endian(&insn, insn_ptr)?; },
            ebpf::BE         => { check_imm_endian(&insn, insn_ptr)?; },

            // BPF_ALU64 class
            ebpf::ADD64_IMM  => {},
            ebpf::ADD64_REG  => {},
            ebpf::SUB64_IMM  => {},
            ebpf::SUB64_REG  => {},
            ebpf::MUL64_IMM  => { check_imm_nonzero(&insn, insn_ptr)?; },
            ebpf::MUL64_REG  => {},
            ebpf::DIV64_IMM  => { check_imm_nonzero(&insn, insn_ptr)?; },
            ebpf::DIV64_REG  => {},
            ebpf::OR64_IMM   => {},
            ebpf::OR64_REG   => {},
            ebpf::AND64_IMM  => {},
            ebpf::AND64_REG  => {},
            ebpf::LSH64_IMM  => {},
            ebpf::LSH64_REG  => {},
            ebpf::RSH64_IMM  => {},
            ebpf::RSH64_REG  => {},
            ebpf::NEG64      => {},
            ebpf::MOD64_IMM  => {},
            ebpf::MOD64_REG  => {},
            ebpf::XOR64_IMM  => {},
            ebpf::XOR64_REG  => {},
            ebpf::MOV64_IMM  => {},
            ebpf::MOV64_REG  => {},
            ebpf::ARSH64_IMM => {},
            ebpf::ARSH64_REG => {},

            // BPF_JMP class
            ebpf::JA         => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JEQ_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JEQ_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JGT_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JGT_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JGE_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JGE_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JLT_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JLT_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JLE_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JLE_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSET_IMM   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSET_REG   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JNE_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JNE_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSGT_IMM   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSGT_REG   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSGE_IMM   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSGE_REG   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSLT_IMM   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSLT_REG   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSLE_IMM   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSLE_REG   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::CALL       => {},
            ebpf::TAIL_CALL  => { unimplemented!() },
            ebpf::EXIT       => {},

            _                => {
                reject(format!("unknown eBPF opcode {:#2x} (insn #{:?})", insn.opc, insn_ptr))?;
            },
        }

        check_registers(&insn, store, insn_ptr)?;

        insn_ptr += 1;
    }

    // insn_ptr should now be equal to number of instructions.
    if insn_ptr != prog.len() / ebpf::INSN_SIZE {
        reject(format!("jumped out of code to #{:?}", insn_ptr))?;
    }

    Ok(())
}
<commit_msg>Fix empty program error message<commit_after>// Derived from uBPF <https://github.com/iovisor/ubpf>
// Copyright 2015 Big Switch Networks, Inc
//      (uBPF: safety checks, originally in C)
// Copyright 2016 6WIND S.A. <quentin.monnet@6wind.com>
//      (Translation to Rust)
//
// Licensed under the Apache License, Version 2.0 <http://www.apache.org/licenses/LICENSE-2.0> or
// the MIT license <http://opensource.org/licenses/MIT>, at your option. This file may not be
// copied, modified, or distributed except according to those terms.

// This “verifier” performs simple checks when the eBPF program is loaded into the VM (before it is
// interpreted or JIT-compiled). It has nothing to do with the much more elaborated verifier inside
// Linux kernel. There is no verification regarding the program flow control (should be a Direct
// Acyclic Graph) or the consistency for registers usage (the verifier of the kernel assigns types
// to the registers and is much stricter).
//
// On the other hand, rbpf is not expected to run in kernel space.
//
// Improving the verifier would be nice, but this is not trivial (and Linux kernel is under GPL
// license, so we cannot copy it).
//
// Contrary to the verifier of the Linux kernel, this one does not modify the bytecode at all.

use ebpf;
use std::io::{Error, ErrorKind};

fn reject<S: AsRef<str>>(msg: S) -> Result<(), Error> {
    let full_msg = format!("[Verifier] Error: {}", msg.as_ref());
    Err(Error::new(ErrorKind::Other, full_msg))
}

fn check_prog_len(prog: &[u8]) -> Result<(), Error> {
    if prog.len() % ebpf::INSN_SIZE != 0 {
        reject(format!("eBPF program length must be a multiple of {:?} octets",
                       ebpf::INSN_SIZE))?;
    }
    if prog.len() > ebpf::PROG_MAX_SIZE {
        reject(format!("eBPF program length limited to {:?}, here {:?}",
                       ebpf::PROG_MAX_INSNS, prog.len() / ebpf::INSN_SIZE))?;
    }

    if prog.is_empty() {
        reject("No program set, call prog_set() to load one".to_string())?;
    }
    let last_insn = ebpf::get_insn(prog, (prog.len() / ebpf::INSN_SIZE) - 1);
    if last_insn.opc != ebpf::EXIT {
        reject("program does not end with “EXIT” instruction".to_string())?;
    }

    Ok(())
}

fn check_imm_nonzero(insn: &ebpf::Insn, insn_ptr: usize) -> Result<(), Error> {
    if insn.imm == 0 {
        reject(format!("division by 0 (insn #{:?})", insn_ptr))?;
    }

    Ok(())
}

fn check_imm_endian(insn: &ebpf::Insn, insn_ptr: usize) -> Result<(), Error> {
    match insn.imm {
        16 | 32 | 64 => Ok(()),
        _ => reject(format!("unsupported argument for LE/BE (insn #{:?})", insn_ptr))
    }
}

fn check_load_dw(prog: &[u8], insn_ptr: usize) -> Result<(), Error> {
    // We know we can reach next insn since we enforce an EXIT insn at the end of program, while
    // this function should be called only for LD_DW insn, that cannot be last in program.
    let next_insn = ebpf::get_insn(prog, insn_ptr + 1);
    if next_insn.opc != 0 {
        reject(format!("incomplete LD_DW instruction (insn #{:?})", insn_ptr))?;
    }

    Ok(())
}

fn check_jmp_offset(prog: &[u8], insn_ptr: usize) -> Result<(), Error> {
    let insn = ebpf::get_insn(prog, insn_ptr);
    if insn.off == -1 {
        reject(format!("infinite loop (insn #{:?})", insn_ptr))?;
    }

    let dst_insn_ptr = insn_ptr as isize + 1 + insn.off as isize;
    if dst_insn_ptr < 0 || dst_insn_ptr as usize >= (prog.len() / ebpf::INSN_SIZE) {
        reject(format!("jump out of code to #{:?} (insn #{:?})", dst_insn_ptr, insn_ptr))?;
    }

    let dst_insn = ebpf::get_insn(prog, dst_insn_ptr as usize);
    if dst_insn.opc == 0 {
        reject(format!("jump to middle of LD_DW at #{:?} (insn #{:?})", dst_insn_ptr, insn_ptr))?;
    }

    Ok(())
}

fn check_registers(insn: &ebpf::Insn, store: bool, insn_ptr: usize) -> Result<(), Error> {
    if insn.src > 10 {
        reject(format!("invalid source register (insn #{:?})", insn_ptr))?;
    }

    match (insn.dst, store) {
        (0 ... 9, _) | (10, true) => Ok(()),
        (10, false) => reject(format!("cannot write into register r10 (insn #{:?})", insn_ptr)),
        (_, _)      => reject(format!("invalid destination register (insn #{:?})", insn_ptr))
    }
}

pub fn check(prog: &[u8]) -> Result<(), Error> {
    check_prog_len(prog)?;

    let mut insn_ptr:usize = 0;
    while insn_ptr * ebpf::INSN_SIZE < prog.len() {
        let insn = ebpf::get_insn(prog, insn_ptr);
        let mut store = false;

        match insn.opc {

            // BPF_LD class
            ebpf::LD_ABS_B   => {},
            ebpf::LD_ABS_H   => {},
            ebpf::LD_ABS_W   => {},
            ebpf::LD_ABS_DW  => {},
            ebpf::LD_IND_B   => {},
            ebpf::LD_IND_H   => {},
            ebpf::LD_IND_W   => {},
            ebpf::LD_IND_DW  => {},

            ebpf::LD_DW_IMM  => {
                store = true;
                check_load_dw(prog, insn_ptr)?;
                insn_ptr += 1;
            },

            // BPF_LDX class
            ebpf::LD_B_REG   => {},
            ebpf::LD_H_REG   => {},
            ebpf::LD_W_REG   => {},
            ebpf::LD_DW_REG  => {},

            // BPF_ST class
            ebpf::ST_B_IMM   => store = true,
            ebpf::ST_H_IMM   => store = true,
            ebpf::ST_W_IMM   => store = true,
            ebpf::ST_DW_IMM  => store = true,

            // BPF_STX class
            ebpf::ST_B_REG   => store = true,
            ebpf::ST_H_REG   => store = true,
            ebpf::ST_W_REG   => store = true,
            ebpf::ST_DW_REG  => store = true,
            ebpf::ST_W_XADD  => { unimplemented!(); },
            ebpf::ST_DW_XADD => { unimplemented!(); },

            // BPF_ALU class
            ebpf::ADD32_IMM  => {},
            ebpf::ADD32_REG  => {},
            ebpf::SUB32_IMM  => {},
            ebpf::SUB32_REG  => {},
            ebpf::MUL32_IMM  => {},
            ebpf::MUL32_REG  => {},
            ebpf::DIV32_IMM  => { check_imm_nonzero(&insn, insn_ptr)?; },
            ebpf::DIV32_REG  => {},
            ebpf::OR32_IMM   => {},
            ebpf::OR32_REG   => {},
            ebpf::AND32_IMM  => {},
            ebpf::AND32_REG  => {},
            ebpf::LSH32_IMM  => {},
            ebpf::LSH32_REG  => {},
            ebpf::RSH32_IMM  => {},
            ebpf::RSH32_REG  => {},
            ebpf::NEG32      => {},
            ebpf::MOD32_IMM  => { check_imm_nonzero(&insn, insn_ptr)?; },
            ebpf::MOD32_REG  => {},
            ebpf::XOR32_IMM  => {},
            ebpf::XOR32_REG  => {},
            ebpf::MOV32_IMM  => {},
            ebpf::MOV32_REG  => {},
            ebpf::ARSH32_IMM => {},
            ebpf::ARSH32_REG => {},
            ebpf::LE         => { check_imm_endian(&insn, insn_ptr)?; },
            ebpf::BE         => { check_imm_endian(&insn, insn_ptr)?; },

            // BPF_ALU64 class
            ebpf::ADD64_IMM  => {},
            ebpf::ADD64_REG  => {},
            ebpf::SUB64_IMM  => {},
            ebpf::SUB64_REG  => {},
            ebpf::MUL64_IMM  => { check_imm_nonzero(&insn, insn_ptr)?; },
            ebpf::MUL64_REG  => {},
            ebpf::DIV64_IMM  => { check_imm_nonzero(&insn, insn_ptr)?; },
            ebpf::DIV64_REG  => {},
            ebpf::OR64_IMM   => {},
            ebpf::OR64_REG   => {},
            ebpf::AND64_IMM  => {},
            ebpf::AND64_REG  => {},
            ebpf::LSH64_IMM  => {},
            ebpf::LSH64_REG  => {},
            ebpf::RSH64_IMM  => {},
            ebpf::RSH64_REG  => {},
            ebpf::NEG64      => {},
            ebpf::MOD64_IMM  => {},
            ebpf::MOD64_REG  => {},
            ebpf::XOR64_IMM  => {},
            ebpf::XOR64_REG  => {},
            ebpf::MOV64_IMM  => {},
            ebpf::MOV64_REG  => {},
            ebpf::ARSH64_IMM => {},
            ebpf::ARSH64_REG => {},

            // BPF_JMP class
            ebpf::JA         => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JEQ_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JEQ_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JGT_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JGT_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JGE_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JGE_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JLT_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JLT_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JLE_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JLE_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSET_IMM   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSET_REG   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JNE_IMM    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JNE_REG    => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSGT_IMM   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSGT_REG   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSGE_IMM   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSGE_REG   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSLT_IMM   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSLT_REG   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSLE_IMM   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::JSLE_REG   => { check_jmp_offset(prog, insn_ptr)?; },
            ebpf::CALL       => {},
            ebpf::TAIL_CALL  => { unimplemented!() },
            ebpf::EXIT       => {},

            _                => {
                reject(format!("unknown eBPF opcode {:#2x} (insn #{:?})", insn.opc, insn_ptr))?;
            },
        }

        check_registers(&insn, store, insn_ptr)?;

        insn_ptr += 1;
    }

    // insn_ptr should now be equal to number of instructions.
    if insn_ptr != prog.len() / ebpf::INSN_SIZE {
        reject(format!("jumped out of code to #{:?}", insn_ptr))?;
    }

    Ok(())
}
<|endoftext|>


<commit_before>
use spv::*;
use spv::op::*;
use spv::types::*;
use spv::raw::*;
use glsl450;

fn read(module: &'static [u8]) -> ReadResult<RawModule> {
    let inst_sets: Vec<Box<ExtInstSet>> = vec![Box::new(glsl450::InstSet)];
    read_module(module, inst_sets)
}

#[test]
fn load_noop() {
    let result = read(include_bytes!("noop.spv"));
    let glsl450 = OpExtInstImport {
        result_id: ResultId(1),
        name: "GLSL.std.450".into(),
    };
    let mem_model = OpMemoryModel {
        addressing_model: AddressingModel::Logical,
        memory_model: MemoryModel::Glsl450,
    };
    let entry_point = OpEntryPoint {
        execution_model: ExecutionModel::GlCompute,
        entry_point: OpId(4),
        name: "main".into(),
        interface: vec![],
    };
    let local_size = ExecutionMode::LocalSize(32, 32, 32);
    let source = OpSource {
        language: SourceLanguage::Glsl,
        version: SourceVersion(430),
        file: None,
        source: None,
    };
    let workgroup = OpDecorate {
        target: OpId(9),
        decoration: Decoration::BuiltIn(BuiltIn::WorkgroupSize),
    };
    let ty_void = Core::OpTypeVoid(OpTypeVoid { result_id: ResultId(2) });
    let ty_main = Core::OpTypeFunction(OpTypeFunction {
        result_id: ResultId(3),
        return_type: OpId(2),
        parameter_types: vec![],
    });
    let ty_int = Core::OpTypeInt(OpTypeInt {
        result_id: ResultId(6),
        width: 32,
        signedness: Signedness::UnsignedOrNone,
    });
    let ty_vec = Core::OpTypeVector(OpTypeVector {
        result_id: ResultId(7),
        component_type: OpId(6),
        component_count: 3,
    });
    let int_32 = Core::OpConstant(OpConstant {
        result_type: OpId(6),
        result_id: ResultId(8),
        value: vec![32],
    });
    let int_32_x3 = vec![OpId(8), OpId(8), OpId(8)];
    let int_32x3 = Core::OpConstantComposite(OpConstantComposite {
        result_type: OpId(7),
        result_id: ResultId(9),
        constituents: int_32_x3,
    });
    let main = OpFunction {
        result_type: OpId(2),
        result_id: ResultId(4),
        function_control: FunctionControl::default(),
        function_type: OpId(3),
    };
    let instructions = vec![Core::OpCapability(OpCapability { capability: Capability::Shader }),
                            Core::OpExtInstImport(glsl450),
                            Core::OpMemoryModel(mem_model),
                            Core::OpEntryPoint(entry_point),
                            Core::OpExecutionMode(OpExecutionMode {
                                entry_point: OpId(4),
                                mode: local_size,
                            }),
                            Core::OpSource(source),
                            Core::OpName(OpName {
                                target: OpId(4),
                                name: "main".to_string(),
                            }),
                            Core::OpDecorate(workgroup),
                            ty_void,
                            ty_main,
                            ty_int,
                            ty_vec,
                            int_32,
                            int_32x3,
                            Core::OpFunction(main),
                            Core::OpLabel(OpLabel { result_id: ResultId(5) }),
                            Core::OpReturn(OpReturn),
                            Core::OpFunctionEnd(OpFunctionEnd)];
    let expected = RawModule {
        version: Version(1, 0),
        generator: Generator {
            tool: Tool::KhronosGlslang,
            version: 1,
        },
        bound: 10,
        instructions: instructions,
    };
    assert_eq!(result, Ok(expected));
}

macro_rules! def_test {
    ($name: ident) => {
        mod $name {

            use super::read;
            use spv::logical::validate;

            const SPV: &'static [u8] = include_bytes!(concat!(stringify!($name), ".spv"));
            const DIS: &'static str = include_str!(concat!(stringify!($name), ".dis"));

            #[test]
            fn disassemble() {
                let raw_module = read(SPV).expect("Failed to load spv");
                let disassembly = format!("{}", raw_module);
                for (dis, expect) in disassembly.lines().zip(DIS.lines()) {
                    assert_eq!(dis, expect);
                }
                assert_eq!(DIS, disassembly);
            }

            #[test]
            fn logical_pass() {
                let raw_module = read(SPV).expect("Failed to load spv");
                let module = validate(raw_module);
                module.unwrap();
            }
        }
    }
}

def_test!(noop);
def_test!(write_multiply);
def_test!(cond_trig);
def_test!(nest_if);
def_test!(loop_simple);
<commit_msg>Add automated tests for find_control_flow<commit_after>
use spv::*;
use spv::op::*;
use spv::types::*;
use spv::raw::*;
use glsl450;

fn read(module: &'static [u8]) -> ReadResult<RawModule> {
    let inst_sets: Vec<Box<ExtInstSet>> = vec![Box::new(glsl450::InstSet)];
    read_module(module, inst_sets)
}

#[test]
fn load_noop() {
    let result = read(include_bytes!("noop.spv"));
    let glsl450 = OpExtInstImport {
        result_id: ResultId(1),
        name: "GLSL.std.450".into(),
    };
    let mem_model = OpMemoryModel {
        addressing_model: AddressingModel::Logical,
        memory_model: MemoryModel::Glsl450,
    };
    let entry_point = OpEntryPoint {
        execution_model: ExecutionModel::GlCompute,
        entry_point: OpId(4),
        name: "main".into(),
        interface: vec![],
    };
    let local_size = ExecutionMode::LocalSize(32, 32, 32);
    let source = OpSource {
        language: SourceLanguage::Glsl,
        version: SourceVersion(430),
        file: None,
        source: None,
    };
    let workgroup = OpDecorate {
        target: OpId(9),
        decoration: Decoration::BuiltIn(BuiltIn::WorkgroupSize),
    };
    let ty_void = Core::OpTypeVoid(OpTypeVoid { result_id: ResultId(2) });
    let ty_main = Core::OpTypeFunction(OpTypeFunction {
        result_id: ResultId(3),
        return_type: OpId(2),
        parameter_types: vec![],
    });
    let ty_int = Core::OpTypeInt(OpTypeInt {
        result_id: ResultId(6),
        width: 32,
        signedness: Signedness::UnsignedOrNone,
    });
    let ty_vec = Core::OpTypeVector(OpTypeVector {
        result_id: ResultId(7),
        component_type: OpId(6),
        component_count: 3,
    });
    let int_32 = Core::OpConstant(OpConstant {
        result_type: OpId(6),
        result_id: ResultId(8),
        value: vec![32],
    });
    let int_32_x3 = vec![OpId(8), OpId(8), OpId(8)];
    let int_32x3 = Core::OpConstantComposite(OpConstantComposite {
        result_type: OpId(7),
        result_id: ResultId(9),
        constituents: int_32_x3,
    });
    let main = OpFunction {
        result_type: OpId(2),
        result_id: ResultId(4),
        function_control: FunctionControl::default(),
        function_type: OpId(3),
    };
    let instructions = vec![Core::OpCapability(OpCapability { capability: Capability::Shader }),
                            Core::OpExtInstImport(glsl450),
                            Core::OpMemoryModel(mem_model),
                            Core::OpEntryPoint(entry_point),
                            Core::OpExecutionMode(OpExecutionMode {
                                entry_point: OpId(4),
                                mode: local_size,
                            }),
                            Core::OpSource(source),
                            Core::OpName(OpName {
                                target: OpId(4),
                                name: "main".to_string(),
                            }),
                            Core::OpDecorate(workgroup),
                            ty_void,
                            ty_main,
                            ty_int,
                            ty_vec,
                            int_32,
                            int_32x3,
                            Core::OpFunction(main),
                            Core::OpLabel(OpLabel { result_id: ResultId(5) }),
                            Core::OpReturn(OpReturn),
                            Core::OpFunctionEnd(OpFunctionEnd)];
    let expected = RawModule {
        version: Version(1, 0),
        generator: Generator {
            tool: Tool::KhronosGlslang,
            version: 1,
        },
        bound: 10,
        instructions: instructions,
    };
    assert_eq!(result, Ok(expected));
}

macro_rules! def_test {
    ($name: ident) => {
        mod $name {

            use super::read;
            use spv::logical::validate;
            use spv::logical::find_control_flow;

            const SPV: &'static [u8] = include_bytes!(concat!(stringify!($name), ".spv"));
            const DIS: &'static str = include_str!(concat!(stringify!($name), ".dis"));

            #[test]
            fn disassemble() {
                let raw_module = read(SPV).expect("Failed to load spv");
                let disassembly = format!("{}", raw_module);
                for (dis, expect) in disassembly.lines().zip(DIS.lines()) {
                    assert_eq!(dis, expect);
                }
                assert_eq!(DIS, disassembly);
            }

            #[test]
            fn logical_pass() {
                let raw_module = read(SPV).expect("Failed to load spv");
                let module = validate(raw_module);
                module.unwrap();
            }

            #[test]
            fn cfa() {
                let raw_module = read(SPV).expect("Failed to load spv");
                let module = validate(raw_module);
                let raw = module.unwrap();
                for func in raw.function_definitions {
                    let chain_res = find_control_flow(func);
                    chain_res.unwrap();
                }
            }
        }
    }
}

def_test!(noop);
def_test!(write_multiply);
def_test!(cond_trig);
def_test!(nest_if);
def_test!(loop_simple);
<|endoftext|>


<commit_before>#![allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411

cfg_io_util! {
    mod async_buf_read_ext;
    pub use async_buf_read_ext::AsyncBufReadExt;

    mod async_read_ext;
    pub use async_read_ext::AsyncReadExt;

    mod async_seek_ext;
    pub use async_seek_ext::AsyncSeekExt;

    mod async_write_ext;
    pub use async_write_ext::AsyncWriteExt;

    mod buf_reader;
    pub use buf_reader::BufReader;

    mod buf_stream;
    pub use buf_stream::BufStream;

    mod buf_writer;
    pub use buf_writer::BufWriter;

    mod chain;

    mod copy;
    pub use copy::copy;

    mod copy_buf;
    pub use copy_buf::copy_buf;

    mod empty;
    pub use empty::{empty, Empty};

    mod flush;

    mod lines;
    pub use lines::Lines;

    mod mem;
    pub use mem::{duplex, DuplexStream};

    mod read;
    mod read_buf;
    mod read_exact;
    mod read_int;
    mod read_line;

    mod read_to_end;
    mod vec_with_initialized;
    cfg_process! {
        pub(crate) use read_to_end::read_to_end;
    }

    mod read_to_string;
    mod read_until;

    mod repeat;
    pub use repeat::{repeat, Repeat};

    mod shutdown;

    mod sink;
    pub use sink::{sink, Sink};

    mod split;
    pub use split::Split;

    mod take;
    pub use take::Take;

    mod write;
    mod write_all;
    mod write_buf;
    mod write_int;

    // used by `BufReader` and `BufWriter`
    // https://github.com/rust-lang/rust/blob/master/src/libstd/sys_common/io.rs#L1
    const DEFAULT_BUF_SIZE: usize = 8 * 1024;
}

cfg_not_io_util! {
    cfg_process! {
        mod vec_with_initialized;
        mod read_to_end;
        // Used by process
        pub(crate) use read_to_end::read_to_end;
    }
}
<commit_msg>io: fix link to std src in util (#3568)<commit_after>#![allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411

cfg_io_util! {
    mod async_buf_read_ext;
    pub use async_buf_read_ext::AsyncBufReadExt;

    mod async_read_ext;
    pub use async_read_ext::AsyncReadExt;

    mod async_seek_ext;
    pub use async_seek_ext::AsyncSeekExt;

    mod async_write_ext;
    pub use async_write_ext::AsyncWriteExt;

    mod buf_reader;
    pub use buf_reader::BufReader;

    mod buf_stream;
    pub use buf_stream::BufStream;

    mod buf_writer;
    pub use buf_writer::BufWriter;

    mod chain;

    mod copy;
    pub use copy::copy;

    mod copy_buf;
    pub use copy_buf::copy_buf;

    mod empty;
    pub use empty::{empty, Empty};

    mod flush;

    mod lines;
    pub use lines::Lines;

    mod mem;
    pub use mem::{duplex, DuplexStream};

    mod read;
    mod read_buf;
    mod read_exact;
    mod read_int;
    mod read_line;

    mod read_to_end;
    mod vec_with_initialized;
    cfg_process! {
        pub(crate) use read_to_end::read_to_end;
    }

    mod read_to_string;
    mod read_until;

    mod repeat;
    pub use repeat::{repeat, Repeat};

    mod shutdown;

    mod sink;
    pub use sink::{sink, Sink};

    mod split;
    pub use split::Split;

    mod take;
    pub use take::Take;

    mod write;
    mod write_all;
    mod write_buf;
    mod write_int;

    // used by `BufReader` and `BufWriter`
    // https://github.com/rust-lang/rust/blob/master/library/std/src/sys_common/io.rs#L1
    const DEFAULT_BUF_SIZE: usize = 8 * 1024;
}

cfg_not_io_util! {
    cfg_process! {
        mod vec_with_initialized;
        mod read_to_end;
        // Used by process
        pub(crate) use read_to_end::read_to_end;
    }
}
<|endoftext|>


<commit_before>fn make_squares_list(n: i32) -> Vec<i32> {
    let mut re: Vec<i32> = vec![];
    for i in 0..n + 1 {
        let temp = i * i;
        if temp > n {
            break;
        } else {
            re.push(temp);
        }
    }

    re
}

fn sum_list(l1: &Vec<i32>, l2: &Vec<i32>) -> Vec<i32> {
    let mut temp: Vec<Vec<i32>> = l1
        .iter()
        .map(|a| l2.iter().map(|b| b + a).collect())
        .collect();

    let mut re: Vec<i32> = vec![];
    for i in temp.iter_mut() {
        re.append(i);
    }

    re
}

fn gen_all_nums(time: &mut u32, basic: &Vec<i32>) -> Vec<i32> {
    let mut re: Vec<i32> = sum_list(basic, basic);
    while (*time > 2) {
        let temp = re.clone();
        re.append(&mut sum_list(basic, &temp));
        *time -= 1;
        //println!("{:?}", re);
    }
    re
}

fn ps(n: i32) -> i32 {
    let basic_list = make_squares_list(n);

    for i in 2..4 {
        if gen_all_nums(&mut (i as u32), &basic_list).contains(&n) {
            return i;
        }
    }

    0
}

fn main() {
    //println!("{:?}", make_squares_list(5));
    //println!("{:?}", sum_list(&vec![1, 2, 3], &vec![4, 5, 6]));
    //println!("{:?}", gen_all_nums(&mut (2 as u32), &vec![1, 2]));

    println!("{:?}", ps(13)); //=> 2
    println!("{:?}", ps(3)); //=> 3
    println!("{:?}", ps(12)); //=> 3
}
<commit_msg>finish rust version<commit_after>// fn make_squares_list(n: i32) -> Vec<i32> {
//     let mut re: Vec<i32> = vec![];
//     for i in 0..n + 1 {
//         let temp = i * i;
//         if temp > n {
//             break;
//         } else {
//             re.push(temp);
//         }
//     }

//     re
// }

// fn sum_list(l1: &Vec<i32>, l2: &Vec<i32>) -> Vec<i32> {
//     let mut temp: Vec<Vec<i32>> = l1
//         .iter()
//         .map(|a| l2.iter().map(|b| b + a).collect())
//         .collect();

//     let mut re: Vec<i32> = vec![];
//     for i in temp.iter_mut() {
//         re.append(i);
//     }

//     re
// }

// fn gen_all_nums(time: &mut u32, basic: &Vec<i32>) -> Vec<i32> {
//     let mut re: Vec<i32> = sum_list(basic, basic);
//     while (*time > 2) {
//         let temp = re.clone();
//         re.append(&mut sum_list(basic, &temp));
//         *time -= 1;
//         //println!("{:?}", re);
//     }
//     re
// }

// fn ps(n: i32) -> i32 {
//     let basic_list = make_squares_list(n);

//     for i in 2..4 {
//         if gen_all_nums(&mut (i as u32), &basic_list).contains(&n) {
//             return i;
//         }
//     }

//     0
// }

pub fn num_squares(n: i32) -> i32 {
    let mut temp = Vec::new();
    for i in 1..=n {
        if i * i == n {
            return 1;
        }
        temp.push(i * i)
    }

    let one = temp.clone();
    let mut flag = 1;
    let mut cache = Vec::new();

    loop {
        for i in temp {
            for o in &one {
                if i + o == n {
                    return flag + 1;
                } else if i + o < n {
                    cache.push(i + o);
                } else {
                    break;
                }
            }
        }
        temp = cache;
        cache = Vec::new();
        flag += 1;
    }
}

fn main() {
    println!("{:?}", num_squares(13)); //=> 2
    println!("{:?}", num_squares(3)); //=> 3
    println!("{:?}", num_squares(12)); //=> 3
}
<|endoftext|>


<commit_before>use super::helpers::fixtures::{get_language, get_property_sheet, get_property_sheet_json};
use lazy_static::lazy_static;
use std::ffi::CString;
use std::{ptr, slice, str};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{c, highlight, highlight_html, Highlight, HighlightEvent, Properties};

lazy_static! {
    static ref JS_SHEET: PropertySheet<Properties> =
        get_property_sheet("javascript", "highlights.json");
    static ref HTML_SHEET: PropertySheet<Properties> =
        get_property_sheet("html", "highlights.json");
    static ref EJS_SHEET: PropertySheet<Properties> =
        get_property_sheet("embedded-template", "highlights-ejs.json");
    static ref SCOPE_CLASS_STRINGS: Vec<String> = {
        let mut result = Vec::new();
        let mut i = 0;
        while let Some(highlight) = Highlight::from_usize(i) {
            result.push(format!("class={:?}", highlight));
            i += 1;
        }
        result
    };
}

#[test]
fn test_highlighting_injected_html_in_javascript() {
    let source = vec!["const s = html `<div>${a < b}</div>`;"].join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
        &[vec![
            ("const", vec![Highlight::Keyword]),
            (" ", vec![]),
            ("s", vec![Highlight::Variable]),
            (" ", vec![]),
            ("=", vec![Highlight::Operator]),
            (" ", vec![]),
            ("html", vec![Highlight::Function]),
            (" ", vec![]),
            ("`<", vec![Highlight::String]),
            ("div", vec![Highlight::String, Highlight::Tag]),
            (">", vec![Highlight::String]),
            (
                "${",
                vec![
                    Highlight::String,
                    Highlight::Embedded,
                    Highlight::PunctuationSpecial
                ]
            ),
            (
                "a",
                vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
            ),
            (" ", vec![Highlight::String, Highlight::Embedded]),
            (
                "<",
                vec![Highlight::String, Highlight::Embedded, Highlight::Operator]
            ),
            (" ", vec![Highlight::String, Highlight::Embedded]),
            (
                "b",
                vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
            ),
            (
                "}",
                vec![
                    Highlight::String,
                    Highlight::Embedded,
                    Highlight::PunctuationSpecial
                ]
            ),
            ("</", vec![Highlight::String]),
            ("div", vec![Highlight::String, Highlight::Tag]),
            (">`", vec![Highlight::String]),
            (";", vec![Highlight::PunctuationDelimiter]),
        ]]
    );
}

#[test]
fn test_highlighting_injected_javascript_in_html() {
    let source = vec![
        "",
        "  ",
        "",
    ]
    .join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("html"), &HTML_SHEET).unwrap(),
        &[
            vec![("<", vec![]), ("body", vec![Highlight::Tag]), (">", vec![]),],
            vec![
                ("  <", vec![]),
                ("script", vec![Highlight::Tag]),
                (">", vec![]),
            ],
            vec![
                ("    ", vec![]),
                ("const", vec![Highlight::Keyword]),
                (" ", vec![]),
                ("x", vec![Highlight::Variable]),
                (" ", vec![]),
                ("=", vec![Highlight::Operator]),
                (" ", vec![]),
                ("new", vec![Highlight::Keyword]),
                (" ", vec![]),
                ("Thing", vec![Highlight::Constructor]),
                ("(", vec![Highlight::PunctuationBracket]),
                (")", vec![Highlight::PunctuationBracket]),
                (";", vec![Highlight::PunctuationDelimiter]),
            ],
            vec![
                ("  </", vec![]),
                ("script", vec![Highlight::Tag]),
                (">", vec![]),
            ],
            vec![
                ("</", vec![]),
                ("body", vec![Highlight::Tag]),
                (">", vec![]),
            ],
        ]
    );
}

#[test]
fn test_highlighting_multiline_nodes_to_html() {
    let source = vec![
        "const SOMETHING = `",
        "  one ${",
        "    two()",
        "  } three",
        "`",
        "",
    ]
    .join("\n");

    assert_eq!(
        &to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
        &[
            "const SOMETHING = `\n".to_string(),
            "  one ${\n".to_string(),
            "    two()\n".to_string(),
            "  } three\n".to_string(),
            "`\n".to_string(),
        ]
    );
}

#[test]
fn test_highlighting_empty_lines() {
    let source = vec![
        "class A {",
        "",
        "  b(c) {",
        "",
        "    d(e)",
        "",
        "  }",
        "",
        "}",
    ]
    .join("\n");

    assert_eq!(
        &to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
        &[
            "class A {\n".to_string(),
            "\n".to_string(),
            "  b(c) {\n".to_string(),
            "\n".to_string(),
            "    d(e)\n".to_string(),
            "\n".to_string(),
            "  }\n".to_string(),
            "\n".to_string(),
            "}\n".to_string(),
        ]
    );
}

#[test]
fn test_highlighting_ejs() {
    let source = vec!["
<% foo() %>
"].join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("embedded-template"), &EJS_SHEET).unwrap(),
        &[[
            ("<", vec![]),
            ("div", vec![Highlight::Tag]),
            (">", vec![]),
            ("<%", vec![Highlight::Keyword]),
            (" ", vec![]),
            ("foo", vec![Highlight::Function]),
            ("(", vec![Highlight::PunctuationBracket]),
            (")", vec![Highlight::PunctuationBracket]),
            (" ", vec![]),
            ("%>", vec![Highlight::Keyword]),
            ("</", vec![]),
            ("div", vec![Highlight::Tag]),
            (">", vec![])
        ]],
    );
}

#[test]
fn test_highlighting_via_c_api() {
    let js_lang = get_language("javascript");
    let html_lang = get_language("html");
    let js_sheet = get_property_sheet_json("javascript", "highlights.json");
    let js_sheet = c_string(&js_sheet);
    let html_sheet = get_property_sheet_json("html", "highlights.json");
    let html_sheet = c_string(&html_sheet);

    let class_tag = c_string("class=tag");
    let class_function = c_string("class=function");
    let class_string = c_string("class=string");
    let class_keyword = c_string("class=keyword");

    let js_scope_name = c_string("source.js");
    let html_scope_name = c_string("text.html.basic");
    let injection_regex = c_string("^(javascript|js)$");
    let source_code = c_string("");

    let attribute_strings = &mut [ptr::null(); Highlight::Unknown as usize + 1];
    attribute_strings[Highlight::Tag as usize] = class_tag.as_ptr();
    attribute_strings[Highlight::String as usize] = class_string.as_ptr();
    attribute_strings[Highlight::Keyword as usize] = class_keyword.as_ptr();
    attribute_strings[Highlight::Function as usize] = class_function.as_ptr();

    let highlighter = c::ts_highlighter_new(attribute_strings.as_ptr());
    let buffer = c::ts_highlight_buffer_new();

    c::ts_highlighter_add_language(
        highlighter,
        html_scope_name.as_ptr(),
        html_lang,
        html_sheet.as_ptr(),
        ptr::null_mut(),
    );
    c::ts_highlighter_add_language(
        highlighter,
        js_scope_name.as_ptr(),
        js_lang,
        js_sheet.as_ptr(),
        injection_regex.as_ptr(),
    );
    c::ts_highlighter_highlight(
        highlighter,
        html_scope_name.as_ptr(),
        source_code.as_ptr(),
        source_code.as_bytes().len() as u32,
        buffer,
        ptr::null_mut(),
    );

    let output_bytes = c::ts_highlight_buffer_content(buffer);
    let output_line_offsets = c::ts_highlight_buffer_line_offsets(buffer);
    let output_len = c::ts_highlight_buffer_len(buffer);
    let output_line_count = c::ts_highlight_buffer_line_count(buffer);

    let output_bytes = unsafe { slice::from_raw_parts(output_bytes, output_len as usize) };
    let output_line_offsets =
        unsafe { slice::from_raw_parts(output_line_offsets, output_line_count as usize) };

    let mut lines = Vec::new();
    for i in 0..(output_line_count as usize) {
        let line_start = output_line_offsets[i] as usize;
        let line_end = output_line_offsets
            .get(i + 1)
            .map(|x| *x as usize)
            .unwrap_or(output_bytes.len());
        lines.push(str::from_utf8(&output_bytes[line_start..line_end]).unwrap());
    }

    assert_eq!(
        lines,
        vec![
            "<script>",
            "const a = b('c');",
            "c.d();",
            "</script>",
        ]
    );

    c::ts_highlighter_delete(highlighter);
    c::ts_highlight_buffer_delete(buffer);
}

fn c_string(s: &str) -> CString {
    CString::new(s.as_bytes().to_vec()).unwrap()
}

fn test_language_for_injection_string<'a>(
    string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {
    match string {
        "javascript" => Some((get_language("javascript"), &JS_SHEET)),
        "html" => Some((get_language("html"), &HTML_SHEET)),
        _ => None,
    }
}

fn to_html<'a>(
    src: &'a str,
    language: Language,
    property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<String>, String> {
    highlight_html(
        src.as_bytes(),
        language,
        property_sheet,
        &test_language_for_injection_string,
        &|highlight| SCOPE_CLASS_STRINGS[highlight as usize].as_str(),
    )
}

fn to_token_vector<'a>(
    src: &'a str,
    language: Language,
    property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<Vec<(&'a str, Vec<Highlight>)>>, String> {
    let mut lines = Vec::new();
    let mut highlights = Vec::new();
    let mut line = Vec::new();
    for event in highlight(
        src.as_bytes(),
        language,
        property_sheet,
        &test_language_for_injection_string,
    )? {
        match event {
            HighlightEvent::HighlightStart(s) => highlights.push(s),
            HighlightEvent::HighlightEnd => {
                highlights.pop();
            }
            HighlightEvent::Source(s) => {
                for (i, l) in s.lines().enumerate() {
                    if i > 0 {
                        lines.push(line);
                        line = Vec::new();
                    }
                    if l.len() > 0 {
                        line.push((l, highlights.clone()));
                    }
                }
            }
        }
    }
    lines.push(line);
    Ok(lines)
}
<commit_msg>Update JS highlighting unit test<commit_after>use super::helpers::fixtures::{get_language, get_property_sheet, get_property_sheet_json};
use lazy_static::lazy_static;
use std::ffi::CString;
use std::{ptr, slice, str};
use tree_sitter::{Language, PropertySheet};
use tree_sitter_highlight::{c, highlight, highlight_html, Highlight, HighlightEvent, Properties};

lazy_static! {
    static ref JS_SHEET: PropertySheet<Properties> =
        get_property_sheet("javascript", "highlights.json");
    static ref HTML_SHEET: PropertySheet<Properties> =
        get_property_sheet("html", "highlights.json");
    static ref EJS_SHEET: PropertySheet<Properties> =
        get_property_sheet("embedded-template", "highlights-ejs.json");
    static ref SCOPE_CLASS_STRINGS: Vec<String> = {
        let mut result = Vec::new();
        let mut i = 0;
        while let Some(highlight) = Highlight::from_usize(i) {
            result.push(format!("class={:?}", highlight));
            i += 1;
        }
        result
    };
}

#[test]
fn test_highlighting_injected_html_in_javascript() {
    let source = vec!["const s = html `<div>${a < b}</div>`;"].join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("javascript"), &JS_SHEET).unwrap(),
        &[vec![
            ("const", vec![Highlight::Keyword]),
            (" ", vec![]),
            ("s", vec![Highlight::Variable]),
            (" ", vec![]),
            ("=", vec![Highlight::Operator]),
            (" ", vec![]),
            ("html", vec![Highlight::Function]),
            (" ", vec![]),
            ("`<", vec![Highlight::String]),
            ("div", vec![Highlight::String, Highlight::Tag]),
            (">", vec![Highlight::String]),
            (
                "${",
                vec![
                    Highlight::String,
                    Highlight::Embedded,
                    Highlight::PunctuationSpecial
                ]
            ),
            (
                "a",
                vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
            ),
            (" ", vec![Highlight::String, Highlight::Embedded]),
            (
                "<",
                vec![Highlight::String, Highlight::Embedded, Highlight::Operator]
            ),
            (" ", vec![Highlight::String, Highlight::Embedded]),
            (
                "b",
                vec![Highlight::String, Highlight::Embedded, Highlight::Variable]
            ),
            (
                "}",
                vec![
                    Highlight::String,
                    Highlight::Embedded,
                    Highlight::PunctuationSpecial
                ]
            ),
            ("</", vec![Highlight::String]),
            ("div", vec![Highlight::String, Highlight::Tag]),
            (">`", vec![Highlight::String]),
            (";", vec![Highlight::PunctuationDelimiter]),
        ]]
    );
}

#[test]
fn test_highlighting_injected_javascript_in_html() {
    let source = vec![
        "",
        "  ",
        "",
    ]
    .join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("html"), &HTML_SHEET).unwrap(),
        &[
            vec![("<", vec![]), ("body", vec![Highlight::Tag]), (">", vec![]),],
            vec![
                ("  <", vec![]),
                ("script", vec![Highlight::Tag]),
                (">", vec![]),
            ],
            vec![
                ("    ", vec![]),
                ("const", vec![Highlight::Keyword]),
                (" ", vec![]),
                ("x", vec![Highlight::Variable]),
                (" ", vec![]),
                ("=", vec![Highlight::Operator]),
                (" ", vec![]),
                ("new", vec![Highlight::Keyword]),
                (" ", vec![]),
                ("Thing", vec![Highlight::Constructor]),
                ("(", vec![Highlight::PunctuationBracket]),
                (")", vec![Highlight::PunctuationBracket]),
                (";", vec![Highlight::PunctuationDelimiter]),
            ],
            vec![
                ("  </", vec![]),
                ("script", vec![Highlight::Tag]),
                (">", vec![]),
            ],
            vec![
                ("</", vec![]),
                ("body", vec![Highlight::Tag]),
                (">", vec![]),
            ],
        ]
    );
}

#[test]
fn test_highlighting_multiline_nodes_to_html() {
    let source = vec![
        "const SOMETHING = `",
        "  one ${",
        "    two()",
        "  } three",
        "`",
        "",
    ]
    .join("\n");

    assert_eq!(
        &to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
        &[
            "const SOMETHING = `\n".to_string(),
            "  one ${\n".to_string(),
            "    two()\n".to_string(),
            "  } three\n".to_string(),
            "`\n".to_string(),
        ]
    );
}

#[test]
fn test_highlighting_empty_lines() {
    let source = vec![
        "class A {",
        "",
        "  b(c) {",
        "",
        "    d(e)",
        "",
        "  }",
        "",
        "}",
    ]
    .join("\n");

    assert_eq!(
        &to_html(&source, get_language("javascript"), &JS_SHEET,).unwrap(),
        &[
            "class A {\n".to_string(),
            "\n".to_string(),
            "  b(c) {\n".to_string(),
            "\n".to_string(),
            "    d(e)\n".to_string(),
            "\n".to_string(),
            "  }\n".to_string(),
            "\n".to_string(),
            "}\n".to_string(),
        ]
    );
}

#[test]
fn test_highlighting_ejs() {
    let source = vec!["
<% foo() %>
"].join("\n");

    assert_eq!(
        &to_token_vector(&source, get_language("embedded-template"), &EJS_SHEET).unwrap(),
        &[[
            ("<", vec![]),
            ("div", vec![Highlight::Tag]),
            (">", vec![]),
            ("<%", vec![Highlight::Keyword]),
            (" ", vec![]),
            ("foo", vec![Highlight::Function]),
            ("(", vec![Highlight::PunctuationBracket]),
            (")", vec![Highlight::PunctuationBracket]),
            (" ", vec![]),
            ("%>", vec![Highlight::Keyword]),
            ("</", vec![]),
            ("div", vec![Highlight::Tag]),
            (">", vec![])
        ]],
    );
}

#[test]
fn test_highlighting_via_c_api() {
    let js_lang = get_language("javascript");
    let html_lang = get_language("html");
    let js_sheet = get_property_sheet_json("javascript", "highlights.json");
    let js_sheet = c_string(&js_sheet);
    let html_sheet = get_property_sheet_json("html", "highlights.json");
    let html_sheet = c_string(&html_sheet);

    let class_tag = c_string("class=tag");
    let class_function = c_string("class=function");
    let class_string = c_string("class=string");
    let class_keyword = c_string("class=keyword");

    let js_scope_name = c_string("source.js");
    let html_scope_name = c_string("text.html.basic");
    let injection_regex = c_string("^(javascript|js)$");
    let source_code = c_string("");

    let attribute_strings = &mut [ptr::null(); Highlight::Unknown as usize + 1];
    attribute_strings[Highlight::Tag as usize] = class_tag.as_ptr();
    attribute_strings[Highlight::String as usize] = class_string.as_ptr();
    attribute_strings[Highlight::Keyword as usize] = class_keyword.as_ptr();
    attribute_strings[Highlight::Function as usize] = class_function.as_ptr();

    let highlighter = c::ts_highlighter_new(attribute_strings.as_ptr());
    let buffer = c::ts_highlight_buffer_new();

    c::ts_highlighter_add_language(
        highlighter,
        html_scope_name.as_ptr(),
        html_lang,
        html_sheet.as_ptr(),
        ptr::null_mut(),
    );
    c::ts_highlighter_add_language(
        highlighter,
        js_scope_name.as_ptr(),
        js_lang,
        js_sheet.as_ptr(),
        injection_regex.as_ptr(),
    );
    c::ts_highlighter_highlight(
        highlighter,
        html_scope_name.as_ptr(),
        source_code.as_ptr(),
        source_code.as_bytes().len() as u32,
        buffer,
        ptr::null_mut(),
    );

    let output_bytes = c::ts_highlight_buffer_content(buffer);
    let output_line_offsets = c::ts_highlight_buffer_line_offsets(buffer);
    let output_len = c::ts_highlight_buffer_len(buffer);
    let output_line_count = c::ts_highlight_buffer_line_count(buffer);

    let output_bytes = unsafe { slice::from_raw_parts(output_bytes, output_len as usize) };
    let output_line_offsets =
        unsafe { slice::from_raw_parts(output_line_offsets, output_line_count as usize) };

    let mut lines = Vec::new();
    for i in 0..(output_line_count as usize) {
        let line_start = output_line_offsets[i] as usize;
        let line_end = output_line_offsets
            .get(i + 1)
            .map(|x| *x as usize)
            .unwrap_or(output_bytes.len());
        lines.push(str::from_utf8(&output_bytes[line_start..line_end]).unwrap());
    }

    assert_eq!(
        lines,
        vec![
            "<script>",
            "const a = b('c');",
            "c.d();",
            "</script>",
        ]
    );

    c::ts_highlighter_delete(highlighter);
    c::ts_highlight_buffer_delete(buffer);
}

fn c_string(s: &str) -> CString {
    CString::new(s.as_bytes().to_vec()).unwrap()
}

fn test_language_for_injection_string<'a>(
    string: &str,
) -> Option<(Language, &'a PropertySheet<Properties>)> {
    match string {
        "javascript" => Some((get_language("javascript"), &JS_SHEET)),
        "html" => Some((get_language("html"), &HTML_SHEET)),
        _ => None,
    }
}

fn to_html<'a>(
    src: &'a str,
    language: Language,
    property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<String>, String> {
    highlight_html(
        src.as_bytes(),
        language,
        property_sheet,
        &test_language_for_injection_string,
        &|highlight| SCOPE_CLASS_STRINGS[highlight as usize].as_str(),
    )
}

fn to_token_vector<'a>(
    src: &'a str,
    language: Language,
    property_sheet: &'a PropertySheet<Properties>,
) -> Result<Vec<Vec<(&'a str, Vec<Highlight>)>>, String> {
    let mut lines = Vec::new();
    let mut highlights = Vec::new();
    let mut line = Vec::new();
    for event in highlight(
        src.as_bytes(),
        language,
        property_sheet,
        &test_language_for_injection_string,
    )? {
        match event {
            HighlightEvent::HighlightStart(s) => highlights.push(s),
            HighlightEvent::HighlightEnd => {
                highlights.pop();
            }
            HighlightEvent::Source(s) => {
                for (i, l) in s.lines().enumerate() {
                    if i > 0 {
                        lines.push(line);
                        line = Vec::new();
                    }
                    if l.len() > 0 {
                        line.push((l, highlights.clone()));
                    }
                }
            }
        }
    }
    lines.push(line);
    Ok(lines)
}
<|endoftext|>


<commit_before>extern crate rulinalg;

use self::rulinalg::vector::Vector;
use self::rulinalg::matrix::Matrix;
use self::rulinalg::matrix::BaseMatrix;
use self::rulinalg::matrix::BaseMatrixMut;

pub struct Ctrnn {
}

impl Ctrnn {
    pub fn new() -> Ctrnn{
        Ctrnn {}
    }

    pub fn activate(&self, steps: usize, gamma_v: &Vec<f64>, delta_t: f64, tau_v: &Vec<f64>, wij_v: &Vec<Vec<f64>>, theta_v: &Vec<f64>, wik_v: &Vec<Vec<f64>>, i_v: &Vec<f64>) -> Vec<f64> {
        let mut gamma = Ctrnn::matrix_from_vector(gamma_v);
        let mut theta = Ctrnn::matrix_from_vector(theta_v);
        let mut wij = Ctrnn::matrix_from_vectors(wij_v);
        let mut wik = Ctrnn::matrix_from_vectors(wik_v);
        let mut i = Ctrnn::matrix_from_vector(i_v);
        let mut tau = Ctrnn::matrix_from_vector(tau_v);
        let mut result = &gamma + (tau.apply( &(|x| 1.0/x) ) * delta_t).elemul(&( (&wij * ( &gamma - &theta ).apply(&Ctrnn::sigmoid)) - &gamma + (&wik * &i)));
        println!("{:?}", result);
        return vec![0.0, 0.0];
    }

    fn sigmoid(y: f64) -> f64 {
        1f64 / (1f64 + (-y).exp())
    }

    fn matrix_from_vector(vector: &Vec<f64>) -> Matrix<f64> {
        Matrix::new(vector.len(), 1, vector.as_slice())
    }

    fn matrix_from_vectors(vectors: &Vec<Vec<f64>>) -> Matrix<f64> {
        Matrix::new(vectors.len(), vectors[1].len(), vectors.iter().flat_map(|s| s.iter().cloned()).collect::<Vec<f64>>())
    }
}

#[cfg(test)]
mod tests {
    use neat::*;

    #[test]
    fn neural_network_activation_should_return_correct_values(){
        let gamma =         vec![0.0, 0.0, 0.0];
        let delta_t =       13.436;
        let tau =           vec![61.694, 10.149, 16.851];
        let wij =           vec![vec![-2.94737, 2.70665, -0.57046],
                            vec![-3.27553,   3.67193, 1.83218],
                            vec![2.32476,   0.24739, 0.58587]];
        let theta =         vec![-0.695126, -0.677891, -0.072129];
        let wik =           vec![vec![-0.10097, -3.04457],
                            vec![-4.86594, 1.79273],
                            vec![-3.45899, -1.27388]];
        let i =             vec![0.98856, 0.31540];

        let ctrnn = Ctrnn::new();

        assert_eq!( ctrnn.activate(1, &gamma, delta_t, &tau, &wij, &theta, &wik, &i),
            vec![-0.33246, -4.03171, -1.43741]);

        assert_eq!( ctrnn.activate(2, &gamma, delta_t, &tau, &wij, &theta, &wik, &i),
            vec![-0.86998, -6.11057, -2.14179]);
        assert_eq!( ctrnn.activate(10, &gamma, delta_t, &tau, &wij, &theta, &wik, &i),
            vec![-1.7698, -4.9550, -3.1953]);
        assert_eq!( ctrnn.activate(30, &gamma, delta_t, &tau, &wij, &theta, &wik, &i),
            vec![-1.7869, -4.9408, -3.2092]);

        //converges
        assert_eq!( ctrnn.activate(30, &gamma, delta_t, &tau, &wij, &theta, &wik, &i),
            ctrnn.activate(100, &gamma, delta_t, &tau, &wij, &theta, &wik, &i));
    }
}
<commit_msg>Ctrnn passes tests<commit_after>extern crate rulinalg;

use self::rulinalg::matrix::Matrix;
use self::rulinalg::matrix::BaseMatrix;
use self::rulinalg::matrix::BaseMatrixMut;

pub struct Ctrnn {
}

impl Ctrnn {
    pub fn new() -> Ctrnn{
        Ctrnn {}
    }

    pub fn activate(&self, steps: usize, gamma_v: &Vec<f64>, delta_t: f64, tau_v: &Vec<f64>, wij_v: &Vec<Vec<f64>>, theta_v: &Vec<f64>, wik_v: &Vec<Vec<f64>>, i_v: &Vec<f64>) -> Vec<f64> {
        let mut gamma = Ctrnn::matrix_from_vector(gamma_v);
        let theta = Ctrnn::matrix_from_vector(theta_v);
        let wij = Ctrnn::matrix_from_vectors(wij_v);
        let wik = Ctrnn::matrix_from_vectors(wik_v);
        let i = Ctrnn::matrix_from_vector(i_v);
        let tau = Ctrnn::matrix_from_vector(tau_v);
        let delta_t_tau = tau.apply( &(|x| 1.0/x) ) * delta_t;
        for _ in 0..steps {
            gamma = &gamma + delta_t_tau.elemul(&( (&wij * ( &gamma - &theta ).apply(&Ctrnn::sigmoid)) - &gamma + (&wik * &i)));
        }
        return gamma.into_vec();
    }

    fn sigmoid(y: f64) -> f64 {
        1f64 / (1f64 + (-y).exp())
    }

    fn matrix_from_vector(vector: &Vec<f64>) -> Matrix<f64> {
        Matrix::new(vector.len(), 1, vector.as_slice())
    }

    fn matrix_from_vectors(vectors: &Vec<Vec<f64>>) -> Matrix<f64> {
        Matrix::new(vectors.len(), vectors[1].len(), vectors.iter().flat_map(|s| s.iter().cloned()).collect::<Vec<f64>>())
    }
}

#[cfg(test)]
mod tests {
    use neat::*;

    #[test]
    fn neural_network_activation_should_return_correct_values(){
        let gamma =         vec![0.0, 0.0, 0.0];
        let delta_t =       13.436;
        let tau =           vec![61.694, 10.149, 16.851];
        let wij =           vec![vec![-2.94737, 2.70665, -0.57046],
                            vec![-3.27553,   3.67193, 1.83218],
                            vec![2.32476,   0.24739, 0.58587]];
        let theta =         vec![-0.695126, -0.677891, -0.072129];
        let wik =           vec![vec![-0.10097, -3.04457],
                            vec![-4.86594, 1.79273],
                            vec![-3.45899, -1.27388]];
        let i =             vec![0.98856, 0.31540];

        let ctrnn = Ctrnn::new();

        assert_eq!( ctrnn.activate(1, &gamma, delta_t, &tau, &wij, &theta, &wik, &i),
            vec![-0.3324610969203632, -4.031708339680356, -1.4374137183957871]);

        assert_eq!( ctrnn.activate(2, &gamma, delta_t, &tau, &wij, &theta, &wik, &i),
            vec![-0.8747989167571583, -6.213498344386237, -2.143379723316894]);
        assert_eq!( ctrnn.activate(10, &gamma, delta_t, &tau, &wij, &theta, &wik, &i),
            vec![-1.7607615233242453, -4.96208021858224, -3.1877336197565627]);
        assert_eq!( ctrnn.activate(30, &gamma, delta_t, &tau, &wij, &theta, &wik, &i),
            vec![-1.7868613827907782, -4.940744845752775, -3.2091966016693183]);

        //converges
        assert_eq!( ctrnn.activate(100, &gamma, delta_t, &tau, &wij, &theta, &wik, &i),
            vec![-1.7868662273977671, -4.940740909854486, -3.2092005096958243]);
    }
}
<|endoftext|>


<commit_before>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: i64,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    pub fn identifier(&self) -> Option<&String> {
        match self {
            &Token::Identifier { ident: ref ident, .. } => Some(&ident),
            _ => None,
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> i64 {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[-?\d+\]$").unwrap();
        }

        if !has_array_brackets(s) {
            return Ok(Token::Identifier { ident: String::from(s), next: None });
        }

        match RE.captures(s) {
            None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
            Some(captures) => {
                match captures.get(0) {
                    None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                    Some(mtch) => {
                        let mtch = without_array_brackets(mtch.as_str());
                        let i : i64 = FromStr::from_str(&mtch).unwrap(); // save because regex
                        Ok(Token::Index {
                            idx: i,
                            next: None,
                        })
                    }
                }
            }
        }
    }

    /// Check whether a str begins with '[' and ends with ']'
    fn has_array_brackets(s: &str) -> bool {
        s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']'
    }

    /// Remove '[' and ']' from a str
    fn without_array_brackets(s: &str) -> String {
        s.replace("[","").replace("]","")
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                if token.len() == 0 {
                    return Err(Error::from(ErrorKind::EmptyIdentifier));
                }
                let mut token = try!(mk_token_object(token));
                try!(build_token_tree(split, &mut token));
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier().unwrap());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier().unwrap());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier().unwrap());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier().unwrap());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    #[test]
    fn test_tokenize_empty_token_after_good_token() {
        let tokens = tokenize_with_seperator(&String::from("a..b"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    quickcheck! {
        fn test_array_index(i: i64) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<commit_msg>Change: Token::Index holds an usize internally, because arrays are indexed with usize<commit_after>/// The tokenizer for the query interpreter

use error::*;
use std::rc::Rc;

#[derive(Debug, PartialEq, Eq)]
pub enum Token {
    Identifier {
        ident: String,
        next: Option<Rc<Token>>
    },

    Index {
        idx: usize,
        next: Option<Rc<Token>>
    }
}

impl Token {

    pub fn set_next(&mut self, token: Token) {
        match self {
            &mut Token::Identifier { next: ref mut next, .. } => *next = Some(Rc::new(token)),
            &mut Token::Index { next: ref mut next, .. }      => *next = Some(Rc::new(token)),
        }
    }

    pub fn identifier(&self) -> Option<&String> {
        match self {
            &Token::Identifier { ident: ref ident, .. } => Some(&ident),
            _ => None,
        }
    }

    #[cfg(test)]
    pub fn idx(&self) -> usize {
        match self {
            &Token::Index { idx: i, .. } => i,
            _ => unreachable!(),
        }
    }
}

pub fn tokenize_with_seperator(query: &String, seperator: char) -> Result<Token> {
    use std::str::Split;

    /// Creates a Token object from a string
    ///
    /// # Panics
    ///
    /// * If the internal regex does not compile (should never happen)
    /// * If the token is non-valid (that is, a array index with a non-i64)
    /// * If the regex does not find anything
    /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
    ///
    /// # Incorrect behaviour
    ///
    /// * If the regex finds multiple captures
    ///
    /// # Returns
    ///
    /// The `Token` object with the correct identifier/index for this token and no next token.
    ///
    fn mk_token_object(s: &str) -> Result<Token> {
        use regex::Regex;
        use std::str::FromStr;

        lazy_static! {
            static ref RE: Regex = Regex::new(r"^\[-?\d+\]$").unwrap();
        }

        if !has_array_brackets(s) {
            return Ok(Token::Identifier { ident: String::from(s), next: None });
        }

        match RE.captures(s) {
            None => return Err(Error::from(ErrorKind::ArrayAccessWithoutIndex)),
            Some(captures) => {
                match captures.get(0) {
                    None => Ok(Token::Identifier { ident: String::from(s), next: None }),
                    Some(mtch) => {
                        let mtch = without_array_brackets(mtch.as_str());
                        let i : usize = FromStr::from_str(&mtch).unwrap(); // save because regex
                        Ok(Token::Index {
                            idx: i,
                            next: None,
                        })
                    }
                }
            }
        }
    }

    /// Check whether a str begins with '[' and ends with ']'
    fn has_array_brackets(s: &str) -> bool {
        s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']'
    }

    /// Remove '[' and ']' from a str
    fn without_array_brackets(s: &str) -> String {
        s.replace("[","").replace("]","")
    }

    fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
        match split.next() {
            None        => { /* No more tokens */ }
            Some(token) => {
                if token.len() == 0 {
                    return Err(Error::from(ErrorKind::EmptyIdentifier));
                }
                let mut token = try!(mk_token_object(token));
                try!(build_token_tree(split, &mut token));
                last.set_next(token);
            }
        }
        Ok(())
    }

    if query.is_empty() {
        return Err(Error::from(ErrorKind::EmptyQueryError));
    }

    let mut tokens = query.split(seperator);

    match tokens.next() {
        None        => Err(Error::from(ErrorKind::EmptyQueryError)),
        Some(token) => {
            if token.len() == 0 {
                return Err(Error::from(ErrorKind::EmptyIdentifier));
            }
            let mut tok = try!(mk_token_object(token));
            let _       = try!(build_token_tree(&mut tokens, &mut tok));
            Ok(tok)
        }
    }
}

#[cfg(test)]
mod test {
    use error::*;
    use super::*;

    use std::ops::Deref;

    #[test]
    fn test_tokenize_empty_query_to_error() {
        let tokens = tokenize_with_seperator(&String::from(""), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyQueryError { .. }));
    }

    #[test]
    fn test_tokenize_seperator_only() {
        let tokens = tokenize_with_seperator(&String::from("."), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    #[test]
    fn test_tokenize_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_with_array_brackets_only() {
        let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_identifiers_in_array_brackets() {
        let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::ArrayAccessWithoutIndex { .. }));
    }

    #[test]
    fn test_tokenize_single_token_query() {
        let tokens = tokenize_with_seperator(&String::from("example"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier {
                ident: ident,
                next: None
            } => {
                assert_eq!(String::from("example"), ident);
                true
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_double_token_query() {
        let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                assert_eq!("b", next.deref().identifier().unwrap());
                match next.deref() {
                    &Token::Identifier { next: None, .. } => true,
                    _ => false
                }
            },
            _ => false,
        });
        assert_eq!("a", tokens.identifier().unwrap());
    }

    #[test]
    fn test_tokenize_ident_then_array_query() {
        let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier().unwrap());
        assert!(match tokens {
            Token::Identifier { next: Some(ref next), .. } => {
                assert_eq!(1, Rc::strong_count(&next));
                match next.deref() {
                    &Token::Index { idx: 0, next: None } => true,
                    _ => false
                }
            },
            _ => false,
        });
    }

    #[test]
    fn test_tokenize_many_idents_then_array_query() {
        use std::ops::Deref;

        let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
        assert!(tokens.is_ok());
        let tokens = tokens.unwrap();

        assert_eq!("a", tokens.identifier().unwrap());

        let expected =
            Token::Identifier {
                ident: String::from("a"),
                next: Some(Rc::new(Token::Identifier {
                    ident: String::from("b"),
                    next: Some(Rc::new(Token::Identifier {
                        ident: String::from("c"),
                        next: Some(Rc::new(Token::Index {
                            idx: 1000,
                            next: None,
                        })),
                    })),
                })),
            };

        assert_eq!(expected, tokens);
    }

    #[test]
    fn test_tokenize_empty_token_after_good_token() {
        let tokens = tokenize_with_seperator(&String::from("a..b"), '.');
        assert!(tokens.is_err());
        let tokens = tokens.unwrap_err();

        let errkind = tokens.kind();
        assert!(is_match!(errkind, &ErrorKind::EmptyIdentifier { .. }));
    }

    quickcheck! {
        fn test_array_index(i: usize) -> bool {
            match tokenize_with_seperator(&format!("[{}]", i), '.') {
                Ok(Token::Index {
                    idx: i,
                    next: None,
                }) => true,
                _ => false,
            }
        }
    }

}
<|endoftext|>