<commit_before>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15677

use prelude::v1::*;

use cmp;
use fmt;
use intrinsics;
use libc::{self, uintptr_t};
use os;
use slice;
use str;
use sync::atomic::{self, Ordering};

/// Dynamically inquire about whether we're running under V.
/// You should usually not use this unless your test definitely
/// can't run correctly un-altered. Valgrind is there to help
/// you notice weirdness in normal, un-doctored code paths!
pub fn running_on_valgrind() -> bool {
    extern {
        fn rust_running_on_valgrind() -> uintptr_t;
    }
    unsafe { rust_running_on_valgrind() != 0 }
}

/// Valgrind has a fixed-sized array (size around 2000) of segment descriptors
/// wired into it; this is a hard limit and requires rebuilding valgrind if you
/// want to go beyond it. Normally this is not a problem, but in some tests, we
/// produce a lot of threads casually.  Making lots of threads alone might not
/// be a problem _either_, except on OSX, the segments produced for new threads
/// _take a while_ to get reclaimed by the OS. Combined with the fact that libuv
/// schedulers fork off a separate thread for polling fsevents on OSX, we get a
/// perfect storm of creating "too many mappings" for valgrind to handle when
/// running certain stress tests in the runtime.
pub fn limit_thread_creation_due_to_osx_and_valgrind() -> bool {
    (cfg!(target_os="macos")) && running_on_valgrind()
}

pub fn min_stack() -> uint {
    static MIN: atomic::AtomicUint = atomic::ATOMIC_UINT_INIT;
    match MIN.load(Ordering::SeqCst) {
        0 => {}
        n => return n - 1,
    }
    let amt = os::getenv("RUST_MIN_STACK").and_then(|s| s.parse());
    let amt = amt.unwrap_or(2 * 1024 * 1024);
    // 0 is our sentinel value, so ensure that we'll never see 0 after
    // initialization has run
    MIN.store(amt + 1, Ordering::SeqCst);
    return amt;
}

/// Get's the number of scheduler threads requested by the environment
/// either `RUST_THREADS` or `num_cpus`.
pub fn default_sched_threads() -> uint {
    match os::getenv("RUST_THREADS") {
        Some(nstr) => {
            let opt_n: Option<uint> = nstr.parse();
            match opt_n {
                Some(n) if n > 0 => n,
                _ => panic!("`RUST_THREADS` is `{}`, should be a positive integer", nstr)
            }
        }
        None => {
            if limit_thread_creation_due_to_osx_and_valgrind() {
                1
            } else {
                os::num_cpus()
            }
        }
    }
}

// Indicates whether we should perform expensive sanity checks, including rtassert!
//
// FIXME: Once the runtime matures remove the `true` below to turn off rtassert,
//        etc.
pub const ENFORCE_SANITY: bool = true || !cfg!(rtopt) || cfg!(rtdebug) ||
                                  cfg!(rtassert);

#[allow(missing_copy_implementations)]
pub struct Stdio(libc::c_int);

#[allow(non_upper_case_globals)]
pub const Stdout: Stdio = Stdio(libc::STDOUT_FILENO);
#[allow(non_upper_case_globals)]
pub const Stderr: Stdio = Stdio(libc::STDERR_FILENO);

impl Stdio {
    pub fn write_bytes(&mut self, data: &[u8]) {
        #[cfg(unix)]
        type WriteLen = libc::size_t;
        #[cfg(windows)]
        type WriteLen = libc::c_uint;
        unsafe {
            let Stdio(fd) = *self;
            libc::write(fd,
                        data.as_ptr() as *const libc::c_void,
                        data.len() as WriteLen);
        }
    }
}

impl fmt::Writer for Stdio {
    fn write_str(&mut self, data: &str) -> fmt::Result {
        self.write_bytes(data.as_bytes());
        Ok(()) // yes, we're lying
    }
}

pub fn dumb_print(args: fmt::Arguments) {
    let _ = Stderr.write_fmt(args);
}

pub fn abort(args: fmt::Arguments) -> ! {
    use fmt::Writer;

    struct BufWriter<'a> {
        buf: &'a mut [u8],
        pos: uint,
    }
    impl<'a> fmt::Writer for BufWriter<'a> {
        fn write_str(&mut self, bytes: &str) -> fmt::Result {
            let left = self.buf.slice_from_mut(self.pos);
            let to_write = &bytes.as_bytes()[0..cmp::min(bytes.len(), left.len())];
            slice::bytes::copy_memory(left, to_write);
            self.pos += to_write.len();
            Ok(())
        }
    }

    // Convert the arguments into a stack-allocated string
    let mut msg = [0u8; 512];
    let mut w = BufWriter { buf: &mut msg, pos: 0 };
    let _ = write!(&mut w, "{}", args);
    let msg = str::from_utf8(&w.buf[0..w.pos]).unwrap_or("aborted");
    let msg = if msg.is_empty() {"aborted"} else {msg};

    // Give some context to the message
    let hash = msg.bytes().fold(0, |accum, val| accum + (val as uint) );
    let quote = match hash % 10 {
        0 => "
It was from the artists and poets that the pertinent answers came, and I
know that panic would have broken loose had they been able to compare notes.
As it was, lacking their original letters, I half suspected the compiler of
having asked leading questions, or of having edited the correspondence in
corroboration of what he had latently resolved to see.",
        1 => "
There are not many persons who know what wonders are opened to them in the
stories and visions of their youth; for when as children we listen and dream,
we think but half-formed thoughts, and when as men we try to remember, we are
dulled and prosaic with the poison of life. But some of us awake in the night
with strange phantasms of enchanted hills and gardens, of fountains that sing
in the sun, of golden cliffs overhanging murmuring seas, of plains that stretch
down to sleeping cities of bronze and stone, and of shadowy companies of heroes
that ride caparisoned white horses along the edges of thick forests; and then
we know that we have looked back through the ivory gates into that world of
wonder which was ours before we were wise and unhappy.",
        2 => "
Instead of the poems I had hoped for, there came only a shuddering blackness
and ineffable loneliness; and I saw at last a fearful truth which no one had
ever dared to breathe before — the unwhisperable secret of secrets — The fact
that this city of stone and stridor is not a sentient perpetuation of Old New
York as London is of Old London and Paris of Old Paris, but that it is in fact
quite dead, its sprawling body imperfectly embalmed and infested with queer
animate things which have nothing to do with it as it was in life.",
        3 => "
The ocean ate the last of the land and poured into the smoking gulf, thereby
giving up all it had ever conquered. From the new-flooded lands it flowed
again, uncovering death and decay; and from its ancient and immemorial bed it
trickled loathsomely, uncovering nighted secrets of the years when Time was
young and the gods unborn. Above the waves rose weedy remembered spires. The
moon laid pale lilies of light on dead London, and Paris stood up from its damp
grave to be sanctified with star-dust. Then rose spires and monoliths that were
weedy but not remembered; terrible spires and monoliths of lands that men never
knew were lands...",
        4 => "
There was a night when winds from unknown spaces whirled us irresistibly into
limitless vacuum beyond all thought and entity. Perceptions of the most
maddeningly untransmissible sort thronged upon us; perceptions of infinity
which at the time convulsed us with joy, yet which are now partly lost to my
memory and partly incapable of presentation to others.",
        _ => "You've met with a terrible fate, haven't you?"
    };
    rterrln!("{}", "");
    rterrln!("{}", quote);
    rterrln!("{}", "");
    rterrln!("fatal runtime error: {}", msg);
    unsafe { intrinsics::abort(); }
}

pub unsafe fn report_overflow() {
    use thread::Thread;

    // See the message below for why this is not emitted to the
    // ^ Where did the message below go?
    // task's logger. This has the additional conundrum of the
    // logger may not be initialized just yet, meaning that an FFI
    // call would happen to initialized it (calling out to libuv),
    // and the FFI call needs 2MB of stack when we just ran out.

    rterrln!("\nthread '{}' has overflowed its stack",
             Thread::current().name().unwrap_or("<unknown>"));
}
<commit_msg>std: Remove fortune cookies from fatal runtime errors<commit_after>// Copyright 2013 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//
// ignore-lexer-test FIXME #15677

use prelude::v1::*;

use cmp;
use fmt;
use intrinsics;
use libc::{self, uintptr_t};
use os;
use slice;
use str;
use sync::atomic::{self, Ordering};

/// Dynamically inquire about whether we're running under V.
/// You should usually not use this unless your test definitely
/// can't run correctly un-altered. Valgrind is there to help
/// you notice weirdness in normal, un-doctored code paths!
pub fn running_on_valgrind() -> bool {
    extern {
        fn rust_running_on_valgrind() -> uintptr_t;
    }
    unsafe { rust_running_on_valgrind() != 0 }
}

/// Valgrind has a fixed-sized array (size around 2000) of segment descriptors
/// wired into it; this is a hard limit and requires rebuilding valgrind if you
/// want to go beyond it. Normally this is not a problem, but in some tests, we
/// produce a lot of threads casually.  Making lots of threads alone might not
/// be a problem _either_, except on OSX, the segments produced for new threads
/// _take a while_ to get reclaimed by the OS. Combined with the fact that libuv
/// schedulers fork off a separate thread for polling fsevents on OSX, we get a
/// perfect storm of creating "too many mappings" for valgrind to handle when
/// running certain stress tests in the runtime.
pub fn limit_thread_creation_due_to_osx_and_valgrind() -> bool {
    (cfg!(target_os="macos")) && running_on_valgrind()
}

pub fn min_stack() -> uint {
    static MIN: atomic::AtomicUint = atomic::ATOMIC_UINT_INIT;
    match MIN.load(Ordering::SeqCst) {
        0 => {}
        n => return n - 1,
    }
    let amt = os::getenv("RUST_MIN_STACK").and_then(|s| s.parse());
    let amt = amt.unwrap_or(2 * 1024 * 1024);
    // 0 is our sentinel value, so ensure that we'll never see 0 after
    // initialization has run
    MIN.store(amt + 1, Ordering::SeqCst);
    return amt;
}

/// Get's the number of scheduler threads requested by the environment
/// either `RUST_THREADS` or `num_cpus`.
pub fn default_sched_threads() -> uint {
    match os::getenv("RUST_THREADS") {
        Some(nstr) => {
            let opt_n: Option<uint> = nstr.parse();
            match opt_n {
                Some(n) if n > 0 => n,
                _ => panic!("`RUST_THREADS` is `{}`, should be a positive integer", nstr)
            }
        }
        None => {
            if limit_thread_creation_due_to_osx_and_valgrind() {
                1
            } else {
                os::num_cpus()
            }
        }
    }
}

// Indicates whether we should perform expensive sanity checks, including rtassert!
//
// FIXME: Once the runtime matures remove the `true` below to turn off rtassert,
//        etc.
pub const ENFORCE_SANITY: bool = true || !cfg!(rtopt) || cfg!(rtdebug) ||
                                  cfg!(rtassert);

#[allow(missing_copy_implementations)]
pub struct Stdio(libc::c_int);

#[allow(non_upper_case_globals)]
pub const Stdout: Stdio = Stdio(libc::STDOUT_FILENO);
#[allow(non_upper_case_globals)]
pub const Stderr: Stdio = Stdio(libc::STDERR_FILENO);

impl Stdio {
    pub fn write_bytes(&mut self, data: &[u8]) {
        #[cfg(unix)]
        type WriteLen = libc::size_t;
        #[cfg(windows)]
        type WriteLen = libc::c_uint;
        unsafe {
            let Stdio(fd) = *self;
            libc::write(fd,
                        data.as_ptr() as *const libc::c_void,
                        data.len() as WriteLen);
        }
    }
}

impl fmt::Writer for Stdio {
    fn write_str(&mut self, data: &str) -> fmt::Result {
        self.write_bytes(data.as_bytes());
        Ok(()) // yes, we're lying
    }
}

pub fn dumb_print(args: fmt::Arguments) {
    let _ = Stderr.write_fmt(args);
}

pub fn abort(args: fmt::Arguments) -> ! {
    use fmt::Writer;

    struct BufWriter<'a> {
        buf: &'a mut [u8],
        pos: uint,
    }
    impl<'a> fmt::Writer for BufWriter<'a> {
        fn write_str(&mut self, bytes: &str) -> fmt::Result {
            let left = self.buf.slice_from_mut(self.pos);
            let to_write = &bytes.as_bytes()[0..cmp::min(bytes.len(), left.len())];
            slice::bytes::copy_memory(left, to_write);
            self.pos += to_write.len();
            Ok(())
        }
    }

    // Convert the arguments into a stack-allocated string
    let mut msg = [0u8; 512];
    let mut w = BufWriter { buf: &mut msg, pos: 0 };
    let _ = write!(&mut w, "{}", args);
    let msg = str::from_utf8(&w.buf[0..w.pos]).unwrap_or("aborted");
    let msg = if msg.is_empty() {"aborted"} else {msg};
    rterrln!("fatal runtime error: {}", msg);
    unsafe { intrinsics::abort(); }
}

pub unsafe fn report_overflow() {
    use thread::Thread;

    // See the message below for why this is not emitted to the
    // ^ Where did the message below go?
    // task's logger. This has the additional conundrum of the
    // logger may not be initialized just yet, meaning that an FFI
    // call would happen to initialized it (calling out to libuv),
    // and the FFI call needs 2MB of stack when we just ran out.

    rterrln!("\nthread '{}' has overflowed its stack",
             Thread::current().name().unwrap_or("<unknown>"));
}
<|endoftext|>


<commit_before>
mod icu {
    type UBool = u8;
    type UProperty = int;
    type UChar32 = char;

    const TRUE : u8 = 1u8;
    const FALSE : u8 = 1u8;

    const UCHAR_ALPHABETIC : UProperty = 0;
    const UCHAR_BINARY_START : UProperty = 0; // = UCHAR_ALPHABETIC
    const UCHAR_ASCII_HEX_DIGIT : UProperty = 1;
    const UCHAR_BIDI_CONTROL : UProperty = 2;

    const UCHAR_BIDI_MIRRORED : UProperty = 3;
    const UCHAR_DASH : UProperty = 4;
    const UCHAR_DEFAULT_IGNORABLE_CODE_POINT : UProperty = 5;
    const UCHAR_DEPRECATED : UProperty = 6;

    const UCHAR_DIACRITIC : UProperty = 7;
    const UCHAR_EXTENDER : UProperty = 8;
    const UCHAR_FULL_COMPOSITION_EXCLUSION : UProperty = 9;
    const UCHAR_GRAPHEME_BASE : UProperty = 10;

    const UCHAR_GRAPHEME_EXTEND : UProperty = 11;
    const UCHAR_GRAPHEME_LINK : UProperty = 12;
    const UCHAR_HEX_DIGIT : UProperty = 13;
    const UCHAR_HYPHEN : UProperty = 14;

    const UCHAR_ID_CONTINUE : UProperty = 15;
    const UCHAR_ID_START : UProperty = 16;
    const UCHAR_IDEOGRAPHIC : UProperty = 17;
    const UCHAR_IDS_BINARY_OPERATOR : UProperty = 18;

    const UCHAR_IDS_TRINARY_OPERATOR : UProperty = 19;
    const UCHAR_JOIN_CONTROL : UProperty = 20;
    const UCHAR_LOGICAL_ORDER_EXCEPTION : UProperty = 21;
    const UCHAR_LOWERCASE : UProperty = 22;

    const UCHAR_MATH : UProperty = 23;
    const UCHAR_NONCHARACTER_CODE_POINT : UProperty = 24;
    const UCHAR_QUOTATION_MARK : UProperty = 25;
    const UCHAR_RADICAL : UProperty = 26;

    const UCHAR_SOFT_DOTTED : UProperty = 27;
    const UCHAR_TERMINAL_PUNCTUATION : UProperty = 28;
    const UCHAR_UNIFIED_IDEOGRAPH : UProperty = 29;
    const UCHAR_UPPERCASE : UProperty = 30;

    const UCHAR_WHITE_SPACE : UProperty = 31;
    const UCHAR_XID_CONTINUE : UProperty = 32;
    const UCHAR_XID_START : UProperty = 33;
    const UCHAR_CASE_SENSITIVE : UProperty = 34;

    const UCHAR_S_TERM : UProperty = 35;
    const UCHAR_VARIATION_SELECTOR : UProperty = 36;
    const UCHAR_NFD_INERT : UProperty = 37;
    const UCHAR_NFKD_INERT : UProperty = 38;

    const UCHAR_NFC_INERT : UProperty = 39;
    const UCHAR_NFKC_INERT : UProperty = 40;
    const UCHAR_SEGMENT_STARTER : UProperty = 41;
    const UCHAR_PATTERN_SYNTAX : UProperty = 42;

    const UCHAR_PATTERN_WHITE_SPACE : UProperty = 43;
    const UCHAR_POSIX_ALNUM : UProperty = 44;
    const UCHAR_POSIX_BLANK : UProperty = 45;
    const UCHAR_POSIX_GRAPH : UProperty = 46;

    const UCHAR_POSIX_PRINT : UProperty = 47;
    const UCHAR_POSIX_XDIGIT : UProperty = 48;
    const UCHAR_CASED : UProperty = 49;
    const UCHAR_CASE_IGNORABLE : UProperty = 50;

    const UCHAR_CHANGES_WHEN_LOWERCASED : UProperty = 51;
    const UCHAR_CHANGES_WHEN_UPPERCASED : UProperty = 52;
    const UCHAR_CHANGES_WHEN_TITLECASED : UProperty = 53;
    const UCHAR_CHANGES_WHEN_CASEFOLDED : UProperty = 54;

    const UCHAR_CHANGES_WHEN_CASEMAPPED : UProperty = 55;
    const UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED : UProperty = 56;
    const UCHAR_BINARY_LIMIT : UProperty = 57;
    const UCHAR_BIDI_CLASS : UProperty = 0x1000;

    const UCHAR_INT_START : UProperty = 0x1000; // UCHAR_BIDI_CLASS
    const UCHAR_BLOCK : UProperty = 0x1001;
    const UCHAR_CANONICAL_COMBINING_CLASS : UProperty = 0x1002;
    const UCHAR_DECOMPOSITION_TYPE : UProperty = 0x1003;

    const UCHAR_EAST_ASIAN_WIDTH : UProperty = 0x1004;
    const UCHAR_GENERAL_CATEGORY : UProperty = 0x1005;
    const UCHAR_JOINING_GROUP : UProperty = 0x1006;
    const UCHAR_JOINING_TYPE : UProperty = 0x1007;

    const UCHAR_LINE_BREAK : UProperty = 0x1008;
    const UCHAR_NUMERIC_TYPE : UProperty = 0x1009;
    const UCHAR_SCRIPT : UProperty = 0x100A;
    const UCHAR_HANGUL_SYLLABLE_TYPE : UProperty = 0x100B;

    const UCHAR_NFD_QUICK_CHECK : UProperty = 0x100C;
    const UCHAR_NFKD_QUICK_CHECK : UProperty = 0x100D;
    const UCHAR_NFC_QUICK_CHECK : UProperty = 0x100E;
    const UCHAR_NFKC_QUICK_CHECK : UProperty = 0x100F;

    const UCHAR_LEAD_CANONICAL_COMBINING_CLASS : UProperty = 0x1010;
    const UCHAR_TRAIL_CANONICAL_COMBINING_CLASS : UProperty = 0x1011;
    const UCHAR_GRAPHEME_CLUSTER_BREAK : UProperty = 0x1012;
    const UCHAR_SENTENCE_BREAK : UProperty = 0x1013;

    const UCHAR_WORD_BREAK : UProperty = 0x1014;
    const UCHAR_INT_LIMIT : UProperty = 0x1015;

    const UCHAR_GENERAL_CATEGORY_MASK : UProperty = 0x2000;
    const UCHAR_MASK_START : UProperty = 0x2000;
    // = UCHAR_GENERAL_CATEGORY_MASK
    const UCHAR_MASK_LIMIT : UProperty = 0x2001;

    const UCHAR_NUMERIC_VALUE : UProperty = 0x3000;
    const UCHAR_DOUBLE_START : UProperty = 0x3000;
    // = UCHAR_NUMERIC_VALUE
    const UCHAR_DOUBLE_LIMIT : UProperty = 0x3001;

    const UCHAR_AGE : UProperty = 0x4000;
    const UCHAR_STRING_START : UProperty = 0x4000; // = UCHAR_AGE
    const UCHAR_BIDI_MIRRORING_GLYPH : UProperty = 0x4001;
    const UCHAR_CASE_FOLDING : UProperty = 0x4002;

    const UCHAR_ISO_COMMENT : UProperty = 0x4003;
    const UCHAR_LOWERCASE_MAPPING : UProperty = 0x4004;
    const UCHAR_NAME : UProperty = 0x4005;
    const UCHAR_SIMPLE_CASE_FOLDING : UProperty = 0x4006;

    const UCHAR_SIMPLE_LOWERCASE_MAPPING : UProperty = 0x4007;
    const UCHAR_SIMPLE_TITLECASE_MAPPING : UProperty = 0x4008;
    const UCHAR_SIMPLE_UPPERCASE_MAPPING : UProperty = 0x4009;
    const UCHAR_TITLECASE_MAPPING : UProperty = 0x400A;

    const UCHAR_UNICODE_1_NAME : UProperty = 0x400B;
    const UCHAR_UPPERCASE_MAPPING : UProperty = 0x400C;
    const UCHAR_STRING_LIMIT : UProperty = 0x400D;

    const UCHAR_SCRIPT_EXTENSIONS : UProperty = 0x7000;
    const UCHAR_OTHER_PROPERTY_START : UProperty = 0x7000;
    // = UCHAR_SCRIPT_EXTENSIONS;
    const UCHAR_OTHER_PROPERTY_LIMIT : UProperty = 0x7001;

    const UCHAR_INVALID_CODE : UProperty = -1;

    #[link_name = "icuuc"]
    #[abi = "cdecl"]
    extern mod libicu {
        pure fn u_hasBinaryProperty(c: UChar32, which: UProperty) -> UBool;
        pure fn u_isdigit(c: UChar32) -> UBool;
        pure fn u_islower(c: UChar32) -> UBool;
        pure fn u_isspace(c: UChar32) -> UBool;
        pure fn u_isupper(c: UChar32) -> UBool;
        pure fn u_tolower(c: UChar32) -> UChar32;
        pure fn u_toupper(c: UChar32) -> UChar32;
    }
}

pure fn is_XID_start(c: char) -> bool {
    return icu::libicu::u_hasBinaryProperty(c, icu::UCHAR_XID_START)
        == icu::TRUE;
}

pure fn is_XID_continue(c: char) -> bool {
    return icu::libicu::u_hasBinaryProperty(c, icu::UCHAR_XID_START)
        == icu::TRUE;
}

/*
Function: is_digit

Returns true if a character is a digit.
*/
pure fn is_digit(c: char) -> bool {
    return icu::libicu::u_isdigit(c) == icu::TRUE;
}

/*
Function: is_lower

Returns true if a character is a lowercase letter.
*/
pure fn is_lower(c: char) -> bool {
    return icu::libicu::u_islower(c) == icu::TRUE;
}

/*
Function: is_space

Returns true if a character is space.
*/
pure fn is_space(c: char) -> bool {
    return icu::libicu::u_isspace(c) == icu::TRUE;
}

/*
Function: is_upper

Returns true if a character is an uppercase letter.
*/
pure fn is_upper(c: char) -> bool {
    return icu::libicu::u_isupper(c) == icu::TRUE;
}

#[cfg(test)]
mod tests {

    #[test]
    fn test_is_digit() {
        assert (unicode::icu::is_digit('0'));
        assert (!unicode::icu::is_digit('m'));
    }

    #[test]
    fn test_is_lower() {
        assert (unicode::icu::is_lower('m'));
        assert (!unicode::icu::is_lower('M'));
    }

    #[test]
    fn test_is_space() {
        assert (unicode::icu::is_space(' '));
        assert (!unicode::icu::is_space('m'));
    }

    #[test]
    fn test_is_upper() {
        assert (unicode::icu::is_upper('M'));
        assert (!unicode::icu::is_upper('m'));
    }
}<commit_msg>Confirm demode of unicode.rs<commit_after>#[forbid(deprecated_mode)];
#[forbid(deprecated_pattern)];

mod icu {
    type UBool = u8;
    type UProperty = int;
    type UChar32 = char;

    const TRUE : u8 = 1u8;
    const FALSE : u8 = 1u8;

    const UCHAR_ALPHABETIC : UProperty = 0;
    const UCHAR_BINARY_START : UProperty = 0; // = UCHAR_ALPHABETIC
    const UCHAR_ASCII_HEX_DIGIT : UProperty = 1;
    const UCHAR_BIDI_CONTROL : UProperty = 2;

    const UCHAR_BIDI_MIRRORED : UProperty = 3;
    const UCHAR_DASH : UProperty = 4;
    const UCHAR_DEFAULT_IGNORABLE_CODE_POINT : UProperty = 5;
    const UCHAR_DEPRECATED : UProperty = 6;

    const UCHAR_DIACRITIC : UProperty = 7;
    const UCHAR_EXTENDER : UProperty = 8;
    const UCHAR_FULL_COMPOSITION_EXCLUSION : UProperty = 9;
    const UCHAR_GRAPHEME_BASE : UProperty = 10;

    const UCHAR_GRAPHEME_EXTEND : UProperty = 11;
    const UCHAR_GRAPHEME_LINK : UProperty = 12;
    const UCHAR_HEX_DIGIT : UProperty = 13;
    const UCHAR_HYPHEN : UProperty = 14;

    const UCHAR_ID_CONTINUE : UProperty = 15;
    const UCHAR_ID_START : UProperty = 16;
    const UCHAR_IDEOGRAPHIC : UProperty = 17;
    const UCHAR_IDS_BINARY_OPERATOR : UProperty = 18;

    const UCHAR_IDS_TRINARY_OPERATOR : UProperty = 19;
    const UCHAR_JOIN_CONTROL : UProperty = 20;
    const UCHAR_LOGICAL_ORDER_EXCEPTION : UProperty = 21;
    const UCHAR_LOWERCASE : UProperty = 22;

    const UCHAR_MATH : UProperty = 23;
    const UCHAR_NONCHARACTER_CODE_POINT : UProperty = 24;
    const UCHAR_QUOTATION_MARK : UProperty = 25;
    const UCHAR_RADICAL : UProperty = 26;

    const UCHAR_SOFT_DOTTED : UProperty = 27;
    const UCHAR_TERMINAL_PUNCTUATION : UProperty = 28;
    const UCHAR_UNIFIED_IDEOGRAPH : UProperty = 29;
    const UCHAR_UPPERCASE : UProperty = 30;

    const UCHAR_WHITE_SPACE : UProperty = 31;
    const UCHAR_XID_CONTINUE : UProperty = 32;
    const UCHAR_XID_START : UProperty = 33;
    const UCHAR_CASE_SENSITIVE : UProperty = 34;

    const UCHAR_S_TERM : UProperty = 35;
    const UCHAR_VARIATION_SELECTOR : UProperty = 36;
    const UCHAR_NFD_INERT : UProperty = 37;
    const UCHAR_NFKD_INERT : UProperty = 38;

    const UCHAR_NFC_INERT : UProperty = 39;
    const UCHAR_NFKC_INERT : UProperty = 40;
    const UCHAR_SEGMENT_STARTER : UProperty = 41;
    const UCHAR_PATTERN_SYNTAX : UProperty = 42;

    const UCHAR_PATTERN_WHITE_SPACE : UProperty = 43;
    const UCHAR_POSIX_ALNUM : UProperty = 44;
    const UCHAR_POSIX_BLANK : UProperty = 45;
    const UCHAR_POSIX_GRAPH : UProperty = 46;

    const UCHAR_POSIX_PRINT : UProperty = 47;
    const UCHAR_POSIX_XDIGIT : UProperty = 48;
    const UCHAR_CASED : UProperty = 49;
    const UCHAR_CASE_IGNORABLE : UProperty = 50;

    const UCHAR_CHANGES_WHEN_LOWERCASED : UProperty = 51;
    const UCHAR_CHANGES_WHEN_UPPERCASED : UProperty = 52;
    const UCHAR_CHANGES_WHEN_TITLECASED : UProperty = 53;
    const UCHAR_CHANGES_WHEN_CASEFOLDED : UProperty = 54;

    const UCHAR_CHANGES_WHEN_CASEMAPPED : UProperty = 55;
    const UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED : UProperty = 56;
    const UCHAR_BINARY_LIMIT : UProperty = 57;
    const UCHAR_BIDI_CLASS : UProperty = 0x1000;

    const UCHAR_INT_START : UProperty = 0x1000; // UCHAR_BIDI_CLASS
    const UCHAR_BLOCK : UProperty = 0x1001;
    const UCHAR_CANONICAL_COMBINING_CLASS : UProperty = 0x1002;
    const UCHAR_DECOMPOSITION_TYPE : UProperty = 0x1003;

    const UCHAR_EAST_ASIAN_WIDTH : UProperty = 0x1004;
    const UCHAR_GENERAL_CATEGORY : UProperty = 0x1005;
    const UCHAR_JOINING_GROUP : UProperty = 0x1006;
    const UCHAR_JOINING_TYPE : UProperty = 0x1007;

    const UCHAR_LINE_BREAK : UProperty = 0x1008;
    const UCHAR_NUMERIC_TYPE : UProperty = 0x1009;
    const UCHAR_SCRIPT : UProperty = 0x100A;
    const UCHAR_HANGUL_SYLLABLE_TYPE : UProperty = 0x100B;

    const UCHAR_NFD_QUICK_CHECK : UProperty = 0x100C;
    const UCHAR_NFKD_QUICK_CHECK : UProperty = 0x100D;
    const UCHAR_NFC_QUICK_CHECK : UProperty = 0x100E;
    const UCHAR_NFKC_QUICK_CHECK : UProperty = 0x100F;

    const UCHAR_LEAD_CANONICAL_COMBINING_CLASS : UProperty = 0x1010;
    const UCHAR_TRAIL_CANONICAL_COMBINING_CLASS : UProperty = 0x1011;
    const UCHAR_GRAPHEME_CLUSTER_BREAK : UProperty = 0x1012;
    const UCHAR_SENTENCE_BREAK : UProperty = 0x1013;

    const UCHAR_WORD_BREAK : UProperty = 0x1014;
    const UCHAR_INT_LIMIT : UProperty = 0x1015;

    const UCHAR_GENERAL_CATEGORY_MASK : UProperty = 0x2000;
    const UCHAR_MASK_START : UProperty = 0x2000;
    // = UCHAR_GENERAL_CATEGORY_MASK
    const UCHAR_MASK_LIMIT : UProperty = 0x2001;

    const UCHAR_NUMERIC_VALUE : UProperty = 0x3000;
    const UCHAR_DOUBLE_START : UProperty = 0x3000;
    // = UCHAR_NUMERIC_VALUE
    const UCHAR_DOUBLE_LIMIT : UProperty = 0x3001;

    const UCHAR_AGE : UProperty = 0x4000;
    const UCHAR_STRING_START : UProperty = 0x4000; // = UCHAR_AGE
    const UCHAR_BIDI_MIRRORING_GLYPH : UProperty = 0x4001;
    const UCHAR_CASE_FOLDING : UProperty = 0x4002;

    const UCHAR_ISO_COMMENT : UProperty = 0x4003;
    const UCHAR_LOWERCASE_MAPPING : UProperty = 0x4004;
    const UCHAR_NAME : UProperty = 0x4005;
    const UCHAR_SIMPLE_CASE_FOLDING : UProperty = 0x4006;

    const UCHAR_SIMPLE_LOWERCASE_MAPPING : UProperty = 0x4007;
    const UCHAR_SIMPLE_TITLECASE_MAPPING : UProperty = 0x4008;
    const UCHAR_SIMPLE_UPPERCASE_MAPPING : UProperty = 0x4009;
    const UCHAR_TITLECASE_MAPPING : UProperty = 0x400A;

    const UCHAR_UNICODE_1_NAME : UProperty = 0x400B;
    const UCHAR_UPPERCASE_MAPPING : UProperty = 0x400C;
    const UCHAR_STRING_LIMIT : UProperty = 0x400D;

    const UCHAR_SCRIPT_EXTENSIONS : UProperty = 0x7000;
    const UCHAR_OTHER_PROPERTY_START : UProperty = 0x7000;
    // = UCHAR_SCRIPT_EXTENSIONS;
    const UCHAR_OTHER_PROPERTY_LIMIT : UProperty = 0x7001;

    const UCHAR_INVALID_CODE : UProperty = -1;

    #[link_name = "icuuc"]
    #[abi = "cdecl"]
    extern mod libicu {
        pure fn u_hasBinaryProperty(c: UChar32, which: UProperty) -> UBool;
        pure fn u_isdigit(c: UChar32) -> UBool;
        pure fn u_islower(c: UChar32) -> UBool;
        pure fn u_isspace(c: UChar32) -> UBool;
        pure fn u_isupper(c: UChar32) -> UBool;
        pure fn u_tolower(c: UChar32) -> UChar32;
        pure fn u_toupper(c: UChar32) -> UChar32;
    }
}

pure fn is_XID_start(c: char) -> bool {
    return icu::libicu::u_hasBinaryProperty(c, icu::UCHAR_XID_START)
        == icu::TRUE;
}

pure fn is_XID_continue(c: char) -> bool {
    return icu::libicu::u_hasBinaryProperty(c, icu::UCHAR_XID_START)
        == icu::TRUE;
}

/*
Function: is_digit

Returns true if a character is a digit.
*/
pure fn is_digit(c: char) -> bool {
    return icu::libicu::u_isdigit(c) == icu::TRUE;
}

/*
Function: is_lower

Returns true if a character is a lowercase letter.
*/
pure fn is_lower(c: char) -> bool {
    return icu::libicu::u_islower(c) == icu::TRUE;
}

/*
Function: is_space

Returns true if a character is space.
*/
pure fn is_space(c: char) -> bool {
    return icu::libicu::u_isspace(c) == icu::TRUE;
}

/*
Function: is_upper

Returns true if a character is an uppercase letter.
*/
pure fn is_upper(c: char) -> bool {
    return icu::libicu::u_isupper(c) == icu::TRUE;
}

#[cfg(test)]
mod tests {

    #[test]
    fn test_is_digit() {
        assert (unicode::icu::is_digit('0'));
        assert (!unicode::icu::is_digit('m'));
    }

    #[test]
    fn test_is_lower() {
        assert (unicode::icu::is_lower('m'));
        assert (!unicode::icu::is_lower('M'));
    }

    #[test]
    fn test_is_space() {
        assert (unicode::icu::is_space(' '));
        assert (!unicode::icu::is_space('m'));
    }

    #[test]
    fn test_is_upper() {
        assert (unicode::icu::is_upper('M'));
        assert (!unicode::icu::is_upper('m'));
    }
}
<|endoftext|>


<commit_before>use std::io::{Read};
use byteorder::{LittleEndian as E, ReadBytesExt};
use gfx;
use gfx::format::I8Norm;
use render::ObjectVertex;

const SCALE: f32 = 1.0 / 256.0;

pub struct Mesh<R: gfx::Resources> {
    pub slice: gfx::Slice<R>,
    pub buffer: gfx::handle::Buffer<R, ObjectVertex>,
}

pub fn load_c3d<I, R, F>(source: &mut I, factory: &mut F) -> Mesh<R> where
    I: ReadBytesExt,
    R: gfx::Resources,
    F: gfx::traits::FactoryExt<R>,
{
    let version = source.read_u32::<E>().unwrap();
    assert_eq!(version, 8);
    let num_positions = source.read_u32::<E>().unwrap();
    let num_normals   = source.read_u32::<E>().unwrap();
    let num_polygons  = source.read_u32::<E>().unwrap();
    let _total_verts  = source.read_u32::<E>().unwrap();

    let coord_max  = [
        source.read_u32::<E>().unwrap() as f32 * SCALE,
        source.read_u32::<E>().unwrap() as f32 * SCALE,
        source.read_u32::<E>().unwrap() as f32 * SCALE,
    ];
    let coord_min  = [
        source.read_u32::<E>().unwrap() as f32 * SCALE,
        source.read_u32::<E>().unwrap() as f32 * SCALE,
        source.read_u32::<E>().unwrap() as f32 * SCALE,
    ];
    let _parent_off = [
        source.read_u32::<E>().unwrap() as f32 * SCALE,
        source.read_u32::<E>().unwrap() as f32 * SCALE,
        source.read_u32::<E>().unwrap() as f32 * SCALE,
    ];
    let _max_radius = source.read_u32::<E>().unwrap();
    let _parent_rot = [source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap()];
    let _extras    = [source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap()];

    info!("\tReading {} positions...", num_positions);
    let mut positions = Vec::with_capacity(num_positions as usize);
    for _ in 0 .. num_positions {
        for _ in 0..3 {
            source.read_u32::<E>().unwrap(); //unknown
        }
        let mut pos = [0u8; 3];
        source.read(&mut pos).unwrap();
        let _sort_info = source.read_u32::<E>().unwrap();
        positions.push(pos);
    }

    info!("\tReading {} normals...", num_normals);
    let mut normals = Vec::with_capacity(num_normals as usize);
    for _ in 0 .. num_normals {
        let mut norm = [0u8; 4];
        source.read(&mut norm).unwrap();
        let _sort_info = source.read_u32::<E>().unwrap();
        normals.push(norm);
    }

    info!("\tReading {} polygons...", num_polygons);
    let mut vertices = Vec::with_capacity(num_polygons as usize * 3);
    for _ in 0 .. num_polygons {
        let num_corners = source.read_u32::<E>().unwrap();
        assert_eq!(num_corners, 3);
        let _sort_info = source.read_u32::<E>().unwrap();
        let color = [source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap()];
        let mut dummy = [0, 8];
        source.read(&mut dummy[..8]).unwrap(); //skip flat normal
        source.read(&mut dummy[..3]).unwrap(); //skip middle point
        for _ in 0..3 {
            let pid = source.read_u32::<E>().unwrap();
            let nid = source.read_u32::<E>().unwrap();
            let v = (positions[pid as usize], normals[nid as usize], color);
            vertices.push(v);
        }
    }

    info!("\tCompacting...");
    vertices.sort();
    vertices.dedup();

    let gpu_verts: Vec<_> = vertices.into_iter().map(|(p, n, c)| ObjectVertex {
        pos: [
            coord_min[0] + p[0] as f32 * (coord_max[0] - coord_min[0]) / 255.0,
            coord_min[1] + p[1] as f32 * (coord_max[1] - coord_min[1]) / 255.0,
            coord_min[2] + p[2] as f32 * (coord_max[2] - coord_min[2]) / 255.0,
            1.0],
        color: c,
        normal: [I8Norm(n[0] as i8), I8Norm(n[1] as i8), I8Norm(n[2] as i8), I8Norm(n[3] as i8)],
    }).collect();
    info!("\tGot {} GPU vertices...", gpu_verts.len());
    let (vbuf, slice) = factory.create_vertex_buffer_with_slice(&gpu_verts, ());

    Mesh {
        slice: slice,
        buffer: vbuf,
    }
}
<commit_msg>Basic M3D loading<commit_after>use std::io::{Read};
use byteorder::{LittleEndian as E, ReadBytesExt};
use gfx;
use gfx::format::I8Norm;
use render::ObjectVertex;

const SCALE: f32 = 1.0 / 256.0;

pub struct Mesh<R: gfx::Resources> {
    pub slice: gfx::Slice<R>,
    pub buffer: gfx::handle::Buffer<R, ObjectVertex>,
}

pub struct Wheel<R: gfx::Resources> {
    pub mesh: Option<Mesh<R>>,
    pub steering: u32,
    pub width: f32,
    pub radius: f32,
}

pub struct Debrie<R: gfx::Resources> {
    pub mesh: Mesh<R>,
}

pub struct Model<R: gfx::Resources> {
    pub body: Mesh<R>,
    pub color: [u32; 2],
    pub wheels: Vec<Wheel<R>>,
    pub debris: Vec<Debrie<R>>,
}

pub fn load_c3d<I, R, F>(source: &mut I, factory: &mut F) -> Mesh<R> where
    I: ReadBytesExt,
    R: gfx::Resources,
    F: gfx::traits::FactoryExt<R>,
{
    let version = source.read_u32::<E>().unwrap();
    assert_eq!(version, 8);
    let num_positions = source.read_u32::<E>().unwrap();
    let num_normals   = source.read_u32::<E>().unwrap();
    let num_polygons  = source.read_u32::<E>().unwrap();
    let _total_verts  = source.read_u32::<E>().unwrap();

    let coord_max  = [
        source.read_u32::<E>().unwrap() as f32 * SCALE,
        source.read_u32::<E>().unwrap() as f32 * SCALE,
        source.read_u32::<E>().unwrap() as f32 * SCALE,
    ];
    let coord_min  = [
        source.read_u32::<E>().unwrap() as f32 * SCALE,
        source.read_u32::<E>().unwrap() as f32 * SCALE,
        source.read_u32::<E>().unwrap() as f32 * SCALE,
    ];
    let _parent_off = [
        source.read_u32::<E>().unwrap(),
        source.read_u32::<E>().unwrap(),
        source.read_u32::<E>().unwrap(),
    ];
    let _max_radius = source.read_u32::<E>().unwrap();
    let _parent_rot = [source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap()];
    let _extras    = [source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap()];

    info!("\tReading {} positions...", num_positions);
    let mut positions = Vec::with_capacity(num_positions as usize);
    for _ in 0 .. num_positions {
        for _ in 0..3 {
            source.read_u32::<E>().unwrap(); //unknown
        }
        let mut pos = [0u8; 3];
        source.read(&mut pos).unwrap();
        let _sort_info = source.read_u32::<E>().unwrap();
        positions.push(pos);
    }

    info!("\tReading {} normals...", num_normals);
    let mut normals = Vec::with_capacity(num_normals as usize);
    for _ in 0 .. num_normals {
        let mut norm = [0u8; 4];
        source.read(&mut norm).unwrap();
        let _sort_info = source.read_u32::<E>().unwrap();
        normals.push(norm);
    }

    info!("\tReading {} polygons...", num_polygons);
    let mut vertices = Vec::with_capacity(num_polygons as usize * 3);
    for _ in 0 .. num_polygons {
        let num_corners = source.read_u32::<E>().unwrap();
        assert_eq!(num_corners, 3);
        let _sort_info = source.read_u32::<E>().unwrap();
        let color = [source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap()];
        let mut dummy = [0, 8];
        source.read(&mut dummy[..8]).unwrap(); //skip flat normal
        source.read(&mut dummy[..3]).unwrap(); //skip middle point
        for _ in 0..3 {
            let pid = source.read_u32::<E>().unwrap();
            let nid = source.read_u32::<E>().unwrap();
            let v = (positions[pid as usize], normals[nid as usize], color);
            vertices.push(v);
        }
    }

    info!("\tCompacting...");
    vertices.sort();
    vertices.dedup();

    let gpu_verts: Vec<_> = vertices.into_iter().map(|(p, n, c)| ObjectVertex {
        pos: [
            coord_min[0] + p[0] as f32 * (coord_max[0] - coord_min[0]) / 255.0,
            coord_min[1] + p[1] as f32 * (coord_max[1] - coord_min[1]) / 255.0,
            coord_min[2] + p[2] as f32 * (coord_max[2] - coord_min[2]) / 255.0,
            1.0],
        color: c,
        normal: [I8Norm(n[0] as i8), I8Norm(n[1] as i8), I8Norm(n[2] as i8), I8Norm(n[3] as i8)],
    }).collect();
    info!("\tGot {} GPU vertices...", gpu_verts.len());
    let (vbuf, slice) = factory.create_vertex_buffer_with_slice(&gpu_verts, ());

    Mesh {
        slice: slice,
        buffer: vbuf,
    }
}

fn load_m3d<I, R, F>(source: &mut I, factory: &mut F) -> Model<R> where
    I: ReadBytesExt,
    R: gfx::Resources,
    F: gfx::traits::FactoryExt<R>,
{
    info!("\tReading the body...");
    let mut model = Model {
        body: load_c3d(source, factory),
        color: [0, 0],
        wheels: Vec::new(),
        debris: Vec::new(),
    };
    let _bounds = [
        source.read_u32::<E>().unwrap(),
        source.read_u32::<E>().unwrap(),
        source.read_u32::<E>().unwrap(),
    ];
    let _max_radius = source.read_u32::<E>().unwrap();
    let num_wheels = source.read_u32::<E>().unwrap();
    let num_debris = source.read_u32::<E>().unwrap();
    model.color = [source.read_u32::<E>().unwrap(), source.read_u32::<E>().unwrap()];
    model.wheels.reserve_exact(num_wheels as usize);
    model.debris.reserve_exact(num_debris as usize);

    info!("\tReading wheels...");
    for _ in 0 .. num_wheels {
        let steering = source.read_u32::<E>().unwrap();
        source.by_ref().bytes().nth(3*8).unwrap().unwrap();
        let width = source.read_u32::<E>().unwrap() as f32 * SCALE;
        let radius = source.read_u32::<E>().unwrap() as f32 * SCALE;
        let _bound_index = source.read_u32::<E>().unwrap();
        model.wheels.push(Wheel {
            mesh: if steering != 0 {
                Some(load_c3d(source, factory))
            } else {None},
            steering: steering,
            width: width,
            radius: radius,
        })
    }

    info!("\tReading debris...");
    for _ in 0 .. num_debris {
        let mesh = load_c3d(source, factory);
        let _phys_bound = load_c3d(source, factory);
        model.debris.push(Debrie {
            mesh: mesh,
        })
    }

    //let _phys_bound = load_c3d(source, factory);
    model
}
<|endoftext|>


<commit_before>use matching::fragment;
use matching::fragment::Fragment;
use std::path::PathBuf;
use std::collections::hash_map::HashMap;

#[derive(PartialEq)]
pub struct Entry {
    pub path: PathBuf,
    pub index: HashMap<char, Vec<usize>>,
}

impl Entry {
    /// Compares the query string to the entry,
    /// and returns a score between 0 and 1.
    pub fn similarity(&self, query: &str) -> f32 {
        // Exact matches produce a perfect score.
        if query == self.path.to_string_lossy() {
            return 1.0;
        }

        // Pre-calculate the path length as we'll be using it frequently.
        let path_length = self.path.to_string_lossy().chars().count();

        // We track fragment/substring matches, which have a greater weight than
        // a simple sum of individual character occurrences in the entry path.
        let mut match_fragments: Vec<Fragment> = Vec::new();

        // This counter is used to track characters in the query that aren't in
        // the entry's path, which has discrete weighting in the final score.
        let mut non_existent_char_count = 0;

        // Look for the query's character in the path's index.
        for query_char in query.chars() {
            match self.index.get(&query_char) {
                Some(occurrences) => {
                    // Initially, we'll assume that none of the occurrences
                    // of this character have been tracked as fragments.
                    let mut unaccounted_occurrences = occurrences.clone();

                    // Extend any existing fragment matches that also
                    // have this char at the end of their existing match.
                    for fragment in match_fragments.iter_mut() {
                        // Get the index at which this character would have to
                        // occur to extend this particular fragment over it.
                        let target_index = fragment.next_index();

                        if occurrences.contains(&target_index) {
                            // Since this character match is covered by this
                            // fragment, remove it from the unaccounted set.
                            match unaccounted_occurrences.iter().position(|&o| o == target_index) {
                                Some(position) => {
                                    unaccounted_occurrences.remove(position);
                                },
                                None => (),
                            }

                            // Bump the fragment's length to cover this char.
                            fragment.increase_length();
                        }
                    }

                    // Create fragment matches for any unaccounted occurrences.
                    for occurrence_index in unaccounted_occurrences.iter() {
                        match_fragments.push(fragment::new(*occurrence_index));
                    }

                },
                None => {
                    // Characters in the query string that
                    // aren't in the path reduce the score;
                    // track these so we can consider them later.
                    non_existent_char_count += 1;
                },
            }
        }

        // Determine the percentage of characters in the query string that
        // are in the entry, using the non-existent count we've calculated.
        let non_existence_penalty =
            // Guard against a potential arithmetic overflow here.
            if non_existent_char_count >= path_length {
                return 0.0f32
            } else {
                (path_length - non_existent_char_count) as f32 / path_length as f32
            };

        // Calculate an exponentially-scaled score based on fragment lengths.
        let fragment_score = match_fragments.iter().fold(0, |acc, ref fragment| {
            acc + fragment.length.pow(2)
        });

        // Calculate and return the similarity value. The path_length division
        // is used to offset the increased fragment score probability for
        // larger entry paths.
        fragment_score as f32 * non_existence_penalty / path_length as f32
    }
}

fn index_path(path: &str) -> HashMap<char, Vec<usize>> {
    let mut index: HashMap<char, Vec<usize>> = HashMap::new();
    for (char_index, path_char) in path.chars().enumerate() {
        if index.contains_key(&path_char) {
            match index.get_mut(&path_char) {
               Some(occurrences) => occurrences.push(char_index),
               None => ()
            }
        } else {
           index.insert(path_char, vec![char_index]);
        }
    }

    index
}

pub fn new(path: String) -> Entry {
    Entry{
        // Build the index before we transfer ownership of path.
        index: index_path(&path),
        path: PathBuf::from(path),
    }
}

#[cfg(test)]
mod tests {
    use super::new;

    #[test]
    fn similarity_correctly_scores_perfect_matches() {
        let entry = new("src/hound.rs".to_string());
        assert_eq!(entry.similarity("src/hound.rs"), 1.0);
    }

    #[test]
    fn similarity_correctly_scores_completely_different_terms() {
        let entry = new("src".to_string());
        assert_eq!(entry.similarity("lib"), 0.0);
    }

    #[test]
    fn similarity_scores_based_on_term_length() {
        let long_entry = new("hound library".to_string());
        let differing_length_score = long_entry.similarity("houn");

        // Don't use a perfect match, since those product a perfect score.
        let short_entry = new("hound".to_string());
        let same_length_score = short_entry.similarity("houn");

        assert!(same_length_score > differing_length_score);
    }

    #[test]
    fn similarity_score_increases_for_consecutive_matches() {
        let entry = new("hound".to_string());

        // Don't use a perfect match, since those product a perfect score.
        let properly_ordered_score = entry.similarity(" houn");

        let improperly_ordered_score = entry.similarity("nuoh");
        assert!(properly_ordered_score > improperly_ordered_score);
    }

    #[test]
    fn similarity_score_decreases_for_non_matching_characters() {
        let entry = new("hound".to_string());

        // Don't use a perfect match, since those product a perfect score.
        let non_matching_score = entry.similarity("houns");

        let subset_score = entry.similarity("houn");
        assert!(subset_score > non_matching_score);
    }

    #[test]
    fn similarity_score_is_zero_for_larger_query_with_no_matching_characters() {
        let entry = new("amp".to_string());
        assert_eq!(entry.similarity("hound"), 0.0f32);
    }
}
<commit_msg>Don't treat perfect matches explicitly.<commit_after>use matching::fragment;
use matching::fragment::Fragment;
use std::path::PathBuf;
use std::collections::hash_map::HashMap;

#[derive(PartialEq)]
pub struct Entry {
    pub path: PathBuf,
    pub index: HashMap<char, Vec<usize>>,
}

impl Entry {
    /// Compares the query string to the entry,
    /// and returns a score between 0 and 1.
    pub fn similarity(&self, query: &str) -> f32 {
        // Pre-calculate the path length as we'll be using it frequently.
        let path_length = self.path.to_string_lossy().chars().count();

        // We track fragment/substring matches, which have a greater weight than
        // a simple sum of individual character occurrences in the entry path.
        let mut match_fragments: Vec<Fragment> = Vec::new();

        // This counter is used to track characters in the query that aren't in
        // the entry's path, which has discrete weighting in the final score.
        let mut non_existent_char_count = 0;

        // Look for the query's character in the path's index.
        for query_char in query.chars() {
            match self.index.get(&query_char) {
                Some(occurrences) => {
                    // Initially, we'll assume that none of the occurrences
                    // of this character have been tracked as fragments.
                    let mut unaccounted_occurrences = occurrences.clone();

                    // Extend any existing fragment matches that also
                    // have this char at the end of their existing match.
                    for fragment in match_fragments.iter_mut() {
                        // Get the index at which this character would have to
                        // occur to extend this particular fragment over it.
                        let target_index = fragment.next_index();

                        if occurrences.contains(&target_index) {
                            // Since this character match is covered by this
                            // fragment, remove it from the unaccounted set.
                            match unaccounted_occurrences.iter().position(|&o| o == target_index) {
                                Some(position) => {
                                    unaccounted_occurrences.remove(position);
                                },
                                None => (),
                            }

                            // Bump the fragment's length to cover this char.
                            fragment.increase_length();
                        }
                    }

                    // Create fragment matches for any unaccounted occurrences.
                    for occurrence_index in unaccounted_occurrences.iter() {
                        match_fragments.push(fragment::new(*occurrence_index));
                    }

                },
                None => {
                    // Characters in the query string that
                    // aren't in the path reduce the score;
                    // track these so we can consider them later.
                    non_existent_char_count += 1;
                },
            }
        }

        // Determine the percentage of characters in the query string that
        // are in the entry, using the non-existent count we've calculated.
        let non_existence_penalty =
            // Guard against a potential arithmetic overflow here.
            if non_existent_char_count >= path_length {
                return 0.0f32
            } else {
                (path_length - non_existent_char_count) as f32 / path_length as f32
            };

        // Calculate an exponentially-scaled score based on fragment lengths.
        let fragment_score = match_fragments.iter().fold(0, |acc, ref fragment| {
            acc + fragment.length.pow(2)
        });

        // Calculate and return the similarity value. The path_length division
        // is used to offset the increased fragment score probability for
        // larger entry paths.
        fragment_score as f32 * non_existence_penalty / path_length as f32
    }
}

fn index_path(path: &str) -> HashMap<char, Vec<usize>> {
    let mut index: HashMap<char, Vec<usize>> = HashMap::new();
    for (char_index, path_char) in path.chars().enumerate() {
        if index.contains_key(&path_char) {
            match index.get_mut(&path_char) {
               Some(occurrences) => occurrences.push(char_index),
               None => ()
            }
        } else {
           index.insert(path_char, vec![char_index]);
        }
    }

    index
}

pub fn new(path: String) -> Entry {
    Entry{
        // Build the index before we transfer ownership of path.
        index: index_path(&path),
        path: PathBuf::from(path),
    }
}

#[cfg(test)]
mod tests {
    use super::new;

    #[test]
    fn similarity_correctly_scores_completely_different_terms() {
        let entry = new("src".to_string());
        assert_eq!(entry.similarity("lib"), 0.0);
    }

    #[test]
    fn similarity_scores_based_on_term_length() {
        let long_entry = new("hound library".to_string());
        let differing_length_score = long_entry.similarity("houn");

        // Don't use a perfect match, since those product a perfect score.
        let short_entry = new("hound".to_string());
        let same_length_score = short_entry.similarity("houn");

        assert!(same_length_score > differing_length_score);
    }

    #[test]
    fn similarity_score_increases_for_consecutive_matches() {
        let entry = new("hound".to_string());

        // Don't use a perfect match, since those product a perfect score.
        let properly_ordered_score = entry.similarity(" houn");

        let improperly_ordered_score = entry.similarity("nuoh");
        assert!(properly_ordered_score > improperly_ordered_score);
    }

    #[test]
    fn similarity_score_decreases_for_non_matching_characters() {
        let entry = new("hound".to_string());

        // Don't use a perfect match, since those product a perfect score.
        let non_matching_score = entry.similarity("houns");

        let subset_score = entry.similarity("houn");
        assert!(subset_score > non_matching_score);
    }

    #[test]
    fn similarity_score_is_zero_for_larger_query_with_no_matching_characters() {
        let entry = new("amp".to_string());
        assert_eq!(entry.similarity("hound"), 0.0f32);
    }
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 Tyler Neely

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
extern crate libc;
use self::libc::{c_char, c_int, c_void, size_t};
use std::ffi::CString;
use std::mem;
use std::ptr;
use std::slice;

use rocksdb_options::{RocksDBOptions};
use rocksdb::{RocksDB, RocksDBResult, RocksDBVector};

pub struct MergeOperatorCallback {
    pub name: CString,
    pub merge_fn: fn (&[u8], Option<&[u8]>, &mut MergeOperands) -> Vec<u8>,
}

pub extern "C" fn destructor_callback(raw_cb: *mut c_void) {
    // turn this back into a local variable so rust will reclaim it
    let _: Box<MergeOperatorCallback> = unsafe {mem::transmute(raw_cb)};

}

pub extern "C" fn name_callback(raw_cb: *mut c_void) -> *const c_char {
    unsafe {
        let cb: &mut MergeOperatorCallback =
            &mut *(raw_cb as *mut MergeOperatorCallback);
        let ptr = cb.name.as_ptr();
        ptr as *const c_char
    }
}

pub extern "C" fn full_merge_callback(
    raw_cb: *mut c_void, raw_key: *const c_char, key_len: size_t,
    existing_value: *const c_char, existing_value_len: size_t,
    operands_list: *const *const c_char, operands_list_len: *const size_t,
    num_operands: c_int,
    success: *mut u8, new_value_length: *mut size_t) -> *const c_char {
    unsafe {
        let cb: &mut MergeOperatorCallback =
            &mut *(raw_cb as *mut MergeOperatorCallback);
        let operands =
            &mut MergeOperands::new(operands_list,
                                    operands_list_len,
                                    num_operands);
        let key: &[u8] = mem::transmute(slice::from_raw_parts(&raw_key, key_len as usize));
        let oldval: &[u8] = mem::transmute(slice::from_raw_parts(&existing_value,
                                  existing_value_len as usize));
        let mut result =
            (cb.merge_fn)(key, Some(oldval), operands);
        result.shrink_to_fit();
        //TODO(tan) investigate zero-copy techniques to improve performance
        let buf = libc::malloc(result.len() as size_t);
        assert!(!buf.is_null());
        *new_value_length = result.len() as size_t;
        *success = 1 as u8;
        ptr::copy(result.as_ptr() as *mut c_void, &mut *buf, result.len());
        buf as *const c_char
    }
}

pub extern "C" fn partial_merge_callback(
    raw_cb: *mut c_void, raw_key: *const c_char, key_len: size_t,
    operands_list: *const *const c_char, operands_list_len: *const size_t,
    num_operands: c_int,
    success: *mut u8, new_value_length: *mut size_t) -> *const c_char {
    unsafe {
        let cb: &mut MergeOperatorCallback =
            &mut *(raw_cb as *mut MergeOperatorCallback);
        let operands = &mut MergeOperands::new(operands_list,
                                               operands_list_len,
                                               num_operands);
        let key: &[u8] = mem::transmute(slice::from_raw_parts(&raw_key, key_len as usize));
        let mut result = (cb.merge_fn)(key, None, operands);
        result.shrink_to_fit();
        //TODO(tan) investigate zero-copy techniques to improve performance
        let buf = libc::malloc(result.len() as size_t);
        assert!(!buf.is_null());
        *new_value_length = 1 as size_t;
        *success = 1 as u8;
        ptr::copy(result.as_ptr() as *mut c_void, &mut *buf, result.len());
        buf as *const c_char
    }
}

pub struct MergeOperands {
    operands_list: *const *const c_char,
    operands_list_len: *const size_t,
    num_operands: usize,
    cursor: usize,
}

impl MergeOperands {
    fn new(operands_list: *const *const c_char,
           operands_list_len: *const size_t,
           num_operands: c_int) -> MergeOperands {
        assert!(num_operands >= 0);
        MergeOperands {
            operands_list: operands_list,
            operands_list_len: operands_list_len,
            num_operands: num_operands as usize,
            cursor: 0,
        }
    }
}

impl<'a> Iterator for &'a mut MergeOperands {
    type Item = &'a [u8];
    fn next(&mut self) -> Option<&'a [u8]> {
        use std::raw::Slice;
        match self.cursor == self.num_operands {
            true => None,
            false => {
                unsafe {
                    let base = self.operands_list as usize;
                    let base_len = self.operands_list_len as usize;
                    let spacing = mem::size_of::<*const *const u8>();
                    let spacing_len = mem::size_of::<*const size_t>();
                    let len_ptr = (base_len + (spacing_len * self.cursor))
                        as *const size_t;
                    let len = *len_ptr as usize;
                    let ptr = base + (spacing * self.cursor);
                    self.cursor += 1;
                    Some(mem::transmute(Slice{data:*(ptr as *const *const u8)
                        as *const u8, len: len}))
                }
            }
        }
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        let remaining = self.num_operands - self.cursor;
        (remaining, Some(remaining))
    }
}

fn test_provided_merge(new_key: &[u8], existing_val: Option<&[u8]>,
    mut operands: &mut MergeOperands) -> Vec<u8> {
    let nops = operands.size_hint().0;
    let mut result: Vec<u8> = Vec::with_capacity(nops);
    match existing_val {
        Some(v) => result.push_all(v),
        None => (),
    }
    for op in operands {
        result.push_all(op);
    }
    result
}

#[allow(dead_code)]
#[test]
fn mergetest() {
    let path = "_rust_rocksdb_mergetest";
    let opts = RocksDBOptions::new();
    opts.create_if_missing(true);
    opts.add_merge_operator("test operator", test_provided_merge);
    let db = RocksDB::open(opts, path).unwrap();
    let p = db.put(b"k1", b"a");
    assert!(p.is_ok());
    db.merge(b"k1", b"b");
    db.merge(b"k1", b"c");
    db.merge(b"k1", b"d");
    db.merge(b"k1", b"efg");
    let m = db.merge(b"k1", b"h");
    assert!(m.is_ok());
    db.get(b"k1").map( |value| {
        match value.to_utf8() {
            Some(v) =>
                println!("retrieved utf8 value: {}", v),
            None =>
                println!("did not read valid utf-8 out of the db"),
        }
    }).on_absent( || { println!("value not present!") })
      .on_error( |e| { println!("error reading value")}); //: {", e) });

    assert!(m.is_ok());
    let r: RocksDBResult<RocksDBVector, &str> = db.get(b"k1");
    assert!(r.unwrap().to_utf8().unwrap() == "abcdefgh");
    assert!(db.delete(b"k1").is_ok());
    assert!(db.get(b"k1").is_none());
    db.close();
    assert!(RocksDB::destroy(opts, path).is_ok());
}
<commit_msg>Fix breakage due to transmutation changes.<commit_after>/*
   Copyright 2014 Tyler Neely

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/
extern crate libc;
use self::libc::{c_char, c_int, c_void, size_t};
use std::ffi::CString;
use std::mem;
use std::ptr;
use std::slice;

use rocksdb_options::{RocksDBOptions};
use rocksdb::{RocksDB, RocksDBResult, RocksDBVector};

pub struct MergeOperatorCallback {
    pub name: CString,
    pub merge_fn: fn (&[u8], Option<&[u8]>, &mut MergeOperands) -> Vec<u8>,
}

pub extern "C" fn destructor_callback(raw_cb: *mut c_void) {
    // turn this back into a local variable so rust will reclaim it
    let _: Box<MergeOperatorCallback> = unsafe {mem::transmute(raw_cb)};

}

pub extern "C" fn name_callback(raw_cb: *mut c_void) -> *const c_char {
    unsafe {
        let cb: &mut MergeOperatorCallback =
            &mut *(raw_cb as *mut MergeOperatorCallback);
        let ptr = cb.name.as_ptr();
        ptr as *const c_char
    }
}

pub extern "C" fn full_merge_callback(
    raw_cb: *mut c_void, raw_key: *const c_char, key_len: size_t,
    existing_value: *const c_char, existing_value_len: size_t,
    operands_list: *const *const c_char, operands_list_len: *const size_t,
    num_operands: c_int,
    success: *mut u8, new_value_length: *mut size_t) -> *const c_char {
    unsafe {
        let cb: &mut MergeOperatorCallback =
            &mut *(raw_cb as *mut MergeOperatorCallback);
        let operands =
            &mut MergeOperands::new(operands_list,
                                    operands_list_len,
                                    num_operands);
        let key: &[u8] = slice::from_raw_parts(raw_key as *const u8, key_len as usize);
        let oldval: &[u8] = slice::from_raw_parts(existing_value as *const u8,
                                  existing_value_len as usize);
        let mut result =
            (cb.merge_fn)(key, Some(oldval), operands);
        result.shrink_to_fit();
        //TODO(tan) investigate zero-copy techniques to improve performance
        let buf = libc::malloc(result.len() as size_t);
        assert!(!buf.is_null());
        *new_value_length = result.len() as size_t;
        *success = 1 as u8;
        ptr::copy(result.as_ptr() as *mut c_void, &mut *buf, result.len());
        buf as *const c_char
    }
}

pub extern "C" fn partial_merge_callback(
    raw_cb: *mut c_void, raw_key: *const c_char, key_len: size_t,
    operands_list: *const *const c_char, operands_list_len: *const size_t,
    num_operands: c_int,
    success: *mut u8, new_value_length: *mut size_t) -> *const c_char {
    unsafe {
        let cb: &mut MergeOperatorCallback =
            &mut *(raw_cb as *mut MergeOperatorCallback);
        let operands = &mut MergeOperands::new(operands_list,
                                               operands_list_len,
                                               num_operands);
        let key: &[u8] = slice::from_raw_parts(raw_key as *const u8, key_len as usize);
        let mut result = (cb.merge_fn)(key, None, operands);
        result.shrink_to_fit();
        //TODO(tan) investigate zero-copy techniques to improve performance
        let buf = libc::malloc(result.len() as size_t);
        assert!(!buf.is_null());
        *new_value_length = 1 as size_t;
        *success = 1 as u8;
        ptr::copy(result.as_ptr() as *mut c_void, &mut *buf, result.len());
        buf as *const c_char
    }
}

pub struct MergeOperands {
    operands_list: *const *const c_char,
    operands_list_len: *const size_t,
    num_operands: usize,
    cursor: usize,
}

impl MergeOperands {
    fn new(operands_list: *const *const c_char,
           operands_list_len: *const size_t,
           num_operands: c_int) -> MergeOperands {
        assert!(num_operands >= 0);
        MergeOperands {
            operands_list: operands_list,
            operands_list_len: operands_list_len,
            num_operands: num_operands as usize,
            cursor: 0,
        }
    }
}

impl<'a> Iterator for &'a mut MergeOperands {
    type Item = &'a [u8];
    fn next(&mut self) -> Option<&'a [u8]> {
        use std::raw::Slice;
        match self.cursor == self.num_operands {
            true => None,
            false => {
                unsafe {
                    let base = self.operands_list as usize;
                    let base_len = self.operands_list_len as usize;
                    let spacing = mem::size_of::<*const *const u8>();
                    let spacing_len = mem::size_of::<*const size_t>();
                    let len_ptr = (base_len + (spacing_len * self.cursor))
                        as *const size_t;
                    let len = *len_ptr as usize;
                    let ptr = base + (spacing * self.cursor);
                    self.cursor += 1;
                    Some(mem::transmute(Slice{data:*(ptr as *const *const u8)
                        as *const u8, len: len}))
                }
            }
        }
    }

    fn size_hint(&self) -> (usize, Option<usize>) {
        let remaining = self.num_operands - self.cursor;
        (remaining, Some(remaining))
    }
}

fn test_provided_merge(new_key: &[u8], existing_val: Option<&[u8]>,
    mut operands: &mut MergeOperands) -> Vec<u8> {
    let nops = operands.size_hint().0;
    let mut result: Vec<u8> = Vec::with_capacity(nops);
    match existing_val {
        Some(v) => result.push_all(v),
        None => (),
    }
    for op in operands {
        result.push_all(op);
    }
    result
}

#[allow(dead_code)]
#[test]
fn mergetest() {
    let path = "_rust_rocksdb_mergetest";
    let opts = RocksDBOptions::new();
    opts.create_if_missing(true);
    opts.add_merge_operator("test operator", test_provided_merge);
    let db = RocksDB::open(opts, path).unwrap();
    let p = db.put(b"k1", b"a");
    assert!(p.is_ok());
    db.merge(b"k1", b"b");
    db.merge(b"k1", b"c");
    db.merge(b"k1", b"d");
    db.merge(b"k1", b"efg");
    let m = db.merge(b"k1", b"h");
    assert!(m.is_ok());
    db.get(b"k1").map( |value| {
        match value.to_utf8() {
            Some(v) =>
                println!("retrieved utf8 value: {}", v),
            None =>
                println!("did not read valid utf-8 out of the db"),
        }
    }).on_absent( || { println!("value not present!") })
      .on_error( |e| { println!("error reading value")}); //: {", e) });

    assert!(m.is_ok());
    let r: RocksDBResult<RocksDBVector, &str> = db.get(b"k1");
    assert!(r.unwrap().to_utf8().unwrap() == "abcdefgh");
    assert!(db.delete(b"k1").is_ok());
    assert!(db.get(b"k1").is_none());
    db.close();
    assert!(RocksDB::destroy(opts, path).is_ok());
}
<|endoftext|>


<commit_before>use super::config::Config;
use std::collections::BTreeMap;
use std::io::{BufRead, Write};

#[derive(Debug, Clone, Copy)]
pub enum OutputFormat {
    LittleEndian,
    BigEndian,
    HexList,
}

struct Replacement {
    // Line for purposes of printing errors.
    line: usize,
    // Segment to add the value to.
    add_segment: usize,
    // The index in the add_segment to add the value.
    index: usize,
    // The tag to retrieve the position to add.
    tag: String,
    // The segment for which we are retrieving the position from the tag.
    pos_segment: usize,
    // The offset we add after retrieving the value from the tag.
    pos_offset: isize,
}

pub struct Parser<'a> {
    config: &'a Config,
    segments: Vec<Vec<u64>>,
    tags: BTreeMap<String, Vec<usize>>,
    replacements: Vec<Replacement>,
}

impl<'a> Parser<'a> {
    pub fn new(config: &'a Config) -> Self {
        Parser {
            config: config,
            segments: {
                let mut v = Vec::new();
                for _ in 0..config.segment_widths.len() {
                    v.push(Vec::new());
                }
                v
            },
            tags: BTreeMap::new(),
            replacements: Vec::new(),
        }
    }

    pub fn link(&mut self) {
        // Iterate through every replacement.
        for r in &self.replacements {
            // Get the tag offset vector corresponding to the replacement.
            let tag = self.tags.get(&r.tag).unwrap_or_else(|| {
                panic!("Error: Tag \"{}\" used on line {} never defined.",
                       r.tag,
                       r.line);
            });
            self.segments[r.add_segment][r.index] =
                (tag[r.pos_segment] as isize + r.pos_offset) as u64;
        }
    }

    pub fn parse<B>(&mut self, bufread: B)
        where B: BufRead
    {
        for (index, line) in bufread.lines().enumerate() {
            let line = line.unwrap_or_else(|e| panic!("Error: Failed to read from buffer: {}", e));

            if self.config.split_whitespace {
                for word in line.split_whitespace() {
                    self.parse_segment(word, index + 1);
                }
            } else {
                self.parse_segment(&line, index + 1);
            }
        }
    }

    pub fn output<W>(&self, format: OutputFormat, segment: usize, w: &mut W)
        where W: Write
    {
        use byteorder::{ByteOrder, LittleEndian, BigEndian};
        match format {
            OutputFormat::LittleEndian => {
                // Allocate enough bytes to store a u64
                let mut bytes = [0; 8];
                let width = self.config.segment_widths[segment];
                for val in &self.segments[segment] {
                    LittleEndian::write_u64(&mut bytes, *val);
                    w.write_all(&bytes[0..width]).unwrap_or_else(|e| {
                        panic!("Error: Writing to output file for segment {} failed: {}",
                               segment,
                               e);
                    });
                }
            }
            OutputFormat::BigEndian => {
                // Allocate enough bytes to store a u64
                let mut bytes = [0; 8];
                let width = self.config.segment_widths[segment];
                for val in &self.segments[segment] {
                    BigEndian::write_u64(&mut bytes, *val);
                    w.write_all(&bytes[0..width]).unwrap_or_else(|e| {
                        panic!("Error: Writing to output file for segment {} failed: {}",
                               segment,
                               e);
                    });
                }
            }
            OutputFormat::HexList => {
                use std::iter::FromIterator;
                // Allocate enough bytes to store a u64
                let mut bytes = [0u8; 8];
                let width = self.config.segment_widths[segment];
                for val in &self.segments[segment] {
                    BigEndian::write_u64(&mut bytes, *val);
                    w.write_all(&(String::from_iter(bytes[(8 - width)..8]
                                .iter()
                                .map(|&b| format!("{:02X}", b))) +
                                     "\n")
                            .as_bytes())
                        .unwrap_or_else(|e| {
                            panic!("Error: Writing to output file for segment {} failed: {}",
                                   segment,
                                   e);
                        });
                }
            }
        }
    }

    pub fn parse_segment(&mut self, segment: &str, line: usize) {
        if self.attempt_tag_create(segment) {
            return;
        }
        if self.attempt_tag_use(segment, line) {
            return;
        }
        if self.attempt_rules(segment) {
            return;
        }
        panic!("Error: Unrecognized symbol \"{}\" on line {}.",
               segment,
               line);
    }

    fn attempt_tag_create(&mut self, segment: &str) -> bool {
        if let Some(caps) = self.config.tag_create.regex.as_ref().unwrap().captures(segment) {
            self.tags.insert(caps.at(1).unwrap().to_string(),
                             self.segments
                                 .iter()
                                 .map(|v| v.len())
                                 .collect());
            true
        } else {
            false
        }
    }

    fn attempt_tag_use(&mut self, segment: &str, line: usize) -> bool {
        let config = &self.config;
        for rule in &config.tag_use_rules {
            if let Some(caps) = rule.regex.as_ref().unwrap().captures(segment) {
                let tag = caps.at(1).unwrap().to_string();
                println!("Captured segment \"{}\" in rule \"{:?}\"", segment, rule);
                for feedback in &rule.feedbacks {
                    let index = self.segments[feedback.add_segment].len();
                    let current_pos_index = self.segments[feedback.pos_segment].len();
                    self.replacements.push(Replacement {
                        line: line,
                        add_segment: feedback.add_segment,
                        index: index,
                        tag: tag.clone(),
                        pos_segment: feedback.pos_segment,
                        pos_offset: if feedback.relative {
                            feedback.pos_offset - current_pos_index as isize
                        } else {
                            feedback.pos_offset
                        },
                    });

                    // Add the padding
                    self.segments[feedback.add_segment].push(0);
                }
                return true;
            }
        }
        false
    }

    fn attempt_rules(&mut self, segment: &str) -> bool {
        let config = &self.config;
        for rule in &config.rules {
            if let Some(caps) = rule.regex.as_ref().unwrap().captures(segment) {
                let mut segvals = rule.segment_values.clone();
                for (index, capture) in rule.captures.iter().enumerate() {
                    use std::mem::transmute;
                    let cap_string = caps.at(index + 1).unwrap();
                    let pval = i64::from_str_radix(cap_string, capture.base).unwrap_or_else(|e| {
                        panic!("Error: Failed to parse captured string \"{}\" from \"{}\": {}",
                               cap_string,
                               segment,
                               e);
                    });
                    let val: u64 = unsafe { transmute(pval) };

                    for feedback in &capture.feedbacks {
                        let shiftval = val << feedback.shift;
                        if feedback.fill {
                            let baseval = segvals[feedback.segment][index];
                            for _ in 0..(shiftval + feedback.fill_offset as u64) {
                                self.segments[feedback.segment].push(baseval);
                            }
                        } else {
                            segvals[feedback.segment][index] += shiftval;
                        }
                    }
                }
                for (segvec, segment) in segvals.iter_mut().zip(self.segments.iter_mut()) {
                    segment.append(segvec);
                }
                return true;
            }
        }
        false
    }
}
<commit_msg>add support for comments with #<commit_after>use super::config::Config;
use std::collections::BTreeMap;
use std::io::{BufRead, Write};

#[derive(Debug, Clone, Copy)]
pub enum OutputFormat {
    LittleEndian,
    BigEndian,
    HexList,
}

struct Replacement {
    // Line for purposes of printing errors.
    line: usize,
    // Segment to add the value to.
    add_segment: usize,
    // The index in the add_segment to add the value.
    index: usize,
    // The tag to retrieve the position to add.
    tag: String,
    // The segment for which we are retrieving the position from the tag.
    pos_segment: usize,
    // The offset we add after retrieving the value from the tag.
    pos_offset: isize,
}

pub struct Parser<'a> {
    config: &'a Config,
    segments: Vec<Vec<u64>>,
    tags: BTreeMap<String, Vec<usize>>,
    replacements: Vec<Replacement>,
}

impl<'a> Parser<'a> {
    pub fn new(config: &'a Config) -> Self {
        Parser {
            config: config,
            segments: {
                let mut v = Vec::new();
                for _ in 0..config.segment_widths.len() {
                    v.push(Vec::new());
                }
                v
            },
            tags: BTreeMap::new(),
            replacements: Vec::new(),
        }
    }

    pub fn link(&mut self) {
        // Iterate through every replacement.
        for r in &self.replacements {
            // Get the tag offset vector corresponding to the replacement.
            let tag = self.tags.get(&r.tag).unwrap_or_else(|| {
                panic!("Error: Tag \"{}\" used on line {} never defined.",
                       r.tag,
                       r.line);
            });
            self.segments[r.add_segment][r.index] =
                (tag[r.pos_segment] as isize + r.pos_offset) as u64;
        }
    }

    pub fn parse<B>(&mut self, bufread: B)
        where B: BufRead
    {
        for (index, line) in bufread.lines().enumerate() {
            let line = line.unwrap_or_else(|e| panic!("Error: Failed to read from buffer: {}", e));

            // Remove everything after the first #, which denotes a comment.
            let line = line.splitn(2, '#').next().unwrap();

            if self.config.split_whitespace {
                for word in line.split_whitespace() {
                    self.parse_segment(word, index + 1);
                }
            } else {
                self.parse_segment(&line, index + 1);
            }
        }
    }

    pub fn output<W>(&self, format: OutputFormat, segment: usize, w: &mut W)
        where W: Write
    {
        use byteorder::{ByteOrder, LittleEndian, BigEndian};
        match format {
            OutputFormat::LittleEndian => {
                // Allocate enough bytes to store a u64
                let mut bytes = [0; 8];
                let width = self.config.segment_widths[segment];
                for val in &self.segments[segment] {
                    LittleEndian::write_u64(&mut bytes, *val);
                    w.write_all(&bytes[0..width]).unwrap_or_else(|e| {
                        panic!("Error: Writing to output file for segment {} failed: {}",
                               segment,
                               e);
                    });
                }
            }
            OutputFormat::BigEndian => {
                // Allocate enough bytes to store a u64
                let mut bytes = [0; 8];
                let width = self.config.segment_widths[segment];
                for val in &self.segments[segment] {
                    BigEndian::write_u64(&mut bytes, *val);
                    w.write_all(&bytes[0..width]).unwrap_or_else(|e| {
                        panic!("Error: Writing to output file for segment {} failed: {}",
                               segment,
                               e);
                    });
                }
            }
            OutputFormat::HexList => {
                use std::iter::FromIterator;
                // Allocate enough bytes to store a u64
                let mut bytes = [0u8; 8];
                let width = self.config.segment_widths[segment];
                for val in &self.segments[segment] {
                    BigEndian::write_u64(&mut bytes, *val);
                    w.write_all(&(String::from_iter(bytes[(8 - width)..8]
                                .iter()
                                .map(|&b| format!("{:02X}", b))) +
                                     "\n")
                            .as_bytes())
                        .unwrap_or_else(|e| {
                            panic!("Error: Writing to output file for segment {} failed: {}",
                                   segment,
                                   e);
                        });
                }
            }
        }
    }

    pub fn parse_segment(&mut self, segment: &str, line: usize) {
        if segment.is_empty() {
            return;
        }
        if self.attempt_tag_create(segment) {
            return;
        }
        if self.attempt_tag_use(segment, line) {
            return;
        }
        if self.attempt_rules(segment) {
            return;
        }
        panic!("Error: Unrecognized symbol \"{}\" on line {}.",
               segment,
               line);
    }

    fn attempt_tag_create(&mut self, segment: &str) -> bool {
        if let Some(caps) = self.config.tag_create.regex.as_ref().unwrap().captures(segment) {
            self.tags.insert(caps.at(1).unwrap().to_string(),
                             self.segments
                                 .iter()
                                 .map(|v| v.len())
                                 .collect());
            true
        } else {
            false
        }
    }

    fn attempt_tag_use(&mut self, segment: &str, line: usize) -> bool {
        let config = &self.config;
        for rule in &config.tag_use_rules {
            if let Some(caps) = rule.regex.as_ref().unwrap().captures(segment) {
                let tag = caps.at(1).unwrap().to_string();
                println!("Captured segment \"{}\" in rule \"{:?}\"", segment, rule);
                for feedback in &rule.feedbacks {
                    let index = self.segments[feedback.add_segment].len();
                    let current_pos_index = self.segments[feedback.pos_segment].len();
                    self.replacements.push(Replacement {
                        line: line,
                        add_segment: feedback.add_segment,
                        index: index,
                        tag: tag.clone(),
                        pos_segment: feedback.pos_segment,
                        pos_offset: if feedback.relative {
                            feedback.pos_offset - current_pos_index as isize
                        } else {
                            feedback.pos_offset
                        },
                    });

                    // Add the padding
                    self.segments[feedback.add_segment].push(0);
                }
                return true;
            }
        }
        false
    }

    fn attempt_rules(&mut self, segment: &str) -> bool {
        let config = &self.config;
        for rule in &config.rules {
            if let Some(caps) = rule.regex.as_ref().unwrap().captures(segment) {
                let mut segvals = rule.segment_values.clone();
                for (index, capture) in rule.captures.iter().enumerate() {
                    use std::mem::transmute;
                    let cap_string = caps.at(index + 1).unwrap();
                    let pval = i64::from_str_radix(cap_string, capture.base).unwrap_or_else(|e| {
                        panic!("Error: Failed to parse captured string \"{}\" from \"{}\": {}",
                               cap_string,
                               segment,
                               e);
                    });
                    let val: u64 = unsafe { transmute(pval) };

                    for feedback in &capture.feedbacks {
                        let shiftval = val << feedback.shift;
                        if feedback.fill {
                            let baseval = segvals[feedback.segment][index];
                            for _ in 0..(shiftval + feedback.fill_offset as u64) {
                                self.segments[feedback.segment].push(baseval);
                            }
                        } else {
                            segvals[feedback.segment][index] += shiftval;
                        }
                    }
                }
                for (segvec, segment) in segvals.iter_mut().zip(self.segments.iter_mut()) {
                    segment.append(segvec);
                }
                return true;
            }
        }
        false
    }
}
<|endoftext|>