<commit_before>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// FIXME: #13996: mark the `allocate` and `reallocate` return value as `noalias`

/// Returns a pointer to `size` bytes of memory.
///
/// Behavior is undefined if the requested size is 0 or the alignment is not a
/// power of 2. The alignment must be no larger than the largest supported page
/// size on the platform.
#[inline]
pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
    imp::allocate(size, align)
}

/// Extends or shrinks the allocation referenced by `ptr` to `size` bytes of
/// memory.
///
/// Behavior is undefined if the requested size is 0 or the alignment is not a
/// power of 2. The alignment must be no larger than the largest supported page
/// size on the platform.
///
/// The `old_size` and `align` parameters are the parameters that were used to
/// create the allocation referenced by `ptr`. The `old_size` parameter may also
/// be the value returned by `usable_size` for the requested size.
#[inline]
pub unsafe fn reallocate(ptr: *mut u8, old_size: uint, size: uint, align: uint) -> *mut u8 {
    imp::reallocate(ptr, old_size, size, align)
}

/// Extends or shrinks the allocation referenced by `ptr` to `size` bytes of
/// memory in-place.
///
/// Returns true if successful, otherwise false if the allocation was not
/// altered.
///
/// Behavior is undefined if the requested size is 0 or the alignment is not a
/// power of 2. The alignment must be no larger than the largest supported page
/// size on the platform.
///
/// The `old_size` and `align` parameters are the parameters that were used to
/// create the allocation referenced by `ptr`. The `old_size` parameter may be
/// any value in range_inclusive(requested_size, usable_size).
#[inline]
pub unsafe fn reallocate_inplace(ptr: *mut u8, old_size: uint, size: uint, align: uint) -> bool {
    imp::reallocate_inplace(ptr, old_size, size, align)
}

/// Deallocates the memory referenced by `ptr`.
///
/// The `ptr` parameter must not be null.
///
/// The `size` and `align` parameters are the parameters that were used to
/// create the allocation referenced by `ptr`. The `size` parameter may also be
/// the value returned by `usable_size` for the requested size.
#[inline]
pub unsafe fn deallocate(ptr: *mut u8, size: uint, align: uint) {
    imp::deallocate(ptr, size, align)
}

/// Returns the usable size of an allocation created with the specified the
/// `size` and `align`.
#[inline]
pub fn usable_size(size: uint, align: uint) -> uint {
    imp::usable_size(size, align)
}

/// Prints implementation-defined allocator statistics.
///
/// These statistics may be inconsistent if other threads use the allocator
/// during the call.
#[unstable]
pub fn stats_print() {
    imp::stats_print();
}

/// An arbitrary non-null address to represent zero-size allocations.
///
/// This preserves the non-null invariant for types like `Box<T>`. The address may overlap with
/// non-zero-size memory allocations.
pub static EMPTY: *mut () = 0x1 as *mut ();

/// The allocator for unique pointers.
#[cfg(not(test))]
#[lang="exchange_malloc"]
#[inline]
unsafe fn exchange_malloc(size: uint, align: uint) -> *mut u8 {
    if size == 0 {
        EMPTY as *mut u8
    } else {
        allocate(size, align)
    }
}

#[cfg(not(test))]
#[lang="exchange_free"]
#[inline]
unsafe fn exchange_free(ptr: *mut u8, size: uint, align: uint) {
    deallocate(ptr, size, align);
}

// The minimum alignment guaranteed by the architecture. This value is used to
// add fast paths for low alignment values. In practice, the alignment is a
// constant at the call site and the branch will be optimized out.
#[cfg(any(target_arch = "arm",
          target_arch = "mips",
          target_arch = "mipsel"))]
static MIN_ALIGN: uint = 8;
#[cfg(any(target_arch = "x86",
          target_arch = "x86_64"))]
static MIN_ALIGN: uint = 16;

#[cfg(jemalloc)]
mod imp {
    use core::option::{None, Option};
    use core::ptr::{RawPtr, null_mut, null};
    use core::num::Int;
    use libc::{c_char, c_int, c_void, size_t};
    use super::MIN_ALIGN;

    #[link(name = "jemalloc", kind = "static")]
    #[cfg(not(test))]
    extern {}

    extern {
        fn je_mallocx(size: size_t, flags: c_int) -> *mut c_void;
        fn je_rallocx(ptr: *mut c_void, size: size_t,
                      flags: c_int) -> *mut c_void;
        fn je_xallocx(ptr: *mut c_void, size: size_t, extra: size_t,
                      flags: c_int) -> size_t;
        fn je_sdallocx(ptr: *mut c_void, size: size_t, flags: c_int);
        fn je_nallocx(size: size_t, flags: c_int) -> size_t;
        fn je_malloc_stats_print(write_cb: Option<extern "C" fn(cbopaque: *mut c_void,
                                                                *const c_char)>,
                                 cbopaque: *mut c_void,
                                 opts: *const c_char);
    }

    // -lpthread needs to occur after -ljemalloc, the earlier argument isn't enough
    #[cfg(all(not(windows), not(target_os = "android")))]
    #[link(name = "pthread")]
    extern {}

    // MALLOCX_ALIGN(a) macro
    #[inline(always)]
    fn mallocx_align(a: uint) -> c_int { a.trailing_zeros() as c_int }

    #[inline(always)]
    fn align_to_flags(align: uint) -> c_int {
        if align <= MIN_ALIGN { 0 } else { mallocx_align(align) }
    }

    #[inline]
    pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
        let flags = align_to_flags(align);
        let ptr = je_mallocx(size as size_t, flags) as *mut u8;
        if ptr.is_null() {
            ::oom()
        }
        ptr
    }

    #[inline]
    pub unsafe fn reallocate(ptr: *mut u8, _old_size: uint, size: uint, align: uint) -> *mut u8 {
        let flags = align_to_flags(align);
        let ptr = je_rallocx(ptr as *mut c_void, size as size_t, flags) as *mut u8;
        if ptr.is_null() {
            ::oom()
        }
        ptr
    }

    #[inline]
    pub unsafe fn reallocate_inplace(ptr: *mut u8, old_size: uint, size: uint,
                                     align: uint) -> bool {
        let flags = align_to_flags(align);
        let new_size = je_xallocx(ptr as *mut c_void, size as size_t, 0, flags) as uint;
        // checking for failure to shrink is tricky
        if size < old_size {
            usable_size(size, align) == new_size as uint
        } else {
            new_size >= size
        }
    }

    #[inline]
    pub unsafe fn deallocate(ptr: *mut u8, size: uint, align: uint) {
        let flags = align_to_flags(align);
        je_sdallocx(ptr as *mut c_void, size as size_t, flags)
    }

    #[inline]
    pub fn usable_size(size: uint, align: uint) -> uint {
        let flags = align_to_flags(align);
        unsafe { je_nallocx(size as size_t, flags) as uint }
    }

    pub fn stats_print() {
        unsafe {
            je_malloc_stats_print(None, null_mut(), null())
        }
    }
}

#[cfg(all(not(jemalloc), unix))]
mod imp {
    use core::cmp;
    use core::ptr;
    use libc;
    use libc_heap;
    use super::MIN_ALIGN;

    extern {
        fn posix_memalign(memptr: *mut *mut libc::c_void,
                          align: libc::size_t,
                          size: libc::size_t) -> libc::c_int;
    }

    #[inline]
    pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
        if align <= MIN_ALIGN {
            libc_heap::malloc_raw(size)
        } else {
            let mut out = 0 as *mut libc::c_void;
            let ret = posix_memalign(&mut out,
                                     align as libc::size_t,
                                     size as libc::size_t);
            if ret != 0 {
                ::oom();
            }
            out as *mut u8
        }
    }

    #[inline]
    pub unsafe fn reallocate(ptr: *mut u8, old_size: uint, size: uint, align: uint) -> *mut u8 {
        if align <= MIN_ALIGN {
            libc_heap::realloc_raw(ptr, size)
        } else {
            let new_ptr = allocate(size, align);
            ptr::copy_memory(new_ptr, ptr as *const u8, cmp::min(size, old_size));
            deallocate(ptr, old_size, align);
            new_ptr
        }
    }

    #[inline]
    pub unsafe fn reallocate_inplace(_ptr: *mut u8, old_size: uint, size: uint,
                                     _align: uint) -> bool {
        size == old_size
    }

    #[inline]
    pub unsafe fn deallocate(ptr: *mut u8, _size: uint, _align: uint) {
        libc::free(ptr as *mut libc::c_void)
    }

    #[inline]
    pub fn usable_size(size: uint, _align: uint) -> uint {
        size
    }

    pub fn stats_print() {}
}

#[cfg(all(not(jemalloc), windows))]
mod imp {
    use libc::{c_void, size_t};
    use libc;
    use libc_heap;
    use core::ptr::RawPtr;
    use super::MIN_ALIGN;

    extern {
        fn _aligned_malloc(size: size_t, align: size_t) -> *mut c_void;
        fn _aligned_realloc(block: *mut c_void, size: size_t,
                            align: size_t) -> *mut c_void;
        fn _aligned_free(ptr: *mut c_void);
    }

    #[inline]
    pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
        if align <= MIN_ALIGN {
            libc_heap::malloc_raw(size)
        } else {
            let ptr = _aligned_malloc(size as size_t, align as size_t);
            if ptr.is_null() {
                ::oom();
            }
            ptr as *mut u8
        }
    }

    #[inline]
    pub unsafe fn reallocate(ptr: *mut u8, _old_size: uint, size: uint, align: uint) -> *mut u8 {
        if align <= MIN_ALIGN {
            libc_heap::realloc_raw(ptr, size)
        } else {
            let ptr = _aligned_realloc(ptr as *mut c_void, size as size_t,
                                       align as size_t);
            if ptr.is_null() {
                ::oom();
            }
            ptr as *mut u8
        }
    }

    #[inline]
    pub unsafe fn reallocate_inplace(_ptr: *mut u8, old_size: uint, size: uint,
                                     _align: uint) -> bool {
        size == old_size
    }

    #[inline]
    pub unsafe fn deallocate(ptr: *mut u8, _size: uint, align: uint) {
        if align <= MIN_ALIGN {
            libc::free(ptr as *mut libc::c_void)
        } else {
            _aligned_free(ptr as *mut c_void)
        }
    }

    #[inline]
    pub fn usable_size(size: uint, _align: uint) -> uint {
        size
    }

    pub fn stats_print() {}
}

#[cfg(test)]
mod test {
    extern crate test;
    use self::test::Bencher;
    use heap;

    #[test]
    fn basic_reallocate_inplace_noop() {
        unsafe {
            let size = 4000;
            let ptr = heap::allocate(size, 8);
            let ret = heap::reallocate_inplace(ptr, size, size, 8);
            heap::deallocate(ptr, size, 8);
            assert!(ret);
        }
    }

    #[bench]
    fn alloc_owned_small(b: &mut Bencher) {
        b.iter(|| {
            box 10i
        })
    }
}
<commit_msg>alloc: Convert statics to constants<commit_after>// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// FIXME: #13996: mark the `allocate` and `reallocate` return value as `noalias`

/// Returns a pointer to `size` bytes of memory.
///
/// Behavior is undefined if the requested size is 0 or the alignment is not a
/// power of 2. The alignment must be no larger than the largest supported page
/// size on the platform.
#[inline]
pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
    imp::allocate(size, align)
}

/// Extends or shrinks the allocation referenced by `ptr` to `size` bytes of
/// memory.
///
/// Behavior is undefined if the requested size is 0 or the alignment is not a
/// power of 2. The alignment must be no larger than the largest supported page
/// size on the platform.
///
/// The `old_size` and `align` parameters are the parameters that were used to
/// create the allocation referenced by `ptr`. The `old_size` parameter may also
/// be the value returned by `usable_size` for the requested size.
#[inline]
pub unsafe fn reallocate(ptr: *mut u8, old_size: uint, size: uint, align: uint) -> *mut u8 {
    imp::reallocate(ptr, old_size, size, align)
}

/// Extends or shrinks the allocation referenced by `ptr` to `size` bytes of
/// memory in-place.
///
/// Returns true if successful, otherwise false if the allocation was not
/// altered.
///
/// Behavior is undefined if the requested size is 0 or the alignment is not a
/// power of 2. The alignment must be no larger than the largest supported page
/// size on the platform.
///
/// The `old_size` and `align` parameters are the parameters that were used to
/// create the allocation referenced by `ptr`. The `old_size` parameter may be
/// any value in range_inclusive(requested_size, usable_size).
#[inline]
pub unsafe fn reallocate_inplace(ptr: *mut u8, old_size: uint, size: uint, align: uint) -> bool {
    imp::reallocate_inplace(ptr, old_size, size, align)
}

/// Deallocates the memory referenced by `ptr`.
///
/// The `ptr` parameter must not be null.
///
/// The `size` and `align` parameters are the parameters that were used to
/// create the allocation referenced by `ptr`. The `size` parameter may also be
/// the value returned by `usable_size` for the requested size.
#[inline]
pub unsafe fn deallocate(ptr: *mut u8, size: uint, align: uint) {
    imp::deallocate(ptr, size, align)
}

/// Returns the usable size of an allocation created with the specified the
/// `size` and `align`.
#[inline]
pub fn usable_size(size: uint, align: uint) -> uint {
    imp::usable_size(size, align)
}

/// Prints implementation-defined allocator statistics.
///
/// These statistics may be inconsistent if other threads use the allocator
/// during the call.
#[unstable]
pub fn stats_print() {
    imp::stats_print();
}

/// An arbitrary non-null address to represent zero-size allocations.
///
/// This preserves the non-null invariant for types like `Box<T>`. The address may overlap with
/// non-zero-size memory allocations.
pub const EMPTY: *mut () = 0x1 as *mut ();

/// The allocator for unique pointers.
#[cfg(not(test))]
#[lang="exchange_malloc"]
#[inline]
unsafe fn exchange_malloc(size: uint, align: uint) -> *mut u8 {
    if size == 0 {
        EMPTY as *mut u8
    } else {
        allocate(size, align)
    }
}

#[cfg(not(test))]
#[lang="exchange_free"]
#[inline]
unsafe fn exchange_free(ptr: *mut u8, size: uint, align: uint) {
    deallocate(ptr, size, align);
}

// The minimum alignment guaranteed by the architecture. This value is used to
// add fast paths for low alignment values. In practice, the alignment is a
// constant at the call site and the branch will be optimized out.
#[cfg(any(target_arch = "arm",
          target_arch = "mips",
          target_arch = "mipsel"))]
static MIN_ALIGN: uint = 8;
#[cfg(any(target_arch = "x86",
          target_arch = "x86_64"))]
static MIN_ALIGN: uint = 16;

#[cfg(jemalloc)]
mod imp {
    use core::option::{None, Option};
    use core::ptr::{RawPtr, null_mut, null};
    use core::num::Int;
    use libc::{c_char, c_int, c_void, size_t};
    use super::MIN_ALIGN;

    #[link(name = "jemalloc", kind = "static")]
    #[cfg(not(test))]
    extern {}

    extern {
        fn je_mallocx(size: size_t, flags: c_int) -> *mut c_void;
        fn je_rallocx(ptr: *mut c_void, size: size_t,
                      flags: c_int) -> *mut c_void;
        fn je_xallocx(ptr: *mut c_void, size: size_t, extra: size_t,
                      flags: c_int) -> size_t;
        fn je_sdallocx(ptr: *mut c_void, size: size_t, flags: c_int);
        fn je_nallocx(size: size_t, flags: c_int) -> size_t;
        fn je_malloc_stats_print(write_cb: Option<extern "C" fn(cbopaque: *mut c_void,
                                                                *const c_char)>,
                                 cbopaque: *mut c_void,
                                 opts: *const c_char);
    }

    // -lpthread needs to occur after -ljemalloc, the earlier argument isn't enough
    #[cfg(all(not(windows), not(target_os = "android")))]
    #[link(name = "pthread")]
    extern {}

    // MALLOCX_ALIGN(a) macro
    #[inline(always)]
    fn mallocx_align(a: uint) -> c_int { a.trailing_zeros() as c_int }

    #[inline(always)]
    fn align_to_flags(align: uint) -> c_int {
        if align <= MIN_ALIGN { 0 } else { mallocx_align(align) }
    }

    #[inline]
    pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
        let flags = align_to_flags(align);
        let ptr = je_mallocx(size as size_t, flags) as *mut u8;
        if ptr.is_null() {
            ::oom()
        }
        ptr
    }

    #[inline]
    pub unsafe fn reallocate(ptr: *mut u8, _old_size: uint, size: uint, align: uint) -> *mut u8 {
        let flags = align_to_flags(align);
        let ptr = je_rallocx(ptr as *mut c_void, size as size_t, flags) as *mut u8;
        if ptr.is_null() {
            ::oom()
        }
        ptr
    }

    #[inline]
    pub unsafe fn reallocate_inplace(ptr: *mut u8, old_size: uint, size: uint,
                                     align: uint) -> bool {
        let flags = align_to_flags(align);
        let new_size = je_xallocx(ptr as *mut c_void, size as size_t, 0, flags) as uint;
        // checking for failure to shrink is tricky
        if size < old_size {
            usable_size(size, align) == new_size as uint
        } else {
            new_size >= size
        }
    }

    #[inline]
    pub unsafe fn deallocate(ptr: *mut u8, size: uint, align: uint) {
        let flags = align_to_flags(align);
        je_sdallocx(ptr as *mut c_void, size as size_t, flags)
    }

    #[inline]
    pub fn usable_size(size: uint, align: uint) -> uint {
        let flags = align_to_flags(align);
        unsafe { je_nallocx(size as size_t, flags) as uint }
    }

    pub fn stats_print() {
        unsafe {
            je_malloc_stats_print(None, null_mut(), null())
        }
    }
}

#[cfg(all(not(jemalloc), unix))]
mod imp {
    use core::cmp;
    use core::ptr;
    use libc;
    use libc_heap;
    use super::MIN_ALIGN;

    extern {
        fn posix_memalign(memptr: *mut *mut libc::c_void,
                          align: libc::size_t,
                          size: libc::size_t) -> libc::c_int;
    }

    #[inline]
    pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
        if align <= MIN_ALIGN {
            libc_heap::malloc_raw(size)
        } else {
            let mut out = 0 as *mut libc::c_void;
            let ret = posix_memalign(&mut out,
                                     align as libc::size_t,
                                     size as libc::size_t);
            if ret != 0 {
                ::oom();
            }
            out as *mut u8
        }
    }

    #[inline]
    pub unsafe fn reallocate(ptr: *mut u8, old_size: uint, size: uint, align: uint) -> *mut u8 {
        if align <= MIN_ALIGN {
            libc_heap::realloc_raw(ptr, size)
        } else {
            let new_ptr = allocate(size, align);
            ptr::copy_memory(new_ptr, ptr as *const u8, cmp::min(size, old_size));
            deallocate(ptr, old_size, align);
            new_ptr
        }
    }

    #[inline]
    pub unsafe fn reallocate_inplace(_ptr: *mut u8, old_size: uint, size: uint,
                                     _align: uint) -> bool {
        size == old_size
    }

    #[inline]
    pub unsafe fn deallocate(ptr: *mut u8, _size: uint, _align: uint) {
        libc::free(ptr as *mut libc::c_void)
    }

    #[inline]
    pub fn usable_size(size: uint, _align: uint) -> uint {
        size
    }

    pub fn stats_print() {}
}

#[cfg(all(not(jemalloc), windows))]
mod imp {
    use libc::{c_void, size_t};
    use libc;
    use libc_heap;
    use core::ptr::RawPtr;
    use super::MIN_ALIGN;

    extern {
        fn _aligned_malloc(size: size_t, align: size_t) -> *mut c_void;
        fn _aligned_realloc(block: *mut c_void, size: size_t,
                            align: size_t) -> *mut c_void;
        fn _aligned_free(ptr: *mut c_void);
    }

    #[inline]
    pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
        if align <= MIN_ALIGN {
            libc_heap::malloc_raw(size)
        } else {
            let ptr = _aligned_malloc(size as size_t, align as size_t);
            if ptr.is_null() {
                ::oom();
            }
            ptr as *mut u8
        }
    }

    #[inline]
    pub unsafe fn reallocate(ptr: *mut u8, _old_size: uint, size: uint, align: uint) -> *mut u8 {
        if align <= MIN_ALIGN {
            libc_heap::realloc_raw(ptr, size)
        } else {
            let ptr = _aligned_realloc(ptr as *mut c_void, size as size_t,
                                       align as size_t);
            if ptr.is_null() {
                ::oom();
            }
            ptr as *mut u8
        }
    }

    #[inline]
    pub unsafe fn reallocate_inplace(_ptr: *mut u8, old_size: uint, size: uint,
                                     _align: uint) -> bool {
        size == old_size
    }

    #[inline]
    pub unsafe fn deallocate(ptr: *mut u8, _size: uint, align: uint) {
        if align <= MIN_ALIGN {
            libc::free(ptr as *mut libc::c_void)
        } else {
            _aligned_free(ptr as *mut c_void)
        }
    }

    #[inline]
    pub fn usable_size(size: uint, _align: uint) -> uint {
        size
    }

    pub fn stats_print() {}
}

#[cfg(test)]
mod test {
    extern crate test;
    use self::test::Bencher;
    use heap;

    #[test]
    fn basic_reallocate_inplace_noop() {
        unsafe {
            let size = 4000;
            let ptr = heap::allocate(size, 8);
            let ret = heap::reallocate_inplace(ptr, size, size, 8);
            heap::deallocate(ptr, size, 8);
            assert!(ret);
        }
    }

    #[bench]
    fn alloc_owned_small(b: &mut Bencher) {
        b.iter(|| {
            box 10i
        })
    }
}
<|endoftext|>


<commit_before>use crate::libyaml::cstr::CStr;
use std::fmt::{self, Debug, Display};
use std::mem::MaybeUninit;
use std::ptr::NonNull;
use unsafe_libyaml as sys;

pub(crate) type Result<T> = std::result::Result<T, Error>;

pub(crate) struct Error {
    kind: sys::yaml_error_type_t,
    problem: CStr<'static>,
    problem_offset: u64,
    problem_mark: Mark,
    context: Option<CStr<'static>>,
    context_mark: Mark,
}

impl Error {
    pub unsafe fn parse_error(parser: *const sys::yaml_parser_t) -> Self {
        Error {
            kind: (*parser).error,
            problem: match NonNull::new((*parser).problem as *mut _) {
                Some(problem) => CStr::from_ptr(problem),
                None => CStr::from_bytes_with_nul(b"libyaml parser failed but there is no error\0"),
            },
            problem_offset: (*parser).problem_offset,
            problem_mark: Mark {
                sys: (*parser).problem_mark,
            },
            context: match NonNull::new((*parser).context as *mut _) {
                Some(context) => Some(CStr::from_ptr(context)),
                None => None,
            },
            context_mark: Mark {
                sys: (*parser).context_mark,
            },
        }
    }

    pub unsafe fn emit_error(emitter: *const sys::yaml_emitter_t) -> Self {
        Error {
            kind: (*emitter).error,
            problem: match NonNull::new((*emitter).problem as *mut _) {
                Some(problem) => CStr::from_ptr(problem),
                None => {
                    CStr::from_bytes_with_nul(b"libyaml emitter failed but there is no error\0")
                }
            },
            problem_offset: 0,
            problem_mark: Mark {
                sys: MaybeUninit::<sys::yaml_mark_t>::zeroed().assume_init(),
            },
            context: None,
            context_mark: Mark {
                sys: MaybeUninit::<sys::yaml_mark_t>::zeroed().assume_init(),
            },
        }
    }

    pub fn mark(&self) -> Mark {
        self.problem_mark
    }
}

impl Display for Error {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "{}", self.problem)?;
        if self.problem_mark.sys.line != 0 || self.problem_mark.sys.column != 0 {
            write!(formatter, " at {}", self.problem_mark)?;
        } else if self.problem_offset != 0 {
            write!(formatter, " at position {}", self.problem_offset)?;
        }
        if let Some(context) = &self.context {
            write!(formatter, ", {}", context)?;
            if (self.context_mark.sys.line != 0 || self.context_mark.sys.column != 0)
                && (self.context_mark.sys.line != self.problem_mark.sys.line
                    || self.context_mark.sys.column != self.problem_mark.sys.column)
            {
                write!(formatter, " at {}", self.context_mark)?;
            }
        }
        Ok(())
    }
}

impl Debug for Error {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        let mut formatter = formatter.debug_struct("Error");
        if let Some(kind) = match self.kind {
            sys::YAML_MEMORY_ERROR => Some("MEMORY"),
            sys::YAML_READER_ERROR => Some("READER"),
            sys::YAML_SCANNER_ERROR => Some("SCANNER"),
            sys::YAML_PARSER_ERROR => Some("PARSER"),
            sys::YAML_COMPOSER_ERROR => Some("COMPOSER"),
            sys::YAML_WRITER_ERROR => Some("WRITER"),
            sys::YAML_EMITTER_ERROR => Some("EMITTER"),
            _ => None,
        } {
            formatter.field("kind", &format_args!("{}", kind));
        }
        if self.problem_mark.sys.line != 0 || self.problem_mark.sys.column != 0 {
            formatter.field("problem_mark", &self.problem_mark);
        } else if self.problem_offset != 0 {
            formatter.field("problem_offset", &self.problem_offset);
        }
        if let Some(context) = &self.context {
            formatter.field("context", context);
            if self.context_mark.sys.line != 0 || self.context_mark.sys.column != 0 {
                formatter.field("context_mark", &self.context_mark);
            }
        }
        formatter.finish()
    }
}

#[derive(Copy, Clone)]
pub(crate) struct Mark {
    pub(super) sys: sys::yaml_mark_t,
}

impl Mark {
    pub fn index(&self) -> u64 {
        self.sys.index
    }

    pub fn line(&self) -> u64 {
        self.sys.line
    }

    pub fn column(&self) -> u64 {
        self.sys.column
    }
}

impl Display for Mark {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        if self.sys.line != 0 || self.sys.column != 0 {
            write!(
                formatter,
                "line {} column {}",
                self.sys.line + 1,
                self.sys.column + 1,
            )
        } else {
            write!(formatter, "position {}", self.sys.index)
        }
    }
}

impl Debug for Mark {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        let mut formatter = formatter.debug_struct("Mark");
        if self.sys.line != 0 || self.sys.column != 0 {
            formatter.field("line", &(self.sys.line + 1));
            formatter.field("column", &(self.sys.column + 1));
        } else {
            formatter.field("index", &self.sys.index);
        }
        formatter.finish()
    }
}
<commit_msg>Fix missing message in Debug of libyaml Error<commit_after>use crate::libyaml::cstr::CStr;
use std::fmt::{self, Debug, Display};
use std::mem::MaybeUninit;
use std::ptr::NonNull;
use unsafe_libyaml as sys;

pub(crate) type Result<T> = std::result::Result<T, Error>;

pub(crate) struct Error {
    kind: sys::yaml_error_type_t,
    problem: CStr<'static>,
    problem_offset: u64,
    problem_mark: Mark,
    context: Option<CStr<'static>>,
    context_mark: Mark,
}

impl Error {
    pub unsafe fn parse_error(parser: *const sys::yaml_parser_t) -> Self {
        Error {
            kind: (*parser).error,
            problem: match NonNull::new((*parser).problem as *mut _) {
                Some(problem) => CStr::from_ptr(problem),
                None => CStr::from_bytes_with_nul(b"libyaml parser failed but there is no error\0"),
            },
            problem_offset: (*parser).problem_offset,
            problem_mark: Mark {
                sys: (*parser).problem_mark,
            },
            context: match NonNull::new((*parser).context as *mut _) {
                Some(context) => Some(CStr::from_ptr(context)),
                None => None,
            },
            context_mark: Mark {
                sys: (*parser).context_mark,
            },
        }
    }

    pub unsafe fn emit_error(emitter: *const sys::yaml_emitter_t) -> Self {
        Error {
            kind: (*emitter).error,
            problem: match NonNull::new((*emitter).problem as *mut _) {
                Some(problem) => CStr::from_ptr(problem),
                None => {
                    CStr::from_bytes_with_nul(b"libyaml emitter failed but there is no error\0")
                }
            },
            problem_offset: 0,
            problem_mark: Mark {
                sys: MaybeUninit::<sys::yaml_mark_t>::zeroed().assume_init(),
            },
            context: None,
            context_mark: Mark {
                sys: MaybeUninit::<sys::yaml_mark_t>::zeroed().assume_init(),
            },
        }
    }

    pub fn mark(&self) -> Mark {
        self.problem_mark
    }
}

impl Display for Error {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        write!(formatter, "{}", self.problem)?;
        if self.problem_mark.sys.line != 0 || self.problem_mark.sys.column != 0 {
            write!(formatter, " at {}", self.problem_mark)?;
        } else if self.problem_offset != 0 {
            write!(formatter, " at position {}", self.problem_offset)?;
        }
        if let Some(context) = &self.context {
            write!(formatter, ", {}", context)?;
            if (self.context_mark.sys.line != 0 || self.context_mark.sys.column != 0)
                && (self.context_mark.sys.line != self.problem_mark.sys.line
                    || self.context_mark.sys.column != self.problem_mark.sys.column)
            {
                write!(formatter, " at {}", self.context_mark)?;
            }
        }
        Ok(())
    }
}

impl Debug for Error {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        let mut formatter = formatter.debug_struct("Error");
        if let Some(kind) = match self.kind {
            sys::YAML_MEMORY_ERROR => Some("MEMORY"),
            sys::YAML_READER_ERROR => Some("READER"),
            sys::YAML_SCANNER_ERROR => Some("SCANNER"),
            sys::YAML_PARSER_ERROR => Some("PARSER"),
            sys::YAML_COMPOSER_ERROR => Some("COMPOSER"),
            sys::YAML_WRITER_ERROR => Some("WRITER"),
            sys::YAML_EMITTER_ERROR => Some("EMITTER"),
            _ => None,
        } {
            formatter.field("kind", &format_args!("{}", kind));
        }
        formatter.field("problem", &self.problem);
        if self.problem_mark.sys.line != 0 || self.problem_mark.sys.column != 0 {
            formatter.field("problem_mark", &self.problem_mark);
        } else if self.problem_offset != 0 {
            formatter.field("problem_offset", &self.problem_offset);
        }
        if let Some(context) = &self.context {
            formatter.field("context", context);
            if self.context_mark.sys.line != 0 || self.context_mark.sys.column != 0 {
                formatter.field("context_mark", &self.context_mark);
            }
        }
        formatter.finish()
    }
}

#[derive(Copy, Clone)]
pub(crate) struct Mark {
    pub(super) sys: sys::yaml_mark_t,
}

impl Mark {
    pub fn index(&self) -> u64 {
        self.sys.index
    }

    pub fn line(&self) -> u64 {
        self.sys.line
    }

    pub fn column(&self) -> u64 {
        self.sys.column
    }
}

impl Display for Mark {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        if self.sys.line != 0 || self.sys.column != 0 {
            write!(
                formatter,
                "line {} column {}",
                self.sys.line + 1,
                self.sys.column + 1,
            )
        } else {
            write!(formatter, "position {}", self.sys.index)
        }
    }
}

impl Debug for Mark {
    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
        let mut formatter = formatter.debug_struct("Mark");
        if self.sys.line != 0 || self.sys.column != 0 {
            formatter.field("line", &(self.sys.line + 1));
            formatter.field("column", &(self.sys.column + 1));
        } else {
            formatter.field("index", &self.sys.index);
        }
        formatter.finish()
    }
}
<|endoftext|>


<commit_before>use std::slice;

use libc::{c_void, c_uint, c_double, c_int, size_t};

use symbols::{ZopfliGetDistExtraBits, ZopfliGetLengthExtraBits, ZopfliGetLengthSymbol, ZopfliGetDistSymbol, ZOPFLI_NUM_LL, ZOPFLI_NUM_D};

const K_INV_LOG2: c_double = 1.4426950408889;  // 1.0 / log(2.0)

/// Cost model which should exactly match fixed tree.
/// type: CostModelFun
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn GetCostFixed(litlen: c_uint, dist: c_uint, _unused: c_void) -> c_double {
    let result = if dist == 0 {
        if litlen <= 143 {
            8
        } else {
            9
        }
    } else {
        let dbits = ZopfliGetDistExtraBits(dist as c_int);
        let lbits = ZopfliGetLengthExtraBits(litlen as c_int);
        let lsym = ZopfliGetLengthSymbol(litlen as c_int);
        let mut cost = 0;
        if lsym <= 279 {
            cost += 7;
        } else {
            cost += 8;
        }
        cost += 5;  // Every dist symbol has length 5.
        cost + dbits + lbits
    };
    result as c_double
}

/// Cost model based on symbol statistics.
/// type: CostModelFun
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn GetCostStat(litlen: c_uint, dist: c_uint, context: *const c_void) -> c_double {
    let stats = unsafe {
        assert!(!context.is_null());
        &*(context as *const SymbolStats)
    };
    if dist == 0 {
        stats.ll_symbols[litlen as usize]
    } else {
        let lsym = ZopfliGetLengthSymbol(litlen as c_int) as usize;
        let lbits = ZopfliGetLengthExtraBits(litlen as c_int) as c_double;
        let dsym = ZopfliGetDistSymbol(dist as c_int) as usize;
        let dbits = ZopfliGetDistExtraBits(dist as c_int) as c_double;
        stats.ll_symbols[lsym] + lbits + stats.d_symbols[dsym] + dbits
    }
}

pub struct RanState {
    m_w: u32,
    m_z: u32,
}

impl RanState {
    pub fn new() -> RanState {
        RanState {
            m_w: 1,
            m_z: 2,
        }
    }

    /// Get random number: "Multiply-With-Carry" generator of G. Marsaglia
    pub fn random_marsaglia(&mut self) -> u32 {
        self.m_z = 36969 * (self.m_z & 65535) + (self.m_z >> 16);
        self.m_w = 18000 * (self.m_w & 65535) + (self.m_w >> 16);
        (self.m_z << 16) + self.m_w // 32-bit result.
    }
}

#[no_mangle]
pub extern fn ran_state_new() -> *mut RanState {
    Box::into_raw(Box::new(RanState::new()))
}

#[repr(C)]
#[derive(Copy)]
pub struct SymbolStats {
  /* The literal and length symbols. */
  litlens: [size_t; ZOPFLI_NUM_LL],
  /* The 32 unique dist symbols, not the 32768 possible dists. */
  dists: [size_t; ZOPFLI_NUM_D],

  /* Length of each lit/len symbol in bits. */
  ll_symbols: [c_double; ZOPFLI_NUM_LL],
  /* Length of each dist symbol in bits. */
  d_symbols: [c_double; ZOPFLI_NUM_D],
}

impl Clone for SymbolStats {
    fn clone(&self) -> Self {
        *self
    }
}

impl SymbolStats {
    pub fn randomize_stat_freqs(&mut self, state: &mut RanState) {
        fn randomize_freqs(freqs: &mut [size_t], state: &mut RanState) {
            let n = freqs.len();
            let mut i: usize = 0;
            let end = n as usize;

            while i < end {
                if (state.random_marsaglia() >> 4) % 3 == 0 {
                    let index = state.random_marsaglia() % n as c_uint;
                    freqs[i] = freqs[index as usize];
                }
                i += 1;
            }
        }
        randomize_freqs(&mut self.litlens, state);
        randomize_freqs(&mut self.dists, state);
        self.litlens[256] = 1; // End symbol.
    }

    /// Calculates the entropy of each symbol, based on the counts of each symbol. The
    /// result is similar to the result of ZopfliCalculateBitLengths, but with the
    /// actual theoritical bit lengths according to the entropy. Since the resulting
    /// values are fractional, they cannot be used to encode the tree specified by
    /// DEFLATE.
    pub fn calculate_entropy(&mut self) {
        fn calculate_and_store_entropy(count_ptr: *const size_t, n: size_t, bitlengths_ptr: *mut c_double) {
            let count = unsafe {
                assert!(!count_ptr.is_null());
                slice::from_raw_parts(count_ptr, n as usize)
            };
            let bitlengths = unsafe {
                assert!(!bitlengths_ptr.is_null());
                slice::from_raw_parts_mut(bitlengths_ptr, n as usize)
            };

            let mut sum = 0;
            for i in 0..(n as usize) {
                sum += count[i];
            }

            let log2sum = (if sum == 0 { n } else { sum } as c_double).ln() * K_INV_LOG2;

            for i in 0..n as usize {
                // When the count of the symbol is 0, but its cost is requested anyway, it
                // means the symbol will appear at least once anyway, so give it the cost as if
                // its count is 1.
                if count[i] == 0 {
                    bitlengths[i] = log2sum;
                } else {
                    bitlengths[i] = log2sum - (count[i] as c_double).ln() * K_INV_LOG2;
                }

                // Depending on compiler and architecture, the above subtraction of two
                // floating point numbers may give a negative result very close to zero
                // instead of zero (e.g. -5.973954e-17 with gcc 4.1.2 on Ubuntu 11.4). Clamp
                // it to zero. These floating point imprecisions do not affect the cost model
                // significantly so this is ok.
                if bitlengths[i] < 0.0 && bitlengths[i] > -1E-5 {
                    bitlengths[i] = 0.0;
                }
                assert!(bitlengths[i] >= 0.0);
            }
        }

        calculate_and_store_entropy(self.litlens.as_ptr(), ZOPFLI_NUM_LL as size_t, self.ll_symbols.as_mut_ptr());
        calculate_and_store_entropy(self.dists.as_ptr(), ZOPFLI_NUM_D as size_t, self.d_symbols.as_mut_ptr());

    }
}

/* Sets everything to 0. */
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn InitStats(stats_ptr: *mut SymbolStats) {
    let stats = unsafe {
        assert!(!stats_ptr.is_null());
        &mut *stats_ptr
    };
    stats.litlens = [0; ZOPFLI_NUM_LL];
    stats.dists = [0; ZOPFLI_NUM_D];
    stats.ll_symbols = [0.0; ZOPFLI_NUM_LL];
    stats.d_symbols = [0.0; ZOPFLI_NUM_D];
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ClearStatFreqs(stats_ptr: *mut SymbolStats) {
    let stats = unsafe {
        assert!(!stats_ptr.is_null());
        &mut *stats_ptr
    };
    stats.litlens = [0; ZOPFLI_NUM_LL];
    stats.dists = [0; ZOPFLI_NUM_D];
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn CopyStats(source_ptr: *mut SymbolStats, dest_ptr: *mut SymbolStats) {
    let source = unsafe {
        assert!(!source_ptr.is_null());
        &mut *source_ptr
    };
    let dest = unsafe {
        assert!(!dest_ptr.is_null());
        &mut *dest_ptr
    };
    *dest = *source;
}

/// Adds the bit lengths.
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn AddWeighedStatFreqs(stats1_ptr: *mut SymbolStats, w1: c_double, stats2_ptr: *mut SymbolStats, w2: c_double, result_ptr: *mut SymbolStats) {
    let stats1 = unsafe {
        assert!(!stats1_ptr.is_null());
        &mut *stats1_ptr
    };
    let stats2 = unsafe {
        assert!(!stats2_ptr.is_null());
        &mut *stats2_ptr
    };
    let result = unsafe {
        assert!(!result_ptr.is_null());
        &mut *result_ptr
    };

    for i in 0..ZOPFLI_NUM_LL {
        result.litlens[i] = (stats1.litlens[i] as c_double * w1 + stats2.litlens[i] as c_double * w2) as size_t;
    }
    for i in 0..ZOPFLI_NUM_D {
        result.dists[i] = (stats1.dists[i] as c_double * w1 + stats2.dists[i] as c_double * w2) as size_t;
    }
    result.litlens[256] = 1; // End symbol.
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn RandomizeStatFreqs(state_ptr: *mut RanState, stats_ptr: *mut SymbolStats) {
    let stats = unsafe {
        assert!(!stats_ptr.is_null());
        &mut *stats_ptr
    };
    let state = unsafe {
        assert!(!state_ptr.is_null());
        &mut *state_ptr
    };

    stats.randomize_stat_freqs(state);
}

/// Calculates the entropy of the statistics
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn CalculateStatistics(stats_ptr: *mut SymbolStats) {
    let stats = unsafe {
        assert!(!stats_ptr.is_null());
        &mut *stats_ptr
    };

    stats.calculate_entropy();
}
<commit_msg>Make calculate_and_store_entropy take a slice instead<commit_after>use libc::{c_void, c_uint, c_double, c_int, size_t};

use symbols::{ZopfliGetDistExtraBits, ZopfliGetLengthExtraBits, ZopfliGetLengthSymbol, ZopfliGetDistSymbol, ZOPFLI_NUM_LL, ZOPFLI_NUM_D};

const K_INV_LOG2: c_double = 1.4426950408889;  // 1.0 / log(2.0)

/// Cost model which should exactly match fixed tree.
/// type: CostModelFun
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn GetCostFixed(litlen: c_uint, dist: c_uint, _unused: c_void) -> c_double {
    let result = if dist == 0 {
        if litlen <= 143 {
            8
        } else {
            9
        }
    } else {
        let dbits = ZopfliGetDistExtraBits(dist as c_int);
        let lbits = ZopfliGetLengthExtraBits(litlen as c_int);
        let lsym = ZopfliGetLengthSymbol(litlen as c_int);
        let mut cost = 0;
        if lsym <= 279 {
            cost += 7;
        } else {
            cost += 8;
        }
        cost += 5;  // Every dist symbol has length 5.
        cost + dbits + lbits
    };
    result as c_double
}

/// Cost model based on symbol statistics.
/// type: CostModelFun
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn GetCostStat(litlen: c_uint, dist: c_uint, context: *const c_void) -> c_double {
    let stats = unsafe {
        assert!(!context.is_null());
        &*(context as *const SymbolStats)
    };
    if dist == 0 {
        stats.ll_symbols[litlen as usize]
    } else {
        let lsym = ZopfliGetLengthSymbol(litlen as c_int) as usize;
        let lbits = ZopfliGetLengthExtraBits(litlen as c_int) as c_double;
        let dsym = ZopfliGetDistSymbol(dist as c_int) as usize;
        let dbits = ZopfliGetDistExtraBits(dist as c_int) as c_double;
        stats.ll_symbols[lsym] + lbits + stats.d_symbols[dsym] + dbits
    }
}

pub struct RanState {
    m_w: u32,
    m_z: u32,
}

impl RanState {
    pub fn new() -> RanState {
        RanState {
            m_w: 1,
            m_z: 2,
        }
    }

    /// Get random number: "Multiply-With-Carry" generator of G. Marsaglia
    pub fn random_marsaglia(&mut self) -> u32 {
        self.m_z = 36969 * (self.m_z & 65535) + (self.m_z >> 16);
        self.m_w = 18000 * (self.m_w & 65535) + (self.m_w >> 16);
        (self.m_z << 16) + self.m_w // 32-bit result.
    }
}

#[no_mangle]
pub extern fn ran_state_new() -> *mut RanState {
    Box::into_raw(Box::new(RanState::new()))
}

#[repr(C)]
#[derive(Copy)]
pub struct SymbolStats {
  /* The literal and length symbols. */
  litlens: [size_t; ZOPFLI_NUM_LL],
  /* The 32 unique dist symbols, not the 32768 possible dists. */
  dists: [size_t; ZOPFLI_NUM_D],

  /* Length of each lit/len symbol in bits. */
  ll_symbols: [c_double; ZOPFLI_NUM_LL],
  /* Length of each dist symbol in bits. */
  d_symbols: [c_double; ZOPFLI_NUM_D],
}

impl Clone for SymbolStats {
    fn clone(&self) -> Self {
        *self
    }
}

impl SymbolStats {
    pub fn randomize_stat_freqs(&mut self, state: &mut RanState) {
        fn randomize_freqs(freqs: &mut [size_t], state: &mut RanState) {
            let n = freqs.len();
            let mut i: usize = 0;
            let end = n as usize;

            while i < end {
                if (state.random_marsaglia() >> 4) % 3 == 0 {
                    let index = state.random_marsaglia() % n as c_uint;
                    freqs[i] = freqs[index as usize];
                }
                i += 1;
            }
        }
        randomize_freqs(&mut self.litlens, state);
        randomize_freqs(&mut self.dists, state);
        self.litlens[256] = 1; // End symbol.
    }

    /// Calculates the entropy of each symbol, based on the counts of each symbol. The
    /// result is similar to the result of ZopfliCalculateBitLengths, but with the
    /// actual theoritical bit lengths according to the entropy. Since the resulting
    /// values are fractional, they cannot be used to encode the tree specified by
    /// DEFLATE.
    pub fn calculate_entropy(&mut self) {
        fn calculate_and_store_entropy(count: &[size_t], bitlengths: &mut [c_double]) {
            let n = count.len();

            let mut sum = 0;
            for i in 0..n {
                sum += count[i];
            }

            let log2sum = (if sum == 0 { n } else { sum } as c_double).ln() * K_INV_LOG2;

            for i in 0..n {
                // When the count of the symbol is 0, but its cost is requested anyway, it
                // means the symbol will appear at least once anyway, so give it the cost as if
                // its count is 1.
                if count[i] == 0 {
                    bitlengths[i] = log2sum;
                } else {
                    bitlengths[i] = log2sum - (count[i] as c_double).ln() * K_INV_LOG2;
                }

                // Depending on compiler and architecture, the above subtraction of two
                // floating point numbers may give a negative result very close to zero
                // instead of zero (e.g. -5.973954e-17 with gcc 4.1.2 on Ubuntu 11.4). Clamp
                // it to zero. These floating point imprecisions do not affect the cost model
                // significantly so this is ok.
                if bitlengths[i] < 0.0 && bitlengths[i] > -1E-5 {
                    bitlengths[i] = 0.0;
                }
                assert!(bitlengths[i] >= 0.0);
            }
        }

        calculate_and_store_entropy(&self.litlens, &mut self.ll_symbols);
        calculate_and_store_entropy(&self.dists, &mut self.d_symbols);

    }
}

/* Sets everything to 0. */
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn InitStats(stats_ptr: *mut SymbolStats) {
    let stats = unsafe {
        assert!(!stats_ptr.is_null());
        &mut *stats_ptr
    };
    stats.litlens = [0; ZOPFLI_NUM_LL];
    stats.dists = [0; ZOPFLI_NUM_D];
    stats.ll_symbols = [0.0; ZOPFLI_NUM_LL];
    stats.d_symbols = [0.0; ZOPFLI_NUM_D];
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn ClearStatFreqs(stats_ptr: *mut SymbolStats) {
    let stats = unsafe {
        assert!(!stats_ptr.is_null());
        &mut *stats_ptr
    };
    stats.litlens = [0; ZOPFLI_NUM_LL];
    stats.dists = [0; ZOPFLI_NUM_D];
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn CopyStats(source_ptr: *mut SymbolStats, dest_ptr: *mut SymbolStats) {
    let source = unsafe {
        assert!(!source_ptr.is_null());
        &mut *source_ptr
    };
    let dest = unsafe {
        assert!(!dest_ptr.is_null());
        &mut *dest_ptr
    };
    *dest = *source;
}

/// Adds the bit lengths.
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn AddWeighedStatFreqs(stats1_ptr: *mut SymbolStats, w1: c_double, stats2_ptr: *mut SymbolStats, w2: c_double, result_ptr: *mut SymbolStats) {
    let stats1 = unsafe {
        assert!(!stats1_ptr.is_null());
        &mut *stats1_ptr
    };
    let stats2 = unsafe {
        assert!(!stats2_ptr.is_null());
        &mut *stats2_ptr
    };
    let result = unsafe {
        assert!(!result_ptr.is_null());
        &mut *result_ptr
    };

    for i in 0..ZOPFLI_NUM_LL {
        result.litlens[i] = (stats1.litlens[i] as c_double * w1 + stats2.litlens[i] as c_double * w2) as size_t;
    }
    for i in 0..ZOPFLI_NUM_D {
        result.dists[i] = (stats1.dists[i] as c_double * w1 + stats2.dists[i] as c_double * w2) as size_t;
    }
    result.litlens[256] = 1; // End symbol.
}

#[no_mangle]
#[allow(non_snake_case)]
pub extern fn RandomizeStatFreqs(state_ptr: *mut RanState, stats_ptr: *mut SymbolStats) {
    let stats = unsafe {
        assert!(!stats_ptr.is_null());
        &mut *stats_ptr
    };
    let state = unsafe {
        assert!(!state_ptr.is_null());
        &mut *state_ptr
    };

    stats.randomize_stat_freqs(state);
}

/// Calculates the entropy of the statistics
#[no_mangle]
#[allow(non_snake_case)]
pub extern fn CalculateStatistics(stats_ptr: *mut SymbolStats) {
    let stats = unsafe {
        assert!(!stats_ptr.is_null());
        &mut *stats_ptr
    };

    stats.calculate_entropy();
}
<|endoftext|>


<commit_before>use std::io;
use std::path::{Path, PathBuf};
use url::{self, Url};

/// Wrapper for URLs which are allowed to be loaded
///
/// SVG files can reference other files (PNG/JPEG images, other SVGs,
/// CSS files, etc.).  This object is constructed by checking whether
/// a specified `href` (a possibly-relative filename, for example)
/// should be allowed to be loaded, given the base URL of the SVG
/// being loaded.
#[derive(Debug, PartialEq)]
pub struct AllowedUrl(Url);

#[derive(Debug, PartialEq)]
pub enum AllowedUrlError {
    /// parsing error from `Url::parse()`
    HrefParseError(url::ParseError),

    /// A base file/uri was not set
    BaseRequired,

    /// Cannot reference a file with a different URI scheme from the base file
    DifferentURISchemes,

    /// Some scheme we don't allow loading
    DisallowedScheme,

    /// The requested file is not in the same directory as the base file,
    /// or in one directory below the base file.
    NotSiblingOrChildOfBaseFile,

    /// Error when obtaining the file path or the base file path
    InvalidPath,

    /// The base file cannot be the root of the file system
    BaseIsRoot,

    /// Error when canonicalizing either the file path or the base file path
    CanonicalizationError,
}

impl AllowedUrl {
    pub fn from_href(href: &str, base_url: Option<&Url>) -> Result<AllowedUrl, AllowedUrlError> {
        let url = Url::options()
            .base_url(base_url)
            .parse(href)
            .map_err(AllowedUrlError::HrefParseError)?;

        // Allow loads of data: from any location
        if url.scheme() == "data" {
            return Ok(AllowedUrl(url));
        }

        // All other sources require a base url
        if base_url.is_none() {
            return Err(AllowedUrlError::BaseRequired);
        }

        let base_url = base_url.unwrap();

        // Deny loads from differing URI schemes
        if url.scheme() != base_url.scheme() {
            return Err(AllowedUrlError::DifferentURISchemes);
        }

        // resource: is allowed to load anything from other resources
        if url.scheme() == "resource" {
            return Ok(AllowedUrl(url));
        }

        // Non-file: isn't allowed to load anything
        if url.scheme() != "file" {
            return Err(AllowedUrlError::DisallowedScheme);
        }

        // We have two file: URIs.  Now canonicalize them (remove .. and symlinks, etc.)
        // and see if the directories match

        let url_path = url
            .to_file_path()
            .map_err(|_| AllowedUrlError::InvalidPath)?;
        let base_path = base_url
            .to_file_path()
            .map_err(|_| AllowedUrlError::InvalidPath)?;

        let base_parent = base_path.parent();
        if base_parent.is_none() {
            return Err(AllowedUrlError::BaseIsRoot);
        }

        let base_parent = base_parent.unwrap();

        let url_canon =
            canonicalize(&url_path).map_err(|_| AllowedUrlError::CanonicalizationError)?;
        let parent_canon =
            canonicalize(&base_parent).map_err(|_| AllowedUrlError::CanonicalizationError)?;

        if url_canon.starts_with(parent_canon) {
            Ok(AllowedUrl(url))
        } else {
            Err(AllowedUrlError::NotSiblingOrChildOfBaseFile)
        }
    }

    pub fn url(&self) -> &Url {
        &self.0
    }
}

// For tests, we don't want to touch the filesystem.  In that case,
// assume that we are being passed canonical file names.
#[cfg(not(test))]
fn canonicalize<P: AsRef<Path>>(path: P) -> Result<PathBuf, io::Error> {
    path.as_ref().canonicalize()
}
#[cfg(test)]
fn canonicalize<P: AsRef<Path>>(path: P) -> Result<PathBuf, io::Error> {
    Ok(path.as_ref().to_path_buf())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn disallows_relative_file_with_no_base_file() {
        assert_eq!(
            AllowedUrl::from_href("foo.svg", None),
            Err(AllowedUrlError::HrefParseError(
                url::ParseError::RelativeUrlWithoutBase
            ))
        );
    }

    #[test]
    fn disallows_different_schemes() {
        assert_eq!(
            AllowedUrl::from_href(
                "file:///etc/passwd",
                Some(Url::parse("http://example.com/malicious.svg").unwrap()).as_ref()
            ),
            Err(AllowedUrlError::DifferentURISchemes)
        );
    }

    #[test]
    fn disallows_base_is_root() {
        assert_eq!(
            AllowedUrl::from_href("foo.svg", Some(Url::parse("file:///").unwrap()).as_ref()),
            Err(AllowedUrlError::BaseIsRoot)
        );
    }

    #[test]
    fn disallows_non_file_scheme() {
        assert_eq!(
            AllowedUrl::from_href(
                "foo.svg",
                Some(Url::parse("http://foo.bar/baz.svg").unwrap()).as_ref()
            ),
            Err(AllowedUrlError::DisallowedScheme)
        );
    }

    #[test]
    fn allows_data_url_with_no_base_file() {
        assert_eq!(
            AllowedUrl::from_href("data:image/jpeg;base64,xxyyzz", None)
                .unwrap()
                .url(),
            &Url::parse("data:image/jpeg;base64,xxyyzz").unwrap(),
        );
    }

    #[test]
    fn allows_relative() {
        assert_eq!(
            AllowedUrl::from_href(
                "foo.svg",
                Some(Url::parse("file:///example/bar.svg").unwrap()).as_ref()
            )
            .unwrap()
            .url(),
            &Url::parse("file:///example/foo.svg").unwrap(),
        );
    }

    #[test]
    fn allows_sibling() {
        assert_eq!(
            AllowedUrl::from_href(
                "file:///example/foo.svg",
                Some(Url::parse("file:///example/bar.svg").unwrap()).as_ref()
            )
            .unwrap()
            .url(),
            &Url::parse("file:///example/foo.svg").unwrap(),
        );
    }

    #[test]
    fn allows_child_of_sibling() {
        assert_eq!(
            AllowedUrl::from_href(
                "file:///example/subdir/foo.svg",
                Some(Url::parse("file:///example/bar.svg").unwrap()).as_ref()
            )
            .unwrap()
            .url(),
            &Url::parse("file:///example/subdir/foo.svg").unwrap(),
        );
    }

    #[test]
    fn disallows_non_sibling() {
        assert_eq!(
            AllowedUrl::from_href(
                "file:///etc/passwd",
                Some(Url::parse("file:///example/bar.svg").unwrap()).as_ref()
            ),
            Err(AllowedUrlError::NotSiblingOrChildOfBaseFile)
        );
    }
}
<commit_msg>impl Error for AllowedUrlError<commit_after>use std::error::{self, Error};
use std::fmt;
use std::io;
use std::path::{Path, PathBuf};
use url::{self, Url};

/// Wrapper for URLs which are allowed to be loaded
///
/// SVG files can reference other files (PNG/JPEG images, other SVGs,
/// CSS files, etc.).  This object is constructed by checking whether
/// a specified `href` (a possibly-relative filename, for example)
/// should be allowed to be loaded, given the base URL of the SVG
/// being loaded.
#[derive(Debug, PartialEq)]
pub struct AllowedUrl(Url);

#[derive(Debug, PartialEq)]
pub enum AllowedUrlError {
    /// parsing error from `Url::parse()`
    HrefParseError(url::ParseError),

    /// A base file/uri was not set
    BaseRequired,

    /// Cannot reference a file with a different URI scheme from the base file
    DifferentURISchemes,

    /// Some scheme we don't allow loading
    DisallowedScheme,

    /// The requested file is not in the same directory as the base file,
    /// or in one directory below the base file.
    NotSiblingOrChildOfBaseFile,

    /// Error when obtaining the file path or the base file path
    InvalidPath,

    /// The base file cannot be the root of the file system
    BaseIsRoot,

    /// Error when canonicalizing either the file path or the base file path
    CanonicalizationError,
}

impl AllowedUrl {
    pub fn from_href(href: &str, base_url: Option<&Url>) -> Result<AllowedUrl, AllowedUrlError> {
        let url = Url::options()
            .base_url(base_url)
            .parse(href)
            .map_err(AllowedUrlError::HrefParseError)?;

        // Allow loads of data: from any location
        if url.scheme() == "data" {
            return Ok(AllowedUrl(url));
        }

        // All other sources require a base url
        if base_url.is_none() {
            return Err(AllowedUrlError::BaseRequired);
        }

        let base_url = base_url.unwrap();

        // Deny loads from differing URI schemes
        if url.scheme() != base_url.scheme() {
            return Err(AllowedUrlError::DifferentURISchemes);
        }

        // resource: is allowed to load anything from other resources
        if url.scheme() == "resource" {
            return Ok(AllowedUrl(url));
        }

        // Non-file: isn't allowed to load anything
        if url.scheme() != "file" {
            return Err(AllowedUrlError::DisallowedScheme);
        }

        // We have two file: URIs.  Now canonicalize them (remove .. and symlinks, etc.)
        // and see if the directories match

        let url_path = url
            .to_file_path()
            .map_err(|_| AllowedUrlError::InvalidPath)?;
        let base_path = base_url
            .to_file_path()
            .map_err(|_| AllowedUrlError::InvalidPath)?;

        let base_parent = base_path.parent();
        if base_parent.is_none() {
            return Err(AllowedUrlError::BaseIsRoot);
        }

        let base_parent = base_parent.unwrap();

        let url_canon =
            canonicalize(&url_path).map_err(|_| AllowedUrlError::CanonicalizationError)?;
        let parent_canon =
            canonicalize(&base_parent).map_err(|_| AllowedUrlError::CanonicalizationError)?;

        if url_canon.starts_with(parent_canon) {
            Ok(AllowedUrl(url))
        } else {
            Err(AllowedUrlError::NotSiblingOrChildOfBaseFile)
        }
    }

    pub fn url(&self) -> &Url {
        &self.0
    }
}

impl error::Error for AllowedUrlError {
    fn description(&self) -> &str {
        match *self {
            AllowedUrlError::HrefParseError(_) => "href parse error",
            AllowedUrlError::BaseRequired => "base required",
            AllowedUrlError::DifferentURISchemes => "different URI schemes",
            AllowedUrlError::DisallowedScheme => "disallowed scheme",
            AllowedUrlError::NotSiblingOrChildOfBaseFile => "not sibling or child of base file",
            AllowedUrlError::InvalidPath => "invalid path",
            AllowedUrlError::BaseIsRoot => "base is root",
            AllowedUrlError::CanonicalizationError => "canonicalization error",
        }
    }
}

impl fmt::Display for AllowedUrlError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match *self {
            AllowedUrlError::HrefParseError(e) => write!(f, "{}: {}", self.description(), e),
            _ => write!(f, "{}", self.description()),
        }
    }
}

// For tests, we don't want to touch the filesystem.  In that case,
// assume that we are being passed canonical file names.
#[cfg(not(test))]
fn canonicalize<P: AsRef<Path>>(path: P) -> Result<PathBuf, io::Error> {
    path.as_ref().canonicalize()
}
#[cfg(test)]
fn canonicalize<P: AsRef<Path>>(path: P) -> Result<PathBuf, io::Error> {
    Ok(path.as_ref().to_path_buf())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn disallows_relative_file_with_no_base_file() {
        assert_eq!(
            AllowedUrl::from_href("foo.svg", None),
            Err(AllowedUrlError::HrefParseError(
                url::ParseError::RelativeUrlWithoutBase
            ))
        );
    }

    #[test]
    fn disallows_different_schemes() {
        assert_eq!(
            AllowedUrl::from_href(
                "file:///etc/passwd",
                Some(Url::parse("http://example.com/malicious.svg").unwrap()).as_ref()
            ),
            Err(AllowedUrlError::DifferentURISchemes)
        );
    }

    #[test]
    fn disallows_base_is_root() {
        assert_eq!(
            AllowedUrl::from_href("foo.svg", Some(Url::parse("file:///").unwrap()).as_ref()),
            Err(AllowedUrlError::BaseIsRoot)
        );
    }

    #[test]
    fn disallows_non_file_scheme() {
        assert_eq!(
            AllowedUrl::from_href(
                "foo.svg",
                Some(Url::parse("http://foo.bar/baz.svg").unwrap()).as_ref()
            ),
            Err(AllowedUrlError::DisallowedScheme)
        );
    }

    #[test]
    fn allows_data_url_with_no_base_file() {
        assert_eq!(
            AllowedUrl::from_href("data:image/jpeg;base64,xxyyzz", None)
                .unwrap()
                .url(),
            &Url::parse("data:image/jpeg;base64,xxyyzz").unwrap(),
        );
    }

    #[test]
    fn allows_relative() {
        assert_eq!(
            AllowedUrl::from_href(
                "foo.svg",
                Some(Url::parse("file:///example/bar.svg").unwrap()).as_ref()
            )
            .unwrap()
            .url(),
            &Url::parse("file:///example/foo.svg").unwrap(),
        );
    }

    #[test]
    fn allows_sibling() {
        assert_eq!(
            AllowedUrl::from_href(
                "file:///example/foo.svg",
                Some(Url::parse("file:///example/bar.svg").unwrap()).as_ref()
            )
            .unwrap()
            .url(),
            &Url::parse("file:///example/foo.svg").unwrap(),
        );
    }

    #[test]
    fn allows_child_of_sibling() {
        assert_eq!(
            AllowedUrl::from_href(
                "file:///example/subdir/foo.svg",
                Some(Url::parse("file:///example/bar.svg").unwrap()).as_ref()
            )
            .unwrap()
            .url(),
            &Url::parse("file:///example/subdir/foo.svg").unwrap(),
        );
    }

    #[test]
    fn disallows_non_sibling() {
        assert_eq!(
            AllowedUrl::from_href(
                "file:///etc/passwd",
                Some(Url::parse("file:///example/bar.svg").unwrap()).as_ref()
            ),
            Err(AllowedUrlError::NotSiblingOrChildOfBaseFile)
        );
    }
}
<|endoftext|>


<commit_before>use runtime::Runtime;
use module::Module;
use module::ModuleResult;
use module::ModuleError;
use std::path::Path;
use std::result::Result;
use clap::ArgMatches;
use regex::Regex;

mod header;

pub struct BMModule {
    path: Option<String>,
}

const CALLNAMES : &'static [&'static str] = &[ "bm", "bookmark" ];

impl Module for BMModule {

    fn new(rt : &Runtime) -> BMModule {
        BMModule {
            path: None
        }
    }

    fn callnames() -> &'static [&'static str] {
        CALLNAMES
    }

    fn name(&self) -> &'static str{
        "Bookmark"
    }

    fn execute(&self, rt : &Runtime) -> ModuleResult {
        let cmd = rt.config.cli_matches.subcommand_matches("bm").unwrap();
        match cmd.subcommand_name() {
            Some("add")     => {
                debug!("Calling 'add'...");
                add(rt, cmd.subcommand_matches("add").unwrap())
            }
            Some("list")    => {
                debug!("Calling 'list'...");
                list(rt, cmd.subcommand_matches("list").unwrap())
            }
            Some("remove")  => {
                debug!("Calling 'remove'...");
                list(rt, cmd.subcommand_matches("remove").unwrap())
            }
            _ => {
                info!("Not calling any of add, list, remove");
                Ok(())
            }
        }
    }

    fn shutdown(&self, rt : &Runtime) -> ModuleResult {
        Ok(())
    }
}

fn add<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> ModuleResult {
    let url = sub.value_of("url").unwrap();
    let tags = get_tags(rt, sub);
    info!("Adding url '{}' with tags '{:?}'", url, tags);

    Ok(())
}

fn list<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> ModuleResult {
    let tags    = get_tags(rt, sub);
    let matcher = get_matcher(rt, sub);

    match matcher {
        Some(reg) => {
            info!("Listing urls with matcher '{}' and with tags {:?}",
                     reg.as_str(),
                     tags);
        }
        None => {
            info!("Listing urls with tags {:?}", tags);
        }
    }

    Ok(())
}

fn remove<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> ModuleResult {
    let tags    = get_tags(rt, sub);
    let matcher = get_matcher(rt, sub);
    let id      = get_id(rt, sub);

    match id {
        Some(idstr) => {
            info!("Removing urls with id '{}'", idstr);
        }
        None => {
            match matcher {
                Some(reg) => {
                    info!("Removing urls with matcher '{}' and with tags {:?}",
                             reg.as_str(), tags);
                }
                None => {
                    info!("Listing urls with tags {:?}", tags);
                }
            }
        }
    }

    Ok(())
}

fn get_tags<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Vec<String> {
    debug!("Fetching tags from commandline");
    sub.value_of("tags").and_then(|tags|
                                  Some(tags.split(",")
                                       .into_iter()
                                       .map(|s| s.to_string())
                                       .filter(|e|
                                            if e.contains(" ") {
                                                warn!("Tag contains spaces: '{}'", e);
                                                false
                                            } else {
                                                true
                                            }).collect()
                                      )
                                 ).or(Some(vec![])).unwrap()

}

fn get_matcher<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Option<Regex> {
    debug!("Fetching matcher from commandline");
    if let Some(s) = sub.value_of("match") {
        if let Ok(r) = Regex::new(s) {
            return Some(r)
        } else {
            error!("Regex error, continuing without regex");
        }
    }
    None

}

fn get_id<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Option<String> {
    debug!("Fetching id from commandline");
    sub.value_of("id").and_then(|s| Some(String::from(s)))
}

<commit_msg>Add BMModule::add() implementation<commit_after>use runtime::Runtime;
use module::Module;
use module::ModuleResult;
use module::ModuleError;
use std::path::Path;
use std::result::Result;
use clap::ArgMatches;
use regex::Regex;

mod header;

use self::header::build_header;
use storage::json::parser::JsonHeaderParser;
use storage::parser::FileHeaderParser;

pub struct BMModule {
    path: Option<String>,
}

const CALLNAMES : &'static [&'static str] = &[ "bm", "bookmark" ];

impl Module for BMModule {

    fn new(rt : &Runtime) -> BMModule {
        BMModule {
            path: None
        }
    }

    fn callnames() -> &'static [&'static str] {
        CALLNAMES
    }

    fn name(&self) -> &'static str{
        "Bookmark"
    }

    fn execute(&self, rt : &Runtime) -> ModuleResult {
        let cmd = rt.config.cli_matches.subcommand_matches("bm").unwrap();
        match cmd.subcommand_name() {
            Some("add")     => {
                debug!("Calling 'add'...");
                add(rt, cmd.subcommand_matches("add").unwrap())
            }
            Some("list")    => {
                debug!("Calling 'list'...");
                list(rt, cmd.subcommand_matches("list").unwrap())
            }
            Some("remove")  => {
                debug!("Calling 'remove'...");
                list(rt, cmd.subcommand_matches("remove").unwrap())
            }
            _ => {
                info!("Not calling any of add, list, remove");
                Ok(())
            }
        }
    }

    fn shutdown(&self, rt : &Runtime) -> ModuleResult {
        Ok(())
    }
}

fn add<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> ModuleResult {
    let url = sub.value_of("url").unwrap();
    let tags = get_tags(rt, sub);
    info!("Adding url '{}' with tags '{:?}'", url, tags);

    let header = build_header(&String::from(url), &tags);
    let jheader = JsonHeaderParser::new(None).write(&header);
    println!("JSON: {:?}", jheader);

    Ok(())
}

fn list<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> ModuleResult {
    let tags    = get_tags(rt, sub);
    let matcher = get_matcher(rt, sub);

    match matcher {
        Some(reg) => {
            info!("Listing urls with matcher '{}' and with tags {:?}",
                     reg.as_str(),
                     tags);
        }
        None => {
            info!("Listing urls with tags {:?}", tags);
        }
    }

    Ok(())
}

fn remove<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> ModuleResult {
    let tags    = get_tags(rt, sub);
    let matcher = get_matcher(rt, sub);
    let id      = get_id(rt, sub);

    match id {
        Some(idstr) => {
            info!("Removing urls with id '{}'", idstr);
        }
        None => {
            match matcher {
                Some(reg) => {
                    info!("Removing urls with matcher '{}' and with tags {:?}",
                             reg.as_str(), tags);
                }
                None => {
                    info!("Listing urls with tags {:?}", tags);
                }
            }
        }
    }

    Ok(())
}

fn get_tags<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Vec<String> {
    debug!("Fetching tags from commandline");
    sub.value_of("tags").and_then(|tags|
                                  Some(tags.split(",")
                                       .into_iter()
                                       .map(|s| s.to_string())
                                       .filter(|e|
                                            if e.contains(" ") {
                                                warn!("Tag contains spaces: '{}'", e);
                                                false
                                            } else {
                                                true
                                            }).collect()
                                      )
                                 ).or(Some(vec![])).unwrap()

}

fn get_matcher<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Option<Regex> {
    debug!("Fetching matcher from commandline");
    if let Some(s) = sub.value_of("match") {
        if let Ok(r) = Regex::new(s) {
            return Some(r)
        } else {
            error!("Regex error, continuing without regex");
        }
    }
    None

}

fn get_id<'a>(rt: &Runtime, sub: &ArgMatches<'a, 'a>) -> Option<String> {
    debug!("Fetching id from commandline");
    sub.value_of("id").and_then(|s| Some(String::from(s)))
}

<|endoftext|>


<commit_before>#[macro_use]
extern crate serde_derive;

extern crate serde;
extern crate serde_json;

use std::error::Error;
use std::fs::File;
use std::path::Path;

#[derive(Serialize, Deserialize, Debug)]
struct PyFunction {
    ast_type: String,
    body: Vec<PyBody>
}

#[derive(Serialize, Deserialize, Debug)]
struct PyBody {
    col_offset: u8,
    lineno: u8
}

fn read_code_from_file<P: AsRef<Path>>(path: P) -> Result<PyFunction, Box<Error>> {
    // Open the file in read-only mode.
    let file = File::open(path)?;

    // Read the JSON contents of the file as an instance of `User`.
    let u = serde_json::from_reader(file)?;

    // Return the `User`.
    Ok(u)
}

fn main() {
    let u = read_code_from_file("../client/file1.json").unwrap();
    println!("{:#?}", u);
}<commit_msg>correctly parsing arbitraty json. beef it up until completion<commit_after>#[macro_use]
extern crate serde_derive;

extern crate serde;
extern crate serde_json;

use std::error::Error;
use std::fs::File;
use std::path::Path;

#[derive(Serialize, Deserialize, Debug)]
struct PyAssign
{
    ast_type: String,
    col_offset: u8,
    lineno: u8
}

#[derive(Serialize, Deserialize, Debug)]
struct PyFunctionDef
{
    //args
    ast_type: String,
    //body
    col_offset: u8,
    //decorator_list
    lineno: u8,
    name: String
}

#[derive(Serialize, Deserialize, Debug)]
struct PyExpr
{
    ast_type: String,
    col_offset: u8,
    lineno: u8,
    //value
}

#[derive(Serialize, Deserialize, Debug)]
struct PyBody {
    ast_type: String,
    col_offset: u8,
    lineno: u8,
    //targets: Vec<PyTarget>
}

#[derive(Serialize, Deserialize, Debug)]
struct PyFunction {
    ast_type: String,
    body: Vec<serde_json::Value>
}

fn read_code_from_file<P: AsRef<Path>>(path: P) -> Result<PyFunction, Box<Error>> {
    // Open the file in read-only mode.
    let file = File::open(path)?;

    // Read the JSON contents of the file as an instance of `User`.
    let first_pass = serde_json::from_reader(file)?;

    // Return the `User`.
    Ok(first_pass)
}

fn main() {
    let nodes = read_code_from_file("../client/file1.json").unwrap();
    println!("Parsed {:?} nodes:\n", nodes.body.len());

    //println!("{:#?}", nodes.body[0]);

    //parse each node
    for i in 0..nodes.body.len()
    {
        //parse each node
        let n: PyBody = serde_json::from_value(nodes.body[i].clone()).unwrap();
        match n.ast_type.as_ref() {
            "Assign" => {
                //println!("parsing Assign node:");
                let assign: PyAssign = serde_json::from_value(nodes.body[i].clone()).unwrap();
                println!("{:?}", assign.ast_type);
            },
            "FunctionDef" => {
                //println!("parsing FunctionDef node:");
                let function_def: PyFunctionDef = serde_json::from_value(nodes.body[i].clone()).unwrap();
                println!("{:?}, {:?}", function_def.ast_type, function_def.name);
            },
            "Expr" => {
                //println!("parsing Expr node:");
                let expr: PyExpr = serde_json::from_value(nodes.body[i].clone()).unwrap();
                println!("{:?}", expr.ast_type);
            },
            _ => println!("matched unknown node, no action taken: {:?}", n.ast_type)
        }
        //println!("{:?}", n);
    }
}<|endoftext|>


<commit_before>///Convert true to 1_i8 and false to 0_i8.
pub fn toi8(b:bool) -> i8 {
    if b { 1 } else { 0 }
}

///Convert true to 1_i16 and false to 0_i16.
pub fn toi16(b:bool) -> i16 {
    if b { 1 } else { 0 }
}

///Convert true to 1_i32 and false to 0_i32.
pub fn toi32(b:bool) -> i32 {
    if b { 1 } else { 0 }
}

///Convert true to 1_i64 and false to 0_i64.
pub fn toi64(b:bool) -> i64 {
    if b { 1 } else { 0 }
}

///Convert true to 1_u8 and false to 0_u8.
pub fn tou8(b:bool) -> u8 {
    if b { 1 } else { 0 }
}

///Convert true to 1_u16 and false to 0_u16.
pub fn tou16(b:bool) -> u16 {
    if b { 1 } else { 0 }
}

///Convert true to 1_u32 and false to 0_u32.
pub fn tou32(b:bool) -> u32 {
    if b { 1 } else { 0 }
}

///Convert true to 1_u64 and false to 0_u64.
pub fn tou64(b:bool) -> u64 {
    if b { 1 } else { 0 }
}

///Convert true to 1_f32 and false to 0_f32.
pub fn tof32(b:bool) -> f32 {
    if b { 1.0 } else { 0.0 }
}

///Convert true to 1_f64 and false to 0_f64.
pub fn tof64(b:bool) -> f64 {
    if b { 1.0 } else { 0.0 }
}

///Convert true to 1_usize and false to 0_usize.
pub fn tousize(b:bool) -> usize {
    if b { 1 } else { 0 }
}

///Convert true to 1_isize and false to 0_isize.
pub fn toisize(b:bool) -> isize {
    if b { 1 } else { 0 }
}
<commit_msg>parse.boolean comments<commit_after>///Converts `true` to `1_i8` and `false` to `0_i8`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::toi8(x), 1_i8);
/// assert_eq!(boolean::toi8(y), 0_i8);
/// ```
pub fn toi8(b:bool) -> i8 {
    if b { 1 } else { 0 }
}

///Converts `true` to `1_i16` and `false` to `0_i16`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::toi16(x), 1_i16);
/// assert_eq!(boolean::toi16(y), 0_i16);
/// ```
pub fn toi16(b:bool) -> i16 {
    if b { 1 } else { 0 }
}

///Converts `true` to `1_i32` and `false` to `0_i32`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::toi32(x), 1_i32);
/// assert_eq!(boolean::toi32(y), 0_i32);
/// ```
pub fn toi32(b:bool) -> i32 {
    if b { 1 } else { 0 }
}

///Converts `true` to `1_i64` and `false` to `0_i64`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::toi64(x), 1_i64);
/// assert_eq!(boolean::toi64(y), 0_i64);
/// ```
pub fn toi64(b:bool) -> i64 {
    if b { 1 } else { 0 }
}

///Converts `true` to `1_u8` and `false` to `0_u8`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::tou8(x), 1_u8);
/// assert_eq!(boolean::tou8(y), 0_u8);
/// ```
pub fn tou8(b:bool) -> u8 {
    if b { 1 } else { 0 }
}

///Converts `true` to `1_u16` and `false` to `0_u16`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::tou16(x), 1_u16);
/// assert_eq!(boolean::tou16(y), 0_u16);
/// ```
pub fn tou16(b:bool) -> u16 {
    if b { 1 } else { 0 }
}

///Converts `true` to `1_u32` and `false` to `0_u32`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::tou32(x), 1_u32);
/// assert_eq!(boolean::tou32(y), 0_u32);
/// ```
pub fn tou32(b:bool) -> u32 {
    if b { 1 } else { 0 }
}

///Converts `true` to `1_u64` and `false` to `0_u64`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::tou64(x), 1_u64);
/// assert_eq!(boolean::tou64(y), 0_u64);
/// ```
pub fn tou64(b:bool) -> u64 {
    if b { 1 } else { 0 }
}

///Converts `true` to `1_f32` and `false` to `0_f32`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::tof32(x), 1_f32);
/// assert_eq!(boolean::tof32(y), 0_f32);
/// ```
pub fn tof32(b:bool) -> f32 {
    if b { 1.0 } else { 0.0 }
}

///Converts `true` to `1_f64` and `false` to `0_f64`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::tof64(x), 1_f64);
/// assert_eq!(boolean::tof64(y), 0_f64);
/// ```
pub fn tof64(b:bool) -> f64 {
    if b { 1.0 } else { 0.0 }
}

///Converts `true` to `1_usize` and `false` to `0_usize`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::tousize(x), 1_usize);
/// assert_eq!(boolean::tousize(y), 0_usize);
/// ```
pub fn tousize(b:bool) -> usize {
    if b { 1 } else { 0 }
}

///Converts `true` to `1_isize` and `false` to `0_isize`.
/// # Example
///
/// ```
/// use rustils::parse::boolean;
///
/// let x = true;
/// let y = false;
///
/// assert_eq!(boolean::toisize(x), 1_isize);
/// assert_eq!(boolean::toisize(y), 0_isize);
/// ```
pub fn toisize(b:bool) -> isize {
    if b { 1 } else { 0 }
}
<|endoftext|>


<commit_before>use super::stream::Stream;
use std::io;

pub struct Parser<'a>
{
   strm: Stream<'a>
}

pub type ParseError = String;

impl<'a> Parser<'a>
{
   pub fn new(input: &str) -> Parser
   {
      Parser {strm: Stream::new(input)}
   }

   pub fn eof(&self) -> bool { self.strm.eof() }

   pub fn skip(&mut self, str: &str) -> Result<(), ParseError>
   {
      if str.is_empty() {
         return Err("Couldn't skip empty str!".to_string());
      }

      self.push_pos();
      let mut strm = Stream::new(str);
      while ! self.eof() && ! strm.eof() {
         if self.next_char_or_fail() != strm.next_char_or_fail() {
            self.pop_and_reset_pos();
            return Err(format!("Couldn't skip str '{}'!", str));
         }

         self.take_char_or_fail();
         strm.take_char_or_fail();
      }

      self.pop_pos();
      Ok(())
   }

   pub fn skip_while(&mut self, test: |char| -> bool)
   {
      while ! self.eof() && test(self.next_char_or_fail()) {
         self.take_char_or_fail();
      }
   }

   pub fn skip_whitespace(&mut self)
   {
      self.skip_while(|c| c.is_whitespace());
   }

   pub fn take_while(&mut self, test: |char| -> bool) -> String
   {
      let mut string = String::new();
      while ! self.eof() {
         if ! test(self.next_char_or_fail()) {
            return string;
         }

         string.push(self.take_char_or_fail());
      }

      string
   }

   pub fn take_uint(&mut self) -> Result<uint, ParseError>
   {
      self.push_pos();
      let mut digits = String::new();
      while ! self.eof() && self.next_char_or_fail().is_digit(10) {
         digits.push(self.take_char_or_fail());
      }

      match from_str::<uint>(digits.as_slice()) {
         Some(uint) => {
            self.pop_pos();
            Ok(uint)
         }

         None => {
            self.pop_and_reset_pos();
            Err("Couldn't take uint!".to_string())
         }
      }
   }

   pub fn take_char(&mut self) -> Result<char, ParseError>
   {
      if self.eof() {
         return Err("Couldn't take char!".to_string());
      }

      Ok(self.take_char_or_fail())
   }

   pub fn take_till_eof(&mut self) -> String
   {
      let mut string = String::new();
      while ! self.eof() {
         string.push(self.take_char_or_fail());
      }

      string
   }

   pub fn consumed(&self) -> &str { self.strm.consumed() }

   pub fn unconsumed(&self) -> &str { self.strm.unconsumed() }

   fn take_char_or_fail(&mut self) -> char { self.strm.take_char_or_fail() }

   fn next_char_or_fail(&self) -> char { self.strm.next_char_or_fail() }

   fn push_pos(&mut self) { self.strm.push_pos(); }

   fn pop_pos(&mut self) { self.strm.pop_pos(); }

   fn pop_and_reset_pos(&mut self) { self.strm.pop_and_reset_pos(); }
}

#[test]
#[cfg(test)]
fn tests()
{
   match parser_tests() {
      Ok(_)    => {}
      Err(err) => {
         let mut stderr = io::stderr();
         let _ = writeln!(stderr, "Parser test error: {}", err);
      }
   }
}

#[cfg(test)]
fn parser_tests() -> Result<(), ParseError>
{
   let mut parser = Parser::new("ssss 21 qqq aswe ");
   assert_eq!(parser.consumed(), "");
   assert_eq!(parser.unconsumed(), "ssss 21 qqq aswe ");

   parser.skip_whitespace();
   assert_eq!(parser.consumed(), "");
   assert_eq!(parser.unconsumed(), "ssss 21 qqq aswe ");

   try!(parser.skip("ss"));
   assert_eq!(parser.consumed(), "ss");
   assert_eq!(parser.unconsumed(), "ss 21 qqq aswe ");

   try!(parser.skip("ss"));
   assert_eq!(parser.consumed(), "ssss");
   assert_eq!(parser.unconsumed(), " 21 qqq aswe ");

   parser.skip_whitespace();
   assert_eq!(parser.consumed(), "ssss ");
   assert_eq!(parser.unconsumed(), "21 qqq aswe ");

   assert_eq!(try!(parser.take_uint()), 21u);
   assert_eq!(parser.consumed(), "ssss 21");
   assert_eq!(parser.unconsumed(), " qqq aswe ");

   parser.skip_whitespace();
   assert_eq!(try!(parser.take_while(|c| c == 'q')), "qqq".to_string());
   assert_eq!(parser.consumed(), "ssss 21 qqq");
   assert_eq!(parser.unconsumed(), " aswe ");

   parser.skip_while(|c| c.is_whitespace());
   assert_eq!(parser.consumed(), "ssss 21 qqq ");
   assert_eq!(parser.unconsumed(), "aswe ");

   assert_eq!(try!(parser.take_char()), 'a');
   assert_eq!(parser.consumed(), "ssss 21 qqq a");
   assert_eq!(parser.unconsumed(), "swe ");

   assert_eq!(parser.eof(), false);
   assert_eq!(try!(parser.take_till_eof()), "swe ".to_string());
   assert_eq!(parser.eof(), true);

   Ok(())
}
<commit_msg>Fix parser tests<commit_after>use super::stream::Stream;
use std::io;

pub struct Parser<'a>
{
   strm: Stream<'a>
}

pub type ParseError = String;

impl<'a> Parser<'a>
{
   pub fn new(input: &str) -> Parser
   {
      Parser {strm: Stream::new(input)}
   }

   pub fn eof(&self) -> bool { self.strm.eof() }

   pub fn skip(&mut self, str: &str) -> Result<(), ParseError>
   {
      if str.is_empty() {
         return Err("Couldn't skip empty str!".to_string());
      }

      self.push_pos();
      let mut strm = Stream::new(str);
      while ! self.eof() && ! strm.eof() {
         if self.next_char_or_fail() != strm.next_char_or_fail() {
            self.pop_and_reset_pos();
            return Err(format!("Couldn't skip str '{}'!", str));
         }

         self.take_char_or_fail();
         strm.take_char_or_fail();
      }

      self.pop_pos();
      Ok(())
   }

   pub fn skip_while(&mut self, test: |char| -> bool)
   {
      while ! self.eof() && test(self.next_char_or_fail()) {
         self.take_char_or_fail();
      }
   }

   pub fn skip_whitespace(&mut self)
   {
      self.skip_while(|c| c.is_whitespace());
   }

   pub fn take_while(&mut self, test: |char| -> bool) -> String
   {
      let mut string = String::new();
      while ! self.eof() {
         if ! test(self.next_char_or_fail()) {
            return string;
         }

         string.push(self.take_char_or_fail());
      }

      string
   }

   pub fn take_uint(&mut self) -> Result<uint, ParseError>
   {
      self.push_pos();
      let mut digits = String::new();
      while ! self.eof() && self.next_char_or_fail().is_digit(10) {
         digits.push(self.take_char_or_fail());
      }

      match from_str::<uint>(digits.as_slice()) {
         Some(uint) => {
            self.pop_pos();
            Ok(uint)
         }

         None => {
            self.pop_and_reset_pos();
            Err("Couldn't take uint!".to_string())
         }
      }
   }

   pub fn take_char(&mut self) -> Result<char, ParseError>
   {
      if self.eof() {
         return Err("Couldn't take char!".to_string());
      }

      Ok(self.take_char_or_fail())
   }

   pub fn take_till_eof(&mut self) -> String
   {
      let mut string = String::new();
      while ! self.eof() {
         string.push(self.take_char_or_fail());
      }

      string
   }

   pub fn consumed(&self) -> &str { self.strm.consumed() }

   pub fn unconsumed(&self) -> &str { self.strm.unconsumed() }

   fn take_char_or_fail(&mut self) -> char { self.strm.take_char_or_fail() }

   fn next_char_or_fail(&self) -> char { self.strm.next_char_or_fail() }

   fn push_pos(&mut self) { self.strm.push_pos(); }

   fn pop_pos(&mut self) { self.strm.pop_pos(); }

   fn pop_and_reset_pos(&mut self) { self.strm.pop_and_reset_pos(); }
}

#[test]
#[cfg(test)]
fn tests()
{
   match parser_tests() {
      Ok(_)    => {}
      Err(err) => {
         let stderr = &mut io::stderr();
         let _ = writeln!(stderr, "Parser test error: {}", err);
      }
   }
}

#[cfg(test)]
fn parser_tests() -> Result<(), ParseError>
{
   let mut parser = Parser::new("ssss 21 qqq aswe ");
   assert_eq!(parser.consumed(), "");
   assert_eq!(parser.unconsumed(), "ssss 21 qqq aswe ");

   parser.skip_whitespace();
   assert_eq!(parser.consumed(), "");
   assert_eq!(parser.unconsumed(), "ssss 21 qqq aswe ");

   try!(parser.skip("ss"));
   assert_eq!(parser.consumed(), "ss");
   assert_eq!(parser.unconsumed(), "ss 21 qqq aswe ");

   try!(parser.skip("ss"));
   assert_eq!(parser.consumed(), "ssss");
   assert_eq!(parser.unconsumed(), " 21 qqq aswe ");

   parser.skip_whitespace();
   assert_eq!(parser.consumed(), "ssss ");
   assert_eq!(parser.unconsumed(), "21 qqq aswe ");

   assert_eq!(try!(parser.take_uint()), 21u);
   assert_eq!(parser.consumed(), "ssss 21");
   assert_eq!(parser.unconsumed(), " qqq aswe ");

   parser.skip_whitespace();
   assert_eq!(parser.take_while(|c| c == 'q'), "qqq".to_string());
   assert_eq!(parser.consumed(), "ssss 21 qqq");
   assert_eq!(parser.unconsumed(), " aswe ");

   parser.skip_while(|c| c.is_whitespace());
   assert_eq!(parser.consumed(), "ssss 21 qqq ");
   assert_eq!(parser.unconsumed(), "aswe ");

   assert_eq!(try!(parser.take_char()), 'a');
   assert_eq!(parser.consumed(), "ssss 21 qqq a");
   assert_eq!(parser.unconsumed(), "swe ");

   assert_eq!(parser.eof(), false);
   assert_eq!(parser.take_till_eof(), "swe ".to_string());
   assert_eq!(parser.eof(), true);

   Ok(())
}
<|endoftext|>