<commit_before>pub type clock_t = i64;
pub type suseconds_t = i64;
pub type dev_t = i32;
pub type sigset_t = ::c_uint;
pub type blksize_t = ::uint32_t;
pub type fsblkcnt_t = ::uint64_t;
pub type fsfilcnt_t = ::uint64_t;
pub type pthread_attr_t = *mut ::c_void;
pub type pthread_mutex_t = *mut ::c_void;
pub type pthread_mutexattr_t = *mut ::c_void;
pub type pthread_cond_t = *mut ::c_void;
pub type pthread_rwlock_t = *mut ::c_void;

s! {
    pub struct dirent {
        pub d_fileno: ::ino_t,
        pub d_off: ::off_t,
        pub d_reclen: u16,
        pub d_type: u8,
        pub d_namelen: u8,
        __d_padding: [u8; 4],
        pub d_name: [::c_char; 256],
    }

    pub struct glob_t {
        pub gl_pathc:   ::c_int,
        __unused1:      ::c_int,
        pub gl_offs:    ::c_int,
        __unused2:      ::c_int,
        pub gl_pathv:   *mut *mut ::c_char,

        __unused3: *mut ::c_void,

        __unused4: *mut ::c_void,
        __unused5: *mut ::c_void,
        __unused6: *mut ::c_void,
        __unused7: *mut ::c_void,
        __unused8: *mut ::c_void,
        __unused9: *mut ::c_void,
    }

    pub struct stat {
        pub st_mode: ::mode_t,
        pub st_dev: ::dev_t,
        pub st_ino: ::ino_t,
        pub st_nlink: ::nlink_t,
        pub st_uid: ::uid_t,
        pub st_gid: ::gid_t,
        pub st_rdev: ::dev_t,
        pub st_atime: ::time_t,
        pub st_atime_nsec: ::c_long,
        pub st_mtime: ::time_t,
        pub st_mtime_nsec: ::c_long,
        pub st_ctime: ::time_t,
        pub st_ctime_nsec: ::c_long,
        pub st_size: ::off_t,
        pub st_blocks: ::blkcnt_t,
        pub st_blksize: ::blksize_t,
        pub st_flags: ::uint32_t,
        pub st_gen: ::uint32_t,
        pub st_birthtime: ::time_t,
        pub st_birthtime_nsec: ::c_long,
    }

    pub struct statvfs {
        pub f_bsize: ::c_ulong,
        pub f_frsize: ::c_ulong,
        pub f_blocks: ::fsblkcnt_t,
        pub f_bfree: ::fsblkcnt_t,
        pub f_bavail: ::fsblkcnt_t,
        pub f_files: ::fsfilcnt_t,
        pub f_ffree: ::fsfilcnt_t,
        pub f_favail: ::fsfilcnt_t,
        pub f_fsid: ::c_ulong,
        pub f_flag: ::c_ulong,
        pub f_namemax: ::c_ulong,
    }

    pub struct addrinfo {
        pub ai_flags: ::c_int,
        pub ai_family: ::c_int,
        pub ai_socktype: ::c_int,
        pub ai_protocol: ::c_int,
        pub ai_addrlen: ::socklen_t,
        pub ai_addr: *mut ::sockaddr,
        pub ai_canonname: *mut ::c_char,
        pub ai_next: *mut ::addrinfo,
    }

    pub struct sockaddr_storage {
        pub ss_len: u8,
        pub ss_family: ::sa_family_t,
        __ss_pad1: [u8; 6],
        __ss_pad2: i64,
        __ss_pad3: [u8; 240],
    }

    pub struct siginfo_t {
        pub si_signo: ::c_int,
        pub si_code: ::c_int,
        pub si_errno: ::c_int,
        pub si_addr: *mut ::c_void,
        __pad: [u8; 116],
    }

    pub struct Dl_info {
        pub dli_fname: *const ::c_char,
        pub dli_fbase: *mut ::c_void,
        pub dli_sname: *const ::c_char,
        pub dli_saddr: *mut ::c_void,
    }
}

pub const O_CLOEXEC: ::c_int = 0x10000;

pub const MS_SYNC : ::c_int = 0x0002;
pub const MS_INVALIDATE : ::c_int = 0x0004;

pub const PTHREAD_STACK_MIN : ::size_t = 2048;

pub const ENOATTR : ::c_int = 83;
pub const EILSEQ : ::c_int = 84;
pub const EOVERFLOW : ::c_int = 87;
pub const ECANCELED : ::c_int = 88;
pub const EIDRM : ::c_int = 89;
pub const ENOMSG : ::c_int = 90;
pub const ENOTSUP : ::c_int = 91;
pub const ELAST : ::c_int = 91;

pub const F_DUPFD_CLOEXEC : ::c_int = 10;

pub const RLIM_NLIMITS: ::c_int = 9;

pub const SO_SNDTIMEO: ::c_int = 0x1005;
pub const SO_RCVTIMEO: ::c_int = 0x1006;

pub const KERN_PROC : ::c_int = 66;
pub const O_DSYNC : ::c_int = 128;

pub const MAP_RENAME : ::c_int = 0x0000;
pub const MAP_NORESERVE : ::c_int = 0x0000;
pub const MAP_HASSEMAPHORE : ::c_int = 0x0000;

pub const EIPSEC : ::c_int = 82;
pub const ENOMEDIUM : ::c_int = 85;
pub const EMEDIUMTYPE : ::c_int = 86;

pub const RUSAGE_THREAD: ::c_int = 1;

pub const MAP_COPY : ::c_int = 0x0002;
pub const MAP_NOEXTEND : ::c_int = 0x0000;

pub const _SC_CLK_TCK : ::c_int = 3;
pub const _SC_IOV_MAX : ::c_int = 51;
pub const _SC_GETGR_R_SIZE_MAX : ::c_int = 100;
pub const _SC_GETPW_R_SIZE_MAX : ::c_int = 101;
pub const _SC_LOGIN_NAME_MAX : ::c_int = 102;
pub const _SC_MQ_PRIO_MAX : ::c_int = 59;
pub const _SC_THREADS : ::c_int = 91;
pub const _SC_THREAD_ATTR_STACKADDR : ::c_int = 77;
pub const _SC_THREAD_ATTR_STACKSIZE : ::c_int = 78;
pub const _SC_THREAD_DESTRUCTOR_ITERATIONS : ::c_int = 80;
pub const _SC_THREAD_KEYS_MAX : ::c_int = 81;
pub const _SC_THREAD_PRIO_INHERIT : ::c_int = 82;
pub const _SC_THREAD_PRIO_PROTECT : ::c_int = 83;
pub const _SC_THREAD_PRIORITY_SCHEDULING : ::c_int = 84;
pub const _SC_THREAD_PROCESS_SHARED : ::c_int = 85;
pub const _SC_THREAD_SAFE_FUNCTIONS : ::c_int = 103;
pub const _SC_THREAD_STACK_MIN : ::c_int = 89;
pub const _SC_THREAD_THREADS_MAX : ::c_int = 90;
pub const _SC_TTY_NAME_MAX : ::c_int = 107;
pub const _SC_ATEXIT_MAX : ::c_int = 46;
pub const _SC_AIO_LISTIO_MAX : ::c_int = 42;
pub const _SC_AIO_MAX : ::c_int = 43;
pub const _SC_ASYNCHRONOUS_IO : ::c_int = 45;
pub const _SC_MAPPED_FILES : ::c_int = 53;
pub const _SC_MEMLOCK : ::c_int = 54;
pub const _SC_MEMLOCK_RANGE : ::c_int = 55;
pub const _SC_MEMORY_PROTECTION : ::c_int = 56;
pub const _SC_MESSAGE_PASSING : ::c_int = 57;
pub const _SC_MQ_OPEN_MAX : ::c_int = 58;
pub const _SC_PRIORITY_SCHEDULING : ::c_int = 61;
pub const _SC_SEMAPHORES : ::c_int = 67;
pub const _SC_SHARED_MEMORY_OBJECTS : ::c_int = 68;
pub const _SC_SYNCHRONIZED_IO : ::c_int = 75;
pub const _SC_TIMERS : ::c_int = 94;
pub const _SC_XOPEN_CRYPT : ::c_int = 117;
pub const _SC_XOPEN_ENH_I18N : ::c_int = 118;
pub const _SC_XOPEN_LEGACY : ::c_int = 119;
pub const _SC_XOPEN_REALTIME : ::c_int = 120;
pub const _SC_XOPEN_REALTIME_THREADS : ::c_int = 121;
pub const _SC_XOPEN_UNIX : ::c_int = 123;
pub const _SC_XOPEN_VERSION : ::c_int = 125;
pub const _SC_SEM_NSEMS_MAX : ::c_int = 31;
pub const _SC_SEM_VALUE_MAX : ::c_int = 32;
pub const _SC_AIO_PRIO_DELTA_MAX : ::c_int = 44;
pub const _SC_DELAYTIMER_MAX : ::c_int = 50;
pub const _SC_PRIORITIZED_IO : ::c_int = 60;
pub const _SC_REALTIME_SIGNALS : ::c_int = 64;
pub const _SC_RTSIG_MAX : ::c_int = 66;
pub const _SC_SIGQUEUE_MAX : ::c_int = 70;
pub const _SC_TIMER_MAX : ::c_int = 93;

pub const SIGSTKSZ: ::size_t = 131072;

pub const FD_SETSIZE: usize = 1024;

pub const ST_NOSUID: ::c_ulong = 2;

pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = 0 as *mut _;
pub const PTHREAD_COND_INITIALIZER: pthread_cond_t = 0 as *mut _;
pub const PTHREAD_RWLOCK_INITIALIZER: pthread_rwlock_t = 0 as *mut _;
pub const PTHREAD_MUTEX_RECURSIVE: ::c_int = 2;

pub const KERN_PROC_ARGS: ::c_int = 55;

// syscall numbers
pub const SYS_getentropy: ::c_int = 7;

extern {
    pub fn getnameinfo(sa: *const ::sockaddr,
                       salen: ::socklen_t,
                       host: *mut ::c_char,
                       hostlen: ::size_t,
                       serv: *mut ::c_char,
                       servlen: ::size_t,
                       flags: ::c_int) -> ::c_int;
    pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int)
                    -> ::c_int;
    pub fn sysctl(name: *const ::c_int,
                  namelen: ::c_uint,
                  oldp: *mut ::c_void,
                  oldlenp: *mut ::size_t,
                  newp: *mut ::c_void,
                  newlen: ::size_t)
                  -> ::c_int;
}
<commit_msg>change sign for `blksize_t` type under openbsd<commit_after>pub type clock_t = i64;
pub type suseconds_t = i64;
pub type dev_t = i32;
pub type sigset_t = ::c_uint;
pub type blksize_t = ::int32_t;
pub type fsblkcnt_t = ::uint64_t;
pub type fsfilcnt_t = ::uint64_t;
pub type pthread_attr_t = *mut ::c_void;
pub type pthread_mutex_t = *mut ::c_void;
pub type pthread_mutexattr_t = *mut ::c_void;
pub type pthread_cond_t = *mut ::c_void;
pub type pthread_rwlock_t = *mut ::c_void;

s! {
    pub struct dirent {
        pub d_fileno: ::ino_t,
        pub d_off: ::off_t,
        pub d_reclen: u16,
        pub d_type: u8,
        pub d_namelen: u8,
        __d_padding: [u8; 4],
        pub d_name: [::c_char; 256],
    }

    pub struct glob_t {
        pub gl_pathc:   ::c_int,
        __unused1:      ::c_int,
        pub gl_offs:    ::c_int,
        __unused2:      ::c_int,
        pub gl_pathv:   *mut *mut ::c_char,

        __unused3: *mut ::c_void,

        __unused4: *mut ::c_void,
        __unused5: *mut ::c_void,
        __unused6: *mut ::c_void,
        __unused7: *mut ::c_void,
        __unused8: *mut ::c_void,
        __unused9: *mut ::c_void,
    }

    pub struct stat {
        pub st_mode: ::mode_t,
        pub st_dev: ::dev_t,
        pub st_ino: ::ino_t,
        pub st_nlink: ::nlink_t,
        pub st_uid: ::uid_t,
        pub st_gid: ::gid_t,
        pub st_rdev: ::dev_t,
        pub st_atime: ::time_t,
        pub st_atime_nsec: ::c_long,
        pub st_mtime: ::time_t,
        pub st_mtime_nsec: ::c_long,
        pub st_ctime: ::time_t,
        pub st_ctime_nsec: ::c_long,
        pub st_size: ::off_t,
        pub st_blocks: ::blkcnt_t,
        pub st_blksize: ::blksize_t,
        pub st_flags: ::uint32_t,
        pub st_gen: ::uint32_t,
        pub st_birthtime: ::time_t,
        pub st_birthtime_nsec: ::c_long,
    }

    pub struct statvfs {
        pub f_bsize: ::c_ulong,
        pub f_frsize: ::c_ulong,
        pub f_blocks: ::fsblkcnt_t,
        pub f_bfree: ::fsblkcnt_t,
        pub f_bavail: ::fsblkcnt_t,
        pub f_files: ::fsfilcnt_t,
        pub f_ffree: ::fsfilcnt_t,
        pub f_favail: ::fsfilcnt_t,
        pub f_fsid: ::c_ulong,
        pub f_flag: ::c_ulong,
        pub f_namemax: ::c_ulong,
    }

    pub struct addrinfo {
        pub ai_flags: ::c_int,
        pub ai_family: ::c_int,
        pub ai_socktype: ::c_int,
        pub ai_protocol: ::c_int,
        pub ai_addrlen: ::socklen_t,
        pub ai_addr: *mut ::sockaddr,
        pub ai_canonname: *mut ::c_char,
        pub ai_next: *mut ::addrinfo,
    }

    pub struct sockaddr_storage {
        pub ss_len: u8,
        pub ss_family: ::sa_family_t,
        __ss_pad1: [u8; 6],
        __ss_pad2: i64,
        __ss_pad3: [u8; 240],
    }

    pub struct siginfo_t {
        pub si_signo: ::c_int,
        pub si_code: ::c_int,
        pub si_errno: ::c_int,
        pub si_addr: *mut ::c_void,
        __pad: [u8; 116],
    }

    pub struct Dl_info {
        pub dli_fname: *const ::c_char,
        pub dli_fbase: *mut ::c_void,
        pub dli_sname: *const ::c_char,
        pub dli_saddr: *mut ::c_void,
    }
}

pub const O_CLOEXEC: ::c_int = 0x10000;

pub const MS_SYNC : ::c_int = 0x0002;
pub const MS_INVALIDATE : ::c_int = 0x0004;

pub const PTHREAD_STACK_MIN : ::size_t = 2048;

pub const ENOATTR : ::c_int = 83;
pub const EILSEQ : ::c_int = 84;
pub const EOVERFLOW : ::c_int = 87;
pub const ECANCELED : ::c_int = 88;
pub const EIDRM : ::c_int = 89;
pub const ENOMSG : ::c_int = 90;
pub const ENOTSUP : ::c_int = 91;
pub const ELAST : ::c_int = 91;

pub const F_DUPFD_CLOEXEC : ::c_int = 10;

pub const RLIM_NLIMITS: ::c_int = 9;

pub const SO_SNDTIMEO: ::c_int = 0x1005;
pub const SO_RCVTIMEO: ::c_int = 0x1006;

pub const KERN_PROC : ::c_int = 66;
pub const O_DSYNC : ::c_int = 128;

pub const MAP_RENAME : ::c_int = 0x0000;
pub const MAP_NORESERVE : ::c_int = 0x0000;
pub const MAP_HASSEMAPHORE : ::c_int = 0x0000;

pub const EIPSEC : ::c_int = 82;
pub const ENOMEDIUM : ::c_int = 85;
pub const EMEDIUMTYPE : ::c_int = 86;

pub const RUSAGE_THREAD: ::c_int = 1;

pub const MAP_COPY : ::c_int = 0x0002;
pub const MAP_NOEXTEND : ::c_int = 0x0000;

pub const _SC_CLK_TCK : ::c_int = 3;
pub const _SC_IOV_MAX : ::c_int = 51;
pub const _SC_GETGR_R_SIZE_MAX : ::c_int = 100;
pub const _SC_GETPW_R_SIZE_MAX : ::c_int = 101;
pub const _SC_LOGIN_NAME_MAX : ::c_int = 102;
pub const _SC_MQ_PRIO_MAX : ::c_int = 59;
pub const _SC_THREADS : ::c_int = 91;
pub const _SC_THREAD_ATTR_STACKADDR : ::c_int = 77;
pub const _SC_THREAD_ATTR_STACKSIZE : ::c_int = 78;
pub const _SC_THREAD_DESTRUCTOR_ITERATIONS : ::c_int = 80;
pub const _SC_THREAD_KEYS_MAX : ::c_int = 81;
pub const _SC_THREAD_PRIO_INHERIT : ::c_int = 82;
pub const _SC_THREAD_PRIO_PROTECT : ::c_int = 83;
pub const _SC_THREAD_PRIORITY_SCHEDULING : ::c_int = 84;
pub const _SC_THREAD_PROCESS_SHARED : ::c_int = 85;
pub const _SC_THREAD_SAFE_FUNCTIONS : ::c_int = 103;
pub const _SC_THREAD_STACK_MIN : ::c_int = 89;
pub const _SC_THREAD_THREADS_MAX : ::c_int = 90;
pub const _SC_TTY_NAME_MAX : ::c_int = 107;
pub const _SC_ATEXIT_MAX : ::c_int = 46;
pub const _SC_AIO_LISTIO_MAX : ::c_int = 42;
pub const _SC_AIO_MAX : ::c_int = 43;
pub const _SC_ASYNCHRONOUS_IO : ::c_int = 45;
pub const _SC_MAPPED_FILES : ::c_int = 53;
pub const _SC_MEMLOCK : ::c_int = 54;
pub const _SC_MEMLOCK_RANGE : ::c_int = 55;
pub const _SC_MEMORY_PROTECTION : ::c_int = 56;
pub const _SC_MESSAGE_PASSING : ::c_int = 57;
pub const _SC_MQ_OPEN_MAX : ::c_int = 58;
pub const _SC_PRIORITY_SCHEDULING : ::c_int = 61;
pub const _SC_SEMAPHORES : ::c_int = 67;
pub const _SC_SHARED_MEMORY_OBJECTS : ::c_int = 68;
pub const _SC_SYNCHRONIZED_IO : ::c_int = 75;
pub const _SC_TIMERS : ::c_int = 94;
pub const _SC_XOPEN_CRYPT : ::c_int = 117;
pub const _SC_XOPEN_ENH_I18N : ::c_int = 118;
pub const _SC_XOPEN_LEGACY : ::c_int = 119;
pub const _SC_XOPEN_REALTIME : ::c_int = 120;
pub const _SC_XOPEN_REALTIME_THREADS : ::c_int = 121;
pub const _SC_XOPEN_UNIX : ::c_int = 123;
pub const _SC_XOPEN_VERSION : ::c_int = 125;
pub const _SC_SEM_NSEMS_MAX : ::c_int = 31;
pub const _SC_SEM_VALUE_MAX : ::c_int = 32;
pub const _SC_AIO_PRIO_DELTA_MAX : ::c_int = 44;
pub const _SC_DELAYTIMER_MAX : ::c_int = 50;
pub const _SC_PRIORITIZED_IO : ::c_int = 60;
pub const _SC_REALTIME_SIGNALS : ::c_int = 64;
pub const _SC_RTSIG_MAX : ::c_int = 66;
pub const _SC_SIGQUEUE_MAX : ::c_int = 70;
pub const _SC_TIMER_MAX : ::c_int = 93;

pub const SIGSTKSZ: ::size_t = 131072;

pub const FD_SETSIZE: usize = 1024;

pub const ST_NOSUID: ::c_ulong = 2;

pub const PTHREAD_MUTEX_INITIALIZER: pthread_mutex_t = 0 as *mut _;
pub const PTHREAD_COND_INITIALIZER: pthread_cond_t = 0 as *mut _;
pub const PTHREAD_RWLOCK_INITIALIZER: pthread_rwlock_t = 0 as *mut _;
pub const PTHREAD_MUTEX_RECURSIVE: ::c_int = 2;

pub const KERN_PROC_ARGS: ::c_int = 55;

// syscall numbers
pub const SYS_getentropy: ::c_int = 7;

extern {
    pub fn getnameinfo(sa: *const ::sockaddr,
                       salen: ::socklen_t,
                       host: *mut ::c_char,
                       hostlen: ::size_t,
                       serv: *mut ::c_char,
                       servlen: ::size_t,
                       flags: ::c_int) -> ::c_int;
    pub fn mprotect(addr: *mut ::c_void, len: ::size_t, prot: ::c_int)
                    -> ::c_int;
    pub fn sysctl(name: *const ::c_int,
                  namelen: ::c_uint,
                  oldp: *mut ::c_void,
                  oldlenp: *mut ::size_t,
                  newp: *mut ::c_void,
                  newlen: ::size_t)
                  -> ::c_int;
}
<|endoftext|>


<commit_before>/// Divide a string into the longest slice that fits within maximum line
/// length and the rest of the string. Place the split before a whitespace or
/// after a hyphen if possible. Any whitespace between the two segments is
/// trimmed. Newlines will cause a segment split when encountered.
pub fn split_line<'a, F>(text: &'a str, char_width: F, max_len: f32) -> (&'a str, &'a str)
    where F: Fn(char) -> f32 {
    assert!(max_len >= 0.0);

    if text.len() == 0 { return (text, text); }

    // Init the split position to 1 because we always want to return at least
    // 1 character in the head partition.
    let mut head_end = 1;
    let mut tail_start = 1;
    // Is the iteration currently consuming whitespace inside a possible
    // break.
    let mut eat_whitespace = false;
    let mut length = 0.0;

    for (i, c) in text.chars().enumerate() {
        length = length + char_width(c);

        // Invariant: head_end and tail_start describe a valid, but possibly
        // suboptimal return value at this point.
        if eat_whitespace {
            if c.is_whitespace() {
                tail_start = i + 1;
                if c == '\n' { return (&text[..head_end], &text[tail_start..]); }
                continue;
            } else {
                eat_whitespace = false;
            }
        }

        // We're either just encountering the first whitespace after a block
        // of text, or over non-whitespace text.
        assert!(!eat_whitespace);

        // Invariant: The length of the string processed up to this point is
        // still short enough to return.

        // Encounter the first whitespace, set head_end marker.
        if c.is_whitespace() {
            head_end = i;
            tail_start = i + 1;
            if c == '\n' { return (&text[..head_end], &text[tail_start..]); }
            eat_whitespace = true;
            continue;
        }

        assert!(!c.is_whitespace());

        // Went over the allowed length.
        if length > max_len {
            if i > 1 && head_end == 1 && tail_start == 1 {
                // Didn't encounter any better cut points, so just place cut
                // in the middle of the word where we're at.
                head_end = i;
                tail_start = i;
            }

            // Use the last good result.
            return (&text[..head_end], &text[tail_start..]);
        }

        // Hyphens are a possible cut point.
        if c == '-' {
            head_end = i;
            tail_start = i;
        }
    }

    (&text, &""[])
}

pub struct Map2DIterator<T> {
    /// Input iterator
    iter: T,
    x: i32,
    y: i32,
}

impl<T: Iterator<Item=char>> Iterator for Map2DIterator<T> {
    type Item = (char, i32, i32);

    fn next(&mut self) -> Option<(char, i32, i32)> {
        loop {
            match self.iter.next() {
                None => { return None }
                Some(c) if c == '\n' => { self.y += 1; self.x = 0; }
                Some(c) if (c as u32) < 32 => { }
                Some(c) => { self.x += 1; return Some((c, self.x - 1, self.y)) }
            }
        }
    }
}

pub trait Map2DUtil {
    fn map2d(self) -> Map2DIterator<Self>;
}

impl<T: Iterator<Item=char>> Map2DUtil for T {
    fn map2d(self) -> Map2DIterator<T> {
        Map2DIterator{ iter: self, x: 0, y: 0 }
    }
}

#[cfg(test)]
mod test {
    #[test]
    fn test_split_line() {
        use super::split_line;

        assert_eq!(("", ""), split_line("", |_| 1.0, 12.0));
        assert_eq!(("a", ""), split_line("a", |_| 1.0, 5.0));
        assert_eq!(("the", "cat"), split_line("the cat", |_| 1.0, 5.0));
        assert_eq!(("the", "cat"), split_line("the     cat", |_| 1.0, 5.0));
        assert_eq!(("the", "cat"), split_line("the  \t cat", |_| 1.0, 5.0));
        assert_eq!(("the", "cat"), split_line("the \ncat", |_| 1.0, 32.0));
        assert_eq!(("the", "   cat"), split_line("the \n   cat", |_| 1.0, 32.0));
        assert_eq!(("the  cat", ""), split_line("the  cat", |_| 1.0, 32.0));
        assert_eq!(("the", "cat sat"), split_line("the cat sat", |_| 1.0, 6.0));
        assert_eq!(("the cat", "sat"), split_line("the cat sat", |_| 1.0, 7.0));
        assert_eq!(("a", "bc"), split_line("abc", |_| 1.0, 0.01));
        assert_eq!(("dead", "beef"), split_line("deadbeef", |_| 1.0, 4.0));
    }
}
<commit_msg>wrap_lines function again<commit_after>/// Divide a string into the longest slice that fits within maximum line
/// length and the rest of the string. Place the split before a whitespace or
/// after a hyphen if possible. Any whitespace between the two segments is
/// trimmed. Newlines will cause a segment split when encountered.
pub fn split_line<'a, F>(text: &'a str, char_width: &F, max_len: f32) -> (&'a str, &'a str)
    where F: Fn(char) -> f32 {
    assert!(max_len >= 0.0);

    if text.len() == 0 { return (text, text); }

    // Init the split position to 1 because we always want to return at least
    // 1 character in the head partition.
    let mut head_end = 1;
    let mut tail_start = 1;
    // Is the iteration currently consuming whitespace inside a possible
    // break.
    let mut eat_whitespace = false;
    let mut length = 0.0;

    for (i, c) in text.chars().enumerate() {
        length = length + char_width(c);

        // Invariant: head_end and tail_start describe a valid, but possibly
        // suboptimal return value at this point.
        if eat_whitespace {
            if c.is_whitespace() {
                tail_start = i + 1;
                if c == '\n' { return (&text[..head_end], &text[tail_start..]); }
                continue;
            } else {
                eat_whitespace = false;
            }
        }

        // We're either just encountering the first whitespace after a block
        // of text, or over non-whitespace text.
        assert!(!eat_whitespace);

        // Invariant: The length of the string processed up to this point is
        // still short enough to return.

        // Encounter the first whitespace, set head_end marker.
        if c.is_whitespace() {
            head_end = i;
            tail_start = i + 1;
            if c == '\n' { return (&text[..head_end], &text[tail_start..]); }
            eat_whitespace = true;
            continue;
        }

        assert!(!c.is_whitespace());

        // Went over the allowed length.
        if length > max_len {
            if i > 1 && head_end == 1 && tail_start == 1 {
                // Didn't encounter any better cut points, so just place cut
                // in the middle of the word where we're at.
                head_end = i;
                tail_start = i;
            }

            // Use the last good result.
            return (&text[..head_end], &text[tail_start..]);
        }

        // Hyphens are a possible cut point.
        if c == '-' {
            head_end = i;
            tail_start = i;
        }
    }

    (&text, &""[])
}

pub fn wrap_lines<F>(mut text: &str, char_width: &F, max_len: f32) -> String
    where F: Fn(char) -> f32 {
    let mut result = String::new();
    loop {
        let (head, tail) = split_line(text, char_width, max_len);
        if head.len() == 0 && tail.len() == 0 { break; }
        result = result + head;
        if tail.len() != 0 { result = result + "\n"; }
        text = tail;
    }
    result
}

pub struct Map2DIterator<T> {
    /// Input iterator
    iter: T,
    x: i32,
    y: i32,
}

impl<T: Iterator<Item=char>> Iterator for Map2DIterator<T> {
    type Item = (char, i32, i32);

    fn next(&mut self) -> Option<(char, i32, i32)> {
        loop {
            match self.iter.next() {
                None => { return None }
                Some(c) if c == '\n' => { self.y += 1; self.x = 0; }
                Some(c) if (c as u32) < 32 => { }
                Some(c) => { self.x += 1; return Some((c, self.x - 1, self.y)) }
            }
        }
    }
}

pub trait Map2DUtil {
    fn map2d(self) -> Map2DIterator<Self>;
}

impl<T: Iterator<Item=char>> Map2DUtil for T {
    fn map2d(self) -> Map2DIterator<T> {
        Map2DIterator{ iter: self, x: 0, y: 0 }
    }
}

#[cfg(test)]
mod test {
    #[test]
    fn test_split_line() {
        use super::split_line;

        assert_eq!(("", ""), split_line("", &|_| 1.0, 12.0));
        assert_eq!(("a", ""), split_line("a", &|_| 1.0, 5.0));
        assert_eq!(("the", "cat"), split_line("the cat", &|_| 1.0, 5.0));
        assert_eq!(("the", "cat"), split_line("the     cat", &|_| 1.0, 5.0));
        assert_eq!(("the", "cat"), split_line("the  \t cat", &|_| 1.0, 5.0));
        assert_eq!(("the", "cat"), split_line("the \ncat", &|_| 1.0, 32.0));
        assert_eq!(("the", "   cat"), split_line("the \n   cat", &|_| 1.0, 32.0));
        assert_eq!(("the  cat", ""), split_line("the  cat", &|_| 1.0, 32.0));
        assert_eq!(("the", "cat sat"), split_line("the cat sat", &|_| 1.0, 6.0));
        assert_eq!(("the cat", "sat"), split_line("the cat sat", &|_| 1.0, 7.0));
        assert_eq!(("a", "bc"), split_line("abc", &|_| 1.0, 0.01));
        assert_eq!(("dead", "beef"), split_line("deadbeef", &|_| 1.0, 4.0));
    }
}
<|endoftext|>


<commit_before>/// Integration tests for the whole project

extern crate zwreec;
use std::path::Path;
use std::fs::File;
use std::error::Error;
use std::io::Cursor;
use std::vec::Vec;

static TESTFOLDER_PASS: &'static str = "./tests/integration/should-compile/";
static TESTFOLDER_FAIL: &'static str = "./tests/integration/should-fail/";

fn test_compile(input_filename: String) {
    let path = Path::new(&input_filename);
    let mut input = match File::open(path) {
        Err(why) => {
            panic!("Couldn't open {}: {}",
                           path.display(), Error::description(&why))
        },
        Ok(file) => {
            file
        }
    };

    let vec: Vec<u8> = vec![];
    let mut output = Cursor::new(vec);

    let cfg = zwreec::config::Config::default_config();

    zwreec::compile(cfg, &mut input, &mut output);
}

#[test]
fn expressions_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "Expressions.twee");
}

#[test]
fn helloworld_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "HelloWorld.twee");
}

#[test]
fn long_text_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "HelloWorld.twee");
}

#[test]
fn zscii_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "ZSCII.twee");
}

#[test]
fn ascii_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "ASCII.twee");
}

#[test]
fn unicode_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "Unicode.twee");
}

#[test]
fn passage_links_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "PassageLinks.twee");
}

#[test]
fn random_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "Random.twee");
}

#[test]
fn random_expanded_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "RandomExpanded.twee");
}

#[test]
fn if_else_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "If-Else.twee");
}

#[test]
fn current_status_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "CurrentStatus.twee");
}

#[test]
#[should_panic]
fn expression_double_operators_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "ExpressionDoubleOperators.twee");
}

#[test]
#[should_panic]
fn expression_wrong_parentheses1_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "ExpressionsWrongParentheses1.twee");
}

#[test]
#[should_panic]
fn expression_wrong_parentheses2_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "ExpressionsWrongParentheses2.twee");
}

#[test]
#[should_panic]
fn duplicate_passages() {
    test_compile(TESTFOLDER_FAIL.to_string() + "DuplicatePassage.twee");
}

#[test]
#[should_panic]
fn invalid_macro_test() {
   test_compile(TESTFOLDER_FAIL.to_string() + "InvalidMacro.twee");
}

#[test]
#[should_panic]
fn multiple_else_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "MultipleElse.twee");
}

#[test]
#[should_panic]
fn multiple_endif_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "MultipleEndIf.twee");
}

#[test]
#[should_panic]
fn no_start_passage_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "NoStartPassage.twee");
}

#[test]
#[should_panic]
fn passage_not_allowed_chars1_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "PassageNotAllowedChars1.twee");
}

#[test]
#[should_panic]
fn passage_not_allowed_chars2_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "PassageNotAllowedChars2.twee");
}

#[test]
#[should_panic]
fn wrong_formatting_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "WrongFormatting.twee");
}
<commit_msg>disable tests which are failing with stable rustlex<commit_after>/// Integration tests for the whole project

extern crate zwreec;
use std::path::Path;
use std::fs::File;
use std::error::Error;
use std::io::Cursor;
use std::vec::Vec;

static TESTFOLDER_PASS: &'static str = "./tests/integration/should-compile/";
static TESTFOLDER_FAIL: &'static str = "./tests/integration/should-fail/";

fn test_compile(input_filename: String) {
    let path = Path::new(&input_filename);
    let mut input = match File::open(path) {
        Err(why) => {
            panic!("Couldn't open {}: {}",
                           path.display(), Error::description(&why))
        },
        Ok(file) => {
            file
        }
    };

    let vec: Vec<u8> = vec![];
    let mut output = Cursor::new(vec);

    let cfg = zwreec::config::Config::default_config();

    zwreec::compile(cfg, &mut input, &mut output);
}

#[test]
fn expressions_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "Expressions.twee");
}

#[test]
fn helloworld_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "HelloWorld.twee");
}

#[test]
fn long_text_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "HelloWorld.twee");
}

#[test]
fn zscii_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "ZSCII.twee");
}

#[test]
fn ascii_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "ASCII.twee");
}

#[test]
fn unicode_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "Unicode.twee");
}

#[test]
fn passage_links_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "PassageLinks.twee");
}

#[test]
fn random_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "Random.twee");
}

#[test]
fn random_expanded_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "RandomExpanded.twee");
}

#[test]
fn if_else_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "If-Else.twee");
}

#[test]
fn current_status_test() {
    test_compile(TESTFOLDER_PASS.to_string() + "CurrentStatus.twee");
}

#[test]
#[should_panic]
fn expression_double_operators_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "ExpressionDoubleOperators.twee");
}

#[test]
#[should_panic]
fn expression_wrong_parentheses1_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "ExpressionsWrongParentheses1.twee");
}

#[test]
#[should_panic]
fn expression_wrong_parentheses2_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "ExpressionsWrongParentheses2.twee");
}

#[test]
#[should_panic]
fn duplicate_passages() {
    test_compile(TESTFOLDER_FAIL.to_string() + "DuplicatePassage.twee");
}

#[test]
#[should_panic]
fn invalid_macro_test() {
   test_compile(TESTFOLDER_FAIL.to_string() + "InvalidMacro.twee");
}

#[test]
#[should_panic]
fn multiple_else_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "MultipleElse.twee");
}

#[test]
#[should_panic]
fn multiple_endif_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "MultipleEndIf.twee");
}

#[test]
#[should_panic]
fn no_start_passage_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "NoStartPassage.twee");
}

//#[test]
//#[should_panic]
//fn passage_not_allowed_chars1_test() {
//    test_compile(TESTFOLDER_FAIL.to_string() + "PassageNotAllowedChars1.twee");
//}

//#[test]
//#[should_panic]
//fn passage_not_allowed_chars2_test() {
//    test_compile(TESTFOLDER_FAIL.to_string() + "PassageNotAllowedChars2.twee");
//}

#[test]
#[should_panic]
fn wrong_formatting_test() {
    test_compile(TESTFOLDER_FAIL.to_string() + "WrongFormatting.twee");
}
<|endoftext|>


<commit_before>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

//! This module implements an in-memory Merkle Accumulator that is similar to what we use in
//! storage. This accumulator will only store a small portion of the tree -- for any subtree that
//! is full, we store only the root. Also we only store the frozen nodes, therefore this structure
//! will always store up to `Log(n)` number of nodes, where `n` is the total number of leaves in
//! the tree.
//!
//! This accumulator is immutable once constructed. If we append new leaves to the tree we will
//! obtain a new accumulator instance and the old one remains unchanged.

#[cfg(test)]
mod accumulator_test;

#[cfg(any(test, feature = "fuzzing"))]
pub mod mock;

use super::MerkleTreeInternalNode;
use crate::proof::definition::{LeafCount, MAX_ACCUMULATOR_LEAVES};
use anyhow::{ensure, format_err, Result};
use aptos_crypto::{
    hash::{
        CryptoHash, CryptoHasher, ACCUMULATOR_PLACEHOLDER_HASH, SPARSE_MERKLE_PLACEHOLDER_HASH,
    },
    HashValue,
};
use serde::{Deserialize, Serialize};
use std::marker::PhantomData;

/// The Accumulator implementation.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct InMemoryAccumulator<H> {
    /// Represents the roots of all the full subtrees from left to right in this accumulator. For
    /// example, if we have the following accumulator, this vector will have two hashes that
    /// correspond to `X` and `e`.
    /// ```text
    ///                 root
    ///                /    \
    ///              /        \
    ///            /            \
    ///           X              o
    ///         /   \           / \
    ///        /     \         /   \
    ///       o       o       o     placeholder
    ///      / \     / \     / \
    ///     a   b   c   d   e   placeholder
    /// ```
    frozen_subtree_roots: Vec<HashValue>,

    /// The total number of leaves in this accumulator.
    num_leaves: LeafCount,

    /// The root hash of this accumulator.
    root_hash: HashValue,

    phantom: PhantomData<H>,
}

impl<H> InMemoryAccumulator<H>
where
    H: CryptoHasher,
{
    /// Constructs a new accumulator with roots of existing frozen subtrees. Returns error if the
    /// number of frozen subtree roots does not match the number of leaves.
    pub fn new(frozen_subtree_roots: Vec<HashValue>, num_leaves: LeafCount) -> Result<Self> {
        ensure!(
            frozen_subtree_roots.len() == num_leaves.count_ones() as usize,
            "The number of frozen subtrees does not match the number of leaves. \
             frozen_subtree_roots.len(): {}. num_leaves: {}.",
            frozen_subtree_roots.len(),
            num_leaves,
        );

        let root_hash = Self::compute_root_hash(&frozen_subtree_roots, num_leaves);

        Ok(Self {
            frozen_subtree_roots,
            num_leaves,
            root_hash,
            phantom: PhantomData,
        })
    }

    pub fn new_empty() -> Self {
        Self {
            frozen_subtree_roots: Vec::new(),
            num_leaves: 0,
            root_hash: *SPARSE_MERKLE_PLACEHOLDER_HASH,
            phantom: PhantomData,
        }
    }

    /// Constructs a new accumulator with given leaves.
    pub fn from_leaves(leaves: &[HashValue]) -> Self {
        Self::default().append(leaves)
    }

    /// Appends a list of new leaves to an existing accumulator. Since the accumulator is
    /// immutable, the existing one remains unchanged and a new one representing the result is
    /// returned.
    pub fn append(&self, leaves: &[HashValue]) -> Self {
        let mut frozen_subtree_roots = self.frozen_subtree_roots.clone();
        let mut num_leaves = self.num_leaves;
        for leaf in leaves {
            Self::append_one(&mut frozen_subtree_roots, num_leaves, *leaf);
            num_leaves += 1;
        }

        Self::new(frozen_subtree_roots, num_leaves).expect(
            "Appending leaves to a valid accumulator should create another valid accumulator.",
        )
    }

    /// Appends one leaf. This will update `frozen_subtree_roots` to store new frozen root nodes
    /// and remove old nodes if they are now part of a larger frozen subtree.
    fn append_one(
        frozen_subtree_roots: &mut Vec<HashValue>,
        num_existing_leaves: LeafCount,
        leaf: HashValue,
    ) {
        // For example, this accumulator originally had N = 7 leaves. Appending a leaf is like
        // adding one to this number N: 0b0111 + 1 = 0b1000. Every time we carry a bit to the left
        // we merge the rightmost two subtrees and compute their parent.
        // ```text
        //       A
        //     /   \
        //    /     \
        //   o       o       B
        //  / \     / \     / \
        // o   o   o   o   o   o   o
        // ```

        // First just append the leaf.
        frozen_subtree_roots.push(leaf);

        // Next, merge the last two subtrees into one. If `num_existing_leaves` has N trailing
        // ones, the carry will happen N times.
        let num_trailing_ones = (!num_existing_leaves).trailing_zeros();

        for _i in 0..num_trailing_ones {
            let right_hash = frozen_subtree_roots.pop().expect("Invalid accumulator.");
            let left_hash = frozen_subtree_roots.pop().expect("Invalid accumulator.");
            let parent_hash = MerkleTreeInternalNode::<H>::new(left_hash, right_hash).hash();
            frozen_subtree_roots.push(parent_hash);
        }
    }

    /// Appends a list of new subtrees to the existing accumulator. This is similar to
    /// [`append`](InMemoryAccumulator::append) except that the new leaves themselves are not known and
    /// they are represented by `subtrees`. As an example, given the following accumulator that
    /// currently has 10 leaves, the frozen subtree roots and the new subtrees are annotated below.
    /// Note that in this case `subtrees[0]` represents two new leaves `A` and `B`, `subtrees[1]`
    /// represents four new leaves `C`, `D`, `E` and `F`, `subtrees[2]` represents four new leaves
    /// `G`, `H`, `I` and `J`, and the last `subtrees[3]` represents one new leaf `K`.
    ///
    /// ```text
    ///                                                                           new_root
    ///                                                                         /          \
    ///                                                                       /              \
    ///                                                                     /                  \
    ///                                                                   /                      \
    ///                                                                 /                          \
    ///                                                               /                              \
    ///                                                             /                                  \
    ///                                                           /                                      \
    ///                                                         /                                          \
    ///                                                       /                                              \
    ///                                                     /                                                  \
    ///                                                   /                                                      \
    ///                                                 /                                                          \
    ///                                         old_root                                                            o
    ///                                        /        \                                                          / \
    ///                                      /            \                                                       /   placeholder
    ///                                    /                \                                                    /
    ///                                  /                    \                                                 /
    ///                                /                        \                                              /
    ///                              /                            \                                           o
    ///                            /                                \                                        / \
    ///                          /                                    \                                     /    \
    ///                        /                                       o                                  /        \
    /// frozen_subtree_roots[0]                                      /   \                              /            \
    ///                    /   \                                    /     \                           /                \
    ///                   /     \                                  /       \                        /                    \
    ///                  o       o                                o         subtrees[1]  subtrees[2]                     o
    ///                 / \     / \                              / \                / \          / \                    / \
    ///                o   o   o   o      frozen_subtree_roots[1]   subtrees[0]    o   o        o   o                  o   placeholder
    ///               / \ / \ / \ / \                         / \           / \   / \ / \      / \ / \                / \
    ///               o o o o o o o o                         o o           A B   C D E F      G H I J  K (subtrees[3]) placeholder
    /// ```
    pub fn append_subtrees(
        &self,
        subtrees: &[HashValue],
        num_new_leaves: LeafCount,
    ) -> Result<Self> {
        ensure!(
            num_new_leaves <= MAX_ACCUMULATOR_LEAVES - self.num_leaves,
            "Too many new leaves. self.num_leaves: {}. num_new_leaves: {}.",
            self.num_leaves,
            num_new_leaves,
        );

        if self.num_leaves == 0 {
            return Self::new(subtrees.to_vec(), num_new_leaves);
        }

        let mut current_subtree_roots = self.frozen_subtree_roots.clone();
        let mut current_num_leaves = self.num_leaves;
        let mut remaining_new_leaves = num_new_leaves;
        let mut subtree_iter = subtrees.iter();

        // Check if we want to combine a new subtree with the rightmost frozen subtree. To do that
        // this new subtree needs to represent `rightmost_frozen_subtree_size` leaves, so we need
        // to have at least this many new leaves remaining.
        let mut rightmost_frozen_subtree_size = 1 << current_num_leaves.trailing_zeros();
        while remaining_new_leaves >= rightmost_frozen_subtree_size {
            // Note that after combining the rightmost frozen subtree of size X with a new subtree,
            // we obtain a subtree of size 2X. If there was already a frozen subtree of size 2X, we
            // need to carry this process further.
            let mut mask = rightmost_frozen_subtree_size;
            let mut current_hash = *subtree_iter
                .next()
                .ok_or_else(|| format_err!("Too few subtrees."))?;
            while current_num_leaves & mask != 0 {
                let left_hash = current_subtree_roots
                    .pop()
                    .expect("This frozen subtree must exist.");
                current_hash = MerkleTreeInternalNode::<H>::new(left_hash, current_hash).hash();
                mask <<= 1;
            }
            current_subtree_roots.push(current_hash);

            current_num_leaves += rightmost_frozen_subtree_size;
            remaining_new_leaves -= rightmost_frozen_subtree_size;
            rightmost_frozen_subtree_size = mask;
        }

        // Now all the new subtrees are smaller than the rightmost frozen subtree. We just append
        // all of them. Note that if the number of new subtrees does not actually match the number
        // of new leaves, `Self::new` below will raise an error.
        current_num_leaves += remaining_new_leaves;
        current_subtree_roots.extend(subtree_iter);

        Self::new(current_subtree_roots, current_num_leaves)
    }

    /// Returns the root hash of the accumulator.
    pub fn root_hash(&self) -> HashValue {
        self.root_hash
    }

    pub fn version(&self) -> u64 {
        if self.num_leaves() == 0 {
            0
        } else {
            self.num_leaves() - 1
        }
    }

    /// Computes the root hash of an accumulator given the frozen subtree roots and the number of
    /// leaves in this accumulator.
    fn compute_root_hash(frozen_subtree_roots: &[HashValue], num_leaves: LeafCount) -> HashValue {
        match frozen_subtree_roots.len() {
            0 => return *ACCUMULATOR_PLACEHOLDER_HASH,
            1 => return frozen_subtree_roots[0],
            _ => (),
        }

        // The trailing zeros do not matter since anything below the lowest frozen subtree is
        // already represented by the subtree roots.
        let mut bitmap = num_leaves >> num_leaves.trailing_zeros();
        let mut current_hash = *ACCUMULATOR_PLACEHOLDER_HASH;
        let mut frozen_subtree_iter = frozen_subtree_roots.iter().rev();

        while bitmap > 0 {
            current_hash = if bitmap & 1 != 0 {
                MerkleTreeInternalNode::<H>::new(
                    *frozen_subtree_iter
                        .next()
                        .expect("This frozen subtree should exist."),
                    current_hash,
                )
            } else {
                MerkleTreeInternalNode::<H>::new(current_hash, *ACCUMULATOR_PLACEHOLDER_HASH)
            }
            .hash();
            bitmap >>= 1;
        }

        current_hash
    }

    /// Returns the set of frozen subtree roots in this accumulator
    pub fn frozen_subtree_roots(&self) -> &Vec<HashValue> {
        &self.frozen_subtree_roots
    }

    /// Returns the total number of leaves in this accumulator.
    pub fn num_leaves(&self) -> LeafCount {
        self.num_leaves
    }

    /// Returns true if this accumulator is empty and has no leaves.
    pub fn is_empty(&self) -> bool {
        self.num_leaves == 0
    }
}

// #[derive(..)] doesn't seem to work b/c of the PhantomData :(
impl<H> std::cmp::Eq for InMemoryAccumulator<H> {}
impl<H> std::cmp::PartialEq for InMemoryAccumulator<H> {
    fn eq(&self, other: &Self) -> bool {
        self.num_leaves == other.num_leaves
            && self.root_hash == other.root_hash
            && self.frozen_subtree_roots == other.frozen_subtree_roots
    }
}

impl<H> Default for InMemoryAccumulator<H>
where
    H: CryptoHasher,
{
    fn default() -> Self {
        Self::new(vec![], 0).expect("Constructing empty accumulator should work.")
    }
}
<commit_msg>[bug] Wrong in-mem accumulator placeholder_hash (#1761)<commit_after>// Copyright (c) Aptos
// SPDX-License-Identifier: Apache-2.0

//! This module implements an in-memory Merkle Accumulator that is similar to what we use in
//! storage. This accumulator will only store a small portion of the tree -- for any subtree that
//! is full, we store only the root. Also we only store the frozen nodes, therefore this structure
//! will always store up to `Log(n)` number of nodes, where `n` is the total number of leaves in
//! the tree.
//!
//! This accumulator is immutable once constructed. If we append new leaves to the tree we will
//! obtain a new accumulator instance and the old one remains unchanged.

#[cfg(test)]
mod accumulator_test;

#[cfg(any(test, feature = "fuzzing"))]
pub mod mock;

use super::MerkleTreeInternalNode;
use crate::proof::definition::{LeafCount, MAX_ACCUMULATOR_LEAVES};
use anyhow::{ensure, format_err, Result};
use aptos_crypto::{
    hash::{CryptoHash, CryptoHasher, ACCUMULATOR_PLACEHOLDER_HASH},
    HashValue,
};
use serde::{Deserialize, Serialize};
use std::marker::PhantomData;

/// The Accumulator implementation.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct InMemoryAccumulator<H> {
    /// Represents the roots of all the full subtrees from left to right in this accumulator. For
    /// example, if we have the following accumulator, this vector will have two hashes that
    /// correspond to `X` and `e`.
    /// ```text
    ///                 root
    ///                /    \
    ///              /        \
    ///            /            \
    ///           X              o
    ///         /   \           / \
    ///        /     \         /   \
    ///       o       o       o     placeholder
    ///      / \     / \     / \
    ///     a   b   c   d   e   placeholder
    /// ```
    frozen_subtree_roots: Vec<HashValue>,

    /// The total number of leaves in this accumulator.
    num_leaves: LeafCount,

    /// The root hash of this accumulator.
    root_hash: HashValue,

    phantom: PhantomData<H>,
}

impl<H> InMemoryAccumulator<H>
where
    H: CryptoHasher,
{
    /// Constructs a new accumulator with roots of existing frozen subtrees. Returns error if the
    /// number of frozen subtree roots does not match the number of leaves.
    pub fn new(frozen_subtree_roots: Vec<HashValue>, num_leaves: LeafCount) -> Result<Self> {
        ensure!(
            frozen_subtree_roots.len() == num_leaves.count_ones() as usize,
            "The number of frozen subtrees does not match the number of leaves. \
             frozen_subtree_roots.len(): {}. num_leaves: {}.",
            frozen_subtree_roots.len(),
            num_leaves,
        );

        let root_hash = Self::compute_root_hash(&frozen_subtree_roots, num_leaves);

        Ok(Self {
            frozen_subtree_roots,
            num_leaves,
            root_hash,
            phantom: PhantomData,
        })
    }

    pub fn new_empty() -> Self {
        Self {
            frozen_subtree_roots: Vec::new(),
            num_leaves: 0,
            root_hash: *ACCUMULATOR_PLACEHOLDER_HASH,
            phantom: PhantomData,
        }
    }

    /// Constructs a new accumulator with given leaves.
    pub fn from_leaves(leaves: &[HashValue]) -> Self {
        Self::default().append(leaves)
    }

    /// Appends a list of new leaves to an existing accumulator. Since the accumulator is
    /// immutable, the existing one remains unchanged and a new one representing the result is
    /// returned.
    pub fn append(&self, leaves: &[HashValue]) -> Self {
        let mut frozen_subtree_roots = self.frozen_subtree_roots.clone();
        let mut num_leaves = self.num_leaves;
        for leaf in leaves {
            Self::append_one(&mut frozen_subtree_roots, num_leaves, *leaf);
            num_leaves += 1;
        }

        Self::new(frozen_subtree_roots, num_leaves).expect(
            "Appending leaves to a valid accumulator should create another valid accumulator.",
        )
    }

    /// Appends one leaf. This will update `frozen_subtree_roots` to store new frozen root nodes
    /// and remove old nodes if they are now part of a larger frozen subtree.
    fn append_one(
        frozen_subtree_roots: &mut Vec<HashValue>,
        num_existing_leaves: LeafCount,
        leaf: HashValue,
    ) {
        // For example, this accumulator originally had N = 7 leaves. Appending a leaf is like
        // adding one to this number N: 0b0111 + 1 = 0b1000. Every time we carry a bit to the left
        // we merge the rightmost two subtrees and compute their parent.
        // ```text
        //       A
        //     /   \
        //    /     \
        //   o       o       B
        //  / \     / \     / \
        // o   o   o   o   o   o   o
        // ```

        // First just append the leaf.
        frozen_subtree_roots.push(leaf);

        // Next, merge the last two subtrees into one. If `num_existing_leaves` has N trailing
        // ones, the carry will happen N times.
        let num_trailing_ones = (!num_existing_leaves).trailing_zeros();

        for _i in 0..num_trailing_ones {
            let right_hash = frozen_subtree_roots.pop().expect("Invalid accumulator.");
            let left_hash = frozen_subtree_roots.pop().expect("Invalid accumulator.");
            let parent_hash = MerkleTreeInternalNode::<H>::new(left_hash, right_hash).hash();
            frozen_subtree_roots.push(parent_hash);
        }
    }

    /// Appends a list of new subtrees to the existing accumulator. This is similar to
    /// [`append`](InMemoryAccumulator::append) except that the new leaves themselves are not known and
    /// they are represented by `subtrees`. As an example, given the following accumulator that
    /// currently has 10 leaves, the frozen subtree roots and the new subtrees are annotated below.
    /// Note that in this case `subtrees[0]` represents two new leaves `A` and `B`, `subtrees[1]`
    /// represents four new leaves `C`, `D`, `E` and `F`, `subtrees[2]` represents four new leaves
    /// `G`, `H`, `I` and `J`, and the last `subtrees[3]` represents one new leaf `K`.
    ///
    /// ```text
    ///                                                                           new_root
    ///                                                                         /          \
    ///                                                                       /              \
    ///                                                                     /                  \
    ///                                                                   /                      \
    ///                                                                 /                          \
    ///                                                               /                              \
    ///                                                             /                                  \
    ///                                                           /                                      \
    ///                                                         /                                          \
    ///                                                       /                                              \
    ///                                                     /                                                  \
    ///                                                   /                                                      \
    ///                                                 /                                                          \
    ///                                         old_root                                                            o
    ///                                        /        \                                                          / \
    ///                                      /            \                                                       /   placeholder
    ///                                    /                \                                                    /
    ///                                  /                    \                                                 /
    ///                                /                        \                                              /
    ///                              /                            \                                           o
    ///                            /                                \                                        / \
    ///                          /                                    \                                     /    \
    ///                        /                                       o                                  /        \
    /// frozen_subtree_roots[0]                                      /   \                              /            \
    ///                    /   \                                    /     \                           /                \
    ///                   /     \                                  /       \                        /                    \
    ///                  o       o                                o         subtrees[1]  subtrees[2]                     o
    ///                 / \     / \                              / \                / \          / \                    / \
    ///                o   o   o   o      frozen_subtree_roots[1]   subtrees[0]    o   o        o   o                  o   placeholder
    ///               / \ / \ / \ / \                         / \           / \   / \ / \      / \ / \                / \
    ///               o o o o o o o o                         o o           A B   C D E F      G H I J  K (subtrees[3]) placeholder
    /// ```
    pub fn append_subtrees(
        &self,
        subtrees: &[HashValue],
        num_new_leaves: LeafCount,
    ) -> Result<Self> {
        ensure!(
            num_new_leaves <= MAX_ACCUMULATOR_LEAVES - self.num_leaves,
            "Too many new leaves. self.num_leaves: {}. num_new_leaves: {}.",
            self.num_leaves,
            num_new_leaves,
        );

        if self.num_leaves == 0 {
            return Self::new(subtrees.to_vec(), num_new_leaves);
        }

        let mut current_subtree_roots = self.frozen_subtree_roots.clone();
        let mut current_num_leaves = self.num_leaves;
        let mut remaining_new_leaves = num_new_leaves;
        let mut subtree_iter = subtrees.iter();

        // Check if we want to combine a new subtree with the rightmost frozen subtree. To do that
        // this new subtree needs to represent `rightmost_frozen_subtree_size` leaves, so we need
        // to have at least this many new leaves remaining.
        let mut rightmost_frozen_subtree_size = 1 << current_num_leaves.trailing_zeros();
        while remaining_new_leaves >= rightmost_frozen_subtree_size {
            // Note that after combining the rightmost frozen subtree of size X with a new subtree,
            // we obtain a subtree of size 2X. If there was already a frozen subtree of size 2X, we
            // need to carry this process further.
            let mut mask = rightmost_frozen_subtree_size;
            let mut current_hash = *subtree_iter
                .next()
                .ok_or_else(|| format_err!("Too few subtrees."))?;
            while current_num_leaves & mask != 0 {
                let left_hash = current_subtree_roots
                    .pop()
                    .expect("This frozen subtree must exist.");
                current_hash = MerkleTreeInternalNode::<H>::new(left_hash, current_hash).hash();
                mask <<= 1;
            }
            current_subtree_roots.push(current_hash);

            current_num_leaves += rightmost_frozen_subtree_size;
            remaining_new_leaves -= rightmost_frozen_subtree_size;
            rightmost_frozen_subtree_size = mask;
        }

        // Now all the new subtrees are smaller than the rightmost frozen subtree. We just append
        // all of them. Note that if the number of new subtrees does not actually match the number
        // of new leaves, `Self::new` below will raise an error.
        current_num_leaves += remaining_new_leaves;
        current_subtree_roots.extend(subtree_iter);

        Self::new(current_subtree_roots, current_num_leaves)
    }

    /// Returns the root hash of the accumulator.
    pub fn root_hash(&self) -> HashValue {
        self.root_hash
    }

    pub fn version(&self) -> u64 {
        if self.num_leaves() == 0 {
            0
        } else {
            self.num_leaves() - 1
        }
    }

    /// Computes the root hash of an accumulator given the frozen subtree roots and the number of
    /// leaves in this accumulator.
    fn compute_root_hash(frozen_subtree_roots: &[HashValue], num_leaves: LeafCount) -> HashValue {
        match frozen_subtree_roots.len() {
            0 => return *ACCUMULATOR_PLACEHOLDER_HASH,
            1 => return frozen_subtree_roots[0],
            _ => (),
        }

        // The trailing zeros do not matter since anything below the lowest frozen subtree is
        // already represented by the subtree roots.
        let mut bitmap = num_leaves >> num_leaves.trailing_zeros();
        let mut current_hash = *ACCUMULATOR_PLACEHOLDER_HASH;
        let mut frozen_subtree_iter = frozen_subtree_roots.iter().rev();

        while bitmap > 0 {
            current_hash = if bitmap & 1 != 0 {
                MerkleTreeInternalNode::<H>::new(
                    *frozen_subtree_iter
                        .next()
                        .expect("This frozen subtree should exist."),
                    current_hash,
                )
            } else {
                MerkleTreeInternalNode::<H>::new(current_hash, *ACCUMULATOR_PLACEHOLDER_HASH)
            }
            .hash();
            bitmap >>= 1;
        }

        current_hash
    }

    /// Returns the set of frozen subtree roots in this accumulator
    pub fn frozen_subtree_roots(&self) -> &Vec<HashValue> {
        &self.frozen_subtree_roots
    }

    /// Returns the total number of leaves in this accumulator.
    pub fn num_leaves(&self) -> LeafCount {
        self.num_leaves
    }

    /// Returns true if this accumulator is empty and has no leaves.
    pub fn is_empty(&self) -> bool {
        self.num_leaves == 0
    }
}

// #[derive(..)] doesn't seem to work b/c of the PhantomData :(
impl<H> std::cmp::Eq for InMemoryAccumulator<H> {}
impl<H> std::cmp::PartialEq for InMemoryAccumulator<H> {
    fn eq(&self, other: &Self) -> bool {
        self.num_leaves == other.num_leaves
            && self.root_hash == other.root_hash
            && self.frozen_subtree_roots == other.frozen_subtree_roots
    }
}

impl<H> Default for InMemoryAccumulator<H>
where
    H: CryptoHasher,
{
    fn default() -> Self {
        Self::new(vec![], 0).expect("Constructing empty accumulator should work.")
    }
}
<|endoftext|>


<commit_before>use elasticsearch::Elasticsearch;
use std::time::Duration;

pub mod configuration;
pub mod explain;
pub(super) mod internal;
pub mod list;
pub mod models;
pub mod query;
pub mod remote;
pub mod status;
pub mod storage;

pub const ES_DEFAULT_TIMEOUT: u64 = 5000; // milliseconds
pub const ES_DEFAULT_VERSION_REQ: &str = ">=7.13.0";

/// A structure wrapping around the elasticsearch's client.
#[derive(Clone, Debug)]
pub struct ElasticsearchStorage {
    /// Elasticsearch client
    pub(crate) client: Elasticsearch,
    /// Timeout used by every call to the server.
    pub timeout: Duration,
}

#[cfg(test)]
pub mod tests {

    use config::Config;
    use serde::Serialize;
    use serial_test::serial;

    use super::*;

    use crate::domain::ports::secondary::remote::Remote;
    use crate::domain::ports::secondary::storage::Storage;
    use crate::utils::docker;
    use common::document::{ContainerDocument, Document};

    // In this test we present an invalid configuration (its actually empty) to the
    // create_container function, and expect the error message to be meaningful
    #[tokio::test]
    #[serial]
    async fn should_return_invalid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");

        let config = Config::builder().build().expect("build empty config");

        let res = client.create_container(config).await;

        assert!(res
            .unwrap_err()
            .to_string()
            .contains("Invalid Elasticsearch Index Configuration"));
    }

    #[tokio::test]
    async fn should_return_invalid_url() {
        let res = remote::connection_pool_url("foobar").await;
        assert!(res
            .unwrap_err()
            .to_string()
            .starts_with("Invalid Elasticsearch URL"));
    }

    #[tokio::test]
    #[serial]
    async fn should_connect_to_elasticsearch() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let _client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");
    }

    #[tokio::test]
    #[serial]
    async fn should_create_index_with_valid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");
        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");
        let config = config::Config::builder()
            .add_source(config::File::from_str(
                r#"{
                    "container": {
                        "name": "foo",
                        "dataset": "bar"
                    },
                    "elasticsearch": {
                        "parameters": {
                            "timeout": "10s",
                            "wait_for_active_shards": "1"
                        },
                        "settings": {
                            "index": {
                                "number_of_shards": 1,
                                "number_of_replicas": 1
                            }
                        },
                        "mappings": {
                            "properties": {
                                "value": {
                                    "type": "text"
                                }
                            }
                        }
                    }
                }"#,
                config::FileFormat::Json,
            ))
            .build()
            .expect("valid configuration");
        let res = client.create_container(config).await;
        assert!(res.is_ok());
    }

    #[tokio::test]
    #[serial]
    async fn should_correctly_report_invalid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");
        let config = config::Config::builder()
            .add_source(config::File::from_str(
                r#"{
                    "container": {
                        "name": "foo",
                        "dataset": "bar"
                    },
                    "elasticsearch": {
                        "parameters": {
                            "timeout": "10s",
                            "wait_for_active_shards": "1"
                        },
                        "settings": {
                            "indx": {
                                "number_of_shards": 1,
                                "number_of_replicas": 1
                            }
                        },
                        "mappings": {
                            "properties": {
                                "value": {
                                    "type": "text"
                                }
                            }
                        }
                    }
                }"#,
                config::FileFormat::Json,
            ))
            .build()
            .expect("valid configuration");
        let res = client.create_container(config).await;
        assert!(res
            .unwrap_err()
            .to_string()
            .starts_with("Container Creation Error"));
    }

    #[derive(Serialize)]
    struct TestObj {
        value: String,
    }

    impl Document for TestObj {
        fn id(&self) -> String {
            self.value.clone()
        }
    }

    impl ContainerDocument for TestObj {
        fn static_doc_type() -> &'static str {
            "test-obj"
        }
        fn default_es_container_config() -> config::Config {
            config::Config::builder()
                .set_default("container.name", Self::static_doc_type())
                .unwrap()
                .set_default("container.dataset", "default")
                .unwrap()
                .set_default("elasticsearch.parameters.timeout", "10s")
                .unwrap()
                .set_default("elasticsearch.parameters.wait_for_active_shards", "1")
                .unwrap()
                .add_source(config::File::from_str(
                    r#"{ "elasticsearch": { "settings": { "index": { "number_of_shards": 1, "number_of_replicas": 1 } } } }"#,
                    config::FileFormat::Json,
                ))
                .add_source(config::File::from_str(
                    r#"{ "elasticsearch": { "mappings": { "properties": { "value": { "type": "text" } } } } }"#,
                    config::FileFormat::Json,
                ))
                .build()
                .expect("invalid container configuration for TestObj")
        }
    }

    #[tokio::test]
    #[serial]
    async fn should_correctly_insert_multiple_documents() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");

        client
            .create_container(TestObj::default_es_container_config())
            .await
            .expect("container creation");

        let documents = vec![
            TestObj {
                value: String::from("obj1"),
            },
            TestObj {
                value: String::from("obj2"),
            },
            TestObj {
                value: String::from("obj3"),
            },
            TestObj {
                value: String::from("obj4"),
            },
            TestObj {
                value: String::from("obj5"),
            },
            TestObj {
                value: String::from("obj6"),
            },
        ];
        let documents = futures::stream::iter(documents);

        let res = client
            .insert_documents(
                String::from("root_obj_dataset_test-index-bulk-insert"),
                documents,
            )
            .await;

        assert_eq!(res.expect("insertion stats").created, 6);
    }

    #[tokio::test]
    #[serial]
    async fn should_detect_invalid_elasticsearch_version() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let client = pool.conn(ES_DEFAULT_TIMEOUT, ">=9.99.99").await;

        assert!(client
            .unwrap_err()
            .to_string()
            .contains("Elasticsearch Invalid version"));
    }
}
<commit_msg>fix elasticsearch internal tests<commit_after>use elasticsearch::Elasticsearch;
use std::time::Duration;

pub mod configuration;
pub mod explain;
pub(super) mod internal;
pub mod list;
pub mod models;
pub mod query;
pub mod remote;
pub mod status;
pub mod storage;

pub const ES_DEFAULT_TIMEOUT: u64 = 5000; // milliseconds
pub const ES_DEFAULT_VERSION_REQ: &str = ">=7.13.0";

/// A structure wrapping around the elasticsearch's client.
#[derive(Clone, Debug)]
pub struct ElasticsearchStorage {
    /// Elasticsearch client
    pub(crate) client: Elasticsearch,
    /// Timeout used by every call to the server.
    pub timeout: Duration,
}

#[cfg(test)]
pub mod tests {

    use config::Config;
    use serde::{Deserialize, Serialize};
    use serial_test::serial;

    use super::*;

    use crate::domain::ports::secondary::remote::Remote;
    use crate::domain::ports::secondary::storage::Storage;
    use crate::utils::docker;
    use common::document::{ContainerDocument, Document};

    // In this test we present an invalid configuration (its actually empty) to the
    // create_container function, and expect the error message to be meaningful
    #[tokio::test]
    #[serial]
    async fn should_return_invalid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");

        let config = Config::builder().build().expect("build empty config");

        let res = client.create_container(config).await;

        assert!(res
            .unwrap_err()
            .to_string()
            .contains("Invalid Elasticsearch Index Configuration"));
    }

    #[tokio::test]
    async fn should_return_invalid_url() {
        let res = remote::connection_pool_url("foobar").await;
        assert!(res
            .unwrap_err()
            .to_string()
            .starts_with("Invalid Elasticsearch URL"));
    }

    #[tokio::test]
    #[serial]
    async fn should_connect_to_elasticsearch() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let _client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");
    }

    #[tokio::test]
    #[serial]
    async fn should_create_index_with_valid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");
        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");
        let config = config::Config::builder()
            .add_source(config::File::from_str(
                r#"{
                    "container": {
                        "name": "foo",
                        "dataset": "bar"
                    },
                    "elasticsearch": {
                        "parameters": {
                            "timeout": "10s",
                            "wait_for_active_shards": "1"
                        },
                        "settings": {
                            "index": {
                                "number_of_shards": 1,
                                "number_of_replicas": 1
                            }
                        },
                        "mappings": {
                            "properties": {
                                "value": {
                                    "type": "text"
                                }
                            }
                        }
                    }
                }"#,
                config::FileFormat::Json,
            ))
            .build()
            .expect("valid configuration");
        let res = client.create_container(config).await;
        assert!(res.is_ok());
    }

    #[tokio::test]
    #[serial]
    async fn should_correctly_report_invalid_configuration() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");
        let config = config::Config::builder()
            .add_source(config::File::from_str(
                r#"{
                    "container": {
                        "name": "foo",
                        "dataset": "bar"
                    },
                    "elasticsearch": {
                        "parameters": {
                            "timeout": "10s",
                            "wait_for_active_shards": "1"
                        },
                        "settings": {
                            "indx": {
                                "number_of_shards": 1,
                                "number_of_replicas": 1
                            }
                        },
                        "mappings": {
                            "properties": {
                                "value": {
                                    "type": "text"
                                }
                            }
                        }
                    }
                }"#,
                config::FileFormat::Json,
            ))
            .build()
            .expect("valid configuration");
        let res = client.create_container(config).await;
        assert!(res
            .unwrap_err()
            .to_string()
            .starts_with("Container Creation Error"));
    }

    #[derive(Deserialize, Serialize)]
    struct TestObj {
        value: String,
    }

    impl Document for TestObj {
        fn id(&self) -> String {
            self.value.clone()
        }
    }

    impl ContainerDocument for TestObj {
        fn static_doc_type() -> &'static str {
            "test-obj"
        }

        fn default_es_container_config() -> config::Config {
            config::Config::builder()
                .set_default("container.name", Self::static_doc_type())
                .unwrap()
                .set_default("container.dataset", "default")
                .unwrap()
                .set_default("elasticsearch.parameters.timeout", "10s")
                .unwrap()
                .set_default("elasticsearch.parameters.wait_for_active_shards", "1")
                .unwrap()
                .add_source(config::File::from_str(
                    r#"{ "elasticsearch": { "settings": { "index": { "number_of_shards": 1, "number_of_replicas": 1 } } } }"#,
                    config::FileFormat::Json,
                ))
                .add_source(config::File::from_str(
                    r#"{ "elasticsearch": { "mappings": { "properties": { "value": { "type": "text" } } } } }"#,
                    config::FileFormat::Json,
                ))
                .build()
                .expect("invalid container configuration for TestObj")
        }
    }

    #[tokio::test]
    #[serial]
    async fn should_correctly_insert_multiple_documents() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let client = pool
            .conn(ES_DEFAULT_TIMEOUT, ES_DEFAULT_VERSION_REQ)
            .await
            .expect("Elasticsearch Connection Established");

        client
            .create_container(TestObj::default_es_container_config())
            .await
            .expect("container creation");

        let documents = vec![
            TestObj {
                value: String::from("obj1"),
            },
            TestObj {
                value: String::from("obj2"),
            },
            TestObj {
                value: String::from("obj3"),
            },
            TestObj {
                value: String::from("obj4"),
            },
            TestObj {
                value: String::from("obj5"),
            },
            TestObj {
                value: String::from("obj6"),
            },
        ];
        let documents = futures::stream::iter(documents);

        let res = client
            .insert_documents(
                String::from("root_obj_dataset_test-index-bulk-insert"),
                documents,
            )
            .await;

        assert_eq!(res.expect("insertion stats").created, 6);
    }

    #[tokio::test]
    #[serial]
    async fn should_detect_invalid_elasticsearch_version() {
        docker::initialize()
            .await
            .expect("elasticsearch docker initialization");

        let pool = remote::connection_test_pool()
            .await
            .expect("Elasticsearch Connection Pool");
        let client = pool.conn(ES_DEFAULT_TIMEOUT, ">=9.99.99").await;

        assert!(client
            .unwrap_err()
            .to_string()
            .contains("Elasticsearch Invalid version"));
    }
}
<|endoftext|>


<commit_before>#[test]
fn dogfood() {
    if option_env!("RUSTC_TEST_SUITE").is_some() {
        return;
    }
    let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
    let clippy_cmd = std::path::Path::new(&root_dir)
        .join("target")
        .join(env!("PROFILE"))
        .join("cargo-clippy");

    let output = std::process::Command::new(clippy_cmd)
        .current_dir(root_dir)
        .env("CLIPPY_DOGFOOD", "1")
        .arg("clippy")
        .arg("--all-targets")
        .arg("--all-features")
        .arg("--")
        .args(&["-D", "clippy::all"])
        .args(&["-D", "clippy::internal"])
        .args(&["-D", "clippy::pedantic"])
        .output()
        .unwrap();
    println!("status: {}", output.status);
    println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
    println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

    assert!(output.status.success());
}

#[test]
fn dogfood_tests() {
    if option_env!("RUSTC_TEST_SUITE").is_some() {
        return;
    }
    let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
    let clippy_cmd = std::path::Path::new(&root_dir)
        .join("target")
        .join(env!("PROFILE"))
        .join("cargo-clippy");

    for d in &[
        "clippy_workspace_tests",
        "clippy_workspace_tests/src",
        "clippy_workspace_tests/subcrate",
        "clippy_workspace_tests/subcrate/src",
        "clippy_dev",
        "rustc_tools_util",
    ] {
        let output = std::process::Command::new(&clippy_cmd)
            .current_dir(root_dir.join(d))
            .env("CLIPPY_DOGFOOD", "1")
            .arg("clippy")
            .arg("--")
            .args(&["-D", "clippy::all"])
            .args(&["-D", "clippy::pedantic"])
            .output()
            .unwrap();
        println!("status: {}", output.status);
        println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
        println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

        assert!(output.status.success());
    }
}
<commit_msg>Auto merge of #4051 - yaahallo:master, r=Manishearth<commit_after>#[test]
fn dogfood() {
    if option_env!("RUSTC_TEST_SUITE").is_some() {
        return;
    }
    let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
    let clippy_cmd = std::path::Path::new(&root_dir)
        .join("target")
        .join(env!("PROFILE"))
        .join("cargo-clippy");

    let output = std::process::Command::new(clippy_cmd)
        .current_dir(root_dir)
        .env("CLIPPY_DOGFOOD", "1")
        .arg("clippy-preview")
        .arg("--all-targets")
        .arg("--all-features")
        .arg("--")
        .args(&["-D", "clippy::all"])
        .args(&["-D", "clippy::internal"])
        .args(&["-D", "clippy::pedantic"])
        .output()
        .unwrap();
    println!("status: {}", output.status);
    println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
    println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

    assert!(output.status.success());
}

#[test]
fn dogfood_tests() {
    if option_env!("RUSTC_TEST_SUITE").is_some() {
        return;
    }
    let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
    let clippy_cmd = std::path::Path::new(&root_dir)
        .join("target")
        .join(env!("PROFILE"))
        .join("cargo-clippy");

    for d in &[
        "clippy_workspace_tests",
        "clippy_workspace_tests/src",
        "clippy_workspace_tests/subcrate",
        "clippy_workspace_tests/subcrate/src",
        "clippy_dev",
        "rustc_tools_util",
    ] {
        let output = std::process::Command::new(&clippy_cmd)
            .current_dir(root_dir.join(d))
            .env("CLIPPY_DOGFOOD", "1")
            .arg("clippy")
            .arg("--")
            .args(&["-D", "clippy::all"])
            .args(&["-D", "clippy::pedantic"])
            .output()
            .unwrap();
        println!("status: {}", output.status);
        println!("stdout: {}", String::from_utf8_lossy(&output.stdout));
        println!("stderr: {}", String::from_utf8_lossy(&output.stderr));

        assert!(output.status.success());
    }
}
<|endoftext|>